diff --git a/.dockerignore b/.dockerignore index cfb63f532..630b8eb4f 100644 --- a/.dockerignore +++ b/.dockerignore @@ -15,3 +15,5 @@ __pycache__ huggingface *.log .system_generated + +**/node_modules diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000..127e70cb4 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +dev/dashboard/package-lock.json linguist-generated=true diff --git a/.gitignore b/.gitignore index 4f732ed3f..a5b84aa9a 100644 --- a/.gitignore +++ b/.gitignore @@ -72,3 +72,7 @@ site/.hugo_build.lock # The site serves this on purpose (design 016). !site/static/llms.txt + +# Dashboard build +node_modules/ +src/server/dashboard/static/app.js diff --git a/Makefile b/Makefile index 6ab6e7557..711bf0827 100644 --- a/Makefile +++ b/Makefile @@ -19,7 +19,7 @@ HOST ?= 127.0.0.1 PORT ?= 9003 # The fully qualified base URL used by local CLI tools and clients BASE_URL ?= http://$(HOST):$(PORT) -UNIT_TESTS ?= tests.test_session_lifecycle tests.test_proto_codec tests.test_forward_only tests.test_fft_batch_failure tests.test_api_server_paths tests.test_accel_timeslicer tests.test_trainer_optimizer_correctness tests.test_trainer_logprob_chunks tests.test_worker_manager tests.test_scheduler_worker_manager tests.test_estimator tests.test_redis_store tests.test_cluster_eval_script tests.test_perf_compare_script tests.test_delta_weight_sync tests.test_diffing_backends tests.test_sampler_weight_rotation tests.test_sampler_batch_limits tests.test_commands tests.test_weight_sync_config tests.test_exclusive_trainer tests.test_delete_model tests.test_automodel_worker tests.test_worker_thread_device tests.test_trainer_data_parallel tests.test_openrl_status tests.test_skill_docs tests.test_trainer_ready tests.test_sampling_store tests.test_lora_snapshots tests.test_prefetch tests.test_pool_accept tests.test_sampling_dispatcher tests.test_vllm_sampler tests.test_vllm_options tests.test_dashboard +UNIT_TESTS ?= tests.test_session_lifecycle tests.test_proto_codec tests.test_forward_only tests.test_fft_batch_failure tests.test_api_server_paths tests.test_accel_timeslicer tests.test_trainer_optimizer_correctness tests.test_trainer_logprob_chunks tests.test_worker_manager tests.test_scheduler_worker_manager tests.test_estimator tests.test_redis_store tests.test_cluster_eval_script tests.test_perf_compare_script tests.test_delta_weight_sync tests.test_diffing_backends tests.test_sampler_weight_rotation tests.test_sampler_batch_limits tests.test_commands tests.test_weight_sync_config tests.test_exclusive_trainer tests.test_delete_model tests.test_automodel_worker tests.test_worker_thread_device tests.test_trainer_data_parallel tests.test_openrl_status tests.test_skill_docs tests.test_trainer_ready tests.test_sampling_store tests.test_lora_snapshots tests.test_prefetch tests.test_pool_accept tests.test_sampling_dispatcher tests.test_vllm_sampler tests.test_vllm_options tests.test_dashboard tests.test_telemetry_faults tests.test_worker_record tests.test_dashboard_facts tests.test_dashboard_lanes # Only forward BASE_URL to e2e when the user supplied it. The Makefile default # is for local CLI usage; e2e should start its own backend by default. TRAINING_TEST_BASE_URL ?= $(if $(filter environment command line,$(origin BASE_URL)),$(BASE_URL),) @@ -386,3 +386,11 @@ push-vm: # Pull changes from the remote VM back to the local workspace pull-vm: rsync -avz --exclude '.git' --exclude '.venv' --exclude '__pycache__' --exclude '*.pyc' --exclude '.DS_Store' --exclude 'scratch' $(REMOTE_HOST):~/open-rl/ ./ + +# Native dashboard assets are built once; Python serves the resulting static files. +.PHONY: dashboard-build dashboard +dashboard-build: + npm --prefix dev/dashboard run build + +dashboard: dashboard-build + uv run --extra cluster python -m server.dashboard --port 8080 diff --git a/dev/dashboard/build.mjs b/dev/dashboard/build.mjs new file mode 100644 index 000000000..40a29818e --- /dev/null +++ b/dev/dashboard/build.mjs @@ -0,0 +1,20 @@ +import { build } from "esbuild"; +import { compile, compileModule } from "svelte/compiler"; +import { readFile } from "node:fs/promises"; +import { dirname } from "node:path"; +import { fileURLToPath } from "node:url"; + +const root = fileURLToPath(new URL("../../", import.meta.url)); +await build({ + entryPoints: [root + "src/server/dashboard/ui/index.js"], + outfile: root + "src/server/dashboard/static/app.js", + bundle: true, minify: true, format: "esm", conditions: ["browser"], + nodePaths: [fileURLToPath(new URL("node_modules", import.meta.url))], + plugins: [{ name: "svelte", setup(build) { + build.onLoad({ filter: /\.svelte(?:\.js)?$/ }, async ({ path }) => { + const source = await readFile(path, "utf8"), options = { filename: path, generate: "client", dev: false }; + const { js, warnings } = path.endsWith(".svelte.js") ? compileModule(source, options) : compile(source, options); + return { contents: js.code, resolveDir: dirname(path), warnings: warnings.map((w) => ({ text: `${w.code}: ${w.message}` })) }; + }); + } }], +}); diff --git a/dev/dashboard/package-lock.json b/dev/dashboard/package-lock.json new file mode 100644 index 000000000..e16736eca --- /dev/null +++ b/dev/dashboard/package-lock.json @@ -0,0 +1,701 @@ +{ + "name": "openrl-dashboard", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "openrl-dashboard", + "devDependencies": { + "esbuild": "0.28.1", + "svelte": "5.57.2" + } + }, + "node_modules/@esbuild/aix-ppc64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.28.1.tgz", + "integrity": "sha512-Svl7tq8k/08+p6CXPpRjQ1fKX+1odH/BQbb48fV6fj3CWHhsoIOoY87w1oHXm0qEpkIK3ZfVgp0hed3XBXzXMQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.28.1.tgz", + "integrity": "sha512-0k2F129Xdio1TdJfzJ8sy1Q47vUD2NnwdhiAf7drUN1EBTfPf4hsFCtmMgu/6m8JSzsBrlmVjudMBQqOfG8usQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.28.1.tgz", + "integrity": "sha512-34EGEbCIAgosYz6goLcopX6Mo7NyGv9tfwEM2/7Ce2VcVRk568iSvniGWcUXIy7wEDR1wzolcxcriFVrWYcwBg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.28.1.tgz", + "integrity": "sha512-dbwY7ltSMDWsRatcRpCnES4F+im88OCUgGZjy52shC7GqHRE/cYlxNbB4Z4UpJswpcc4Qxd2oE/ufM0p61IKng==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.28.1.tgz", + "integrity": "sha512-TZbWkQY7kvTAXbXUT7uVACR5cMHsDiSz9z7ZKAX/RTq/WJEk3QyRr0wZpNhBDX+/0CtdqUIJlOiodQcta6tY3Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.28.1.tgz", + "integrity": "sha512-zfdzgK9ACBNZLI/CyHTOx81SyNbM6YXn7rxSgX97VjyiPl9W1i4Ka4fgKECEoFCKGpvBj5qArWIGgQjOwkgskQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.28.1.tgz", + "integrity": "sha512-wG2EA8ENdEI0qhkSZMjfqrdY+ziCYCPMmtZjjIwOmXFjmyzEHn+UUxk5of+SYsjtfs3VpnlC7QLzSI5hY/rOAw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.28.1.tgz", + "integrity": "sha512-i7dZ9vQgnvSCzi/rYCXNgtF/U+eKZNJBzu3eTQbRgHnM7tNSizLOkRFAl3qzVc/Op/u5YkHHa4pf/3DOYHthLQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.28.1.tgz", + "integrity": "sha512-qVXBOHQS+d5Y722GwJzJUtOLlX7km3CraOaGormF1pDtPd2C/l1SHRPgjLunLGe51Sh5YYWKMFDyV4SxgMQYTQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.28.1.tgz", + "integrity": "sha512-yHs+0uc8+nvEAfAfxrWQKK5peSNzBc4PegcMO0EJ2hT71uA7vB8Ihg2e77R2P7SG5uYjPbHlLLmve4LLLRCf0g==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.28.1.tgz", + "integrity": "sha512-d1z4ZuP0ajrfz/FhGT4vv278rX8KnPPJx8i5+AtK7TYbx9Le9F1hyzurZpkEyjkGa9dUGhQow4C1NmeGvqxN2w==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.28.1.tgz", + "integrity": "sha512-M5sRjUVZrkm1OAPR3dlOYzNmN+loZKGVi1VUQGrwuqLcbR6qeAz+famMhjASeH3YVKvZz+zT1jlh/keC3Rj/lg==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.28.1.tgz", + "integrity": "sha512-mRObBZeHh2OxcBFPWE/FjylkRgZdYuiTR3vaTozquCGOH14iP9oN4x4Ge81CoIDYQrXmIxpFumJBu5MtZpnQJQ==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.28.1.tgz", + "integrity": "sha512-slScBsMAb3GFDcdrCgLwZtPYRoH2H/youv10QiZyRjmsP48fznoveWytSgCI/R0ZcUgpc0ZhIUEx6LHts8yrfQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.28.1.tgz", + "integrity": "sha512-kw0owk1o0GFETUJyW0jc0G4Yzs0BHZn0JDZ8JRT088vjJYX777BAs1fDGxAC+q831qOs2DTC96mNsG2opdfyyQ==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.28.1.tgz", + "integrity": "sha512-/lAIjX8aYFRByhh6L5rYtPEDRqa9de/4V/juOXcta5frjvzXO4/sqEtyytse0g3zZFuWu5cDN0MkLz2qRDD2Ag==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.28.1.tgz", + "integrity": "sha512-u/anNYF2mmVOEDwLtnQ1wOr3EZ9sTNGLWrsYGYwHWzGA3Si84IOkHXlbWTD1NB+9/1lcnweYKO54uhxZydNzfA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.28.1.tgz", + "integrity": "sha512-oks0DYbLwWMmaakTsCb+zL4E+aHRVLom9IJZOAthMQEPiQmydXHkziYEsGYRx0uNV/IjEKGAV941JzH02pflqw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.28.1.tgz", + "integrity": "sha512-aeL6lAnN89Hz43Mlh1G8ARasbuoYvSITDEx0tHh5b7jJnHcssqgjy9Yx430GDpmCa6OyrKoS0aNRjKundRizGg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.28.1.tgz", + "integrity": "sha512-MEFJe5C3R8pwXdZ5Y21oo6m7ePiS0d9pWucn99O/wvyJZChoIQKrQDxKrGeW8F5+T0okTHesAmDeiHDTIq0V/Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.28.1.tgz", + "integrity": "sha512-i/ZLIOafE0Z8cI/XANJAixoJL/uRAoS2xOA3rb0xN+KK0K177cMAsQYkzHtBrtMXAKuAc7HGgcWiZ/sRC1Nxgw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openharmony-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.28.1.tgz", + "integrity": "sha512-ge+Z7EXFNt2BO1oAMsVpiQ8EwndV9i1xXerAeTIK7AtPs3bKFXQM7nlRxDSIUIMeueR1CNXxqztLzdNeReKBJg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.28.1.tgz", + "integrity": "sha512-BEjgtECkL3vY+SaSQ6nzVfiALUeFxpawyp8Jmf5PtYhf1Ug40N1h/hxlhts+f1FvSvarEigdxS3BlSMI2PJLcQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.28.1.tgz", + "integrity": "sha512-lCv9eK/H6ZJWbE7bh2nw54CZ9M2nupBxJcTsdk/QQnWkdSjKGuxmmH8/GWrlT1eMmZfn4dGcCjRte397WqfQXA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.28.1.tgz", + "integrity": "sha512-zvb/mB2bSCoJOpoCBgYKKpX6YM6mJBlBUVUtVj41DlZJVEB6/0CKlRYxP5wWl1C1ILiCoAU5wZZ4q1P3qeS6Eg==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.28.1.tgz", + "integrity": "sha512-bm4Mowrv+GXMlpWX++EcXw/iLyd1o3+bJkC2DkWXYVvgZCqD/bSj9ctZeAMC3cIxgjRVR2Dufaiu4YPxr5gW1A==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@jridgewell/gen-mapping": { + "version": "0.3.13", + "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.13.tgz", + "integrity": "sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.5.0", + "@jridgewell/trace-mapping": "^0.3.24" + } + }, + "node_modules/@jridgewell/remapping": { + "version": "2.3.5", + "resolved": "https://registry.npmjs.org/@jridgewell/remapping/-/remapping-2.3.5.tgz", + "integrity": "sha512-LI9u/+laYG4Ds1TDKSJW2YPrIlcVYOwi2fUC6xB43lueCjgxV4lffOCZCtYFiH6TNOX+tQKXx97T4IKHbhyHEQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/gen-mapping": "^0.3.5", + "@jridgewell/trace-mapping": "^0.3.24" + } + }, + "node_modules/@jridgewell/resolve-uri": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", + "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.6.0.tgz", + "integrity": "sha512-T7jf+5zgsZHwNJ4lvQ7/aezbyk0nNX+zJVWpmHA7VYsEx7a7qr5Rg5IbtJFqkgze5Y2sruq1RUY8Q837Od7iFw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@jridgewell/trace-mapping": { + "version": "0.3.31", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.31.tgz", + "integrity": "sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/resolve-uri": "^3.1.0", + "@jridgewell/sourcemap-codec": "^1.4.14" + } + }, + "node_modules/@sveltejs/acorn-typescript": { + "version": "1.0.13", + "resolved": "https://registry.npmjs.org/@sveltejs/acorn-typescript/-/acorn-typescript-1.0.13.tgz", + "integrity": "sha512-wgKggnhZVL9Bfx1OaKKTrYY9BFRk6C8UAkQNUcIv1+llzYrIqy+RZm5HPKzn0NpEBvTVhTqB4kQyllZywsRBRQ==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "acorn": "^8.9.0" + } + }, + "node_modules/@types/estree": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.9.tgz", + "integrity": "sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg==", + "dev": true, + "license": "MIT" + }, + "node_modules/acorn": { + "version": "8.19.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.19.0.tgz", + "integrity": "sha512-oJlA3XiRm7Cyk6qFD2Jn8ak9B7jSy0qy00ADO3+8dpT0LSjFihQYv4C02LFCSYJV3Q37xYwwRy5m+IpIiUzqWw==", + "dev": true, + "license": "MIT", + "bin": { + "acorn": "bin/acorn" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/aria-query": { + "version": "5.3.1", + "resolved": "https://registry.npmjs.org/aria-query/-/aria-query-5.3.1.tgz", + "integrity": "sha512-Z/ZeOgVl7bcSYZ/u/rh0fOpvEpq//LZmdbkXyc7syVzjPAhfOa9ebsdTSjEBDU4vs5nC98Kfduj1uFo0qyET3g==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/axobject-query": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/axobject-query/-/axobject-query-4.1.0.tgz", + "integrity": "sha512-qIj0G9wZbMGNLjLmg1PT6v2mE9AH2zlnADJD/2tC6E00hgmhUOfEB6greHPAfLRSufHqROIUTkw6E+M3lH0PTQ==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/clsx": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/clsx/-/clsx-2.1.1.tgz", + "integrity": "sha512-eYm0QWBtUrBWZWG0d386OGAw16Z995PiOVo2B7bjWSbHedGl5e0ZWaq65kOGgUSNesEIDkB9ISbTg/JK9dhCZA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/devalue": { + "version": "5.9.4", + "resolved": "https://registry.npmjs.org/devalue/-/devalue-5.9.4.tgz", + "integrity": "sha512-sPAT4pztbu6586/hrhOnMKS17IJrvg12mXiSPSS3W5qDeN2RGgvZ0diZCm31dBbnevfVmujNO3IM2wrS4Y2Rhg==", + "dev": true, + "license": "MIT" + }, + "node_modules/esbuild": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.28.1.tgz", + "integrity": "sha512-HrJrvZv5ayxBzPfwphOoNzkzOIIlifzk0KJrGK2c8R4+LKpMtpYLQeUdjnwjWv/LZlkH2laZk+4w78pi99D4Vw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.28.1", + "@esbuild/android-arm": "0.28.1", + "@esbuild/android-arm64": "0.28.1", + "@esbuild/android-x64": "0.28.1", + "@esbuild/darwin-arm64": "0.28.1", + "@esbuild/darwin-x64": "0.28.1", + "@esbuild/freebsd-arm64": "0.28.1", + "@esbuild/freebsd-x64": "0.28.1", + "@esbuild/linux-arm": "0.28.1", + "@esbuild/linux-arm64": "0.28.1", + "@esbuild/linux-ia32": "0.28.1", + "@esbuild/linux-loong64": "0.28.1", + "@esbuild/linux-mips64el": "0.28.1", + "@esbuild/linux-ppc64": "0.28.1", + "@esbuild/linux-riscv64": "0.28.1", + "@esbuild/linux-s390x": "0.28.1", + "@esbuild/linux-x64": "0.28.1", + "@esbuild/netbsd-arm64": "0.28.1", + "@esbuild/netbsd-x64": "0.28.1", + "@esbuild/openbsd-arm64": "0.28.1", + "@esbuild/openbsd-x64": "0.28.1", + "@esbuild/openharmony-arm64": "0.28.1", + "@esbuild/sunos-x64": "0.28.1", + "@esbuild/win32-arm64": "0.28.1", + "@esbuild/win32-ia32": "0.28.1", + "@esbuild/win32-x64": "0.28.1" + } + }, + "node_modules/esm-env": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/esm-env/-/esm-env-1.2.2.tgz", + "integrity": "sha512-Epxrv+Nr/CaL4ZcFGPJIYLWFom+YeV1DqMLHJoEd9SYRxNbaFruBwfEX/kkHUJf55j2+TUbmDcmuilbP1TmXHA==", + "dev": true, + "license": "MIT" + }, + "node_modules/esrap": { + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/esrap/-/esrap-2.4.0.tgz", + "integrity": "sha512-CQPcneEHw/czj32uwQC52nYnax8H4SXt25FCM+LIwGZ49cMIxpOj3bP8wmzcI+Z0HIy5KkClaPhRLfCySzwHPA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.4.15" + }, + "peerDependencies": { + "@typescript-eslint/types": "^8.2.0" + }, + "peerDependenciesMeta": { + "@typescript-eslint/types": { + "optional": true + } + } + }, + "node_modules/is-reference": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/is-reference/-/is-reference-3.0.3.tgz", + "integrity": "sha512-ixkJoqQvAP88E6wLydLGGqCJsrFUnqoH6HnaczB8XmDH1oaWU+xxdptvikTgaEhtZ53Ky6YXiBuUI2WXLMCwjw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.6" + } + }, + "node_modules/locate-character": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/locate-character/-/locate-character-3.0.0.tgz", + "integrity": "sha512-SW13ws7BjaeJ6p7Q6CO2nchbYEc3X3J6WrmTTDto7yMPqVSZTUyY5Tjbid+Ab8gLnATtygYtiDIJGQRRn2ZOiA==", + "dev": true, + "license": "MIT" + }, + "node_modules/magic-string": { + "version": "0.30.21", + "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.21.tgz", + "integrity": "sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.5.5" + } + }, + "node_modules/svelte": { + "version": "5.57.2", + "resolved": "https://registry.npmjs.org/svelte/-/svelte-5.57.2.tgz", + "integrity": "sha512-1YiPCAXT0Ct6g1vOMmSLa0Nsc3QRMvtDF32ihgglGVrugIuIK38RyDiFNXVDhtQ81TJjSO57NdMw6DyYLqSGDQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/remapping": "^2.3.5", + "@jridgewell/sourcemap-codec": "^1.6.0", + "@sveltejs/acorn-typescript": "^1.0.13", + "@types/estree": "^1.0.9", + "acorn": "^8.18.0", + "aria-query": "5.3.1", + "axobject-query": "^4.1.0", + "clsx": "^2.1.1", + "devalue": "^5.9.2", + "esm-env": "^1.2.1", + "esrap": "^2.3.6", + "is-reference": "^3.0.3", + "locate-character": "^3.0.0", + "magic-string": "^0.30.11", + "zimmerframe": "^1.1.2" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/zimmerframe": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/zimmerframe/-/zimmerframe-1.1.5.tgz", + "integrity": "sha512-msJxIvYDYcoNL+PJsu+7qmpDWsYmAxTY+2TNYXXF0hzBzBk0BMecOqDOG/EckUoKCuKwObfbugIl8QpqHDXeFA==", + "dev": true, + "license": "MIT" + } + } +} diff --git a/dev/dashboard/package.json b/dev/dashboard/package.json new file mode 100644 index 000000000..8a8e888af --- /dev/null +++ b/dev/dashboard/package.json @@ -0,0 +1,12 @@ +{ + "name": "openrl-dashboard", + "private": true, + "type": "module", + "scripts": { + "build": "node build.mjs" + }, + "devDependencies": { + "esbuild": "0.28.1", + "svelte": "5.57.2" + } +} diff --git a/docs/briefs/pr383-dashboard-trial/findings.md b/docs/briefs/pr383-dashboard-trial/findings.md new file mode 100644 index 000000000..9809ffe48 --- /dev/null +++ b/docs/briefs/pr383-dashboard-trial/findings.md @@ -0,0 +1,505 @@ +# PR #383 (run telemetry and dashboard): trial on a fresh GKE cluster + +Date: 2026-10-27 (evening PDT, 2026-10-28 01:00-01:30 UTC) +PR: https://github.com/gke-labs/open-rl/pull/383 (ShubyM, `dashboard/main`, head `c7dbd3f`) +Cluster: `open-rl-dra-use5`, us-east5, 1 x a3-highgpu-8g (8 x H100 80GB, Spot), nothing of OpenRL installed before +Companion files: `notes.md` (step-by-step log), `shots/` (dashboard screenshots) + +## Summary + +The PR does what its description says. On a cluster brought up from nothing, +the rebased branch built, deployed, ran a five-step GSM8K RL job, and the +telemetry, the inspection API and the dashboard all showed that run +correctly: placements with GPU UUIDs, per-op records, step grouping, +activity lanes, client metrics and client logs. + +Thirteen things need attention before this is useful on a GKE cluster as +shipped. Two are in the facts the API computes (goodput attribution, a +negative age), four are deployment gaps (nothing scrapes the scheduler +metric, no DCGM exporter in the bundle, no IAM or Workload Identity story, +client-directory depth does not match the repo's own e2e job), and two are +small. None blocks merging the records and the read API; the deployment +gaps decide whether the GPU charts show anything on a real cluster. Two +more came from a second run that diverged: duty on a shared GPU is shown +per lane but is per device, and a run whose client died on a 400 still +read as healthy on the overview. + +## What was tested + +| Item | Value | +|---|---| +| Branch | PR head `c7dbd3f` rebased onto upstream main `0644063` (#384); clean rebase; `py_compile` and `make -C scheduler/controller vet test` pass | +| Images | api-server, server, client, scheduler built with Cloud Build from the rebased branch, tag `pr383-844424a`, pushed to a new regional repo `us-east5-docker.pkg.dev/cdrollouts-sunilarora/open-rl` | +| Deployment | `k8s/deploy/fft` + the PR's `k8s/deploy/dashboard`, applied server-side; shared volume on Filestore Zonal (1 TiB) | +| GPUs | NVIDIA driver installer DaemonSet + NVIDIA DRA driver 25.8.0 (Helm), ResourceSlices for 8 GPUs | +| Run | `make cluster-e2e E2E_SCENARIO=fft-gsm8k-rl E2E_ARGS="base_model=Qwen/Qwen2.5-0.5B-Instruct steps=5"`, in-cluster client Job | +| Result | 5 steps, env/all/correct 0.41, 0.36, 0.44, 0.66, 0.81; step wall time 99 s then 47-51 s; trainer and sampler on one GPU (binpack), the other seven idle | + +## What worked + +**Scheduler.** Workload status carried `placedAt` and `gpuUUIDs` within +seconds of binding. The scheduler logged one `placement placed` line per +Workload and exported `openrl_gpu_held{UUID,node,pod,role,run_id}` on its +metrics port for both workers. + +**Records.** Trainer and sampler printed `telemetry op|popped|answered` +JSON lines with the Cloud Logging labels. Redis held every key design 018 +lists: `operations:` and `:samples`, `sampled`, `sampled_at`, `steps`, +`step_times`, `ledger`, a `worker:` record per process, +`session_record`, `placements:`. After cleanup the placement +records had `released_at` and the activity lanes reported `ended: true`. + +**Read API.** `/runs` showed lifecycle `placed` during the run and `ended` +after, 5 steps, 0 failures, 0 relaunches. `/runs/{id}/summary`, +`/runs/{id}/steps`, `/runs/{id}/commands?role=sampler` (one record per +served group: counts 64, 1, 63 at the time of the call), `/activity`, +`/logs?pod=client` and `/inventory` all answered. Errors from sources that +could not be read came back as text in the response, not as empty data. + +**Dashboard.** Overview, Run (Activity, Metrics, Logs), Nodes and Health +render. The Node build stage inside `Dockerfile.api_server` worked the first +time on Cloud Build. Screenshots: `shots/overview.png`, +`shots/run-activity.png`, `shots/run-metrics-client.png`, +`shots/run-logs-client.png`, `shots/nodes.png`, `shots/health.png`. + +## Findings + +### 1. Goodput charges server dispatch time to the client + +Summary for the finished run: wall 335 s, train 11 s, checkpoint 28 s, +rollout 7 s, client 290 s, goodput 0.032. + +The client's own numbers for the same five steps: sampling 128 s, +train_step 144 s. The sampler's served groups took 1-1.5 s each and the +trainer's ops 10-12 s per step. So about 40 s of every 48 s step is spent +between a request leaving the client and a worker popping it, and the facts +call that "client". + +The PR already lists "when each request was enqueued isn't recorded" as +open. On this run that gap is 85% of wall time, so until enqueue times exist +the goodput block points at the wrong party. Suggest either recording the +enqueue time (the API server has it) and adding a "queued" bucket, or +labelling the bucket "not in a worker" until then. + +Separately, and not the PR's problem: 16 s of client-side sampling for +1.5 s of vLLM work is worth a look in the dispatch path. + +### 2. `since_last_step_s` can be negative + +Observed `-32` on `/runs/{id}/summary` at 01:24:42 UTC when the last +`optim_step` was at 01:24:37. `facts.py` subtracts the step time from +`state["observed_at"]`, which is when the cached fleet snapshot was read, +while the step comes from the run's ops cache, which refreshes on its own +schedule. The line below clamps `since_last_op_s` with `max(0, ...)`, which +hides the same thing. Clocks are not the cause: dashboard pod, trainer pod +and laptop agreed to the second. Suggest using the request time for ages. + +### 3. The client-directory search does not reach the repo's own e2e layout + +`k8s/eval/e2e-client-job.yaml` sets `log_dir=/mnt/shared/open-rl/runs/` +and the recipe writes `open-rl-tmp/` under it. That is five levels +below `/mnt/shared`; `client_runs.DEPTH = 3`. Result: `client_directory` +null, Metrics tab "0 metrics", no client logs. + +Setting `OPEN_RL_CLIENT_RUNS_ROOT=/mnt/shared/open-rl/runs` on the dashboard +Deployment fixed it: 53 client metrics and `logs.log` paging appeared. The +dashboard manifest and the e2e job should agree on one layout, or the depth +should be larger. + +### 4. Nothing scrapes `openrl_gpu_held` + +The scheduler exports it, but neither `scheduler/deploy/base` nor +`k8s/deploy/dashboard` adds a `PodMonitoring` for the scheduler's `metrics` +port (8080), and the docs do not mention one. On GKE with managed +collection the PromQL join in design 018 has no right-hand side. Suggest +shipping the `PodMonitoring` with the dashboard manifests. + +### 5. The bundle has no DCGM exporter, and GKE's managed one skips DRA pools + +The PR edits `k8s/deploy/distributed-fft-timeslice/10-dcgm-monitoring.yaml` +(adds `DCGM_FI_PROF_GR_ENGINE_ACTIVE`, 10 s interval), but `k8s/deploy/fft` +does not include that file. GKE's managed `dcgm-exporter` DaemonSet requires +`cloud.google.com/gke-gpu-driver-version in (default, latest)`, which a pool +created with `gpu-driver-version=disabled` (what DRA needs) never has; it +had 0 desired pods on this cluster and on `open-rl-dra`. So GPU duty, the +Nodes page and `/accelerators` stay empty on a DRA cluster unless the +exporter is applied by hand. Suggest moving the exporter manifest into the +bundle the dashboard is deployed with. + +### 6. GKE reads need IAM the bundle does not set up or document + +Every Prometheus and Cloud Logging read returned 403 here. The cluster has +no Workload Identity, the node service account has no +`roles/monitoring.viewer`, and its `logging.write` scope cannot read logs +at all. `docs/dashboard.md` says "monitoring.viewer on the workload +identity" in a manifest comment only. Suggest a short setup section: +Workload Identity or node roles, `roles/monitoring.viewer`, +`roles/logging.viewer`, and what the audit-log lookup for relaunches needs. + +### 7. Node GPU inventory comes only from DCGM + +With the 403, `/inventory` reported the H100 node as `gpu_capacity: 0, +devices: []` although its ResourceSlice lists eight GPUs and the run's +placement names a UUID. The Nodes page was an empty table under the error. +ResourceSlices, or the placements already in Redis, could seed the node +list when metrics are unavailable. + +### 8. Trailing empty step + +After the final `save_state`, `/runs/{id}/steps` lists a step 5 with no +client metrics while the client ran steps 0-4. The post-step ops form a +group of their own. + +### 9. Duty on a time-sliced GPU is the GPU's, shown on every lane + +With trainer and sampler sharing GPU 6 (binpack placement, the +time-slicer alternating them), both Activity lanes showed Duty 3 %. The +value is the device's `DCGM_FI_PROF_GR_ENGINE_ACTIVE` share; the join +reduces `openrl_gpu_held` per UUID, as design 018 says, so it cannot be +split by holder. Two lanes at 3 % is one 3 %. Suggest splitting the GPU's +duty by each holder's recorded op intervals (the lanes already have them), +or labelling the column as the GPU's duty when more than one lane holds +the same UUID. + +### 10. A run whose client died on a 400 still reads as healthy in `/runs` + +Our second run diverged: step 6 recorded loss 8.5 and grad_norm 0.0, at +step 7 the sparse delta was 100 % changed and full of NaN, vLLM rejected +the weight update, and the client exited on HTTP 400. Three minutes later +`/runs` still said `lifecycle placed, failures 0, relaunches 0, +unanswered 0`. `failures` counts failed trainer ops only. The signals were +all recorded elsewhere: 64 failed sampler records (`RuntimeError`, no +message text kept), `samplers.failed: 64` in the summary, and the SDK's +error events in the session summary. Suggest surfacing sampler failures +and SDK errors on the overview line, keeping the error text on failed +sample records, and flagging a live run whose only session has reported +SDK errors and gone quiet. To be fair to the UI: once the workers were +released, the Overview labelled the diverged run "Ended" while the clean +runs read "Completed" (`shots/overview-four-runs.png`), which is the right +distinction; the gap is the two to three minutes while its workers were +still up, and the failure count. + +### 11. Workers sanitise NaN to 0.0 before telemetry sees it + +Root cause of the diverged run: a non-finite gradient at step 6; with no +`grad_clip_norm` set, `clip_grad_norm_` spread the NaN to every gradient, +Adam made every parameter NaN, the step-7 delta was 100 % NaN, vLLM +rejected it, the client got a 400. The summary's `optimizer` block exists +to catch this, but `trainer_worker.sanitize_float` maps NaN to 0.0 and inf +to +-9999 before the op returns, so the record said `grad_norm:mean 0.0` +and `nonfinite []`. Suggest the telemetry hook read the raw metrics, or +the workers mark sanitised values, so a NaN grad norm is reported as +non-finite. The missing non-finite guard before `optimizer.step()` is an +OpenRL issue to file alongside. + +### 12. Placement records are per run on a shared runtime, and 30 s behind + +Building a live fleet view on `/inventory` showed two limits. A shared +LoRA runtime with four adapters appears as four placement records, one +per run, all naming the same pod and GPU, so a consumer cannot tell one +shared pod from four pods without grouping by pod itself. And placement +records are written by the API server's 30 s reaper sweep, so a new +placement can trail by half a minute, while the scheduler stamps +`placedAt` and `gpuUUIDs` on Workload status within seconds. For a live +view we read Workloads directly. Suggest one record per Workload with a +list of runs, and reading placement from Workload status (or a watch) +rather than the sweep. + +### 13. The delta's density is computed and returned, but never recorded + +Under the delta weight-sync strategy the FFT trainer computes, for every +`save_weights_for_sampler`, how many elements changed and what share of +the model that is (`density_pct`, also written to the delta's +`metadata.json` and printed as `Saved sparse delta (3.831% changed +elements, 195570520/5104297504) to …`). The save returns +`{"path", "density_pct"}`, but `command_record` keeps only +`result["metrics"]`, so the op record has `metrics: {}` and +`/runs/{id}/steps` cannot show it. Putting the figure under `metrics` +(for example `delta/density_pct`, `delta/changed_elements`, +`delta/total_elements`, and the save's collect and write seconds, which +the trainer also measures) would make "how much of the model moved this +step" a first-class step metric, with no new plumbing. Observed values: +Qwen3-0.6B 7–100 % (100 % at a NaN step), gemma-4-e2b around 4 % of 5.1 B +elements per step. + +Done on the trial branch as commit `e38f54d` ("Record a sparse delta's +density and size on the save op"), offered for the PR or as its own +change: `BaseTrainerWorker.last_save_metrics`, set by the FFT worker after +a delta save (`delta/density_pct`, `delta/changed_elements`, +`delta/total_elements`, `delta/layers`, `time/delta_collect`, +`time/delta_write`) and cleared after a full save; the processor spreads +it into save results; `translate_future_result` drops `metrics` from the +three save types so the SDK's responses are unchanged. Two unit tests. +Verified on the cluster: a three-step Qwen3-0.6B run reported 11.6 %, +7.8 % and 6.0 % of 596 M elements per step through `/runs/{id}/steps`, +and the recipe client ran unchanged. + +### 14. Off-GPU ops are painted as GPU occupancy + +**Seen:** on the accelerator timeline for gpu-4 during a 14-job sweep, a +0.7 s turn of the gemma trainer (its `save_weights_for_sampler`) sits +inside another trainer's 2.6 s hold of the same GPU. Two holders at once +on a time-sliced claim. + +**Checked:** the time-slicer daemon's log (with `--timestamps`) has the +claim held by one worker from 01:44:29.04 to 01:44:33.66 with no hand-off +in between, so the inner turn is not a real hold. The three trainers +share one claim (`claim-fft-…-93124282`, the scheduler moved two of them +onto it). An FFT trainer that is not exclusive runs with CPU offload +(`api_server.py`: `cpu_offload = … or not settings.exclusive`), and +`handle_batch` then runs saves without a time-slicer turn. The saves are +real ops, just not on the GPU. + +**Cause:** `fleet.activity` builds a lane's `intervals` as +`merge(turns + every op span)`, so an op outside every turn counts as +occupancy, and `exact` is true as soon as the lane has any turn. + +**Fix on the review branch (886bd93):** a lane with turns is occupied +only during them; its ops still show. A lane without turns keeps the ops +as the stand-in. Test `tests/test_dashboard_lanes.py`. The same reasoning +applies to the per-holder shares and the union in any view built on +`intervals`. + +### Small things + +- For up to a minute after the dashboard pod starts, `/inventory` can + report "No GPU metrics source: not on GKE" because the metadata-server + read uses a 1 s timeout and retries a failed read after 60 s. The text + should say the read is pending and will retry. + +- The Metrics tab's step charts label the x axis "0 1 2 2 3" for a + four-step run: tick values are rounded without de-duplication. + +- The Node stage adds `node:22-bookworm-slim` and an `npm ci` to every + api-server build. About a minute on Cloud Build; fine. +- The readiness probe hits `/openapi.json` every few seconds and each hit + is an INFO log line. + +## Bringing up a fresh GKE cluster for this (what we hit) + +Not PR findings, but anyone repeating this will meet them. + +- The NVIDIA DRA driver chart 25.8.0 refuses its own default unless + `--set resources.gpus.enabled=true --set gpuResourcesEnabledOverride=true` + are given. +- Its kubelet-plugin DaemonSet ships no tolerations and uses + `system-node-critical`. On a pool tainted `nvidia.com/gpu=present:NoSchedule` + it schedules zero pods, and GKE's critical-pod quota rejects the priority + class outside `kube-system`. Fix: `--set 'kubeletPlugin.tolerations[0].operator=Exists' + --set kubeletPlugin.priorityClassName=""`. `open-rl-dra` was set up this + way; `scripts/cell_up.sh` on `feat/cold-start-measure` lacks all four + values and would fail the same way. +- Filestore Zonal provisioning took about seven minutes and one transient + CSI polling error before the PVC bound; the retry succeeded on its own. +- A Cloud Build into an empty regional registry can seed its layer cache + from another registry's `:buildcache` with a second `--cache-from`; the + CUDA worker image took about 17 minutes that way. + +## How to reproduce + +```sh +# branch +git fetch https://github.com/ShubyM/open-rl.git dashboard/main:pr-383 +git worktree add -b try/pr-383-rebased ../open-rl-pr383 pr-383 +git -C ../open-rl-pr383 rebase upstream/main + +# images (one per Dockerfile; scheduler uses scheduler/controller as context) +gcloud builds submit --config cloudbuild.yaml \ + --substitutions=_IMAGE=/open-rl-api-server,_DOCKERFILE=src/server/Dockerfile.api_server,_TAG= ../open-rl-pr383 + +# deploy (overlay: ../fft + ../dashboard, images rewritten, PVC on zonal-rwx) +kubectl apply --server-side --force-conflicts -k ../open-rl-pr383/k8s/deploy/gke-use5 +kubectl -n openrl-system set env deployment/open-rl-dashboard OPEN_RL_CLIENT_RUNS_ROOT=/mnt/shared/open-rl/runs + +# run and look +make cluster-e2e E2E_NAMESPACE=openrl-system E2E_SCENARIO=fft-gsm8k-rl \ + E2E_IMAGE=/open-rl-client: E2E_ARGS="base_model=Qwen/Qwen2.5-0.5B-Instruct steps=5" +kubectl -n openrl-system port-forward svc/open-rl-dashboard 8080:8000 +open http://127.0.0.1:8080/dashboard +``` + +## Observation about OpenRL placement, not the PR + +In both runs the scheduler put the FFT trainer and the FFT sampler on the +same GPU (GPU 6, `GPU-876ee4e3`) while the other seven H100s on the node +sat idle. That is the default `binpack` placement booking the sampler's +seat on the ledger the trainer already held, and the time-slicer then +alternating the two processes on one device: vLLM sleeps and frees its +70 GiB, the trainer reloads its weights from pinned CPU memory, runs its +op, offloads again, and the sampler wakes. + +This is surprising for someone who expects trainer and sampler to be +separate processes on separate accelerators, and it means a disaggregated +trainer/sampler setup is not possible under `binpack` on a node with free +GPUs: the second worker is co-located by design, and nothing in the +request asks for a GPU of its own. The dashboard made the co-location +visible immediately (both lanes on GPU 6, the Nodes page with one of eight +claimed), which is a point in its favour. The duty figure it shows is then +the shared device's, see finding 9. + +A second placement observation from the twenty-job mix under `spread`: +a shared LoRA runtime is two Workloads placed independently. Both LoRA +trainers got a GPU each, loaded, created five adapters, and then sat idle +for the whole run because the two LoRA samplers had no seat while ten FFT +jobs held the other six GPUs. Two H100s held and idle, ten jobs waiting. +Gang-placing the pair, or seating the LoRA sampler on its own trainer's +GPU, would avoid this. + +Why the half-placed runtime happens: each worker is its own Workload, +placed on its own; LoRA claims are exclusive by design and never book a +seat on another ledger, so a LoRA sampler can only take a whole free GPU +while its trainer already holds one. Fixes, cheapest first: (1) the +design's own future-work item, owner-affinity, extended with a LoRA-pair +exception so a runtime's trainer and sampler may share one claim when the +second cannot get its own (needs the LoRA trainer to honour the +time-slicer's suspend/resume as the FFT trainer does); (2) pair-aware +placement in the controller, committing both halves' assignments in one +pass or leaving both pending, which keeps the standing pending claim as +the scale-up signal and never lets one half hoard; (3) Kubernetes-level +gang scheduling (coscheduling plugin, Kueue), held until the upstream +feature settles because of its interaction with DRA claims. Without code +changes: submit LoRA jobs before FFT so their samplers take GPUs while +the fleet is empty, or keep the shared LoRA runtimes warm before a sweep. + +Worth raising as its own OpenRL issue: a placement hint or policy that +keeps trainer and sampler seats on different claims when capacity allows, +so time-slicing is a fallback for scarcity rather than the default on an +idle node. The seat cap on `feat/cold-start-measure` limits sharing but is +a different control. + +## Identity follow-up (same evening) + +We enabled Workload Identity on the cluster and the GKE metadata server on +the default pool, gave the dashboard its own Kubernetes service account +(`open-rl-dashboard`), and bound that account's federated principal +directly to `roles/monitoring.viewer` and `roles/logging.viewer`. No +Google service account, no key, nothing granted to the node identity. + +Result: `/inventory` reports no errors, `/logs` returns worker lines from +Cloud Logging (200 in the first page), and the Prometheus query succeeds. +It returns no GPUs, which confirms findings 4 and 5: the 403s were only +IAM, and the GPU charts stay empty on a DRA cluster until a DCGM exporter +runs on the GPU nodes and a `PodMonitoring` scrapes the scheduler. + +One thing the new Logs tab showed straight away: the sampler segfaults on +shutdown (`cuCtxSynchronize` during `torch.accelerator` teardown, two +seconds after SIGTERM, screenshot `shots/run-logs-gke.png`). The run was +already complete, so no failure was counted and the client saw a clean +exit. Not the PR's problem; worth its own issue. + +Two things to note for the PR: + +- The dashboard should have its own service account in the manifests. + Reusing the API server's `open-rl-sa` means any cloud role the dashboard + needs is also held by the API server. +- A node pool update to turn on the metadata server replaces nodes; one of + ours could not be replaced for a while because the zone was out of + e2-standard-4 capacity. A fresh cluster should be created with Workload + Identity on from the start, which the setup docs could say. + +## Metrics backend follow-up (same evening) + +We added to our overlay a copy of the PR's `10-dcgm-monitoring.yaml` +(kustomize cannot reference a file outside the overlay), a `PodMonitoring` +for the scheduler's `metrics` port, and a `PodMonitoring` for sampler pods +on the `http` port the PR adds (9095, selector `openrl.io/worker` exists; +trainers have no such port). Five new objects, nothing else changed. DCGM +series reached Managed Prometheus about four minutes after the exporter +started; the `PodMonitoring` status fields stayed empty the whole time. + +With that in place, during a second eight-step run: + +- `/inventory` listed the H100 node with eight devices and no errors. +- `/query?name=gpu_busy&run_id=...` returned one series for the run's GPU, + so the `openrl_gpu_held` join works. `gpu_util` and + `vllm_generation_tokens_per_second` (118 to 384 tokens/s) answered too. +- The Activity lanes showed a DCGM duty of 3 % for trainer and sampler; + the Nodes page showed all eight GPUs, GPU 6 claimed, held spans drawn. +- The live summary showed an unanswered `forward_backward` with its age + and that the holder was alive, which is the exact question the PR set + out to answer. + +So findings 4 and 5 are confirmed and have a small, mechanical fix: ship +these three scrape objects and the exporter with `k8s/deploy/dashboard`. +The 3 % duty on an H100 for a 0.5B GRPO run is the same story as finding +1 seen from the GPU: most of each step is spent with the request in the +dispatch path, not on the device. + +## Stability check after the NaN + +Two concurrent 10-step FFT runs on Qwen3-0.6B with seeds 1 and 2 (separate +client Jobs, since the `-x2` scenario shares one `extra` string) both +completed cleanly: no NaN, no failed ops, entropy and KL flat, accuracy +trending up over the last five steps, completions shortening. Gradient +norms stayed at 10k-22k unclipped. The dashboard listed both as +`ended, steps 10, failures 0` next to the two earlier runs. So the stack +is stable here; the NaN was one run's divergence, which the trainer +should have caught (finding 11 and the missing guard). + +Placement note: this time binpack crossed the pairs, trainer of one run +with sampler of the other on each of two GPUs, six idle. + +## A setting we had been missing + +None of the runs in this report set `openrl.recipe=rl`, the session tag +that makes the API server start the sampler together with the trainer. +Without it the sampler starts on the first sample request, 22 s after the +trainer on an idle H100 here. The e2e harness never sent it; it now does +for RL scenarios (commit `8cda9c0` on the trial branch), and the demo +app's Jobs set it. So the first-step times and the client-side sampling +waits quoted above include a sampler start that the tag removes. Finding +1's point stands either way: the dispatch wait is not recorded. + +## Rebase onto sampling pools (2026-10-29) + +Upstream main moved to #386, which rewrote the sampler around pools. The +trial stack was rebased on it and re-verified on the cluster with a mixed +LoRA (routed `vllm serve` pool behind a dispatcher pod) and FFT +(in-process sampler) sweep: samples recorded on both paths, lanes with +turns and ops for both, the delta metric per step, the demo app +unchanged. #383's hooks had to move: per-request and per-group sample +records now come from `sampling_dispatcher.Dispatcher`, GPU turns from +`holding_gpu`, the worker record from `run_sampling_worker`; the +`TelemetryRecordsTest` fixture pins the torch backend as upstream's +in-memory tests do. These are on the review branch for #383's author. + +### Second rebase, onto 163533d + +Upstream then landed its own first dashboard pieces (#388 inventory, +#389 Kubernetes reads) under the same package, `src/server/dashboard`, +plus the DCGM bundle move (#391) and checkpoint prefetch (#380). The +stack was rebased again. What this means for #383: + +- **Same package, two designs.** Upstream's `fleet.py` and `router.py` + are a subset of #383's (one route, runs with lifecycle and + client_status). The rebase takes #383's files; upstream's + `kubernetes.py` stays next to #383's `sources.py`, which reads the same + pods, nodes, Workloads and ResourceSlices its own way. One of the two + has to go when #383 merges, and upstream's two inventory tests need + porting to #383's `fleet.join`. +- **The API server now serves `/api/v1/inspect` unconditionally** (#388). + #383 gated that on `OPEN_RL_INSPECTION_API=true` and tested that the + default server returns 404 there. The rebase keeps upstream's mount, + turns the gate into a documented no-op, and drops that test. Whether the + model-capability API should list every run is a question for the two + authors; #383's RBAC note (node reads need `k8s/deploy/dashboard`) now + applies to every API server on GKE. +- **DCGM.** Finding 5 is resolved upstream: the lora bundle includes + `k8s/deploy/dcgm`, so every GPU bundle ships the exporter and its + scraper. The overlay's copy is gone. Finding 4 stands: the scheduler + and sampler PodMonitorings still come from nowhere in the tree. +- **`store.py`.** Both sides added `scan_values`; the merge needs one copy. + +Unit tests (352), lint and the scheduler suite pass on the result. + +## Open on our side + +- Rerun the default-pool metadata update when us-east5-c has capacity; one + node (api-server only) is still on the old config. +- File the sampler shutdown segfault as its own issue. +- Decide whether to offer the overlay's exporter and `PodMonitoring` + objects to the PR as a commit on `k8s/deploy/dashboard`. +- The pooled-sampling dispatcher (`SAMPLING_BACKEND=vllm-pools`) was not + exercised; the PR says it records no samples yet. +- The trial branch `try/pr-383-rebased` and the overlay `k8s/deploy/gke-use5` + are local and unpushed. diff --git a/docs/briefs/pr383-dashboard-trial/notes.md b/docs/briefs/pr383-dashboard-trial/notes.md new file mode 100644 index 000000000..1b6ad61b3 --- /dev/null +++ b/docs/briefs/pr383-dashboard-trial/notes.md @@ -0,0 +1,1094 @@ +# PR #383 (run telemetry and dashboard) on a fresh cluster + +Trial of https://github.com/gke-labs/open-rl/pull/383 on `open-rl-dra-use5` +(us-east5), a cluster with no OpenRL stack on it. Started 2026-10-27 evening +PDT (2026-10-28 01:00 UTC). This file is the running log; findings about the +PR itself are collected at the end. + +## Branch + +- PR head `c7dbd3f` (ShubyM/open-rl `dashboard/main`), five commits, base + `3eb0c9e` (#368). Upstream main was two commits ahead (#384, shared volume + storage class and size at render time). +- Rebased onto upstream main `0644063` as local branch `try/pr-383-rebased` + in worktree `../open-rl-pr383`, head `844424a`. Rebase was clean. +- Checks on the rebased branch: `py_compile` of every changed Python file + passes; `make -C scheduler/controller vet test` passes. + +## Cluster as found + +| Item | State | +|---|---| +| Context | `gke_cdrollouts-sunilarora_us-east5_open-rl-dra-use5` | +| Control plane | 1.35.8-gke.1796000, Managed Prometheus on, DCGM metrics component on, logging SYSTEM+WORKLOADS | +| default-pool | 3 x e2-standard-4 | +| kubecon-h100-8g | 1 x a3-highgpu-8g Spot (8 x H100 80GB), 16 local SSDs, GPU driver install `INSTALLATION_DISABLED`, labels `openrl.io/{enabled,trainer,sampler}=true`, `nvidia.com/gpu.present=true`, `gke-no-default-nvidia-gpu-device-plugin=true`, `group.timeslice.io/trainers=true` | +| sandbox-pool | gVisor pool, 0 nodes at the time | +| DRA | no NVIDIA DRA driver, no ResourceSlices; only `mrdma.google.com` and `netdev.google.com` device classes | +| OpenRL | nothing: no `openrl-system` namespace, no CRDs, no PVC | +| Workload Identity | not enabled (same as `open-rl-dra`). Node SA is the default compute SA with scopes `monitoring`, `logging.write`, `devstorage.read_only`; project roles artifactregistry.reader, cloudtrace.agent, storage.objectViewer | +| Filestore CSI | enabled; classes `standard-rwx`, `zonal-rwx`, `regional-rwx`, `enterprise-rwx` present | + +## Decisions + +- Images go to a new Artifact Registry repo in the cluster's region: + `us-east5-docker.pkg.dev/cdrollouts-sunilarora/open-rl` (created + 2026-10-28 01:03 UTC). Tag `pr383-844424a`. +- Shared volume: Filestore Zonal (`zonal-rwx`) instead of the bundle's + `standard-rwx` Basic HDD, created in-region by the PVC. +- Built with Cloud Build from the worktree. First build into the new repo + seeds its layer cache from the us-central1 repo's `:buildcache` refs + (config `/tmp/cloudbuild-seed.yaml`, the repo's `cloudbuild.yaml` plus a + second `--cache-from`). +- Scheduler image built from `scheduler/controller` as the build context + (`_DOCKERFILE=Dockerfile`): the PR changes the Workload CRD and the + controller, so the upstream GHCR scheduler tag would not do. + +## Builds (Cloud Build, project cdrollouts-sunilarora) + +| Image | Build ID | Result | +|---|---|---| +| open-rl-api-server | 2912dfe5 | SUCCESS, pushed | +| open-rl-server | 4fbd3392 | SUCCESS, pushed (about 17 min with the seeded cache) | +| open-rl-client | 760f3505 | SUCCESS, pushed | +| open-rl-scheduler | be62534c | SUCCESS, pushed | + +The first four submissions (807c67e5, 883ea021, 9a88b8a7, 7bf0719c) failed +with `invalid tag`: a zsh `for spec in ...; set -- $spec` loop does not +word-split, so each image name carried the whole spec. Resubmitted 01:12 UTC. + +## Steps + +1. 01:03 UTC: registry created, builds submitted. +2. GPU drivers: the NVIDIA driver installer DaemonSet and the NVIDIA DRA + driver Helm chart (25.8.0, `nvidiaDriverRoot=/home/kubernetes/bin/nvidia/`), + same as `scripts/cell_up.sh` on `feat/cold-start-measure`. Pending: the + session's permission classifier blocked these two cluster-wide installs; + handed to the user. The driver installer DaemonSet applied at 01:16 UTC. + The chart refuses its own default (`resources.gpus.enabled=true`) unless + `gpuResourcesEnabledOverride=true` is also set; `open-rl-dra` was + installed with both values, and `scripts/cell_up.sh` on + `feat/cold-start-measure` lacks them and would fail the same way. + Installed 01:10 UTC (user). The kubelet-plugin DaemonSet then had 0 + desired pods: the chart ships no tolerations, and the GPU pool carries + `nvidia.com/gpu=present:NoSchedule`; its `system-node-critical` priority + class also trips GKE's critical-pod quota outside `kube-system` + (`insufficient quota to match these scopes`, visible on the controller + Deployment, which stays 0/1 on `open-rl-dra` too). `open-rl-dra`'s + DaemonSet has `tolerations: [{operator: Exists}]` and no priority class, + so the fix is `--set 'kubeletPlugin.tolerations[0].operator=Exists' + --set kubeletPlugin.priorityClassName=""`. +4. Bundle apply (`kubectl apply --server-side --force-conflicts -k + ../open-rl-pr383/k8s/deploy/gke-use5`) also handed to the user; the + classifier blocked it. Applied 01:13 UTC: 35 objects. Scheduler and Redis + up within seconds; api-server and dashboard Pending on the zonal PVC; + time-slicer in ImagePullBackOff until the worker image push finished at + 01:21 UTC. +3. Overlay `k8s/deploy/gke-use5/kustomization.yaml` (untracked, copied into + the worktree so it can reach `../dashboard`): `../fft` + `../dashboard`, + images rewritten to the us-east5 repo, PVC patched to `zonal-rwx`. + Renders 35 objects; ConfigMap `open-rl-images` picks up the rewritten + worker and dispatcher images. + +5. 01:21 UTC: all pods Running (api-server, dashboard, scheduler, redis, + time-slicer, snapshot-agent). Dashboard port-forwarded to + `http://127.0.0.1:8080/dashboard` (loop in the background). +6. 01:21 UTC: `make cluster-e2e E2E_NAMESPACE=openrl-system + E2E_SCENARIO=fft-gsm8k-rl E2E_IMAGE= + E2E_ARGS="base_model=Qwen/Qwen2.5-0.5B-Instruct steps=5"`. Completed + 01:27 UTC, 5 steps, accuracy 0.41 -> 0.36 -> 0.44 -> 0.66 -> 0.81, step + wall time 99 s then 47-51 s (client: sampling 16-18 s, train_step 28 s, + save 3-8 s). Trainer and sampler were both placed on the same GPU + (`GPU-876ee4e3`) with seven idle; binpack default, not the PR's doing. +7. 01:27 UTC: dashboard Deployment env `OPEN_RL_CLIENT_RUNS_ROOT= + /mnt/shared/open-rl/runs` (see finding 3). Screenshots in `shots/`. + +## Run under test + +| Item | Value | +|---|---| +| run_id | `31b33c82-39f5-49ac-a2e6-da35865a47fd` | +| session | `sess-b313188013eb` | +| workers | `orw-fft-31b33c82-...-trainer`, `orw-fft-31b33c82-...-sampler`, both on GPU `GPU-876ee4e3-fa1b-b40f-21a9-808cc6f615cc` | +| client dir | `/mnt/shared/open-rl/runs/fft-gsm8k-rl/open-rl-tmp/fft_gsm8k_rl` | + +## Findings about the PR + +### What worked as described + +- **Scheduler stamps placements.** Workload status carried `placedAt` and + `gpuUUIDs` within seconds of binding; the scheduler logged one + `placement placed` line per Workload; `/metrics` on port 8080 exported + `openrl_gpu_held{UUID,node,pod,role,run_id}` for both workers. +- **Records.** Trainer and sampler print `telemetry op|popped|answered` + JSON lines with the Cloud Logging labels. Redis held every key the + design lists: `operations:` and `:samples`, `sampled`, `sampled_at`, + `steps`, `step_times`, `ledger`, two `worker:` records, + `session_record`, `placements:`. Placement records had + `released_at` after cleanup, and the activity lanes showed `ended: true`. +- **Read API.** `/runs` (lifecycle `placed` during the run, `ended` after, + 5 steps, 0 failures, 0 relaunches), `/runs/{id}/summary` (steps, step + seconds, last op, last metrics, goodput, ledger counters, saves, grad + norms, sampler sends and median), `/runs/{id}/steps` (ops per step with + metric spread), `/runs/{id}/commands?role=sampler` (one record per served + group: counts 64, 1, 63), `/activity` (lanes, ops, waits), `/logs` + (`pod=client` pages `logs.log`), `/inventory`. +- **Dashboard.** Overview, Run (Activity lanes, Metrics with client and + trainer charts, Logs), Nodes and Health render; a source that cannot be + read shows its error text, as the docs promise. Build inside the api-server + image's Node stage worked first time. +- **Rebase.** Clean on upstream main `0644063`; scheduler tests pass. + +### Needs attention + +1. **"client" time is mostly server dispatch time.** Summary goodput for + the finished run: wall 335 s, train 11 s, checkpoint 28 s, rollout 7 s, + client 290 s (goodput 0.032). The client measured sampling 128 s and + train_step 144 s over the five steps, while the sampler's served groups + took 1-1.5 s each and the trainer's ops 10-12 s per step. The 40 s per + step between a request leaving the client and a worker popping it is + attributed to the client. The PR lists "when each request was enqueued + isn't recorded" as open; on this run that gap is 85% of wall time, so the + goodput block answers the wrong question until enqueue times exist. + (Separately worth a look in OpenRL itself: 16 s client-side sampling for + 1.5 s of vLLM work.) +2. **`since_last_step_s` can be negative.** Observed `-32` on a live run + (`/runs/{id}/summary` at 01:24:42 UTC, last `optim_step` at 01:24:37). + `facts.py` subtracts the step time from `state["observed_at"]`, the time + the cached fleet snapshot was read, while the step comes from the run's + ops cache, which refreshes separately. `since_last_op_s` is clamped with + `max(0, ...)` one line below, which hides the same thing. Clocks were + checked: dashboard, trainer and laptop agree to the second. +3. **The repo's own e2e layout is too deep for the client-directory search.** + `k8s/eval/e2e-client-job.yaml` sets `log_dir=/mnt/shared/open-rl/runs/ + ` and the recipe writes `open-rl-tmp/` under it, five + levels below `/mnt/shared`; `client_runs.DEPTH = 3`, so + `client_directory` was null and the Metrics tab showed "0 metrics". + Setting `OPEN_RL_CLIENT_RUNS_ROOT=/mnt/shared/open-rl/runs` on the + dashboard Deployment fixed it (53 client metrics, `logs.log` paged). The + dashboard manifest or the e2e job should agree on one layout. +4. **Nothing scrapes `openrl_gpu_held`.** The scheduler exports it, but + neither `scheduler/deploy/base` nor `k8s/deploy/dashboard` adds a + `PodMonitoring` for the scheduler's `metrics` port, and the docs do not + mention one. On GKE with managed collection the PromQL join in design + 018 therefore has no right-hand side. +5. **No DCGM exporter in the bundle the dashboard ships with.** The PR + edits `k8s/deploy/distributed-fft-timeslice/10-dcgm-monitoring.yaml` + (adds `DCGM_FI_PROF_GR_ENGINE_ACTIVE`, 10 s interval), but `k8s/deploy/fft` + does not include that file, and GKE's managed `dcgm-exporter` DaemonSet + requires `cloud.google.com/gke-gpu-driver-version in (default, latest)`, + which a pool with `gpu-driver-version=disabled` (needed for DRA) never + has: it had 0 desired pods here and on `open-rl-dra`. So GPU duty, the + Nodes page and `/accelerators` stay empty on a DRA cluster unless the + exporter is applied by hand. +6. **GKE reads need IAM the bundle does not set up.** Every Prometheus and + Cloud Logging read returned 403: this cluster has no Workload Identity, + the node SA lacks `monitoring.viewer`, and its `logging.write` scope + cannot read logs at all. The docs say "monitoring.viewer on the workload + identity" in a manifest comment only. Worth a short setup section: + Workload Identity (or node roles), `roles/monitoring.viewer`, + `roles/logging.viewer`, and what the audit-log lookup needs. +7. **Node GPU inventory comes only from DCGM.** With the 403, the H100 node + showed `gpu_capacity: 0, devices: []` although its ResourceSlice listed + eight GPUs and the run's placement named a UUID. The Nodes page was an + empty table under the error. ResourceSlices (or the placements) could + seed the node list when metrics are unavailable. +8. **Trailing empty step.** After the final `save_state`, `/runs/{id}/steps` + lists a step 5 with no client metrics while the client ran steps 0-4; the + post-step ops form a group of their own. + +### Small things + +- The Dockerfile's Node stage adds `node:22-bookworm-slim` and an `npm ci` + to every api-server build (fine on Cloud Build, about a minute). +- The dashboard pod's readiness probe hits `/openapi.json` every few + seconds and logs each hit at INFO. + +## Identity (2026-10-28, after the first report) + +User chose Workload Identity over node-SA roles: workload pool on the +cluster, GKE metadata server on `default-pool` (rolls its three nodes; +the H100 pool untouched), a dedicated KSA `open-rl-dashboard` (overlay +`dashboard-sa.yaml` + patches: Deployment serviceAccountName, second +subject on the PR's RoleBinding and ClusterRoleBinding), and direct +principal bindings to `roles/monitoring.viewer` and `roles/logging.viewer` +(no Google service account). The dashboard reads credentials with +`google.auth.default` and the cluster name from the metadata server, so +the GKE metadata server serves both. All four commands run by the user; +the classifier blocked them here. + +Timeline (UTC, 2026-10-28): +- 01:46 cluster `--workload-pool` update started (the user's first + invocation was interrupted but the operation kept running; a rerun + returned CLUSTER_ALREADY_HAS_OPERATION). Done by 01:55. +- 01:49-01:51 IAM: `roles/monitoring.viewer` and `roles/logging.viewer` + bound to the principal + `principal://iam.googleapis.com/projects/288171024361/locations/global/workloadIdentityPools/cdrollouts-sunilarora.svc.id.goog/subject/ns/openrl-system/sa/open-rl-dashboard`. + The project already had a federated principal for `ns/default/sa/open-rl-sa` + with `roles/cloudtrace.agent`, so this pattern was used here before. +- 01:58 `node-pools update default-pool --workload-metadata=GKE_METADATA`. + Nodes hlr9 (us-east5-a) and o9ku (us-east5-b) replaced and labelled + `iam.gke.io/gke-metadata-server-enabled=true`; Redis, scheduler and + dashboard rescheduled onto them. The third node, 45i1 in us-east5-c, + could not be replaced: `ZONE_RESOURCE_POOL_EXHAUSTED_WITH_DETAILS` for + e2-standard-4 (the MIG shows the same error on 2026-10-25). The command + exited 1 at 02:07, pool status RUNNING_WITH_ERROR, old node still + serving (api-server only). Rerun the same command later to finish. + Mitigation: the overlay pins the dashboard with + `nodeSelector: iam.gke.io/gke-metadata-server-enabled: "true"`. +- 02:12 overlay applied. The new ServiceAccount landed in `default`: + the overlay kustomization has no `namespace:` of its own (the bundles + below it set theirs), so a resource added at the overlay level keeps the + file's namespace. The ReplicaSet reported `serviceaccount + "open-rl-dashboard" not found` for three minutes. Fixed by writing + `namespace: openrl-system` into `dashboard-sa.yaml`, deleting the stray + account, re-applying (02:16). The ReplicaSet controller was then in + backoff after 16 failures; `kubectl rollout restart` got the new pod up + at 02:20 (`open-rl-dashboard-68b8ddb5dd-7h7dj`, sa `open-rl-dashboard`, + node o9ku). +- 02:22 verified with the new identity: `/inventory` errors `[]`; + `/logs?run_id=...&since=2h` available with 200 records from Cloud + Logging (sampler shutdown lines, worker pods by owner label); + `/accelerators` no error but `gpus: []`, and node `devices: []`. So IAM + was the only thing behind the 403s, and the empty GPU data that remains + is findings 4 and 5 (no DCGM exporter on the DRA pool, nothing scraping + `openrl_gpu_held`). + +## Metrics backend (2026-10-28 02:20 UTC) + +Added to the overlay (user applied 02:23): a copy of +`distributed-fft-timeslice/10-dcgm-monitoring.yaml` as `dcgm-monitoring.yaml` +(kustomize cannot reference files outside the overlay), moved into +`openrl-system` by an overlay-level `namespace:`; `monitoring.yaml` with a +`PodMonitoring` for the scheduler (`app=open-rl-scheduler`, port `metrics`) +and one for sampler pods (`openrl.io/worker` exists, port `http` = 9095, +which the PR adds to sampler pod templates; trainers have no such port). +The kustomize diff showed exactly five new objects. The exporter pod +started on the H100 node within 30 s (image from the us-central1 GKE +release registry). Second run (8 steps) launched at 02:25 to exercise the +GPU-to-run join. + +02:28 UTC, about four minutes after the exporter started: `/accelerators` +lists eight `NVIDIA H100 80GB HBM3` with `memory_total_mib 81559`, no +error; `/inventory` shows the H100 node with `gpu_capacity 8, devices 8` +and `errors []`; `placements` lists the second run's trainer on +`GPU-876ee4e3`. So finding 7 (node inventory only from DCGM) holds, and +with DCGM present the inventory is complete. PodMonitoring `.status` +stayed empty throughout; Managed Prometheus scraped anyway. +Second run `d034c113-c3aa-4e18-82af-32fcae2b455b`: trainer placed +02:24:22, sampler 02:24:42, both on `GPU-876ee4e3` again (binpack). + +02:29-02:30 UTC, run d034c113 at 5 of 8 steps, every GPU route answers: +- `/query?name=gpu_busy&run_id=...`: one series for `GPU-876ee4e3` + (gpu 6), the join through the placement UUID works; values 0 -> 1.3 %. +- `/query?name=gpu_util&run_id=...`: `DCGM_FI_DEV_GPU_UTIL` 0-8.7 %. +- `/query?name=vllm_generation_tokens_per_second&run_id=...`: 118 -> + 384 tokens/s, from the sampler PodMonitoring on port 9095. +- `/runs/{id}/summary`: live `requests.unanswered` showed one + forward_backward, age 1 s, holder alive; ledger enqueued 19 / popped 18 / + answered 17 / lost 0; samplers 384 sends, median 1.0 s. Goodput again + 0.036 with client 270 of 307 s. +- UI: Activity lanes show Duty 3 % for trainer and sampler + (`shots/run-activity-live-dcgm.png`); Nodes page shows the H100 node, + "1/8 claimed", GPU 6 with held spans, 0 % duty on the node + (`shots/nodes-dcgm.png`). +The 3 % duty on an H100 for a 0.5B GRPO run is the same story as the +goodput split: the GPU is idle most of each step while requests sit in +the dispatch path. + +## Second run: diverged at step 7 (02:32 UTC) + +The 8-step run d034c113 died with the client getting HTTP 400 +`RequestFailedResponse: vLLM weight update failed: ... Sparse checkpoint +patches contain NaN values`. Trainer log at step 7: `Saved sparse delta +(100.0% changed elements, 494032768/494032768)`. Client accuracy per step: +0.31, 0.39, 0.33, 0.59, 0.73, 0.48, 0.38. `/runs/{id}/steps`: step 5 loss +3.56, grad_norm 8160; step 6 loss 8.51, grad_norm 0.0; step 7 no trainer +metrics. `nonfinite` stayed `[]` because the recorded op metrics were +finite; the NaNs were in the weights. Not the PR's problem (the first +run, 5 steps, was fine; this is RL instability with the recipe's defaults +for the 0.5B model) but a good test of the diagnostics: +- `/runs` line: `lifecycle placed, steps 7, failures 0, relaunches 0, + trainer_requests_unanswered 0` three minutes after the client died. + `failures` counts failed trainer ops only; the sampler had 64 failed + records (`RuntimeError`, no message text in the record) and the API + server had ingested the SDK's error events (`telemetry sdk_event` + severity ERROR, visible in `/sessions/sess-610ce40746a5/summary` as + `sdk_errors.count/last`). So the overview said healthy-ish for a dead + run. -> finding 10. +- `/runs/{id}/summary` did show `samplers.failed: 64` and the ledger + balanced (23/23/23, lost 0). Workers stayed Running until the session + reaper released them (~2 min after the client exit). + +Root cause (02:40 UTC, from Cloud Logging via `/logs?q=NaN`, the client's +`metrics.jsonl` and the trainer code), chain in order: +1. Step 6 `forward_backward` returned a finite loss (loss:sum 544 vs 228 + at step 5) but a non-finite gradient. Completions had been getting + longer every step (ac_tokens/turn 217 -> 281 -> 298 toward max_tokens + 512) and reward had fallen from 0.73 to 0.37 over two steps; the exact + NaN source (bf16 importance-sampling loss on long sequences, most + likely) is not recoverable without the batch. +2. `fft_trainer_worker.optim_step`: the recipe sets no `grad_clip_norm`, so + `max_grad_norm = inf` and `torch.nn.utils.clip_grad_norm_` runs with a + NaN total norm; its coefficient multiplies every gradient, so every + gradient became NaN, then `optimizer.step()` made every parameter NaN. + Hence "100.0 % changed elements" in the step-7 delta. +3. `save_weights_for_sampler` wrote that delta; vLLM's patch loader + rejected it (`Sparse checkpoint patches contain NaN values or + out-of-range indices`, sampler log 02:31:40); the API server returned + 400 `RequestFailedResponse` to the client's sample call; the recipe + raised and the Job failed. Unclipped grad norms of 8k-14k through the + whole run (and run 1) show how little margin this configuration has. +4. Why telemetry did not flag it: `trainer_worker.sanitize_float` turns + NaN into 0.0 and inf into +-9999 before the metric leaves the worker, + so the op record held `grad_norm:mean 0.0`, `nonfinite []`, and the + summary's `optimizer` block (built for exactly this) stayed empty. + -> finding 11. The trainer also has no non-finite guard before + `optimizer.step()` (OpenRL issue, not the PR). + +## Duty on a shared GPU (user question, 02:35 UTC) + +Both lanes of run d034c113 show Duty 3 % because both processes hold +GPU 6 (binpack placed the sampler on the trainer's ledger; the +time-slicer alternates them: vLLM sleep mode frees 70 GiB, trainer +reloads/offloads to pinned CPU memory, ~0.3 s each way). DCGM measures +the device, and design 018 reduces `openrl_gpu_held` per UUID before the +join, so the figure is the GPU's engine-active share, not the lane's; two +lanes showing 3 % is one 3 %, not 6 %. -> finding 9. + +## Co-location of FFT trainer and sampler (user, 02:45 UTC) + +Both runs: trainer and sampler on GPU 6 with seven GPUs idle, from the +default `binpack` placement plus the time-slicer. The user flags this as +surprising and as a blocker for a disaggregated trainer/sampler setup +under `binpack`: there is no way to ask for separate accelerators when a +seat is free on the trainer's ledger. Recorded in findings.md as an +OpenRL placement observation (not a PR finding) with a proposed issue: a +hint or policy that places trainer and sampler seats on different claims +when capacity allows. The dashboard showed the co-location at a glance. + +## Stability check: two Qwen3-0.6B FFT runs, seeds 1 and 2, 10 steps (02:50 UTC) + +User, after the NaN: run the FFT GSM8K scenario on Qwen3-0.6B with two +seeds for 10 steps to see whether OpenRL itself is stable, and to give the +dashboard two concurrent runs. The `fft-gsm8k-rl-x2` scenario passes one +`extra` string to both jobs, so seeds cannot differ there; instead two +single-job client Jobs were rendered with `run_cluster_e2e.render_manifest` +and launched side by side: `open-rl-e2e-client-seed1` / `-seed2`, +`log_dir=/mnt/shared/open-rl/runs/fft-gsm8k-rl-seed{1,2}`, +args `base_model=Qwen/Qwen3-0.6B steps=10 extra=seed={1,2}` (the recipe +`examples/autoresearch/recipes/math_rl/train.py` has `seed: int = 0`). +Manifests in `/tmp/e2e-seeds/`. The failed `open-rl-e2e-client` Job from +the diverged run was deleted first. + +Result (03:04 UTC): both Jobs Complete, 11 min each, 10 steps, no NaN, +no failed ops, clean exits. Runs `97df6712` (seed 1) and `4a3b708f` +(seed 2). + +| step | seed 1 correct | seed 2 correct | seed 1 grad_norm | seed 2 grad_norm | seed 1 ac_tok | seed 2 ac_tok | +|---|---|---|---|---|---|---| +| 0 | 0.56 | 0.17 | 19712 | 13504 | 462 | 502 | +| 1 | 0.42 | 0.52 | 19456 | 17536 | 466 | 468 | +| 2 | 0.59 | 0.31 | 21760 | 14592 | 447 | 484 | +| 3 | 0.45 | 0.27 | 19328 | 15744 | 426 | 460 | +| 4 | 0.39 | 0.63 | 21248 | 15360 | 453 | 423 | +| 5 | 0.47 | 0.39 | 17024 | 15680 | 390 | 447 | +| 6 | 0.58 | 0.63 | 10944 | 9984 | 357 | 345 | +| 7 | 0.66 | 0.59 | 17664 | 16640 | 368 | 362 | +| 8 | 0.64 | 0.67 | 16512 | 11584 | 349 | 389 | +| 9 | 0.52 | 0.66 | 13888 | 15232 | 306 | 357 | + +Entropy 0.34-0.58 throughout, KL(sample, train) 3e-4 to 9e-4, no +monotone drift. Accuracy on 64 samples is noisy but both seeds trend up +over the last five steps and completions shorten from ~480 to ~330 +tokens (the 0.5B run went the other way, lengthening toward the 512 cap +before it diverged). Gradient norms 10k-22k, again unclipped. Step wall +time 45-74 s with the two runs concurrent, against 48 s for one run +alone, so the second run cost little. + +Placement this time: binpack crossed them. GPU `876ee4e3` held seed 1's +trainer and seed 2's sampler; GPU `03954844` held seed 2's trainer and +seed 1's sampler. Two GPUs busy, six idle. Dashboard `/runs` lists all +four runs of the evening, the two Qwen3 runs `ended, steps 10, failures 0`. + +Reading: OpenRL's FFT path is stable on Qwen3-0.6B for 10 steps across +two seeds; the earlier NaN is specific to the Qwen2.5-0.5B-Instruct run +(lengthening completions, falling reward, then a non-finite gradient at +step 6) and to the missing non-finite guard and clip default rather than +to the stack. One diverged run out of four is still enough to want the +guard. + +## Something the dashboard surfaced + +The GKE logs tab for the run (`shots/run-logs-gke.png`) shows the sampler +segfaulting during shutdown at 01:27:23 UTC: `!!!!!!! Segfault +encountered !!!!!!!` in `cuCtxSynchronize` from `torch.accelerator` +teardown, right after the EngineCore received SIGTERM at 01:27:21. The run +had already finished and the API server had requested cleanup, so nothing +counted it as a failure or relaunch, and the client saw a clean exit. It +is an OpenRL sampler shutdown issue, not the PR's, but it is the kind of +thing this view exists for. Worth an issue with the log excerpt. + +## Cluster state at the end + +`open-rl-dra-use5` runs the FFT bundle plus dashboard from +`try/pr-383-rebased` (`844424a`), images `pr383-844424a` in the us-east5 +repo, NVIDIA driver installer + DRA driver 25.8.0, Filestore Zonal 1 TiB +PVC `open-rl-shared-pvc`, dashboard env `OPEN_RL_CLIENT_RUNS_ROOT= +/mnt/shared/open-rl/runs`, dedicated KSA `open-rl-dashboard` pinned to +metadata-server nodes, overlay-level `namespace: openrl-system`, DCGM +exporter DaemonSet (copy of the legacy manifest) and PodMonitorings for +the exporter, the scheduler and sampler pods. Workload Identity on; default-pool 2 of 3 nodes +on GKE_METADATA (45i1 in us-east5-c pending a capacity retry, pool +RUNNING_WITH_ERROR); IAM: monitoring.viewer + logging.viewer on the +dashboard KSA principal only. The e2e Job `open-rl-e2e-client` is left +(TTL 24 h). Port-forward loop to 8080 may still be running locally. + +## Turning the KubeCon demo mock into a real demo (assessment, 03:20 UTC) + +Mock: `decks/kubecon-keynote/demo-mock.html` (canvas, three scenes on a +100 s clock, pure function of time; `render.sh` -> `mp4/demo-mock.mp4`, +87.5 s at 12 fps; slide 13 embeds it in an iframe). Scene 1 terminal +submits 20 GRPO jobs (gemma-4-e2b + qwen3-0.6b, lora/full, reward v1-v5); +scene 2 fleet view (counters, job table with state and steps bar, 6 GPU +tiles with LoRA shared runtimes and FFT seats, waiting lane, pool-busy +gauge, end card) driven by an invented PLAN; scene 3 Harvey reward curve +0.57 -> 0.72 (numbers from examples/harvey_labs/README.md). + +What the PR gives for scene 2: `/inventory` (runs with lifecycle +ended/waiting/placed, nodes with GPUs, placements held now with role, pod, +UUIDs, pods with problems), `/runs` (steps, since-last, failures), +`/activity` (lanes with phase, GPU turns, op intervals), `/query` +(gpu_busy etc.). Enough for every element except: the job's intended +step count (the demo spec knows it), a display name (the SDK sends only +`recipe_name` and `git_rev`; `tag_metadata` keeps only `openrl.*` tags, +so no `name` reaches the server; map run -> job from the Job's client log +instead), and the state split starting/running (worker phase from +`/activity`, or steps > 0). + +Pieces to reuse: `scripts/demo_jobs.py` + `demo_rehearsal.sh` + +`demo_report.py` on `feat/cold-start-measure` (N Jobs from kind:model:count +specs, labels `openrl.io/demo-run`); PR #385's `scripts/perf_compare.py` +`job_manifest`/`load_workload` and `k8s/perf/workload.conf` (23-job mix with +`openrl.exclusive=true` tags, run labels, per-job log dirs, client-log +parsing). Scheduler on the PR build has `OPEN_RL_PLACEMENT_STRATEGY=spread` +(dedicated claim first, seat only when none can be cut), which is how to +make 20 jobs use all 8 GPUs without the feature branch's seat cap. + +Blockers on use5: gemma-4-e2b not in the HF cache (only Qwen2.5-0.5B- +Instruct, Qwen3-0.6B, gsm8k); the seeding bucket +`gs://cdrollouts-sunilarora-openrl-cache` no longer exists; `open-rl-dra` +still has `models--google--gemma-4-e2b` (9.6 GB) to re-seed from with +`scripts/cell_seed_hf.sh` (recreates the bucket). Reward variants v1-v5 do +not exist in the recipe; use seeds or learning rates and say so. + +## Demo app direction (user, 03:25 UTC) + +Decisions: the demo app runs in the cluster (no laptop traffic or +connection loss during a screen share), has a basic spec editor for the +job mix with start / stop / stop-all, reads the dashboard's inspect API, +and leaves the dashboard UI untouched until the shape is understood (the +dashboard is a separate PR). Stop must also POST +`/api/v1/delete_model {model_id}` to the API server so GPUs free at once +instead of after the session reaper. Gemma cache: warm it with a one-step +Gemma FFT job through the normal path rather than copying files +(`open-rl-warm-gemma`, launched 03:27 UTC). Scheduler gets +`OPEN_RL_PLACEMENT_STRATEGY=spread` via the overlay so a sweep spreads over +the eight GPUs; apply pending. + +## Demo app v1 written (03:40 UTC) + +`k8s/deploy/gke-use5/demo-app/` (untracked; copy in the worktree): +`app.py` (FastAPI: `/` page, `/api/inspect/{path}` proxy to the dashboard +Service, `/api/sweep/presets|jobs|start|stop|stop_all`; Jobs rendered +from the e2e client template with labels `app=openrl-demo`, +`openrl.io/demo-run`, per-job log dir `/mnt/shared/open-rl/runs/demo//`, +`extra=seed=N`; run id read from the client pod log with the regex the e2e +cleanup uses; stop = POST `/api/v1/delete_model` then delete the Job), +`index.html` (Sweep tab: spec editor rows kind/model/count/steps/seed, +presets keynote-20 / qwen-8 / smoke-2, start, stop, stop-all, a streaming +submit log; Fleet tab: the mock's scene 2 canvas fed by `/inventory`, +`/runs?ended=true` and `/query?name=gpu_busy`; Harvey tab: the mock's +scene 3 unchanged), `demo-app.yaml` (ServiceAccount `open-rl-demo`, Role +for jobs create/delete/list and pods + pods/log get/list, Deployment on +the api-server image with the code from a generated ConfigMap, Service +`open-rl-demo:8000`), `kustomization.yaml` (configMapGenerator). Wired +into the overlay; render 47 objects; diff = the demo objects plus the +scheduler's `OPEN_RL_PLACEMENT_STRATEGY=spread`. + +## Gemma warm-up result (04:12 UTC) + +`open-rl-warm-gemma` (fft-gsm8k-rl, google/gemma-4-e2b, 1 step) completed: +download 9.6 GB without a token (HF only warned about rate limits), one +step trained, `Training completed successfully`. Cache now holds +gemma-4-e2b, Qwen3-0.6B, Qwen2.5-0.5B-Instruct, gsm8k. Flag: step 0 +`env/all/correct 0.03` and the printed trajectory answered a different +problem than the GSM8K prompt (the e2e picks `recipes.math_rl.train_gemma` +with renderer `gemma4`); check the Gemma renderer/model pairing before +quoting Gemma numbers in the demo. Worker pods for the warm-up were +recreated around 04:03 (ages 7 min and 3 min at 04:10), cause not +investigated; the job still succeeded. + +Demo app deployed 04:09 UTC (user applied; scheduler now +`OPEN_RL_PLACEMENT_STRATEGY=spread`). Port-forward `svc/open-rl-demo +8090:8000`; presets, jobs and the inspect proxy answer. Smoke sweep +`smoke-1` (2 x FFT Qwen3-0.6B, 2 steps) started from the API at 04:11. +Added URL-hash tab selection to index.html (pending re-apply). + +## Smoke sweep through the demo app (04:11-04:16 UTC) + +`smoke-1`: 2 x FFT Qwen3-0.6B, 2 steps, seeds 1 and 2, started with POST +`/api/sweep/start`. Jobs appeared in the submit log, run ids were read +from the client logs (`35090c1a`, `181a6291`), states went +submitting -> starting -> running -> done, steps 2/2, Jobs Succeeded. +Spread placement: the four workers took four different GPUs +(`03954844`, `93184bf2`, `56732dec`, `7ceb0f4d`); under binpack earlier +runs shared one. Two fixes after the first look: the page sent the +PromQL window as `300s`, which the inspect API rejects (ISO or Unix +seconds only), so the pool gauge stayed at "waiting for DCGM"; and the +gauge now accumulates GPU time held by a job from the polled placements +(the mock's meaning) with DCGM duty as a second line, since duty on +0.6B models is about 1 %. Tabs now follow the URL hash. Re-apply pending. + +## Faster refresh and live placements (04:20-04:30 UTC) + +User: refresh more aggressively on all tabs. Measured: each proxied route +answers in ~0.3 s, so the page interval was not the limit. The lags are +upstream: the dashboard caches the fleet snapshot 5 s and run ops 10 s, +and the API server records placements only on its 30 s reaper sweep, so +`/inventory.placements` trails a placement by up to 30 s. Changes: the +demo app now reads placements straight from Workload status +(`/api/sweep/placements`: role, owner, pod, node, gpuUUIDs, placedAt, +shared = name starts with `lora-`; Role gained `workloads` get/list); +the page polls jobs and placements every 1 s and runs/inventory every +1-1.5 s, each source on its own timer so a slow one never blocks the +rest; pool tiles and held-time use the live placements, run labels come +from the dashboard's records. Two page bugs from the user's screenshot of +the qwen8 sweep: done jobs showed 0/2 once the dashboard dropped the run +from `/runs` (now fall back to the job's own step count), and a shared +LoRA runtime showed as one chip per run because the dashboard records one +placement per run on a shared runtime (now one chip per Workload, "N +adapters"). qwen8-1 (4 LoRA + 4 FFT Qwen3-0.6B, 5 steps) under spread: +8/8 GPUs busy at +11:30, LoRA sampler on gpu-3 and trainer on gpu-6, FFT +pairs on the other six, nothing waiting. +Finding for the PR (12): on a shared runtime the placement records are +per run, so a consumer cannot tell one shared pod from several; and the +30 s placement sweep is too slow for a live view, Workload status is +seconds behind. + +04:33 UTC: qwen8-1 finished, 8 of 8 Jobs Succeeded (LoRA 5/5 each, FFT +5/5 each). After the demo pod restarted it had lost its job-to-run map +and the re-read only scanned the last 400 log lines, so job 10 showed +"submitting" with its workers placed under a bare run id. Fix: scan the +whole client log once and store the run id as the Job annotation +`openrl.io/demo-run-id` (Role gained `patch` on jobs); all ten Jobs now +carry it. Note for port-forward loops: `pkill -f 'port-forward svc/...'` +also kills the `sh -c while` wrapper because its command line contains +the same string; anchor the pattern with `^kubectl`. + +## Keynote-20 mix live through the demo app (04:37 UTC, user-started, sweep demo-043751) + +Page fixes on the way: jobs newest first, numbered within their sweep; +Fleet tab scoped to the latest sweep (clock, gauge, counters reset per +sweep); tile chips pack into two columns above three holders, six shown +then "+n"; Workload spec fields are `modelID` / `ownerID` (not `owner`), +which is why samplers had shown their phase instead of a job number. +At +3:59: 10 FFT jobs starting/running with their 20 workers spread over +gpu-0..5 (three or four per GPU, time-sliced), the two LoRA trainers +(gemma, qwen; 5 adapters each) on gpu-6 and gpu-7, and the 10 LoRA jobs +waiting because their shared samplers have no seat: 8/8 GPUs busy, 100 % +of GPU time held, DCGM duty 2.3 %. This is the queue-and-backfill scene +the mock invented, now real. Screenshot `shots/demo-fleet-keynote.png`. + +04:48 UTC, user question on the keynote mix: "if the LoRA samplers are +pending, why do gpu-6 and gpu-7 show them occupying GPUs?" Verified: the +GPUs are held by the two shared LoRA *trainers* (gemma on `876ee4e3`, +qwen on `03954844`), which are Running, loaded, and have created and +saved five adapters each (jobs 01-05 and 06-10). The two LoRA *samplers* +are Pending with no GPU, so the ten LoRA jobs sit in lifecycle `waiting`. +The chip said "T 5 adapters", read as "job 5"; relabelled "T lora 01–05". +OpenRL observation (not the PR): a runtime's trainer and sampler are +placed independently, so one half can hold a GPU idle for the whole wait +while the other half has no seat. Gang placement of the pair, or letting +the LoRA sampler seat on its trainer's GPU, would have put ten jobs to +work on two otherwise idle H100s. + +05:00 UTC, two sweeps in parallel (user): chip text spilled out of the +tiles (long unsorted job lists) and job numbers were ambiguous because +each sweep numbers from 01. Fixes: chip text is clipped to the tile with +an ellipsis; numbers are sorted and collapse to a range; workers that +belong to another sweep's jobs draw as dim "other" chips (the pool stays +honest about what is held while the Fleet tab shows one sweep); shared +chips drop the word "lora" since the teal stroke already says it. A run +id that maps to no known job falls back to its first four hex chars +(seen as "T 06e9"): that happens when the job's annotation is not yet +written; it self-heals on the next poll. + +## Keynote-20 sweep timeline (demo-043751, started 04:37 UTC) + +- +0:00 20 Jobs submitted from the Sweep tab (5 gemma LoRA, 5 qwen LoRA, + 5 gemma FFT, 5 qwen FFT; LoRA 8 steps, FFT 4 steps, seeds 1-5). +- +4:00 10 FFT jobs starting/running, 20 FFT workers over gpu-0..5, LoRA + trainers on gpu-6/7, LoRA samplers Pending, 10 LoRA jobs waiting, + 8/8 GPUs held (`shots/demo-fleet-keynote.png`). +- +12:47 first three FFT jobs done (qwen 17, 19, 20), gpu-2 and gpu-5 + down to one worker each, LoRA still waiting + (`shots/demo-fleet-keynote-backfill.png`). +- ~+25:00 (05:02) all 10 FFT done; LoRA samplers placed on freed GPUs; + LoRA jobs backfilled: 3 done by 05:03, 7 running. Per-job timings to + follow from the client metrics (watcher running). +- The user started a second sweep (demo-044850, 4 LoRA + 4 FFT qwen) at + 04:48 while the first was still going; the Fleet tab switched to it + (latest sweep) and showed the first sweep's workers as "other". A sweep + selector on the Fleet tab would help when several run at once; not + built yet. DCGM duty across the pool during the overlap: 12 %. +Takeaway for the demo narration: with ten FFT pairs the pool fills at +once and the LoRA half of the mix waits ~25 min for seats; for a +one-minute time-lapse the mix should be lighter on FFT (or FFT steps +fewer) so the backfill happens within the recording, and a LoRA sampler +should be able to land next to its trainer (see the placement +observation). + +05:05 UTC keynote sweep final: 14 of 20 Jobs Succeeded; the user stopped +jobs 01-05 (gemma LoRA) and 10 (qwen LoRA) by hand while they were +waiting or just started. From metrics.jsonl per job (first step = time +from client start to the end of step 0; later = last step's time/total): +qwen LoRA 06-09: 8/8 steps, first step 1094-1107 s (sampler seat wait), +later 47-64 s, accuracy 0.19-0.72. gemma LoRA 01-04: 2-3 steps before +stop, first step 1309-1317 s, later 86-111 s, accuracy 0.02-0.13. +gemma FFT 11-15: 4/4, first step 250-344 s, later 102-274 s, accuracy +0.02-0.09. qwen FFT 16-20: 4/4, first step 112-230 s, later 143-306 s, +accuracy 0.17-0.78. +Flags: (1) gemma-4-e2b accuracy is near zero for both LoRA and FFT while +Qwen reaches 0.7; consistent with the warm-up's off-topic sample. Check +whether `google/gemma-4-e2b` is the base checkpoint (vs an instruction- +tuned one) or whether the `gemma4` renderer in `recipes.math_rl.train_gemma` +matches it, before Gemma appears in the demo mix. (2) Every LoRA job +waited 18-22 min for a sampler seat, then trained normally: the +half-placed runtime issue is the main cost in this mix. + +05:07-05:10 UTC, "GPUs still occupied" after the sweep: three shared LoRA +Workloads (gemma sampler, qwen sampler + trainer) stayed Running until the +session reaper released them at 05:07:36 and 05:08:06, about 2.5 min +after the last client exited at 05:05 (session TTL 120 s + 30 s sweep). +Normal for shared runtimes: `delete_model` on one adapter cannot free a +runtime other sessions still use, so a stopped LoRA job frees its GPU +only when the last session on that runtime is gone; a stopped FFT job's +dedicated pair goes at once. Did the app's stop call delete_model? The +API server logged 14 `POST /api/v1/delete_model` between 05:04:31 and +05:05:43, matching the 14 jobs that finished on their own (the recipe's +cleanup); none at the time of the six manual stops. Either they were +`kubectl delete job` or the app's stop lost the run id (its cache is +in-memory and the demo pod had restarted). Access logs were off, so it +cannot be told apart. Fixes deployed 05:10: stop reads the run id from +the Job annotation before falling back to the pod log, every stop prints +a JSON `event: stop` line with `released`, and uvicorn access logging is +on. Direct test of the path: `delete_model` with a full run id returns +200 `{"status":"ok"}`; with an 8-char prefix it is 404 `Unknown model`. + +## Timeline drill-down (05:15-05:25 UTC) + +User asked for drill-down views: an accelerator's timeline (which jobs +take turns on it, in colours) and a job's timeline (its trainer and +sampler on their accelerators, interleaved when they share one), and for +live views to scroll like a monitor trace. Built on the inspection API's +`/activity` route (`node_id=` for an accelerator, lanes filtered by GPU +UUID; `run_id=` for a job): one row per worker (accelerator view) or one +row per device with the trainer and sampler lanes stacked (job view); +faint bar = span held, solid = GPU turns (`intervals`), dim = `waits`, +ticks on the top edge = ops by type; colour per job number; an +"any worker holding the GPU" union row with the busy share of the window +for an accelerator; the window slides with the clock, lanes fetched every +1.5 s (the API caches 5 s / 10 s), a pulsing live edge. Reached from a +selector (accelerator, job, window 15 m / 30 m / 1 h / this sweep), from +clicking a GPU tile or a job row on the Fleet canvas (hit rects), or by +URL `#timeline?gpu=` / `#timeline?job=`. +First look at gpu-6 (`shots/demo-timeline-gpu.png`): the gemma LoRA +trainer runtime with five adapter lanes from the stopped sweep (shown by +run prefix since their Jobs are gone) and five from the user's next sweep; +turns are short bursts around each op; union 16 % of the window. + +05:27 UTC: the per-worker lane rows were confusing (user); replaced by +one occupancy trace per accelerator: a step at 1 while a worker holds a +turn, coloured by the holder (job number, or role in the job view), 0 +when idle; a thin dim band at the base for a worker that has the claim +but no turn; per-trace "occupied N % of the window · now: "; a +"held by" legend with each holder's share. gpu-0 over 30 min: 51 % +occupied by six workers in turn (T 11 gemma 22 %, S 20 qwen 15 %, ...); +gpu-6: 16 %, the gemma LoRA trainer's bursts. Screenshots +`shots/demo-timeline-gpu0-trace.png`, `shots/demo-timeline-gpu-trace.png`. + +05:35 UTC: timeline tooltip (user request). Hover draws a crosshair with +the time at the cursor; over a block: holder (job number, model, colour), +worker role and whether it is a shared runtime, job (sweep, number, kind, +seed), run id, pod, the turn's start, end and duration, and the ops that +ran inside it; over the claim band: who holds the claim, since when, and +how many waits are recorded; over nothing: "idle". `#timeline?...&hover= +fx,fy` places the cursor for stills (like the mock's `?still=`). Screenshot +`shots/demo-timeline-tooltip.png`. Demo app files are in +`k8s/deploy/gke-use5/demo-app/` (main repo, untracked) and mirrored in the +worktree; every change is applied through the overlay. + +05:43 UTC: job-view overlay (user request): a toggle "overlay other jobs +on these GPUs" draws, under the job's own turns, every other worker's +turns on the same GPUs (muted, in their job colours; the node's lanes +fetched alongside the run's); the occupancy line reads "this job N % of +the window · M % with K other workers"; the legend lists the others +dimmed; the tooltip labels them "other job". `&others=1` on the URL turns +it on. Job 15 (gemma FFT): trainer on gpu-4 27 % alone, 41 % with two +others (T 14, T 18); sampler on gpu-3 14 % alone, 41 % with S 17 and +S 12. Screenshot `shots/demo-timeline-job-others.png`. Also: the last +tick label is skipped when it would collide with "now". + +05:50 UTC: drag-to-select on the timeline (user request, for exact +occupancy numbers). Dragging across the trace selects a window; the +view freezes on it (no live sliding), the fetch is bounded with +`since`/`until`, the axis re-ticks down to 5 s, and every percentage +(this job, with others, legend shares) is computed over exactly that +span; a yellow banner shows the selected range and duration; Escape or +double-click returns to the live window. `&sel=,` on the +URL opens a selection for stills. + +05:55 UTC: user asked how to solve trainers placed / samplers pending and +what gang scheduling would mean. Analysis added to findings.md (placement +section): root cause is per-Workload placement plus LoRA exclusivity; fix +order owner-affinity with a LoRA-pair exception -> pair-aware placement +in the controller -> Kubernetes gang scheduling later; operational +workarounds are LoRA-first submission order and warm LoRA runtimes. + +## Weights tab: delta sync in action (2026-10-28 ~17:00 UTC) + +User: show the share of weights that change per step for full fine- +tuning, as a tab or link from a job; ideally recorded natively in the +backend. Built a Weights tab (job selector; "weights →" link on the job +timeline; double-click a job row on the Fleet canvas) with headline tiles +(average % changed per step, parameters, deltas published, elements per +sync), one bar per sampler weight version, and a line comparing a full +checkpoint per step with the average delta. Data: `/api/sweep/deltas` +parses the trainer's `Saved sparse delta (...)` lines from the inspection +API's logs route (pages through `next_cursor`), a stopgap. The native fix +is finding 13: the save already returns `density_pct` but the op record +keeps only `result["metrics"]`, so one change in the trainer (or the +telemetry hook) makes it a step metric. Also noted: the session clock +jumped (user away); it is 16:55 UTC, so "since" windows must be long +enough to reach the 05:xx sweeps. + +## Delta density as a native op metric (2026-10-28 ~17:10 UTC) + +User: if it is easier, add the telemetry in the save op. Done on +`try/pr-383-rebased` as commit e38f54d "Record a sparse delta's density +and size on the save op": `BaseTrainerWorker.last_save_metrics` (empty +by default); the FFT worker sets `delta/density_pct`, +`delta/changed_elements`, `delta/total_elements`, `delta/layers`, +`time/delta_collect`, `time/delta_write` after a sparse-delta save and +clears them after a full save; the processor spreads them into the +`save_state` / `save_weights_for_sampler` results, which is what the op +record keeps; `translate_future_result` pops `metrics` from the three +save types so the SDK's save responses are unchanged (forward_backward +keeps its metrics). Two unit tests added (processor result carries the +metrics; API server strips them for saves only). Affected modules pass +(test_worker_manager, test_trainer_optimizer_correctness, +test_telemetry_faults, test_fft_batch_failure); ruff clean. Images +building: `pr383-delta-e38f54d` server (9e37824c) and api-server +(9b32f73a). The Weights tab now reads `/runs/{id}/steps` first +(`save_weights_for_sampler.metrics['delta/density_pct']`) and falls back +to the trainer log parse; the footer names the source. Note for the PR +author: the dashboard's Metrics tab will show these as trainer charts +automatically once the worker image carries the change. + +17:22 UTC: verified on the new images (`pr383-delta-e38f54d`, applied +17:14; api-server, dashboard, demo app, time-slicer and the worker +ConfigMap rolled). Run `delta-1` (FFT Qwen3-0.6B, 3 steps, seed 7, run +2f304942) finished at 17:19. `/runs/{id}/steps` now shows per step under +`save_weights_for_sampler.metrics`: step 0 density 0 (empty delta before +the first step), step 1 11.6 % (69.2 M of 596 M elements, 263 layers, +write 5.7 s), step 2 7.8 % (46.4 M, 2.9 s), step 3 6.0 % (35.7 M, 2.7 s). +`/runs/{id}/summary.last_metrics` carries them too. Finding 13 is +therefore fixed on the trial branch; the client SDK was unaffected (the +run completed through the normal recipe). + +17:25 UTC: screenshots `shots/demo-weights-native.png` (Weights tab, +"source: op metrics via /runs/{id}/steps": 8.5 % average, 596 M +parameters, 50.4 M elements per sync, 12x fewer than a full checkpoint) +and `shots/dashboard-metrics-delta.png` (the PR's own Metrics tab charts +delta/density_pct, delta/changed_elements, delta/layers, +delta/total_elements, time/delta_collect, time/delta_write for both +save_weights_for_sampler and save_state, with no dashboard change). Two +side observations: (1) the dashboard showed "Client metrics: 0" for demo +runs because the demo app wrote `runs/demo///open-rl-tmp/...`, +five levels under the dashboard's root; the demo app now writes +`runs//...` (three levels), deployed 17:26, so the next sweep's +client metrics appear on the Run page; (2) the dashboard's step charts +label the x axis "0 1 2 2 3" for a four-step run (tick rounding), a small +thing for the PR. + +## Sampler started with the trainer (2026-10-28 17:30 UTC) + +User asked whether runs set `openrl.recipe=rl`, which makes the API +server start the sampler alongside the trainer (`launch_worker_and_enqueue +(start_sampler=meta.recipe == "rl")`). They did not: session metadata had +only fine_tuning_type, recipe_name and git_rev, and on run 2f304942 the +sampler was placed 22 s after the trainer (lazy start on the first sample +request). Fixes: (1) the demo app's Jobs now set `TINKER_TAGS= +openrl.recipe=rl` (the e2e script merges its own settings into it); +verified on run 56a28037: trainer and sampler placedAt both 17:27:43, and +the session carries `openrl.recipe: rl`. (2) Commit 8cda9c0 on +`try/pr-383-rebased`: `scripts/run_training_e2e.py` sends +`openrl.recipe=rl` for scenarios whose name carries `rl` or `textsql`, so +`make cluster-e2e` runs get it too; client image `pr383-delta-8cda9c0` +building (26111397). Every earlier timing in this file (first step +100-125 s for a 0.6B FFT job, the 17 s client-side sampling waits) was +measured without this tag. + +## Sweep page links, sweep-agnostic Fleet view (2026-10-28 17:35 UTC) + +User: make jobs on the Sweep page clickable for historic jobs, and make +the Fleet view agnostic of sweeps (anchoring to one sweep breaks when +several are active). Done: every Sweep-page row has timeline / weights / +fleet buttons; opening a finished job frames the timeline to the job's +lifetime (Jobs now carry started_at / completed_at) instead of the empty +live window; the Fleet tab shows all sweeps by default with a selector +to filter to one; sweeps get a letter by recency (A = newest) so tags +read A01, D15 on rows, tiles and the waiting lane; active jobs sort +first in submission order, finished ones after, newest first; the clock +reads "first submit to last finish" once everything shown has ended. +First-step comparison from the recipe tag (same model, solo on an idle +H100): without `openrl.recipe=rl` first step 67 s with 39 s of sampling +wait; with it 48 s and 19 s. +17:40 UTC: demo app Jobs now use client image `pr383-delta-8cda9c0` +(e2e harness with the recipe tag default); for `make cluster-e2e` pass +`E2E_IMAGE=us-east5-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-client:pr383-delta-8cda9c0`. +Polish: the all-sweeps Fleet clock reads "idle · last finish HH:MM:SSZ" +when nothing shown is active; the sweep legend is clipped; the submit +log no longer replays the backlog on page load. + +## Branch for review (2026-10-28 17:50 UTC) + +Pushed `try/pr-383-rebased` to the fork as `droot/open-rl:feat/dashboard- +trial-use5`: the five #383 commits rebased on main 0644063, then e38f54d +(delta metrics on save ops), 8cda9c0 (e2e recipe tag), ed434f5 (demo app ++ use5 overlay under k8s/deploy/gke-use5, report under +docs/briefs/pr383-dashboard-trial with findings.md, notes.md and shots/, +4.8 MB; `runs/` is gitignored so the report lives in docs/briefs for the +PR). The repo-scoped token cannot create pull requests +("Resource not accessible by personal access token"), so the PR is opened +by hand from the compare URL; the description is in +runs/2026-10-27_pr383_dashboard_use5/pr-body.md. The demo app's app.py +was reformatted for ruff before the commit (two long lines split). + +## Rebase onto upstream with sampling pools (2026-10-29) + +Upstream main moved to 8d17acb: #385 (perf_compare) and #386 "sample +through sampling pools" (each model's requests go to a Redis stream per +pool; one-replica pools run vllm_sampler in process, multi-replica and +shared-LoRA pools run `vllm serve` Workloads behind a dispatcher pod with +Envoy and the llm-d EPP; the queue sampler and its patch are deleted; +vllm_sampler.py rewritten, 504 lines changed). PR #383 itself has not +moved (head still c7dbd3f). Rebased our stack on a new local branch +`try/pr-383-rebased-2`: #383's five commits, then ours. Conflicts and +how they were resolved: +- "Record run telemetry where it happens": Makefile test list (union, + minus the deleted tests.test_sampler_patch), api_server imports (both), + delete_model (upstream's pool-aware body plus `await sweep_placements()` + before the release), the sample route (upstream's `accept_pool_sample`), + scheduler_worker_manager (both imports; ops settings added to the + passed env; pod identity env via `downward()`; both method sets), + training_requests_processor imports (both), vllm_sampler (took + upstream's file and re-inserted the hooks: `gpu_turn` around + `time_slicer.acquire` in `holding_gpu`, `sampled_group` after a + time-sliced batch, `serve_metrics()` and `serving("sampler", pool)` + around the worker body, `set_phase("ready")` after READY_FILE; the + per-request `observe_operation` moved into `sampling_dispatcher. + Dispatcher.send` as `op_finished` on success and on error, plus a + group record per dispatch for the exclusive in-process sampler; + `telemetry.run_of` now also reads the pool record's `job`). The routed + dispatcher pod (api-server image) records ops too, without pod identity. +- "Serve run facts...": Makefile test list again (dashboard tests). +- "Read client metrics...": docs/configuration.md, both sections kept. +- our delta-metric commit: processor's sampler save returns upstream's + dict plus `**self.save_metrics()`. +Then ruff --fix and format folded into the telemetry commit. Scheduler +vet/test pass. Python unit tests running (38 modules). Images building as +`pr383-pools-884ddca` (server d1c6139f, api-server c85fd203, client +85aec954); the scheduler image is unchanged (pr383-844424a). #386 needs +`OPEN_RL_TMP_DIR` on the API server (the bundle sets it), Redis with AOF +(bundle), and list on deployments/networkpolicies in the worker RBAC +(bundle). Branch head 884ddca. + +Unit tests on the rebased tree: 341 ran, all pass (25 skipped, GPU-only). +One fix was needed in #383's own `TelemetryRecordsTest`: it deletes a +model through the in-memory store, and under #386 the default sampler +backend is vLLM, whose delete path needs Redis (`store.redis`); upstream's +own tests pin `SAMPLING_BACKEND=torch` for in-memory stores, so the +fixture now does the same. Folded into the telemetry commit by autosquash; +branch head is bed05e5 (commits 703dec6 delta metrics, 1e18e1e e2e tag, +bed05e5 demo app). The images tagged `pr383-pools-884ddca` were built +from the pre-fixup tree, which differs only in that test file, so they +are what runs; the tag name is one hash behind the branch. + +19:25 UTC: pools stack deployed (api-server, dashboard, demo, time-slicer, +worker ConfigMap on `pr383-pools-884ddca`; the first apply had used the +worktree's reset overlay and left the old tags, re-applied from the +synced copy). Smoke sweep `pools-1` (2 LoRA + 2 FFT Qwen3-0.6B, 2 steps): +all four Succeeded in ~3 min. Under #386 the shared LoRA pool ran as a +`vllm serve` Workload (`lora-qwen-qwen3-0-6b-fb68f5a6-0-sampler`) behind a +dispatcher Deployment `sampler-qwen-qwen3-0-6b-fb68f5a6` (containers +dispatcher = api-server image, envoy, llm-d EPP); FFT pools ran the +in-process sampler. Telemetry on the new paths: every run reports +`samplers.sends 128`, median group 1.8 s (in-process) and 6.1 s (routed); +the dispatcher pod prints `telemetry op` group lines; FFT sampler worker +records exist with turns and GPU UUIDs. Limitation: a routed `vllm serve` +replica runs no worker record, so its lane has no turns (DCGM duty still +covers it). Regression from my merge: `workloads()` and +`pod_terminations()` had landed nested under module-level `sampler_state` +instead of inside `SchedulerWorkerManager`, so the API server logged +"Placement sweep skipped: ... no attribute 'workloads'" and wrote no +placement records (activity lanes empty for the pools-1 runs). Fixed and +folded into the telemetry commit; overlay tags committed too; branch head +d4d7d58; api-server image `pr383-pools-d4d7d58` building (78694880); +server and client stay on `pr383-pools-884ddca` (same source). Branch +force-pushed to `droot/open-rl:feat/dashboard-trial-use5`. + +19:30 UTC: api-server `pr383-pools-d4d7d58` applied. Sweep `pools-2` +(1 LoRA + 1 FFT, 2 steps) finished in ~2.5 min; placement records are back: +the FFT run has sampler and trainer lanes with turns (3 and 8) and ops; +the LoRA run's lanes are the shared runtime's (`...fb68f5a6-0-sampler`, +`-0-trainer`), both with turns and ops, so the routed pool's `vllm serve` +replica does appear with turns after all (the dispatcher's group records +land on the pool's sampler Workload pod). "Placement sweep skipped" count +since the fix: 0. Fleet tab on the pools stack: `shots/demo-fleet-pools.png` +(the shared LoRA trainer and sampler Workloads stay up between sweeps, +holding gpu-6 and gpu-7, as before). New: the dashboard's inventory +reported "GPU inventory unavailable: No GPU metrics source: not on GKE and +OPEN_RL_PROMETHEUS_URL is unset" right after its pod was replaced, though +the pod sits on a metadata-server node and can read the metadata server; +investigating whether discovery is one-shot at startup. + +19:33 UTC: the "not on GKE" inventory error cleared by itself: `discover()` +reads the metadata server with a 1 s timeout and retries a failed read +after a minute, so the first requests after a dashboard pod starts can +report "not on GKE" for up to a minute. Small thing for the PR: the +message should say the metadata read is pending and will retry. + +19:41 UTC: user saw negative occupancy percentages ("this job -205 % of +the window") in a selected window. Cause: the sums kept every turn with +`b > t0`, so a turn starting after a selected window's end was clamped to +the end and contributed a negative length. Fix: only turns overlapping +the window count (`b > t0 && a < t1`) for the job's own, the overlaid and +the union sums. Deployed; the same selection now reads 0 % / 5 % / 8 %. + +## Second rebase: upstream 163533d (2026-10-29, evening) + +Upstream main gained five commits after 8d17acb: #388 (an inspection +inventory from Redis, `src/server/dashboard/{fleet,router}.py`, mounted on +the API server unconditionally), #389 (dashboard Kubernetes reads, +`dashboard/kubernetes.py`), #391 (DCGM exporter moved to `k8s/deploy/dcgm` +and included by the lora bundle, so fft and distributed-fft-timeslice get +it too), #380 (trainer checkpoint prefetch, `training/prefetch.py`), and +a Makefile test-list change. Upstream has therefore started its own +dashboard under the same package name as #383's. + +Rebased `try/pr-383-rebased-2` (d4d7d58) onto 163533d as +`try/pr-383-rebased-3`. Stops and how each was resolved: + +- "Record run telemetry...": `store.py`, both sides kept (upstream's + `scan_values` plus #383's `set_value_sync`, samples, counters). #383 had + added an identical `scan_values` further down the file, so each class + ended with two copies (ruff F811); the upstream copies were dropped. + Makefile test list unioned. +- "Serve run facts...": add/add on `dashboard/fleet.py`, `dashboard/router.py` + and `tests/test_dashboard.py`. Took #383's versions (upstream's inventory + is a subset of #383's: run_id, lifecycle, client_status; #383 adds + placements, workers, nodes and the other routes). Upstream's + `dashboard/kubernetes.py` and `tests/test_dashboard_kubernetes.py` stay + untouched next to #383's `sources.py`, which reads the same objects its + own way; one of the two should go when #383 lands. Upstream's two + inventory tests were lost with its `test_dashboard.py` (they are written + against upstream's `fleet.join` signature). `api_server.py` auto-merged + into a double mount: upstream's unconditional `include_router(router)` + plus #383's `OPEN_RL_INSPECTION_API=true` block. Kept upstream's; the + gate is now a no-op and its doc row says so. #383's test + `test_default_gateway_does_not_expose_fleet_inspection` (expects 404 on + `/api/v1/inspect/inventory`) contradicts #388 and was removed. +- "Read client metrics...": `docs/configuration.md`, upstream's + "Inspection API" section kept above #383's "Run telemetry" section. +- Our overlay: `dcgm-monitoring.yaml` copy removed, the lora bundle brings + `k8s/deploy/dcgm` now; the overlay keeps the scheduler and sampler + PodMonitorings (`kubectl kustomize` renders 47 objects, one DCGM + DaemonSet, three PodMonitorings). + +Lint clean. Scheduler fmt/vet/manifests/test pass (upstream did not touch +`scheduler/`, so the scheduler image stays `pr383-844424a`). Unit tests: +353 ran, the one failure was the removed test; 352 pass after it. Images +`pr383-pools-c0cd426` building for server, api-server and client +(c0cd426 is the head before the test removal; the trees differ only in +that test file). + +01:28 UTC (2026-10-30): overlay applied from the rebased-3 worktree; all +rollouts clean (api-server, dashboard, demo and time-slicer on +`pr383-pools-c0cd426`, worker ConfigMap too; DCGM DaemonSet now the lora +bundle's `k8s/deploy/dcgm` copy, same image). `GET /api/v1/inspect/inventory` +on the API server itself answers 200 now (#388's mount). + +01:30 UTC: smoke sweep `pools-3` (1 LoRA + 1 FFT Qwen3-0.6B, 2 steps): +LoRA done in 91 s, FFT in 121 s, both Succeeded. Checks on the new stack: + +| Check | LoRA 18e20d46 | FFT 6b8f36ed | +|---|---|---| +| inventory lifecycle / client_status | ended / completed | ended / completed | +| summary steps, sampler sends | 2, 128 (median 3.7 s, routed pool) | 2, 128 (median 1.6 s, in-process) | +| goodput | 0.18 (wall 68 s, client 38 s) | 0.09 (wall 101 s, client 70 s) | +| placement records | 2 (shared runtime's lanes) | 2 | +| activity lanes | sampler 2 intervals, trainer 11, `exact: false` | sampler 3, trainer 8, `exact: true`, trainer ops incl. 4 saves | +| gpu_busy (DCGM) | sampler 3.1 %, trainer 2.5 % | trainer 1.5 %, sampler 1.4 % | +| delta density per step | n/a | step 0: 0 %, step 1: 11.8 % (263 layers), step 2: 7.8 % (261 layers) | + +Same shape as pools-2 before the rebase; the trailing third step row and +the routed replica's inexact lanes are as noted earlier. Demo app's job +list carries both run ids (annotation path intact). `/runs` lists only +live runs unless `ended=true`, which is why the two old "placed" runs +(2577a30d, 1a4aed73: the shared LoRA runtime's owners from 10-28 and +10-29, since_last_op 44 h) are the only entries there; nothing new. + +01:34 UTC: one difference from the pools-2 picture: the shared LoRA +runtime (`lora-qwen-qwen3-0-6b-fb68f5a6-0-{trainer,sampler}`) no longer +stays up between sweeps. Its Workloads were cut fresh at 01:30:18 for +pools-3 (the api-server restart on the new image had dropped the old +ones) and released at 01:33:49, two minutes after the LoRA run deleted +its model: `[API_SERVER] No live session uses qwen-qwen3-0-6b-fb68f5a6; +tearing its workers down`. That is the existing session-lifecycle +teardown, not a change in #383 or the new upstream commits; before, the +two stale owners of that runtime (2577a30d from 10-28, 1a4aed73 from +10-29, clients that died without delete_model) still had sessions in +Redis and kept it alive. Those sessions are gone now (`open_rl:session:*` +empty), so the Fleet tab shows all eight GPUs idle after a sweep, and +both stale runs read `ended` in the inventory while their +`client_status` stays `active` (finding 10). While pools-3's LoRA run +held the shared runtime they read `placed`, which is finding 12 from the +other side: a shared runtime's placement counts for every run that ever +owned it. + +Screenshots on the new stack: `shots/demo-weights-pools3.png` (deltas 12 % +and 7.8 %, 58.4 M of 596 M elements per sync), `shots/demo-timeline-pools3.png` +(sampler on gpu-1, trainer on gpu-7, spread placement), `shots/demo-fleet-pools3.png` +(30 of 30 done, five sweeps lettered A–E). + +01:50 UTC: user ran a 14-job sweep (`demo-014306`) and saw two trainers' +turns overlapping on gpu-4 in the accelerator timeline (A14 hold +01:44:29.04 → 01:44:31.65 with a 0.7 s A11 gemma turn inside it at +01:44:30.68). The daemon's timestamped log settles it: Snapshot A13 at +28.04, checkpointed 29.04, A14 granted; Snapshot A14 at 31.65, +checkpointed 33.66. One holder throughout. The inner "turn" is the gemma +trainer's `save_weights_for_sampler` op, run off the GPU because +non-exclusive FFT trainers are CPU-offloaded and `handle_batch` saves +without a turn then; `fleet.activity` unions op spans into `intervals`. +Same for the 0.014 s "turns" of the other two trainers' saves. Finding 14. +Fixed in `fleet.activity` (turns only when the lane has turns), test +`tests/test_dashboard_lanes.py`, commit 886bd93; api-server image +`pr383-pools-886bd93` building. + +01:55 UTC: api-server `pr383-pools-886bd93` applied (api-server, dashboard, +demo). The same gpu-4 window re-read through the fixed activity route: +three holds, one per trainer (01:43:45.6–01:44:22.2, 01:44:24.2–28.0, +01:44:29.0–31.7), no overlaps; the saves remain as ops on each lane. +`shots/demo-timeline-gpu4-fixed.png`. diff --git a/docs/configuration.md b/docs/configuration.md index b4ccf07e2..d359b4f10 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -153,3 +153,34 @@ gcloud projects add-iam-policy-binding --role=roles/logging.viewer --m `` is the project number. If you read logs through a log bucket view, also grant `roles/logging.viewAccessor` on that view. + +## Run telemetry + +Trainers, samplers and the API server record each run in Redis. +`python -m server.dashboard` serves the records read only under +`/api/v1/inspect`; `/docs` describes each route. Design Doc 018 explains them. + +| Env var | Default | What it does | +| --- | --- | --- | +| `OPEN_RL_OPS_SAMPLE_LIMIT` | `2000` | Newest command records kept per run, for trainer ops and samples separately. The scheduler passes it to worker pods. | +| `OPEN_RL_OPS_TTL_DAYS` | `7` | Days a record lives after its last write. `0` keeps records. The scheduler passes it to worker pods. | +| `OPEN_RL_INSPECTION_API` | unset | No longer read: since #388 the API server mounts `/api/v1/inspect` itself. It lists every run, so keep that server away from clients. On GKE it reads nodes cluster-wide, which `k8s/deploy/base` does not grant; apply `k8s/deploy/dashboard`, whose RBAC binds the same service account. | +| `OPEN_RL_PROMETHEUS_URL` | unset | Prometheus for GPU and vLLM metrics. Unset, the dashboard uses Managed Prometheus on GKE. | +| `OPEN_RL_GKE_PROJECT` / `OPEN_RL_GKE_CLUSTER` / `OPEN_RL_GKE_LOCATION` | from the metadata server | The cluster whose metrics and logs the dashboard reads. `OPEN_RL_GKE_TELEMETRY=false` turns GKE reads off. | +| `OPEN_RL_LOG_VIEWS` | the project | Comma-separated Cloud Logging views to read run logs from, such as a long-retention bucket's view. | +| `OPEN_RL_CLIENT_RUNS_ROOT` | `/mnt/shared` | Where the dashboard looks for each run's client directory, three levels deep. | + +### Client metrics and logs + +The dashboard reads a run's client metrics and logs from the directory the +tinker-cookbook client writes, its `log_path`. The client sends nothing extra. +On GKE, set each run's `log_path` on the shared volume, such as +`/mnt/shared//`, which the API server also mounts at +`/mnt/shared`. A run whose directory the server cannot read has no client +metrics or logs. + +A directory is a run's when its `checkpoints.jsonl` names `tinker:///`. +Before the first checkpoint, it is a run's when its `config.json` was written +within 60 seconds of the run's creation and names the run's base model. When +several directories match a run and the time does not single one out, none +is used. diff --git a/docs/dashboard.md b/docs/dashboard.md new file mode 100644 index 000000000..0f953917f --- /dev/null +++ b/docs/dashboard.md @@ -0,0 +1,76 @@ +# Cluster dashboard and inspection API + +`/dashboard` is an operator UI over a read-only JSON API under +`/api/v1/inspect`. Agents use the same routes; `/docs` and `/openapi.json` +describe them, and [Design Doc 018](designs/018-run-diagnostics.md) explains +what each answers. + +## Run it + +```sh +npm --prefix dev/dashboard ci # once; installs Svelte and esbuild +make dashboard # builds static/app.js and serves on port 8080 +``` + +`make dashboard` runs `python -m server.dashboard`. Point it at the API +server's Redis with `REDIS_URL`; without it the dashboard has its own empty +store. It reads Kubernetes in `OPEN_RL_WORKER_NAMESPACE`, else +`POD_NAMESPACE`, else `openrl-system`. Run `make dashboard-build` after changing +`src/server/dashboard/ui` and before `uv build`. + +On GKE, `kubectl apply -k k8s/deploy/dashboard` deploys it into +`openrl-system` with the API server's image, which builds the UI in a Node +stage. To open it from a laptop: + +```sh +kubectl -n openrl-system port-forward svc/open-rl-dashboard 8080:8000 +``` + +Then open `http://127.0.0.1:8080/dashboard`. + +The API lists every run in the fleet, so the API server does not mount it by +default. For a trusted local server, run `make dashboard-build`, then +`OPEN_RL_INSPECTION_API=true make server`, and open +`http://127.0.0.1:9003/dashboard`. + +Logs need Cloud Logging, nodes and pods need Kubernetes, and GPU and sampler +charts need a metrics backend. A source that cannot be read shows as an +error, never as an empty cluster. [Run telemetry](configuration.md#run-telemetry) +lists the settings that choose the sources. Logs come from the project's +default buckets, usually kept 30 days. To keep them longer, route the +namespace's logs to a bucket and set `OPEN_RL_LOG_VIEWS` to its view, which +needs `roles/logging.viewAccessor`. + +## Pages + +- **Overview** lists runs with a status label, search, and status filters. +- **Run** shows status, settings, an activity timeline with GPU rows and duty, + metrics by step, sampler charts, and logs. Click the timeline to open the + logs around that time. +- **Nodes** shows which process held which GPU and when, each GPU's duty cycle, + and GPU utilization and memory. Drag or press A/D to pan; use Ctrl/⌘ + wheel, + pinch, or W/S to zoom. +- **Health** lists pod and node problems and sources that could not be read. + +## Lifecycle + +Each run in `/inventory` has a `lifecycle`, which says where it is, not whether +it is healthy: + +- **ended**: the API server marked it finished, its workers stopped, or it has + no workers and no live session. +- **waiting**: a worker is not placed yet, or the server has not created one + for a live session. +- **placed**: every worker is placed. +- **active**: the scheduler cannot be read, so placement is unknown. + +## Client metrics and logs + +The Run page charts the metrics a tinker-cookbook client writes to +`metrics.jsonl` in its `log_path`, and its logs' `client` source pages the +client's `logs.log`. The client sends nothing extra. On GKE, set the run's +`log_path` on the shared volume, such as `/mnt/shared//`, which +`k8s/deploy/dashboard` mounts read only. Locally, set +`OPEN_RL_CLIENT_RUNS_ROOT` to the directory above the client's `log_path`. +[Client metrics and logs](configuration.md#client-metrics-and-logs) says how a +directory is matched to a run. diff --git a/docs/designs/018-run-diagnostics.md b/docs/designs/018-run-diagnostics.md new file mode 100644 index 000000000..3e68e5e6e --- /dev/null +++ b/docs/designs/018-run-diagnostics.md @@ -0,0 +1,165 @@ +# 018: Run diagnostics + +**Status:** Proposed. + +A person or an agent asks one question of a run: why is it not behaving? OpenRL +cannot answer it today from the server: + +- Command records carry no step, and sample requests evict trainer ops. At the + default cap of 2,000 records, a run with eight samplers keeps under two steps. +- Where a worker ran is rebuilt from pods, device slices and Workloads, not + recorded. Samplers record nothing. +- A request lost between the Redis push and the trainer's pop looks the same + as a client that sends no work. +- Nothing records which worker holds a request, so a request held by a dead + worker looks like a slow one. + +This design records where each process ran, which step each command belongs +to, and which requests are outstanding. A read-only API under +`/api/v1/inspect` joins those records into facts. The dashboard UI and agents +use the same routes. + +## Records + +Every key is in the API server's Redis. Each record expires +`OPEN_RL_OPS_TTL_DAYS` (default 7) after its last write; `0` keeps it. + +| Record | Key | Writer | Holds | +|---|---|---|---| +| Command record | `open_rl:operations:` for trainer ops, `open_rl:operations::samples` for samples | Trainer, one per op. Sampler, one per failed sample only | `operation`, `status`, `error_type`, `started_at`, `at`, `elapsed_seconds`, `request_id`, `role`, `runtime_id`, `pod`, `pod_uid`, `node`, finite `metrics`, the names of non-finite ones in `nonfinite`, and `step` on trainer ops. The newest `OPEN_RL_OPS_SAMPLE_LIMIT` (2,000) of each list | +| Sample count and time | `open_rl:sampled:`, `open_rl:sampled_at:` | Sampler and torch trainer, on every sample | The run's samples sent, and when the last one finished. A vLLM sampler keeps one command record per group of samples it served, not one per sample. A torch trainer's samples keep theirs | +| Step counter and step times | `open_rl:steps:`, `open_rl:step_times:` | Trainer, on each successful `optim_step` | The run's step count, and the times of its last 64 steps | +| Request counters | `open_rl:ledger:`, one hash | API server (`enqueued`), trainer (`popped`, `answered`, `failed`) | Trainer requests that reached each point | +| Worker record | `open_rl:worker:`, one per process | Every trainer and sampler, rewritten every 5 s by a thread | Pod, pod UID, node, role, runtime, GPU UUIDs (`null` when NVML is unreadable), phase, recent GPU turns, outstanding requests by op and run with the oldest's age, and the settings env with secrets left out. A record not rewritten for 30 s belongs to a gone process | +| Session record | `open_rl:session_record:` | API server, on create, heartbeat and model attach | Created and last-seen times, the session's models, and its redacted metadata | +| SDK errors | `open_rl:sdk_errors:` | API server, from the SDK's telemetry | The last 200 error events, without credentials | +| Placement | `open_rl:placements:`, one record per Workload placement | API server, which lists the scheduler's Workloads every 30 s and before each release | Workload, role, pod, node, GPU UUIDs, placed and released times | +| Workload status | `status.gpuUUIDs`, `status.placedAt`, `status.releasedAt` | Scheduler, on bind and release | The source of placement records. The scheduler also exports `openrl_gpu_held{run_id, role, pod, node, UUID}` while a GPU is held and logs one `openrl.placement` line per change | + +Each event a writer records also prints one JSON line, so Cloud Logging keeps a +history longer than the Redis TTL. The line carries the Cloud Logging labels +`openrl_source=telemetry` and `openrl_run_id=` or `openrl_session_id=`, +and its `message` is `telemetry `. A run's kinds are `enqueued`, `popped`, +`answered` and `op` (a command record), all written through +`telemetry.record(run_id, kind, **fields)`. Other kinds are `placement`, +`session_created`, `model_attached` and `sdk_event`. An answered vLLM sample +and a session heartbeat print no line. Fields are redacted before they print. +No route reads these lines. + +A run's display name comes from the `name`, `run_name`, `recipe_name` and +`git_rev` keys of its `user_metadata`, kept in its model metadata. + +`openrl_gpu_held` joins DCGM series to runs. A time-sliced GPU has one series +per holder, so reduce it per UUID before the join: + +```promql +100 * DCGM_FI_PROF_GR_ENGINE_ACTIVE * on(UUID) group_left() max by (UUID) (openrl_gpu_held{run_id="$run", role="trainer"}) +``` + +Utilization comes from `DCGM_FI_PROF_GR_ENGINE_ACTIVE`, which averages the exporter's whole 10 s collection interval. +`DCGM_FI_DEV_GPU_UTIL` covers only about the last second of each interval, so a load with the same period aliases. On an +A100 running 5 s on and 5 s off, it read 88% while engine activity read 51%. The read API falls back to it where engine +activity is missing. + +## Routes + +`python -m server.dashboard` serves these routes. The API server serves them +too when `OPEN_RL_INSPECTION_API=true`. Every route is a `GET` under +`/api/v1/inspect`. + +| Route | Answers | +|---|---| +| `/runs?ended=` | Which run to open: every live run in one line (lifecycle, steps, seconds since its last op and step, median step time, unanswered requests and how many a dead worker holds, samples queued, failures, relaunches), longest since its last op first. Built from the counters, the last sample time and the last record of each list, never the command history | +| `/runs/{id}/summary` | What is wrong with this run: the run facts below, and GPU duty by role over the run's span | +| `/runs/{id}/steps?last=&fields=` | What each step did: trainer ops grouped by step, with each op's count, failures, seconds and metric spread, and in `client` the metrics the client wrote for the step, kept by the `fields` prefixes. `client_directory` names the client directory, or is null when none matched | +| `/runs/{id}/commands?since=&until=&role=` | The raw command records, newest first. A vLLM sampler's samples appear as the groups it served, with a count, and one record per failed sample | +| `/sessions/{id}/summary` | What a client did: its session record, whether it is live, its SDK errors, and a line per model over the session's span | +| `/inventory` | What exists now: runs with their lifecycle (`ended`, `waiting`, `placed`, `active`), nodes with their GPUs, the GPUs held now, pods with problems, and one error per source that could not be read | +| `/activity?run_id=&node_id=&since=&until=` | Who ran where and when: one lane per placement with its GPU turns and recorded op intervals. A sampler lane's samples are the groups it served | +| `/accelerators?run_id=&node_id=&since=&until=` | What each GPU is doing: node, slot, model, every placement holding it, utilization now and over the window, memory used of total | +| `/query?name=&run_id=&node_id=&since=&until=&points=` | A named PromQL query (`gpu_util`, `gpu_util_now`, `gpu_busy`, `gpu_memory_mib`, `gpu_memory_total_mib`, `vllm_*`) over a run's or node's GPUs and pods, or the fleet's GPUs | +| `/logs?run_id=&pod=&q=&severity=&since=&until=&cursor=` | The run's lines, newest first. From Cloud Logging, its worker pods by the scheduler's owner label and the lines an in-cluster client labels `openrl_run_id=` and `openrl_source=client`, without telemetry's own lines; a stderr line takes the level it prints. With `pod=client` and a client directory, its `logs.log`, whose lines carry no time, so the window does not apply and the cursor is a byte offset | + +Bounds are ISO-8601 or a span such as `1h`, at most seven days, and default to +the last 30 minutes. + +A run's client metrics and logs come from its client directory, the +tinker-cookbook's `log_path`, read where the server mounts it +(`OPEN_RL_CLIENT_RUNS_ROOT`, `/mnt/shared` by default). The client sends +nothing. `metrics.jsonl` holds one line per step with its `step`, read from the +offset the last read stopped at. `logs.log` is paged back from its end. A +directory is a run's when its `checkpoints.jsonl` names `tinker:///`, or +before the first save when its `config.json` was written within 60 s of the +run's creation for the run's base model. When the match is ambiguous, the run +has no client directory. `docs/configuration.md` gives the rules. +`timing_spans.jsonl` in the same directory is a later source for trace export. +Some runs keep client metrics in Redis under `open_rl:run_metrics:`; +`metrics.jsonl` wins a step both have. + +The `/runs/{id}` routes, `/activity` and `/logs` also take a session id in place +of a run id. A session with one run answers for that run. A session with several +runs answers 409 and names them. + +### Run facts + +`/runs/{id}/summary` returns each block only when it has content. An absent +block means nothing of its kind was recorded. + +| Block | Holds | +|---|---| +| `steps`, `step_seconds`, `since_last_step_s`, `last_op`, `last_metrics` | Optimizer steps, the median and last gap between them, and the newest trainer op | +| `since_last_op_s`, `trainer_idle_s` | For a live run, seconds since any op and since the trainer's last op | +| `goodput` | The share of the run's span spent training, the rest split into checkpoint, forward-only, rollout (a group of samples being served, or a recorded torch sample in flight) and client (nothing in flight) seconds | +| `requests` | Unanswered trainer requests with the holding pod and whether its worker still reports; the counters, with `lost = enqueued - popped - queued`; samples waiting in the sampler queue | +| `failures` | Failed trainer ops: the count, the 5 newest, and a log read around the newest | +| `relaunches` | Workers replaced in the same pod during the run, with a log read around the first. A clean exit, or a shared runtime's idle pod recreated under its name, is not one | +| `saves` | `save_state` calls, failures, the last success and the steps since | +| `optimizer` | Ops with non-finite metrics and the first of them, and the last 5 grad norms | +| `samplers` | Sends (the sample count), failures (the failed records), and the median time of the groups its samplers served | +| `config` | Workers whose echoed settings differ from what the run asked for, or `no echo` | +| `placement` | Placements recorded, and Workloads with no node yet | +| `incidents` | The 5 newest Kubernetes terminations and scheduling failures on the run's pods | +| `gpu_busy` | Mean GPU duty by role. A GPU that time-sliced roles shared counts under the roles joined, such as `sampler+trainer` | + +A log read is a route and its arguments, such as +`{"read": "logs", "args": {"run_id": "…", "pod": "…", "since": "…", "until": "…"}}`, +so an agent follows a failure to its traceback in one call. + +## Rules + +- **Facts, not verdicts.** No route says whether a run is healthy. There are + no thresholds; the reader draws the conclusion. +- **Telemetry never raises or blocks.** Every hook suppresses its own errors. + Every store write runs in the background and is dropped after 5 s. An op + returns or raises exactly as it would without telemetry, even when the store + raises, hangs or is missing. `tests/test_telemetry_faults.py` checks this. +- **Reads are bounded and shared.** The fleet snapshot is cached for 5 s and a + run's command records for 10 s. A stale entry is refreshed by the next + caller, and every caller gets the last good result until the refresh lands, + unless that result is older than both 60 s and six windows. A failed read is + retried by the next caller. The 32 most recently used entries are kept. +- **Logs stay in their run.** A log read takes only pods with the run's owner + label, so a run ID cannot open another run's logs. +- **The client API does not list the fleet.** The API server mounts the routes + only with `OPEN_RL_INSPECTION_API=true`. + +## Known gaps + +- Workers return a NaN loss or grad norm as 0 and infinity as ±9999 + (`sanitize_float`), so those read as numbers in command records. Other + non-finite metrics are named in `nonfinite`. +- A local stack has no Cloud Logging and no Workloads, so its runs have no + logs or placement blocks. +- Runs recorded before step stamps, step times or the failure counter leave + those fields null, and their steps are derived from the order of their ops. +- Sampler latency is the time a group of samples took, not each request's. Runs + recorded before group records, which kept neither, have no sampler latency, + rollout split or sample bars. vLLM's per-pod charts stay on `/query`. +- Client metrics and logs need the client's directory on a volume the server + reads. A run whose `log_path` is not on the shared volume, and a client on a + laptop, have none. Runs started together on one base model match no directory + by time, so each has none until its first checkpoint. +- Facts are computed on request. Alerts would need a timer and thresholds. +- Not recorded yet: cgroup OOM kills and peak memory in the worker record, NVML + utilization for GPU facts without Prometheus, and the request's holder + incarnation in the counters. diff --git a/k8s/deploy/dashboard/dashboard.yaml b/k8s/deploy/dashboard/dashboard.yaml new file mode 100644 index 000000000..3626b26aa --- /dev/null +++ b/k8s/deploy/dashboard/dashboard.yaml @@ -0,0 +1,68 @@ +# The dashboard as its own process (python -m server.dashboard), so it can be +# restarted, scaled, or broken without touching the API server. It reuses the +# slim API-server image: same dependencies, different entrypoint. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: open-rl-dashboard + labels: + app: open-rl-dashboard +spec: + replicas: 1 + selector: + matchLabels: + app: open-rl-dashboard + template: + metadata: + labels: + app: open-rl-dashboard + spec: + serviceAccountName: open-rl-sa + containers: + - name: dashboard + image: ghcr.io/gke-labs/open-rl/api-server:latest + command: ["uv", "run", "python", "-m", "server.dashboard", "--host", "0.0.0.0", "--port", "8000"] + ports: + - name: http + containerPort: 8000 + env: + - name: REDIS_URL + value: redis://redis-service:6379 + - name: OPEN_RL_WORKER_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + # GKE metrics use Cloud Monitoring's PromQL API (monitoring.viewer on + # the workload identity). Set OPEN_RL_PROMETHEUS_URL for another backend. + # A long run's activity keeps the process busy for seconds; the default + # one-second timeout took the pod out of its Service meanwhile. + readinessProbe: + httpGet: + path: /openapi.json + port: 8000 + timeoutSeconds: 5 + resources: + requests: + cpu: 250m + memory: 512Mi + # Each run's client directory, its tinker-cookbook log_path, holds its metrics and logs. + volumeMounts: + - name: shared-storage + mountPath: /mnt/shared + readOnly: true + volumes: + - name: shared-storage + persistentVolumeClaim: + claimName: open-rl-shared-pvc + readOnly: true +--- +apiVersion: v1 +kind: Service +metadata: + name: open-rl-dashboard +spec: + selector: + app: open-rl-dashboard + ports: + - port: 8000 + targetPort: 8000 diff --git a/k8s/deploy/dashboard/kustomization.yaml b/k8s/deploy/dashboard/kustomization.yaml new file mode 100644 index 000000000..266472852 --- /dev/null +++ b/k8s/deploy/dashboard/kustomization.yaml @@ -0,0 +1,9 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: openrl-system +resources: + - rbac.yaml + - dashboard.yaml +images: + - name: ghcr.io/gke-labs/open-rl/api-server + newTag: latest diff --git a/k8s/deploy/dashboard/rbac.yaml b/k8s/deploy/dashboard/rbac.yaml new file mode 100644 index 000000000..79db03a2e --- /dev/null +++ b/k8s/deploy/dashboard/rbac.yaml @@ -0,0 +1,48 @@ +# The dashboard lists pods and Workloads in the namespace and nodes cluster-wide. It runs as the API server's account, +# so an API server with OPEN_RL_INSPECTION_API=true needs this applied too; base grants no node reads. +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: open-rl-dashboard +rules: +- apiGroups: [""] + resources: ["pods"] + verbs: ["list"] +- apiGroups: ["openrl.io"] + resources: ["workloads"] + verbs: ["list"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: open-rl-dashboard +subjects: +- kind: ServiceAccount + name: open-rl-sa +roleRef: + kind: Role + name: open-rl-dashboard + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: open-rl-dashboard +rules: +- apiGroups: [""] + resources: ["nodes"] + verbs: ["list"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: open-rl-dashboard +subjects: +- kind: ServiceAccount + name: open-rl-sa + # A cluster binding's subject needs its namespace; kustomize does not fill it in. + namespace: openrl-system +roleRef: + kind: ClusterRole + name: open-rl-dashboard + apiGroup: rbac.authorization.k8s.io diff --git a/k8s/deploy/dcgm/10-dcgm-monitoring.yaml b/k8s/deploy/dcgm/10-dcgm-monitoring.yaml index 27a536a92..bfd0ebbda 100644 --- a/k8s/deploy/dcgm/10-dcgm-monitoring.yaml +++ b/k8s/deploy/dcgm/10-dcgm-monitoring.yaml @@ -30,6 +30,10 @@ data: DCGM_FI_DEV_FB_FREE, gauge, Framebuffer memory free (in MiB). # FieldID 252 DCGM_FI_DEV_FB_USED, gauge, Framebuffer memory used (in MiB). + + # Profiling metrics, averaged over the collection interval,, + # FieldID 1001 + DCGM_FI_PROF_GR_ENGINE_ACTIVE, gauge, Ratio of time the graphics engine is active. --- apiVersion: apps/v1 kind: DaemonSet @@ -78,6 +82,9 @@ spec: value: ":9400" - name: DISABLE_STARTUP_VALIDATE value: "true" + # A reading every 10 s, as often as it is scraped; the 30 s default left two of every three scrapes repeating it. + - name: DCGM_EXPORTER_INTERVAL + value: "10000" - name: LD_LIBRARY_PATH value: "/usr/local/nvidia/lib64" resources: diff --git a/k8s/deploy/gke-use5/dashboard-sa.yaml b/k8s/deploy/gke-use5/dashboard-sa.yaml new file mode 100644 index 000000000..5e9554326 --- /dev/null +++ b/k8s/deploy/gke-use5/dashboard-sa.yaml @@ -0,0 +1,9 @@ +# Dedicated identity for the dashboard. Bound through Workload Identity +# Federation directly (no Google service account in between) to +# roles/monitoring.viewer and roles/logging.viewer on the project: +# principal://iam.googleapis.com/projects/288171024361/locations/global/workloadIdentityPools/cdrollouts-sunilarora.svc.id.goog/subject/ns/openrl-system/sa/open-rl-dashboard +apiVersion: v1 +kind: ServiceAccount +metadata: + name: open-rl-dashboard + namespace: openrl-system diff --git a/k8s/deploy/gke-use5/demo-app/app.py b/k8s/deploy/gke-use5/demo-app/app.py new file mode 100644 index 000000000..e59a85016 --- /dev/null +++ b/k8s/deploy/gke-use5/demo-app/app.py @@ -0,0 +1,382 @@ +"""OpenRL demo app: start and stop a sweep of RL client Jobs from a web page, and show the fleet from the +dashboard's inspection API. Runs in the cluster on the API server image; the code comes from a ConfigMap. + +Routes + GET / the page + GET /api/inspect/{path} proxy to the dashboard's /api/v1/inspect/{path} + GET /api/sweep/presets built-in job mixes + GET /api/sweep/jobs the demo Jobs with their state and the run each created + POST /api/sweep/start {"run": "...", "jobs": [{"kind","model","count","steps","seed"}]} + POST /api/sweep/stop {"name": ""}: release the run's workers, then delete the Job + POST /api/sweep/stop_all the same for every demo Job +""" + +import asyncio +import json +import os +import re +import time +from pathlib import Path + +import httpx +from fastapi import FastAPI, HTTPException, Request +from fastapi.responses import HTMLResponse, JSONResponse +from kubernetes import client, config + +NAMESPACE = os.getenv("POD_NAMESPACE", "openrl-system") +DASHBOARD = os.getenv("DASHBOARD_URL", "http://open-rl-dashboard:8000").rstrip("/") +API_SERVER = os.getenv("API_SERVER_URL", "http://open-rl-api-server-service:8000").rstrip("/") +CLIENT_IMAGE = os.getenv("CLIENT_IMAGE", "ghcr.io/gke-labs/open-rl/client:latest") +PVC = os.getenv("SHARED_PVC", "open-rl-shared-pvc") +APP_LABEL = "openrl-demo" +RUN_LABEL = "openrl.io/demo-run" +MODEL_ID = re.compile(r"(?:TrainingClient|ServiceClient) initialized for (?:model|session) ([a-f0-9-]{36})") +HERE = Path(__file__).parent + +PRESETS = { + "keynote-20": { + "steps": 4, + "jobs": [ + {"kind": "lora", "model": "google/gemma-4-e2b", "count": 5, "steps": 8}, + {"kind": "lora", "model": "Qwen/Qwen3-0.6B", "count": 5, "steps": 8}, + {"kind": "fft", "model": "google/gemma-4-e2b", "count": 5, "steps": 4}, + {"kind": "fft", "model": "Qwen/Qwen3-0.6B", "count": 5, "steps": 4}, + ], + }, + "qwen-8": { + "steps": 5, + "jobs": [ + {"kind": "lora", "model": "Qwen/Qwen3-0.6B", "count": 4, "steps": 5}, + {"kind": "fft", "model": "Qwen/Qwen3-0.6B", "count": 4, "steps": 5}, + ], + }, + "smoke-2": {"steps": 2, "jobs": [{"kind": "fft", "model": "Qwen/Qwen3-0.6B", "count": 2, "steps": 2}]}, +} + +config.load_incluster_config() +batch = client.BatchV1Api() +core = client.CoreV1Api() +custom = client.CustomObjectsApi() +app = FastAPI(title="OpenRL demo") +run_ids: dict[str, str] = {} # job name -> run id, once seen in the client's log + + +def slug(model: str) -> str: + return re.sub(r"[^a-z0-9]+", "-", model.split("/")[-1].lower()).strip("-") + + +def job_manifest(run: str, job: str, kind: str, model: str, steps: int, seed: int) -> dict: + scenario = f"{kind}-gsm8k-rl" + labels = {"app": APP_LABEL, RUN_LABEL: run, "openrl.io/demo-kind": kind, "openrl.io/demo-model": slug(model), "openrl.io/demo-seed": str(seed)} + return { + "apiVersion": "batch/v1", + "kind": "Job", + "metadata": { + "name": job, + "namespace": NAMESPACE, + "labels": labels, + "annotations": {"openrl.io/demo-steps": str(steps), "openrl.io/demo-base-model": model}, + }, + "spec": { + "backoffLimit": 0, + "ttlSecondsAfterFinished": 86400, + "template": { + "metadata": {"labels": labels}, + "spec": { + "restartPolicy": "Never", + "serviceAccountName": "open-rl-sa", + "tolerations": [{"key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule"}], + "containers": [ + { + "name": "client", + "image": CLIENT_IMAGE, + "imagePullPolicy": "IfNotPresent", + "args": [ + f"scenario={scenario}", + "uv_extra=cpu", + f"base_url={API_SERVER}", + # one directory per job directly under runs/, so the dashboard's client-directory search + # (three levels below its root, set to /mnt/shared/open-rl/runs here) finds the recipe's metrics + f"log_dir=/mnt/shared/open-rl/runs/{job}", + f"base_model={model}", + f"steps={steps}", + "group_size=8", + "groups_per_batch=8", + "max_tokens=512", + f"extra=seed={seed}", + ], + # openrl.recipe=rl makes the API server start the sampler with the trainer instead of on the first sample + # request; the e2e script merges its own openrl. settings into this variable. + "env": [ + {"name": "PYTHONUNBUFFERED", "value": "1"}, + {"name": "HF_HOME", "value": "/mnt/shared/open-rl/huggingface"}, + {"name": "TINKER_TAGS", "value": "openrl.recipe=rl"}, + ], + "resources": {"requests": {"cpu": "1", "memory": "4Gi"}, "limits": {"memory": "12Gi"}}, + "volumeMounts": [{"name": "shared-storage", "mountPath": "/mnt/shared"}], + } + ], + "volumes": [{"name": "shared-storage", "persistentVolumeClaim": {"claimName": PVC}}], + }, + }, + }, + } + + +def job_state(job) -> str: + st = job.status + if st.succeeded: + return "succeeded" + if st.failed: + return "failed" + if st.active: + return "running" + return "pending" + + +def pods_by_job() -> dict[str, object]: + pods = core.list_namespaced_pod(NAMESPACE, label_selector=f"app={APP_LABEL}").items + return {p.metadata.labels.get("job-name"): p for p in pods} + + +RUN_ID_ANNOTATION = "openrl.io/demo-run-id" + + +def find_run_id(job: str, pod, annotations: dict | None = None) -> str | None: + """The run a Job's client created: from memory, else the Job's annotation, else the client log (whole log, once), + which is then written back as an annotation so a restart of this app does not lose it.""" + if job in run_ids: + return run_ids[job] + if annotations and (found := annotations.get(RUN_ID_ANNOTATION)): + run_ids[job] = found + return found + if pod is None or pod.status.phase == "Pending": + return None + try: + text = core.read_namespaced_pod_log(pod.metadata.name, NAMESPACE) + except Exception: + return None + if m := MODEL_ID.search(text): + run_ids[job] = m.group(1) + try: + batch.patch_namespaced_job(job, NAMESPACE, {"metadata": {"annotations": {RUN_ID_ANNOTATION: m.group(1)}}}) + except Exception: + pass + return m.group(1) + return None + + +def list_jobs() -> list[dict]: + jobs = batch.list_namespaced_job(NAMESPACE, label_selector=f"app={APP_LABEL}").items + pods = pods_by_job() + out = [] + for j in sorted(jobs, key=lambda j: j.metadata.creation_timestamp, reverse=True): # newest first + labels, ann = j.metadata.labels or {}, j.metadata.annotations or {} + pod = pods.get(j.metadata.name) + out.append( + { + "name": j.metadata.name, + "run": labels.get(RUN_LABEL), + "kind": labels.get("openrl.io/demo-kind"), + "model": ann.get("openrl.io/demo-base-model"), + "seed": labels.get("openrl.io/demo-seed"), + "steps": int(ann.get("openrl.io/demo-steps", "0") or 0), + "created_at": j.metadata.creation_timestamp.timestamp(), + "started_at": j.status.start_time.timestamp() if j.status.start_time else None, + "completed_at": j.status.completion_time.timestamp() if j.status.completion_time else None, + "state": job_state(j), + "pod": pod.metadata.name if pod else None, + "pod_phase": pod.status.phase if pod else None, + "run_id": find_run_id(j.metadata.name, pod, ann), + } + ) + return out + + +async def release(run_id: str) -> bool: + async with httpx.AsyncClient(timeout=20) as http: + try: + r = await http.post(f"{API_SERVER}/api/v1/delete_model", json={"model_id": run_id}) + return r.status_code < 300 + except httpx.HTTPError: + return False + + +async def stop_job(name: str) -> dict: + run_id = run_ids.get(name) + if run_id is None: + try: + job = batch.read_namespaced_job(name, NAMESPACE) + run_id = (job.metadata.annotations or {}).get(RUN_ID_ANNOTATION) + except Exception: + run_id = None + if run_id is None: + try: + pod = pods_by_job().get(name) + run_id = await asyncio.to_thread(find_run_id, name, pod) + except Exception: + run_id = None + released = await release(run_id) if run_id else False + try: + batch.delete_namespaced_job(name, NAMESPACE, propagation_policy="Background") + deleted = True + except client.ApiException as exc: + deleted = exc.status == 404 + print(json.dumps({"event": "stop", "job": name, "run_id": run_id, "released": released, "deleted": deleted}), flush=True) + return {"name": name, "run_id": run_id, "released": released, "deleted": deleted} + + +@app.get("/", response_class=HTMLResponse) +async def index() -> str: + return (HERE / "index.html").read_text(encoding="utf-8") + + +@app.get("/api/inspect/{path:path}") +async def inspect(path: str, request: Request): + async with httpx.AsyncClient(timeout=120) as http: + try: + r = await http.get(f"{DASHBOARD}/api/v1/inspect/{path}", params=dict(request.query_params)) + except httpx.HTTPError as exc: + raise HTTPException(502, f"dashboard unreachable: {exc}") from exc + is_json = r.headers.get("content-type", "").startswith("application/json") + return JSONResponse(status_code=r.status_code, content=r.json() if is_json else {"text": r.text}) + + +def list_placements() -> list[dict]: + """Live placements from Workload status, which the scheduler stamps within seconds; the dashboard's own + placement records trail by up to 30 s because the API server sweeps Workloads on its reaper interval.""" + items = custom.list_namespaced_custom_object("openrl.io", "v1alpha1", NAMESPACE, "workloads").get("items", []) + out = [] + for w in items: + spec, st = w.get("spec", {}), w.get("status", {}) + if not st.get("nodeName") or st.get("releasedAt"): + continue + out.append( + { + "name": w["metadata"]["name"], + "role": spec.get("role"), + "owner": spec.get("modelID") or spec.get("ownerID"), + "phase": st.get("phase"), + "kind": spec.get("trainingKind"), + "exclusive": spec.get("exclusive"), + "node": st.get("nodeName"), + "pod": st.get("podName"), + "devices": st.get("gpuUUIDs") or [], + "placed_at": st.get("placedAt"), + "shared": w["metadata"]["name"].startswith("lora-"), + } + ) + return out + + +@app.get("/api/sweep/placements") +async def placements(): + return {"placements": await asyncio.to_thread(list_placements), "now": time.time()} + + +DELTA_LINE = re.compile(r"Saved sparse delta \(([0-9.]+)% changed elements, (\d+)/(\d+)\) to (\S+)") +ADAPTER_LINE = re.compile(r"Auto-saved adapter '([0-9a-f-]+)' to (\S+)") + + +@app.get("/api/sweep/deltas") +async def deltas(run_id: str, since: str = "7d", until: str | None = None, kind: str = "fft"): + """Each sparse delta the run's trainer published, from its log lines through the dashboard's logs route: + when, which sampler version, the share of weights that changed, and the counts. LoRA runs report adapter saves. + Stopgap: the trainer returns density_pct from the save but the op record keeps only result["metrics"], so the + figure is not in /runs/{id}/steps yet (see the findings report).""" + out, cursor, pages = [], None, 0 + async with httpx.AsyncClient(timeout=120) as http: + while pages < 20: + params = {"run_id": run_id, "since": since, "q": "Saved sparse delta" if kind == "fft" else "Auto-saved adapter"} + if kind == "fft": + params["pod"] = f"orw-fft-{run_id}-trainer" + if until: + params["until"] = until + if cursor: + params["cursor"] = cursor + try: + r = await http.get(f"{DASHBOARD}/api/v1/inspect/logs", params=params) + body = r.json() + except (httpx.HTTPError, ValueError) as exc: + raise HTTPException(502, f"dashboard logs unreachable: {exc}") from exc + if not body.get("available", True): + raise HTTPException(502, body.get("error") or "logs unavailable") + for rec in body.get("records") or []: + msg = rec.get("message") or "" + if m := DELTA_LINE.search(msg): + path = m.group(4) + out.append( + { + "time": rec.get("timestamp"), + "kind": "delta", + "version": path.rsplit("/", 1)[-1], + "density_pct": float(m.group(1)), + "changed": int(m.group(2)), + "total": int(m.group(3)), + "final": path.endswith("/final"), + } + ) + elif m := ADAPTER_LINE.search(msg): + if run_id.startswith(m.group(1)[:8]): + out.append({"time": rec.get("timestamp"), "kind": "adapter", "version": m.group(2).rsplit("/", 1)[-1]}) + cursor, pages = body.get("next_cursor"), pages + 1 + if not cursor: + break + out.sort(key=lambda x: x["time"] or "") + return {"run_id": run_id, "saves": out, "pages": pages} + + +@app.get("/api/sweep/presets") +async def presets(): + return PRESETS + + +@app.get("/api/sweep/jobs") +async def jobs(): + return {"jobs": await asyncio.to_thread(list_jobs), "now": time.time()} + + +@app.post("/api/sweep/start") +async def start(body: dict): + run = re.sub(r"[^a-z0-9-]", "-", str(body.get("run") or time.strftime("demo-%H%M%S")).lower())[:30].strip("-") + specs = body.get("jobs") or [] + if not specs: + raise HTTPException(400, "no jobs") + existing = {j.metadata.name for j in batch.list_namespaced_job(NAMESPACE, label_selector=f"app={APP_LABEL}").items} + created, n = [], 0 + for spec in specs: + kind, model = spec.get("kind", "fft"), spec.get("model", "Qwen/Qwen3-0.6B") + if kind not in ("lora", "fft"): + raise HTTPException(400, f"kind must be lora or fft, not {kind!r}") + steps, count, seed0 = int(spec.get("steps") or body.get("steps") or 3), int(spec.get("count") or 1), int(spec.get("seed") or 1) + for i in range(count): + n += 1 + name = f"{run}-{n:02d}-{kind}-{slug(model)}"[:63].rstrip("-") + if name in existing: + continue + manifest = job_manifest(run, name, kind, model, steps, seed0 + i) + await asyncio.to_thread(batch.create_namespaced_job, NAMESPACE, manifest) + created.append({"name": name, "kind": kind, "model": model, "steps": steps, "seed": seed0 + i}) + await asyncio.sleep(0.3) # one every third of a second, so the submit log reads as a stream + return {"run": run, "created": created} + + +@app.post("/api/sweep/stop") +async def stop(body: dict): + name = body.get("name") + if not name: + raise HTTPException(400, "name required") + return await stop_job(name) + + +@app.post("/api/sweep/stop_all") +async def stop_all(): + names = [j.metadata.name for j in batch.list_namespaced_job(NAMESPACE, label_selector=f"app={APP_LABEL}").items] + return {"stopped": [await stop_job(n) for n in names]} + + +if __name__ == "__main__": + import uvicorn + + # Access log on, so every start/stop/stop_all call is auditable in the pod log (the polling GETs are noisy but cheap). + uvicorn.run(app, host="0.0.0.0", port=int(os.getenv("PORT", "8000")), log_level="info") diff --git a/k8s/deploy/gke-use5/demo-app/demo-app.yaml b/k8s/deploy/gke-use5/demo-app/demo-app.yaml new file mode 100644 index 000000000..0e06aa8a0 --- /dev/null +++ b/k8s/deploy/gke-use5/demo-app/demo-app.yaml @@ -0,0 +1,100 @@ +# The demo app: a sweep page that starts and stops RL client Jobs, and a fleet +# view fed by the dashboard's inspection API. Runs on the API server image with +# its code mounted from a ConfigMap, so there is no image to build. It is the +# only thing in the namespace with write access to Jobs; the dashboard stays +# read-only. +apiVersion: v1 +kind: ServiceAccount +metadata: + name: open-rl-demo +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: open-rl-demo +rules: +- apiGroups: ["batch"] + resources: ["jobs"] + verbs: ["create", "delete", "get", "list", "watch", "patch"] +- apiGroups: [""] + resources: ["pods", "pods/log"] + verbs: ["get", "list"] +- apiGroups: ["openrl.io"] + resources: ["workloads"] + verbs: ["get", "list"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: open-rl-demo +subjects: +- kind: ServiceAccount + name: open-rl-demo +roleRef: + kind: Role + name: open-rl-demo + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: open-rl-demo + labels: + app: open-rl-demo +spec: + replicas: 1 + selector: + matchLabels: + app: open-rl-demo + template: + metadata: + labels: + app: open-rl-demo + spec: + serviceAccountName: open-rl-demo + containers: + - name: demo + image: ghcr.io/gke-labs/open-rl/api-server:latest + command: ["sh", "-c", "cd /app && uv run python /demo/app.py"] + ports: + - name: http + containerPort: 8000 + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: DASHBOARD_URL + value: http://open-rl-dashboard:8000 + - name: API_SERVER_URL + value: http://open-rl-api-server-service:8000 + - name: CLIENT_IMAGE + value: us-east5-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-client:pr383-pools-c0cd426 + readinessProbe: + httpGet: + path: /api/sweep/presets + port: 8000 + timeoutSeconds: 5 + resources: + requests: + cpu: 100m + memory: 256Mi + volumeMounts: + - name: code + mountPath: /demo + readOnly: true + volumes: + - name: code + configMap: + name: open-rl-demo-code +--- +apiVersion: v1 +kind: Service +metadata: + name: open-rl-demo +spec: + selector: + app: open-rl-demo + ports: + - port: 8000 + targetPort: 8000 diff --git a/k8s/deploy/gke-use5/demo-app/index.html b/k8s/deploy/gke-use5/demo-app/index.html new file mode 100644 index 000000000..2932e2971 --- /dev/null +++ b/k8s/deploy/gke-use5/demo-app/index.html @@ -0,0 +1,635 @@ + + + + + +OpenRL demo + + + +
+

OpenRL demo

+ +
connecting
+
+
+
+
+ + presets + + + + +
+
+
+
typemodelcountstepsfirst seed
+ +

Each row becomes count Jobs, seeds first seed and up. Every Job runs the GSM8K GRPO recipe against the in-cluster API server; LoRA jobs on one base model share a runtime, full fine-tuning jobs get their own trainer and sampler.

+
+
+
+

Jobs

+
sweep#jobmodeltypeseedstatestepsrunviews
+
+
+
Click a GPU tile for its timeline, a job row for the job's timeline, double-click a job row for its weights.
+

Jobs and placements from Kubernetes every second; runs and steps from the inspection API, which caches for 5 to 10 s. DCGM duty is about a minute behind.

+
+
+ + + + + Pick one. Clicking a GPU tile or a job row on the Fleet tab opens it here. +
+
+

One trace per accelerator: 1 while a worker holds the GPU, coloured by who holds it; 0 when idle. The thin band at the base is a worker that has the claim but no turn. Drag across the trace to select a window: the percentages are then computed over exactly that span; Escape or double-click returns to the live sliding window. Turns come from each worker's own record through the inspection API's activity route, fetched every 1.5 s (the API caches 5 to 10 s).

+
+
+
+ + Full fine-tuning publishes a sparse delta to the sampler after every optimizer step: only the weights that changed. This shows how much of the model that was, step by step. +
+
+

Read from the trainer's log lines through the inspection API's logs route (Cloud Logging, a few seconds behind). A LoRA job publishes adapter files instead and has no delta.

+
+

A previous run, from examples/harvey_labs: held-out reward 0.57 to 0.72 in 30 steps. Press r to replay.

+
+ + + diff --git a/k8s/deploy/gke-use5/demo-app/kustomization.yaml b/k8s/deploy/gke-use5/demo-app/kustomization.yaml new file mode 100644 index 000000000..dec10f912 --- /dev/null +++ b/k8s/deploy/gke-use5/demo-app/kustomization.yaml @@ -0,0 +1,10 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: openrl-system +resources: + - demo-app.yaml +configMapGenerator: + - name: open-rl-demo-code + files: + - app.py + - index.html diff --git a/k8s/deploy/gke-use5/kustomization.yaml b/k8s/deploy/gke-use5/kustomization.yaml new file mode 100644 index 000000000..cec8240b4 --- /dev/null +++ b/k8s/deploy/gke-use5/kustomization.yaml @@ -0,0 +1,114 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +# Trial of PR #383 (run telemetry and dashboard) on open-rl-dra-use5 +# (us-east5): the FFT bundle plus the PR's dashboard deployment, with images +# built from the rebased PR branch (try/pr-383-rebased-3) into the +# us-east5 Artifact Registry repo. Shared volume on Filestore Zonal. +# +# Render from the PR worktree, which holds k8s/deploy/dashboard: +# kubectl apply --server-side --force-conflicts -k ../open-rl-pr383/k8s/deploy/gke-use5 +# Notes: runs/2026-10-27_pr383_dashboard_use5/notes.md +# Overlay-level resources take this namespace too (dashboard-sa.yaml would +# otherwise keep the file's). +namespace: openrl-system +resources: + - ../fft + - ../dashboard + - dashboard-sa.yaml + # Scrape targets for the dashboard's GPU and sampler charts. The DCGM + # exporter itself comes with the lora bundle since #391 (k8s/deploy/dcgm); + # k8s/deploy/dashboard still ships neither of these (finding 4). + - monitoring.yaml + # The demo app (sweep page + fleet view over the inspect API). + - demo-app + +images: + - name: ghcr.io/gke-labs/open-rl/server + newName: us-east5-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server + newTag: pr383-pools-c0cd426 # try/pr-383-rebased-3: #383 + our commits rebased on upstream 163533d (#388 #389 #391 #380) + - name: ghcr.io/gke-labs/open-rl/api-server + newName: us-east5-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-api-server + newTag: pr383-pools-886bd93 # carries the dashboard's lane-occupancy fix (finding 14); server and client unchanged since c0cd426 + - name: ghcr.io/gke-labs/open-rl/scheduler + newName: us-east5-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-scheduler + newTag: pr383-844424a + +patches: + # Filestore Zonal for the shared volume (the bundle default is Basic HDD). + - target: + kind: PersistentVolumeClaim + name: open-rl-shared-pvc + patch: |- + apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: open-rl-shared-pvc + spec: + storageClassName: zonal-rwx + + # The dashboard runs as its own service account so Workload Identity can + # give it, and only it, monitoring.viewer and logging.viewer. The PR's + # manifests reuse the API server's open-rl-sa; the bindings below add the + # dedicated account as a second subject so the PR's RBAC stays untouched. + - target: + kind: Deployment + name: open-rl-dashboard + patch: |- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: open-rl-dashboard + spec: + template: + spec: + serviceAccountName: open-rl-dashboard + # Workload Identity needs the GKE metadata server; one default-pool + # node could not be replaced (us-east5-c stockout) and lacks it. + nodeSelector: + iam.gke.io/gke-metadata-server-enabled: "true" + containers: + - name: dashboard + env: + - name: OPEN_RL_CLIENT_RUNS_ROOT + value: /mnt/shared/open-rl/runs + - target: + kind: RoleBinding + name: open-rl-dashboard + patch: |- + - op: add + path: /subjects/- + value: + kind: ServiceAccount + name: open-rl-dashboard + namespace: openrl-system + - target: + kind: ClusterRoleBinding + name: open-rl-dashboard + patch: |- + - op: add + path: /subjects/- + value: + kind: ServiceAccount + name: open-rl-dashboard + namespace: openrl-system + + # Demo: one worker per GPU while GPUs are free; seats on an existing claim + # only when none can be cut. The default binpack co-locates trainer and + # sampler on one GPU with the rest idle. + - target: + kind: Deployment + name: open-rl-scheduler + patch: |- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: open-rl-scheduler + spec: + template: + spec: + containers: + - name: manager + env: + - name: OPEN_RL_PLACEMENT_STRATEGY + value: spread diff --git a/k8s/deploy/gke-use5/monitoring.yaml b/k8s/deploy/gke-use5/monitoring.yaml new file mode 100644 index 000000000..722d0b65e --- /dev/null +++ b/k8s/deploy/gke-use5/monitoring.yaml @@ -0,0 +1,30 @@ +# Managed Prometheus scrape targets the dashboard's PromQL expects: +# openrl_gpu_held from the scheduler, which joins DCGM series to runs. +apiVersion: monitoring.googleapis.com/v1 +kind: PodMonitoring +metadata: + name: open-rl-scheduler +spec: + selector: + matchLabels: + app: open-rl-scheduler + endpoints: + - port: metrics + interval: 10s +--- +# vLLM metrics from sampler pods: the PR adds a port named http (9095) to +# every sampler pod, served by the worker's own Prometheus registry. +# Worker pods carry the openrl.io/worker label; trainers have no http port, +# so only samplers match the endpoint. +apiVersion: monitoring.googleapis.com/v1 +kind: PodMonitoring +metadata: + name: open-rl-samplers +spec: + selector: + matchExpressions: + - key: openrl.io/worker + operator: Exists + endpoints: + - port: http + interval: 10s diff --git a/pyproject.toml b/pyproject.toml index 804694c58..299fe3bea 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -81,6 +81,9 @@ dev = [ where = ["src"] include = ["server*", "training*", "accel_timeslicer*"] +[tool.setuptools.package-data] +"server.dashboard" = ["static/*"] + [tool.uv] override-dependencies = [ # vLLM still carries an older transformers constraint; keep the project on diff --git a/scheduler/controller/api/v1alpha1/workload_types.go b/scheduler/controller/api/v1alpha1/workload_types.go index 57c3d8cbf..9496116a9 100644 --- a/scheduler/controller/api/v1alpha1/workload_types.go +++ b/scheduler/controller/api/v1alpha1/workload_types.go @@ -171,6 +171,20 @@ type WorkloadStatus struct { // the node is genuinely unknown. NodeName string `json:"nodeName,omitempty"` + // GPUUUIDs are the UUIDs of the claim's allocated devices, which GPU + // metrics are keyed by. + // +optional + GPUUUIDs []string `json:"gpuUUIDs,omitempty"` + + // PlacedAt is when the controller observed the allocation and set NodeName. + // +optional + PlacedAt *metav1.Time `json:"placedAt,omitempty"` + + // ReleasedAt is when the worker stopped holding its devices. Cleared by + // the next placement. + // +optional + ReleasedAt *metav1.Time `json:"releasedAt,omitempty"` + // Reason is the most recent human-readable explanation of Phase. Reason string `json:"reason,omitempty"` diff --git a/scheduler/controller/api/v1alpha1/zz_generated.deepcopy.go b/scheduler/controller/api/v1alpha1/zz_generated.deepcopy.go index ef6ccd2c7..e5c84407a 100644 --- a/scheduler/controller/api/v1alpha1/zz_generated.deepcopy.go +++ b/scheduler/controller/api/v1alpha1/zz_generated.deepcopy.go @@ -200,6 +200,19 @@ func (in *WorkloadSpec) DeepCopy() *WorkloadSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *WorkloadStatus) DeepCopyInto(out *WorkloadStatus) { *out = *in + if in.GPUUUIDs != nil { + in, out := &in.GPUUUIDs, &out.GPUUUIDs + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.PlacedAt != nil { + in, out := &in.PlacedAt, &out.PlacedAt + *out = (*in).DeepCopy() + } + if in.ReleasedAt != nil { + in, out := &in.ReleasedAt, &out.ReleasedAt + *out = (*in).DeepCopy() + } if in.Conditions != nil { in, out := &in.Conditions, &out.Conditions *out = make([]v1.Condition, len(*in)) diff --git a/scheduler/controller/go.mod b/scheduler/controller/go.mod index 1ba836367..4b5328e76 100644 --- a/scheduler/controller/go.mod +++ b/scheduler/controller/go.mod @@ -3,6 +3,7 @@ module github.com/gke-labs/open-rl/scheduler/controller go 1.26.0 require ( + github.com/prometheus/client_golang v1.23.2 k8s.io/api v0.36.3 k8s.io/apimachinery v0.36.3 k8s.io/client-go v0.36.3 @@ -32,7 +33,6 @@ require ( github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - github.com/prometheus/client_golang v1.23.2 // indirect github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.67.5 // indirect github.com/prometheus/procfs v0.19.2 // indirect diff --git a/scheduler/controller/internal/controller/fleet.go b/scheduler/controller/internal/controller/fleet.go index f805aa46b..0fd79919a 100644 --- a/scheduler/controller/internal/controller/fleet.go +++ b/scheduler/controller/internal/controller/fleet.go @@ -3,11 +3,14 @@ package controller import ( "context" "fmt" + "path" "sort" + "strings" corev1 "k8s.io/api/core/v1" resourcev1 "k8s.io/api/resource/v1" "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/log" @@ -250,6 +253,43 @@ func latestCompletePools(ctx context.Context, slices []resourcev1.ResourceSlice, return keep } +// gpuUUIDs resolves the devices allocated to a claim to the UUIDs their +// ResourceSlices publish, in allocation order. A device without a UUID +// attribute is left out; an empty result is retried on a later reconcile. +func (r *WorkloadReconciler) gpuUUIDs(ctx context.Context, claimName string) ([]string, error) { + var claim resourcev1.ResourceClaim + if err := r.Get(ctx, types.NamespacedName{Namespace: r.Namespace, Name: claimName}, &claim); err != nil { + return nil, client.IgnoreNotFound(err) + } + if claim.Status.Allocation == nil { + return nil, nil + } + var slices resourcev1.ResourceSliceList + if err := r.List(ctx, &slices); err != nil { + return nil, fmt.Errorf("list resourceslices: %w", err) + } + type deviceKey struct{ driver, pool, device string } + published := map[deviceKey]string{} + for _, i := range latestCompletePools(ctx, slices.Items, r.DeviceDriver) { + spec := slices.Items[i].Spec + for _, device := range spec.Devices { + for name, attr := range device.Attributes { + // NVIDIA's driver publishes "uuid"; match any domain and case. + if attr.StringValue != nil && strings.EqualFold(path.Base(string(name)), "uuid") { + published[deviceKey{spec.Driver, spec.Pool.Name, device.Name}] = *attr.StringValue + } + } + } + } + var uuids []string + for _, result := range claim.Status.Allocation.Devices.Results { + if uuid, ok := published[deviceKey{result.Driver, result.Pool, result.Device}]; ok { + uuids = append(uuids, uuid) + } + } + return uuids, nil +} + // claimFrom reads a managed claim back into the fleet: identity from // metadata, shape from DRA's allocation. An unallocated claim has no shape // yet -- the tiers were alternatives, and which one holds is DRA's answer. diff --git a/scheduler/controller/internal/controller/metrics.go b/scheduler/controller/internal/controller/metrics.go new file mode 100644 index 000000000..57ef4c75a --- /dev/null +++ b/scheduler/controller/internal/controller/metrics.go @@ -0,0 +1,50 @@ +package controller + +import ( + "context" + "time" + + "github.com/prometheus/client_golang/prometheus" + "sigs.k8s.io/controller-runtime/pkg/client" + + openrlv1alpha1 "github.com/gke-labs/open-rl/scheduler/controller/api/v1alpha1" +) + +// The UUID label is spelled as DCGM spells it, so GPU metrics join on it. +var gpuHeldDesc = prometheus.NewDesc( + "openrl_gpu_held", + "1 for each GPU a placed workload holds. Present from placement to release, so its presence over time is the hold interval.", + []string{"run_id", "role", "pod", "node", "UUID"}, nil, +) + +// gpuHeldCollector reads workload status at scrape time, so a released or +// deleted workload's series disappears with it. +type gpuHeldCollector struct { + reader client.Reader + namespace string +} + +func (c gpuHeldCollector) Describe(ch chan<- *prometheus.Desc) { + ch <- gpuHeldDesc +} + +func (c gpuHeldCollector) Collect(ch chan<- prometheus.Metric) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + var workers openrlv1alpha1.WorkloadList + if err := c.reader.List(ctx, &workers, client.InNamespace(c.namespace)); err != nil { + ch <- prometheus.NewInvalidMetric(gpuHeldDesc, err) + return + } + for i := range workers.Items { + worker := &workers.Items[i] + status := worker.Status + if status.NodeName == "" || status.ReleasedAt != nil { + continue + } + for _, uuid := range status.GPUUUIDs { + ch <- prometheus.MustNewConstMetric(gpuHeldDesc, prometheus.GaugeValue, 1, + worker.Spec.ModelID, string(worker.Spec.Role), status.PodName, status.NodeName, uuid) + } + } +} diff --git a/scheduler/controller/internal/controller/metrics_test.go b/scheduler/controller/internal/controller/metrics_test.go new file mode 100644 index 000000000..91eb21b8c --- /dev/null +++ b/scheduler/controller/internal/controller/metrics_test.go @@ -0,0 +1,112 @@ +package controller + +import ( + "context" + "fmt" + "slices" + "testing" + "time" + + "github.com/prometheus/client_golang/prometheus" + corev1 "k8s.io/api/core/v1" + resourcev1 "k8s.io/api/resource/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + + openrlv1alpha1 "github.com/gke-labs/open-rl/scheduler/controller/api/v1alpha1" +) + +// heldGPUs scrapes openrl_gpu_held as the metrics endpoint does, one +// "run role pod node UUID=value" string per series. +func heldGPUs(t *testing.T, r *WorkloadReconciler) []string { + t.Helper() + registry := prometheus.NewPedanticRegistry() + registry.MustRegister(gpuHeldCollector{reader: r.Client, namespace: testNamespace}) + families, err := registry.Gather() + must(t, err) + var series []string + for _, family := range families { + for _, metric := range family.GetMetric() { + l := map[string]string{} + for _, pair := range metric.GetLabel() { + l[pair.GetName()] = pair.GetValue() + } + series = append(series, fmt.Sprintf("%s %s %s %s %s=%v", l["run_id"], l["role"], l["pod"], l["node"], l["UUID"], metric.GetGauge().GetValue())) + } + } + return series +} + +func must(t *testing.T, err error) { + t.Helper() + if err != nil { + t.Fatal(err) + } +} + +// placedWorker returns a reconciler whose worker w-a holds gpu-0, and its claim. +// The observed allocation stamps the GPUs and the time, and the metric reports +// the hold. +func placedWorker(t *testing.T) (*WorkloadReconciler, string) { + t.Helper() + r := newReconciler(t, append(enabledNode(), trainerWorker("w-a", "model-a"))...) + settle(t, r, "w-a") + claimName := claimOf(t, r, "w-a") + allocateClaim(t, r, claimName) + runReconcile(t, r, "w-a") + placed := getWorker(t, r, "w-a").Status + if placed.NodeName != testNode || !slices.Equal(placed.GPUUUIDs, []string{"GPU-gpu-0"}) || placed.PlacedAt == nil || placed.ReleasedAt != nil { + t.Fatalf("status = node %q uuids %v placedAt %v releasedAt %v, want node-a, GPU-gpu-0, placed, not released", + placed.NodeName, placed.GPUUUIDs, placed.PlacedAt, placed.ReleasedAt) + } + if got, want := heldGPUs(t, r), []string{"model-a trainer orw-w-a node-a GPU-gpu-0=1"}; !slices.Equal(got, want) { + t.Fatalf("series = %v, want %v", got, want) + } + return r, claimName +} + +func expectReleased(t *testing.T, r *WorkloadReconciler) { + t.Helper() + status := getWorker(t, r, "w-a").Status + if status.Phase != openrlv1alpha1.PhasePending || status.ReleasedAt == nil || status.NodeName != "" || len(status.GPUUUIDs) != 0 { + t.Fatalf("status = phase %q node %q uuids %v releasedAt %v, want Pending, released, no node or GPUs", + status.Phase, status.NodeName, status.GPUUUIDs, status.ReleasedAt) + } + if got := heldGPUs(t, r); len(got) != 0 { + t.Fatalf("series after release = %v, want none", got) + } +} + +// A placed worker stops holding its GPUs as soon as the controller sees its +// claim abandoned or gone, whether re-placement waits on the old claim's +// deletion or finds no capacity. +func TestLostClaimReleasesBeforeRePlacement(t *testing.T) { + ctx := context.Background() + cases := map[string]func(t *testing.T, r *WorkloadReconciler, claimName string){ + "wedged claim abandoned": func(t *testing.T, r *WorkloadReconciler, _ string) { + markUnschedulable(t, r, "w-a", time.Now().Add(-2*wedgeGracePeriod)) + }, + "old claim still terminating": func(t *testing.T, r *WorkloadReconciler, claimName string) { + var claim resourcev1.ResourceClaim + must(t, r.Get(ctx, types.NamespacedName{Namespace: testNamespace, Name: claimName}, &claim)) + claim.Finalizers = append(claim.Finalizers, "test.openrl.io/hold") + must(t, r.Update(ctx, &claim)) + must(t, r.Delete(ctx, &claim)) + }, + "no capacity left": func(t *testing.T, r *WorkloadReconciler, claimName string) { + must(t, r.Delete(ctx, &resourcev1.ResourceClaim{ObjectMeta: metav1.ObjectMeta{Namespace: testNamespace, Name: claimName}})) + var node corev1.Node + must(t, r.Get(ctx, types.NamespacedName{Name: testNode}, &node)) + node.Labels[NodeLabelEnabled] = "false" + must(t, r.Update(ctx, &node)) + }, + } + for name, loseClaim := range cases { + t.Run(name, func(t *testing.T) { + r, claimName := placedWorker(t) + loseClaim(t, r, claimName) + runReconcile(t, r, "w-a") + expectReleased(t, r) + }) + } +} diff --git a/scheduler/controller/internal/controller/workload_controller.go b/scheduler/controller/internal/controller/workload_controller.go index d8dfe8856..319d29fcb 100644 --- a/scheduler/controller/internal/controller/workload_controller.go +++ b/scheduler/controller/internal/controller/workload_controller.go @@ -23,6 +23,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/metrics" "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" @@ -201,6 +202,16 @@ func (r *WorkloadReconciler) repairPlacementAssignment(ctx context.Context, scop return placementPhaseResult{}, err } scope.claimName = "" + // Record the loss now. A re-cut reuses the claim's name, and the + // later Pending or Failed patches touch neither claim nor node, + // so this is the only transition that releases the placement. + if err := r.patchStatus(ctx, worker, func(status *openrlv1alpha1.WorkloadStatus) { + status.Phase = openrlv1alpha1.PhasePending + status.ClaimName, status.AssignmentID, status.NodeName = "", "", "" + status.Reason = "ReplacingLostClaim: the assigned claim no longer exists" + }); err != nil { + return placementPhaseResult{}, err + } } } // One seat per worker. Any other seat under this name is stale, left by @@ -512,6 +523,14 @@ func (r *WorkloadReconciler) teardown(ctx context.Context, worker *openrlv1alpha return ctrl.Result{}, err } } + if err := r.patchStatus(ctx, worker, func(status *openrlv1alpha1.WorkloadStatus) { + if status.PlacedAt != nil && status.ReleasedAt == nil { + now := metav1.Now() + status.ReleasedAt = &now + } + }); err != nil { + return ctrl.Result{}, err + } controllerutil.RemoveFinalizer(worker, workerFinalizer) return ctrl.Result{}, r.Update(ctx, worker) } @@ -694,9 +713,19 @@ func (r *WorkloadReconciler) reportPod(ctx context.Context, worker *openrlv1alph node = claim.Node } } + var uuids []string + if node != "" && len(worker.Status.GPUUUIDs) == 0 { + var err error + if uuids, err = r.gpuUUIDs(ctx, claimName); err != nil { + return err + } + } return r.patchStatus(ctx, worker, func(status *openrlv1alpha1.WorkloadStatus) { status.Phase, status.ClaimName, status.PodName, status.NodeName, status.Reason = phase, claimName, podName, node, reason + if len(uuids) > 0 { + status.GPUUUIDs = uuids + } // The footprint is DRA's answer, recordable only once the // allocation is observed: which tier held, and what each device // carries under it. @@ -715,6 +744,7 @@ func (r *WorkloadReconciler) reportPod(ctx context.Context, worker *openrlv1alph func (r *WorkloadReconciler) patchStatus(ctx context.Context, worker *openrlv1alpha1.Workload, mutate func(*openrlv1alpha1.WorkloadStatus)) error { before := worker.Status.DeepCopy() mutate(&worker.Status) + trackPlacement(before, &worker.Status, metav1.Now()) worker.Status.ObservedGeneration = worker.Generation // Skip no-op writes: they would re-trigger the watch forever. if apiequality.Semantic.DeepEqual(before, &worker.Status) { @@ -727,9 +757,43 @@ func (r *WorkloadReconciler) patchStatus(ctx context.Context, worker *openrlv1al } return fmt.Errorf("patch status of %s: %w", worker.Name, err) } + logPlacement(ctx, worker, before) return nil } +// trackPlacement stamps the placement interval on every status write. A worker +// is placed when NodeName is set and released when NodeName clears or its claim +// changes. Teardown releases it explicitly, since nothing else changes there. +func trackPlacement(before, after *openrlv1alpha1.WorkloadStatus, now metav1.Time) { + held := before.NodeName != "" && before.PlacedAt != nil && before.ReleasedAt == nil + if held && (after.NodeName == "" || after.ClaimName != before.ClaimName) { + after.NodeName, after.GPUUUIDs, after.ReleasedAt = "", nil, &now + return + } + if after.NodeName != "" && (after.NodeName != before.NodeName || after.PlacedAt == nil) { + after.PlacedAt, after.ReleasedAt = &now, nil + } +} + +// logPlacement writes one JSON line per persisted bind or release. GKE turns +// it into a structured entry the diagnostics read by kind. +func logPlacement(ctx context.Context, worker *openrlv1alpha1.Workload, before *openrlv1alpha1.WorkloadStatus) { + after := &worker.Status + event, held, at := "placed", after, after.PlacedAt + switch { + case after.PlacedAt != nil && !after.PlacedAt.Equal(before.PlacedAt): + case after.ReleasedAt != nil && before.ReleasedAt == nil: + // A release may clear the node and devices, so report what was held. + event, held, at = "released", before, after.ReleasedAt + default: + return + } + log.FromContext(ctx).Info("placement "+event, + "kind", "openrl.placement", "event", event, "run_id", worker.Spec.ModelID, "role", worker.Spec.Role, + "workload", worker.Name, "pod", held.PodName, "node", held.NodeName, "uuids", held.GPUUUIDs, + "time", at.UTC().Format(time.RFC3339)) +} + func (r *WorkloadReconciler) markPending(ctx context.Context, worker *openrlv1alpha1.Workload, reason string) error { return r.patchStatus(ctx, worker, func(status *openrlv1alpha1.WorkloadStatus) { status.Phase, status.Reason = openrlv1alpha1.PhasePending, reason @@ -792,6 +856,9 @@ func managedClaims() predicate.Predicate { // SetupWithManager registers the reconciler. func (r *WorkloadReconciler) SetupWithManager(mgr ctrl.Manager) error { r.reader = mgr.GetAPIReader() + if err := metrics.Registry.Register(gpuHeldCollector{reader: mgr.GetClient(), namespace: r.Namespace}); err != nil { + return err + } // Capacity changes are fleet-wide: a freed claim might unblock any // pending worker, so these events wake every worker rather than one. diff --git a/scheduler/controller/internal/controller/workload_controller_test.go b/scheduler/controller/internal/controller/workload_controller_test.go index a9658d6db..c1d044fb5 100644 --- a/scheduler/controller/internal/controller/workload_controller_test.go +++ b/scheduler/controller/internal/controller/workload_controller_test.go @@ -79,10 +79,12 @@ func enabledNode() []client.Object { nodeName := testNode product := "NVIDIA RTX PRO 6000 Blackwell" device := func(name string) resourcev1.Device { + uuid := "GPU-" + name return resourcev1.Device{ Name: name, Attributes: map[resourcev1.QualifiedName]resourcev1.DeviceAttribute{ "productName": {StringValue: &product}, + "uuid": {StringValue: &uuid}, }, Capacity: map[resourcev1.QualifiedName]resourcev1.DeviceCapacity{ "memory": {Value: resource.MustParse("96Gi")}, diff --git a/scheduler/deploy/base/00-workload-crd.yaml b/scheduler/deploy/base/00-workload-crd.yaml index 2042ce6bb..9eb73f66d 100644 --- a/scheduler/deploy/base/00-workload-crd.yaml +++ b/scheduler/deploy/base/00-workload-crd.yaml @@ -8773,6 +8773,13 @@ spec: the pools registered. format: int32 type: integer + gpuUUIDs: + description: |- + GPUUUIDs are the UUIDs of the claim's allocated devices, which GPU + metrics are keyed by. + items: + type: string + type: array memoryPerDevice: description: |- MemoryPerDevice is what each of those devices must provide, i.e. Memory @@ -8797,6 +8804,11 @@ spec: - Succeeded - Failed type: string + placedAt: + description: PlacedAt is when the controller observed the allocation + and set NodeName. + format: date-time + type: string podName: description: PodName is the worker pod, once created. type: string @@ -8804,6 +8816,12 @@ spec: description: Reason is the most recent human-readable explanation of Phase. type: string + releasedAt: + description: |- + ReleasedAt is when the worker stopped holding its devices. Cleared by + the next placement. + format: date-time + type: string type: object required: - spec diff --git a/scripts/run_training_e2e.py b/scripts/run_training_e2e.py index 7e76d2a27..37067c904 100755 --- a/scripts/run_training_e2e.py +++ b/scripts/run_training_e2e.py @@ -359,7 +359,10 @@ def examples_env(config: RunConfig) -> dict[str, str]: weight_sync = token.split("=", 1)[1] weight_sync = config.weight_sync_strategy or weight_sync fine_tuning_type = "full" if config.scenario.startswith("fft") or "fft" in config.scenario else None - set_settings(env, fine_tuning_type=fine_tuning_type, weight_sync=weight_sync) + # An RL scenario samples from its first step, so the API server should start the sampler with the trainer + # (openrl.recipe=rl) rather than on the first sample request. + recipe = "rl" if {"rl", "textsql"} & set(config.scenario.split("-")) else None + set_settings(env, fine_tuning_type=fine_tuning_type, weight_sync=weight_sync, recipe=recipe) existing_path = env.get("PYTHONPATH", "") env["PYTHONPATH"] = f"examples:{existing_path}" if existing_path else "examples" return env diff --git a/src/server/Dockerfile.api_server b/src/server/Dockerfile.api_server index 6cd6207ce..b7eac5b8d 100644 --- a/src/server/Dockerfile.api_server +++ b/src/server/Dockerfile.api_server @@ -1,3 +1,11 @@ +FROM node:22-bookworm-slim AS dashboard-build +WORKDIR /build +COPY dev/dashboard/package*.json dev/dashboard/ +RUN npm --prefix dev/dashboard ci +COPY dev/dashboard/build.mjs dev/dashboard/ +COPY src/server/dashboard/ui src/server/dashboard/ui +RUN npm --prefix dev/dashboard run build + # Use lightweight standard Python image (No CUDA) FROM python:3.12-slim @@ -35,6 +43,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \ # Copy source code COPY . . +COPY --from=dashboard-build /build/src/server/dashboard/static/app.js src/server/dashboard/static/app.js # Expose source code via PYTHONPATH instead of modifying .venv after COPY . . ENV PYTHONPATH="/app/src:/app:$PYTHONPATH" diff --git a/src/server/api_server.py b/src/server/api_server.py index 0024cb89e..56d4f0b16 100644 --- a/src/server/api_server.py +++ b/src/server/api_server.py @@ -32,6 +32,7 @@ from server.model_metadata import ( TrainingModelMetadata, WeightSyncConfig, + display_metadata, get_model_metadata, persist_model_metadata, pool_name, @@ -40,6 +41,7 @@ from server.sampling_store import SamplingStore from server.session_registry import SessionRegistry from server.store import get_state_store, get_store +from server.telemetry import ingest_sdk_events, record, record_placements, write from server.vllm_options import split_stop from server.worker_manager import LocalWorkerManager, WorkerManager, create_worker_manager, default_metadata, owner_id, owner_of, runtime_of from training import commands @@ -70,16 +72,34 @@ async def bind_session(session_id: str | None, model_id: str) -> None: if meta is not None and session_id not in meta.session_ids: meta.session_ids.append(session_id) await persist_model_metadata(state, model_id, meta) + # A sampling session on a bare base model names no run. + if meta is not None: + session_registry.note(session_id, model_id=model_id) # On the owner, for the reaper: workers stay while any attached session lives. if worker_manager is not None: await session_registry.attach(session_id, owner) +async def sweep_placements() -> None: + """Placement records from the scheduler's Workloads, also taken before a release deletes them. A failed sweep is retried.""" + try: + workloads = await asyncio.wait_for(asyncio.to_thread(worker_manager.workloads), timeout=5) + except Exception as exc: + print(f"[API_SERVER] Placement sweep skipped: {exc!r}") + return + try: + terminations = await asyncio.wait_for(asyncio.to_thread(worker_manager.pod_terminations), timeout=5) + except Exception: # placements stand without their pods' exit reasons + terminations = {} + write(record_placements(state, workloads, terminations), timeout=SESSION_REAP_INTERVAL_SEC) + + async def reap_owner(owner: str) -> None: async with owner_locks[owner]: if await session_registry.in_use(owner): return print(f"[API_SERVER] No live session uses {owner}; tearing its workers down") + await sweep_placements() for model in await asyncio.to_thread(worker_manager.release_owner, owner): await state.delete_values(f"open_rl:trainer_ready:{model}") if get_sampler_backend() == "vllm": @@ -484,7 +504,8 @@ async def _extract_and_persist_model_metadata( lora_config = req.lora_config.model_dump() # The model's own user_metadata wins over what the session was opened with. - settings = resolve_settings(req.user_metadata or {}, await session_registry.user_metadata(req.session_id)) + session_meta = await session_registry.user_metadata(req.session_id) + settings = resolve_settings(req.user_metadata or {}, session_meta) if checkpoint_kind is not None and settings.fine_tuning_type not in (None, checkpoint_kind): raise ValueError(f"{SETTINGS_PREFIX}fine_tuning_type={settings.fine_tuning_type} does not match the {checkpoint_kind} checkpoint") fine_tuning_type = settings.fine_tuning_type or checkpoint_kind or "lora" @@ -528,6 +549,7 @@ async def _extract_and_persist_model_metadata( trainer_gpus=settings.trainer_gpus, trainer_cp=settings.trainer_cp, recipe=settings.recipe, + **display_metadata({**session_meta, **(req.user_metadata or {})}), ) if meta_obj.sampler_replicas > 1 and meta_obj.shares_gpu(): raise ValueError("openrl.sampler_replicas above 1 needs samplers on GPUs of their own; set openrl.exclusive=true") @@ -562,9 +584,8 @@ async def enqueue(command: Command) -> str: active_set_id = await _resolve_active_set_id(command.model_id) await store.put_request(commands.wire(command.model_copy(update={"trace_context": carrier})), active_set_id=active_set_id) - # One line per training request so a request that never reaches a worker can - # be traced end to end (the workers log the same id when they pop it). - print(f"[API_SERVER] enqueued op={command.op} request_id={command.request_id} model_id={command.model_id} active_set={active_set_id}") + # Its log line lets a request that never reaches a worker be traced by its id, which the worker logs when it pops it. + record(command.model_id, "enqueued", request_id=command.request_id, op=command.op, active_set=active_set_id) return command.request_id @@ -642,6 +663,8 @@ def translate_future_result(result: dict) -> dict: response["type"] = public_type_by_internal_type[result_type] if result_type == "state_saved" and isinstance(response.get("path"), str): response["path"] = tinker_state_path(response["path"]) + if result_type in ("state_saved", "sampler_weights_saved", "weights_saved"): + response.pop("metrics", None) # recorded by telemetry on the worker; not part of the SDK's save responses return response return result @@ -661,6 +684,7 @@ async def reap_dead_sessions(): while True: await asyncio.sleep(SESSION_REAP_INTERVAL_SEC) # Nothing in a sweep may end the loop. A failed call is retried next sweep. + await sweep_placements() try: owners = await session_registry.owners() except Exception: @@ -1006,6 +1030,7 @@ async def delete_model(req: ModelRequest): result = {"type": "RequestFailedResponse", "error_message": f"{model_id} was deleted"} await SamplingStore(store.redis).fail_job(meta.pool(model_id), model_id, result) if worker_manager is not None and not meta.shares_runtime(): + await sweep_placements() # placement records before the release deletes the Workloads await asyncio.to_thread(worker_manager.release, model_id) if get_sampler_backend() == "vllm": await SamplingStore(store.redis).delete_if_empty(meta.pool(model_id)) @@ -1416,5 +1441,6 @@ async def list_adapters(): @app.post("/api/v1/telemetry") -async def telemetry(_: dict): +async def telemetry(body: dict): + ingest_sdk_events(state, body) return {"status": "accepted"} diff --git a/src/server/dashboard/__main__.py b/src/server/dashboard/__main__.py new file mode 100644 index 000000000..13d0eed27 --- /dev/null +++ b/src/server/dashboard/__main__.py @@ -0,0 +1,23 @@ +"""The dashboard as its own process, without the client API's queues or worker manager: `python -m server.dashboard`.""" + +import argparse + +import uvicorn +from fastapi import FastAPI +from fastapi.middleware.gzip import GZipMiddleware +from fastapi.responses import RedirectResponse + +from server.dashboard.router import router + +app = FastAPI(title="Open-RL Inspection API") +app.add_middleware(GZipMiddleware, minimum_size=4096, compresslevel=1) # a port-forward carries large JSON slowly +app.include_router(router) +app.get("/", include_in_schema=False)(lambda: RedirectResponse("/dashboard")) + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + # Flags only: Kubernetes injects OPEN_RL_DASHBOARD_PORT=tcp://... for a Service named open-rl-dashboard. + parser.add_argument("--host", default="0.0.0.0") + parser.add_argument("--port", type=int, default=8080) + args = parser.parse_args() + uvicorn.run(app, host=args.host, port=args.port) diff --git a/src/server/dashboard/client_runs.py b/src/server/dashboard/client_runs.py new file mode 100644 index 000000000..35e37e66c --- /dev/null +++ b/src/server/dashboard/client_runs.py @@ -0,0 +1,132 @@ +"""A run's client directory: the tinker-cookbook's log_path on a volume the server also mounts, where the client writes +config.json, metrics.jsonl, checkpoints.jsonl and logs.log. The client does nothing for the server. Every read is +blocking, so callers run it in a thread.""" + +import json +import os +import re + +from server.telemetry import finite + +DEPTH = 3 # the root's run directories, such as /mnt/shared// or one level deeper +MATCH_SECONDS = 60 # config.json is written just before the client creates its model +SCAN_BYTES = 4 << 20 # of logs.log per page +MESSAGE_CHARS = 16384 +TINKER_PATH = re.compile(r'tinker://([^/"]+)/') +# The cookbook's file format, "%(name)s:%(lineno)d [%(levelname)s] %(message)s"; other lines continue the record above. +HEADER = re.compile(r"\S+:\d+ \[(DEBUG|INFO|WARNING|ERROR|CRITICAL)\] ") +listings: dict[str, tuple[int, list[str], bool]] = {} # directory -> (mtime, subdirectories, is a run directory) +parsed: dict[str, tuple[tuple[int, int], object]] = {} # file -> ((mtime, size), parsed contents) +progress: dict[str, tuple[int, int, dict[int, dict[str, float]]]] = {} # metrics.jsonl -> (inode, bytes read, metrics by step) + + +def root() -> str: + return os.getenv("OPEN_RL_CLIENT_RUNS_ROOT", "/mnt/shared") + + +def directories(path: str | None = None, depth: int = DEPTH) -> list[str]: + """Every directory under the root, to DEPTH, that holds config.json or checkpoints.jsonl. A run directory is not + descended into, and a directory is listed again only when its mtime changes.""" + path = path or root() + try: + mtime = os.stat(path).st_mtime_ns + if (listing := listings.get(path)) is None or listing[0] != mtime: + with os.scandir(path) as scan: + entries = list(scan) + run = any(e.name in ("config.json", "checkpoints.jsonl") for e in entries) + below = [] if run else [e.path for e in entries if not e.name.startswith(".") and e.name != "lost+found" and e.is_dir(follow_symlinks=False)] + listing = listings[path] = (mtime, below, run) + except OSError: # no root here, or a directory removed while it was scanned + return [] + if listing[2]: + return [path] + return [found for below in listing[1] for found in directories(below, depth - 1)] if depth > 0 else [] + + +def read(path: str, parse) -> tuple[float, object] | None: + """The file's mtime and parsed contents, read again only when its mtime or size changes. None when it is absent, as + checkpoints.jsonl is until the first save, or is not what the cookbook writes.""" + try: + stat = os.stat(path) + if (entry := parsed.get(path)) is None or entry[0] != (stat.st_mtime_ns, stat.st_size): + with open(path, encoding="utf-8", errors="replace") as f: + entry = parsed[path] = ((stat.st_mtime_ns, stat.st_size), parse(f.read())) + except (OSError, ValueError): + return None + return entry[0][0] / 1e9, entry[1] + + +def started(directory: str, run: dict) -> bool: + """Whether config.json was written within MATCH_SECONDS of the run's creation, for its base model.""" + if not (found := read(os.path.join(directory, "config.json"), json.loads)) or not isinstance(config := found[1], dict): + return False + return abs(found[0] - run["created_at"]) <= MATCH_SECONDS and config.get("model_name") == run["model"] + + +def checkpoint_runs(directory: str) -> set[str]: + found = read(os.path.join(directory, "checkpoints.jsonl"), lambda text: set(TINKER_PATH.findall(text))) + return found[1] if found else set() + + +def match(run: dict, candidates: list[str]) -> str | None: + """The run's directory: the one whose checkpoints.jsonl names tinker:///, else the one started() for it. Of + several, only one that was also started() for it is taken; otherwise none, never a guess.""" + exact = [d for d in candidates if run["run_id"] in checkpoint_runs(d)] + if len(exact) == 1: + return exact[0] + timed = [d for d in exact or candidates if started(d, run)] + return timed[0] if len(timed) == 1 else None + + +def metrics(directory: str) -> dict[int, dict[str, float]]: + """metrics.jsonl's finite numbers by each line's "step", reading only what was appended since the last call.""" + path = os.path.join(directory, "metrics.jsonl") + try: + stat = os.stat(path) + except FileNotFoundError: + return {} # no step has finished yet + inode, done, rows = progress.get(path, (stat.st_ino, 0, {})) + if inode != stat.st_ino or stat.st_size < done: # replaced or truncated + done, rows = 0, {} + with open(path, "rb") as f: + f.seek(done) + data = f.read(stat.st_size - done) + data = data[: data.rfind(b"\n") + 1] # a line still being written waits for the next read + rows = dict(rows) + for row in map(json.loads, data.splitlines()): + if isinstance(step := row.pop("step", None), int): # a call to log_metrics without a step has none + rows[step] = rows.get(step, {}) | finite(row) + progress[path] = (stat.st_ino, done + len(data), rows) + return rows + + +def log_page(directory: str, before: int | None, limit: int, q: str = "", severity: str | None = None) -> dict: + """logs.log, newest first, in the shape of a Cloud Logging page. Its lines carry no time, so a record is a header line + and the lines under it, such as a traceback, and next_cursor is the byte offset the next older page ends at.""" + path = os.path.join(directory, "logs.log") + with open(path, "rb") as f: + size = f.seek(0, os.SEEK_END) + end = size if before is None else min(before, size) + start = max(0, end - SCAN_BYTES) + f.seek(start) + block = f.read(end - start) + if start and (cut := block.find(b"\n") + 1): # the first line began before the block + block, start = block[cut:], start + cut + lines, at = [], start + for raw in block.splitlines(keepends=True): + lines.append((at, raw.decode("utf-8", "replace").rstrip("\r\n"))) + at += len(raw) + records, under, cursor = [], [], None + for at, text in reversed(lines): + under.append(text) + # The file's first line starts a record, and so does a block's first line when the block has no header at all. + if not (header := HEADER.match(text)) and at and not (at == start and cursor is None): + continue + message, under, cursor = "\n".join(reversed(under)), [], at + level = header[1] if header else "UNKNOWN" + if (not severity or level == severity) and q.lower() in message.lower(): + record = {"id": f"logs.log:{at}", "timestamp": None, "pod": "client", "role": "client", "node": None, "severity": level} + records.append(record | {"message": message[:MESSAGE_CHARS], "message_truncated": len(message) > MESSAGE_CHARS}) + if len(records) == limit: + break + return {"records": records, "next_cursor": str(cursor) if cursor else None} diff --git a/src/server/dashboard/facts.py b/src/server/dashboard/facts.py new file mode 100644 index 000000000..582d370af --- /dev/null +++ b/src/server/dashboard/facts.py @@ -0,0 +1,295 @@ +"""Facts about a run, joined from its records, with no verdicts. A block appears only when it has content, so an absent +block means nothing of its kind was recorded. A log pointer is a read against /api/v1/inspect to run as given.""" + +import asyncio +import statistics +from datetime import UTC, datetime + +from server.dashboard import fleet +from server.store import get_state_store +from server.telemetry import alive, key + +BUCKETS = {"train": {"forward_backward", "optim_step"}, "checkpoint": {"save_state", "save_weights_for_sampler"}, "forward": {"forward"}} + + +async def context(state: dict, run: dict, low: float = 0, high: float = float("inf")) -> dict: + """What the facts read about one run: its ops in [low, high], its workers, request and sample counters and queue.""" + store, run_id, finished = get_state_store(), run["run_id"], run["lifecycle"] == "ended" + ops, counts, queued, samples_queued, sampled, sampled_at = await asyncio.gather( + fleet.operations(run_id), + store.read_counts(key("ledger", run_id)), + store.list_length(request_queue(run_id)), + queued_samples(store, run, finished), + store.get_value(key("sampled", run_id)), + store.get_value(key("sampled_at", run_id)), + ) + ops = [s for s in ops if low <= s["at"] <= high] + trainer = [s for s in ops if s["operation"] != "sample"] # a single-process trainer answers samples too + ctx = { + "run": run, + "ops": ops, + "trainer": trainer, + "steps": [s for s in trainer if s["operation"] == "optim_step" and s.get("status") == "succeeded"], + } + ctx.update(ledger=counts, queued=queued, samples_queued=samples_queued, now=state["observed_at"], finished=finished, window=(low, high)) + ctx.update(sampled=int(sampled) if sampled else None, sampled_at=float(sampled_at) if sampled_at else None) + ctx.update(lanes=fleet.lanes(state, run_id), workers=workers_of(state, run)) + return ctx + + +def request_queue(run_id: str) -> str: + return f"open_rl:queue:{run_id}" # RedisStore's names for the queues it pushes to + + +def sampler_queue(runtime_id: str) -> str: + return f"open_rl:sampler_queue:{runtime_id}" + + +def workers_of(state: dict, run: dict) -> list[dict]: + return [w for w in state["workers"] if w.get("runtime_id") in (run["runtime_id"], run["run_id"])] + + +async def queued_samples(store, run: dict, finished: bool) -> int | None: + # Samples wait in the runtime's queue, which co-tenant runs share; a finished run's share of it is gone. + return None if finished else await store.list_length(sampler_queue(run["runtime_id"])) + + +def read(route: str, **args) -> dict: + return {"read": route, "args": {k: v for k, v in args.items() if v is not None}} + + +def iso(at: float) -> str: + return datetime.fromtimestamp(at, UTC).isoformat(timespec="seconds") + + +def present(**blocks) -> dict: + return {k: v for k, v in blocks.items() if v not in (None, [], {})} + + +def unanswered(ctx: dict) -> list[dict]: + """Trainer requests the client waits on, with the pod holding each and whether its worker still reports.""" + found = [] + for w in ctx["workers"]: + for g in (g for g in w.get("outstanding") or [] if g.get("run_id") == ctx["run"]["run_id"] and g["op"] != "sample"): + found.append({"op": g["op"], "count": g["count"], "age_s": round(ctx["now"] - g["since"]), "pod": w.get("pod") or w["id"]}) + found[-1]["holder_alive"] = alive(w, ctx["now"]) + return found + + +def failures(ctx: dict) -> dict | None: + if not (failed := [s for s in ctx["trainer"] if s.get("status") == "failed"]): + return None + ops = [{"operation": s["operation"], "error_type": s.get("error_type"), "at": iso(s["at"]), "pod": s.get("pod")} for s in reversed(failed[-5:])] + last, run_id = failed[-1], ctx["run"]["run_id"] + logs = read("logs", run_id=run_id, pod=last.get("pod"), q=last.get("error_type"), since=iso(last["started_at"] - 60), until=iso(last["at"] + 60)) + return {"count": len(failed), "ops": ops, "logs": logs} + + +def relaunches(ctx: dict) -> list[dict]: + """Worker processes that replaced an earlier one in the same pod during this run: a container restarted in place, or a + new pod after one died holding requests. A clean exit, or a shared runtime's idle pod recreated, is not a crash.""" + by_pod: dict[str, list[dict]] = {} + low, high = (ctx["run"]["created_at"] or 0) - 60, ctx["run"].get("completed_at") or float("inf") + for w in sorted(ctx["workers"], key=lambda w: w["started_at"]): + if w.get("pod") and low <= w["started_at"] <= high: + by_pod.setdefault(w["pod"], []).append(w) + found = [ + {"pod": b["pod"], "stopped_at": a.get("heartbeat_at", b["started_at"]), "started_at": b["started_at"]} + for p in by_pod.values() + for a, b in zip(p, p[1:]) + if a.get("phase") != "exited" and (a.get("pod_uid") == b.get("pod_uid") or any(g["op"] != "sample" for g in a.get("outstanding") or [])) + ] + return sorted(found, key=lambda p: p["started_at"]) + + +def saves(ctx: dict) -> dict | None: + if not (found := [s for s in ctx["trainer"] if s["operation"] == "save_state"]) and not ctx["steps"]: + return None + last = max((s["at"] for s in found if s.get("status") == "succeeded"), default=None) + return present( + count=len(found), + failed=sum(s.get("status") == "failed" for s in found), + last_at=iso(last) if last else None, + steps_since_last=sum(s["at"] > (last or 0) for s in ctx["steps"]), + slowest_s=round(max(s.get("elapsed_seconds", 0) for s in found)) if found else None, + ) + + +def sends(ctx: dict) -> int: + """The samples the run sent. A run recorded before the sample count kept a record per send.""" + return ctx["sampled"] if ctx["sampled"] is not None else sum(s["operation"] == "sample" for s in ctx["ops"]) + + +def answered(ctx: dict) -> list[dict]: + """The answered samples that kept a record: a torch trainer's, or a group a vLLM sampler served, timed as a whole.""" + return [s for s in ctx["ops"] if s["operation"] == "sample" and s.get("status") == "succeeded"] + + +def samplers(ctx: dict) -> dict | None: + """Sends and failures from the run's own count and records, and the median time of the groups its samplers served.""" + if not (sent := sends(ctx)): + return None + records = [s for s in ctx["ops"] if s["operation"] == "sample"] + found = { + "sends": sent, + "cancelled": sum(s.get("status") == "cancelled" for s in records), + "failed": sum(s.get("status") == "failed" for s in records), + } + if recorded := answered(ctx): + found["median_s"] = round(statistics.median(s.get("elapsed_seconds", 0) for s in recorded), 2) + return found + + +def config(ctx: dict) -> list[dict] | str | None: + """Workers whose echoed settings differ from what the run asked for.""" + if not (echoed := [w for w in ctx["workers"] if w.get("config")]): + return "no echo" + settings, found = ctx["run"]["settings"], [] + for w in echoed: + asked, running = settings.get("openrl.fine_tuning_type"), w["config"].get("env", {}).get("OPEN_RL_FINE_TUNING_TYPE") + if asked and running and asked != running: + found.append({"pod": w.get("pod"), "setting": "openrl.fine_tuning_type", "asked": asked, "running": running}) + # gpus is None when the worker could not count them. + gpus, wanted = w.get("gpus"), settings.get("openrl.trainer_gpus") + if w.get("role") == "trainer" and wanted and gpus is not None and len(gpus) != int(wanted): + found.append({"pod": w.get("pod"), "setting": "openrl.trainer_gpus", "asked": wanted, "running": len(gpus)}) + return found or None + + +def trainer_span(ctx: dict) -> tuple[float, float] | None: + """From the run's first trainer op to its last once the run has ended, else to now.""" + if not (ops := ctx["trainer"]): + return None + start = ops[0]["started_at"] + return start, start + max((ops[-1]["at"] if ctx["finished"] else ctx["now"]) - start, 1.0) + + +def goodput(ctx: dict) -> dict | None: + """Share of the run's span the trainer spent training, and where the rest went (Google ML goodput).""" + if not (span := trainer_span(ctx)): + return None + ops, (start, end) = ctx["trainer"], span + wall = end - start + spent = {kind: sum(s.get("elapsed_seconds", 0) for s in ops if s["operation"] in names) for kind, names in BUCKETS.items()} + idle = max(0.0, wall - sum(spent.values())) + split = {"other": round(idle)} + # Trainer idle time with a sample in flight is rollout; with none the server waits on the client. + if running := [(s["started_at"], s["at"]) for s in answered(ctx)]: + busy = [(s["started_at"], s["at"]) for s in ops] + rollout = min(idle, max(0.0, covered(busy + running, start, end) - covered(busy, start, end))) + split = {"rollout": round(rollout), "client": round(idle - rollout)} + first = round(ctx["steps"][0]["at"] - start) if ctx["steps"] else None + return { + "goodput": round(spent["train"] / wall, 3), + "seconds": {"wall": round(wall), **{k: round(v) for k, v in spent.items()}, **split}, + "first_step_after_s": first, + } + + +def covered(spans: list, low: float, high: float) -> float: + """Seconds of [low, high] inside at least one of the spans.""" + return sum(b - a for a, b in fleet.merge([(max(a, low), min(b, high)) for a, b in spans if b > low and a < high])) + + +def last_op_at(ctx: dict) -> float | None: + """When the run last did anything in the context's window: its last recorded op, or its last sample.""" + low, high = ctx["window"] + sampled = [at for at in [ctx["sampled_at"]] if at is not None and low <= at <= high] + return max([s["at"] for s in ctx["ops"][-1:]] + sampled, default=None) + + +def summary(ctx: dict) -> dict: + steps, ops, run, ledger = ctx["steps"], ctx["trainer"], ctx["run"], dict(ctx["ledger"]) + gaps = [b["at"] - a["at"] for a, b in zip(steps, steps[1:])] + if "enqueued" in ledger and ctx["queued"] is not None: + ledger["lost"] = ledger["enqueued"] - ledger.get("popped", 0) - ctx["queued"] + poisoned = [s for s in ops if s.get("nonfinite")] + first = present(operation=poisoned[0]["operation"], step=poisoned[0].get("step"), metrics=poisoned[0]["nonfinite"]) if poisoned else None + norms = [float(f"{s['metrics']['grad_norm:mean']:.4g}") for s in steps if "grad_norm:mean" in s.get("metrics", {})] + restarts = None + if relaunched := relaunches(ctx): + p = relaunched[0] + logs = read("logs", run_id=run["run_id"], pod=p["pod"], since=iso(p["stopped_at"] - 120), until=iso(p["started_at"] + 60)) + restarts = { + "workers": [{"pod": p["pod"], "stopped_reporting_at": iso(p["stopped_at"]), "restarted_at": iso(p["started_at"])} for p in relaunched], + "logs": logs, + # Who created or deleted the run's Workloads and pods around the relaunches, from the cluster's audit log. + "cluster_changes": ctx.get("changes"), + } + unplaced = [ + {"name": w["name"], "role": w["role"], "age_s": round(w["age_seconds"] or 0), "message": w["message"]} for w in run["workloads"] if not w["node"] + ] + # A live pod's restarts, and the terminations placement records kept after their pods were gone. + kept = [ + {"at": t["at"], "pod": lane["pod"], "reason": t.get("reason") or "Terminated", "message": exit_text(t)} + for lane in ctx["lanes"] + for t in lane.get("terminations") or [] + ] + found = {(i["pod"], round(i["at"])): i for i in [*kept, *(i for pod in run["pods"] for i in pod["incidents"])]} + incidents = sorted(found.values(), key=lambda i: -i["at"])[:5] + return present( + lifecycle=run["lifecycle"], + steps=len(steps), + step_seconds={"median": round(statistics.median(gaps), 1), "last": round(gaps[-1], 1)} if gaps else None, + since_last_step_s=round(ctx["now"] - steps[-1]["at"]) if steps and not ctx["finished"] else None, + since_last_op_s=max(0, round(ctx["now"] - last)) if (last := last_op_at(ctx)) is not None and not ctx["finished"] else None, + trainer_idle_s=max(0, round(ctx["now"] - ops[-1]["at"])) if ops and not ctx["finished"] else None, + last_op={"operation": ops[-1]["operation"], "status": ops[-1].get("status"), "at": iso(ops[-1]["at"])} if ops else None, + last_metrics={k: float(f"{v:.6g}") for s in ops[-50:] for k, v in s.get("metrics", {}).items()}, + goodput=goodput(ctx), + requests=present(unanswered=unanswered(ctx), ledger=ledger, samples_queued=ctx["samples_queued"] or None), + failures=failures(ctx), + relaunches=restarts, + saves=saves(ctx), + optimizer=present(nonfinite={"count": len(poisoned), "first": first} if poisoned else None, grad_norm_last=norms[-5:]), + samplers=samplers(ctx), + config=config(ctx), + placement=present(recorded=len(ctx["lanes"]), unplaced=unplaced) or None, + incidents=[present(at=iso(i["at"]), pod=i["pod"], reason=i["reason"], message=(i.get("message") or "")[:200]) for i in incidents], + ) + + +def exit_text(termination: dict) -> str: + signal = f", signal {termination['signal']}" if termination.get("signal") else "" + return f"{termination.get('container') or 'container'} exited with code {termination.get('exit_code')}{signal}" + + +def line(ctx: dict) -> dict: + """A run in one line from its full context, for a session over the session's span.""" + failures = sum(s.get("status") == "failed" for s in ctx["trainer"]) + return one_line(ctx, len(ctx["steps"]), [s["at"] for s in ctx["steps"]], last_op_at(ctx), failures) + + +async def fleet_line(state: dict, run: dict) -> dict: + """A run in one line for /runs from a few small records, never its command history. A run recorded before step times + and the failure count existed leaves those null.""" + store, run_id, finished = get_state_store(), run["run_id"], run["lifecycle"] == "ended" + counts, last_ops, last_samples, times, samples_queued, sampled_at = await asyncio.gather( + store.read_counts(key("ledger", run_id)), + store.read_samples(key("operations", run_id), last=1), + store.read_samples(key("operations", run_id, "samples"), last=1), + store.read_samples(key("step_times", run_id)), + queued_samples(store, run, finished), + store.get_value(key("sampled_at", run_id)), + ) + ctx = {"run": run, "workers": workers_of(state, run), "now": state["observed_at"], "samples_queued": samples_queued} + last = max([r["at"] for r in last_ops + last_samples] + ([float(sampled_at)] if sampled_at else []), default=None) + return one_line(ctx, run["steps"], times, last, counts.get("failed", 0) if counts else None) + + +def one_line(ctx: dict, steps: int, step_times: list[float], last_op_at: float | None, failures: int | None) -> dict: + """A run in one line, enough to see which run to open. ctx holds the run, its workers, now and its queued samples.""" + held, now = unanswered(ctx), ctx["now"] + return { + "lifecycle": ctx["run"]["lifecycle"], + "steps": steps, + # The snapshot time lags records written after it, so a fresh op can look a few seconds in the future. + "since_last_op_s": max(0, round(now - last_op_at)) if last_op_at is not None else None, + "since_last_step_s": round(now - step_times[-1]) if step_times else None, + "median_step_s": round(statistics.median(b - a for a, b in zip(step_times, step_times[1:]))) if len(step_times) > 1 else None, + "trainer_requests_unanswered": sum(o["count"] for o in held), + "held_by_dead_worker": sum(o["count"] for o in held if not o["holder_alive"]), + "samples_queued": ctx["samples_queued"], + "failures": failures, + "relaunches": len(relaunches(ctx)), + } diff --git a/src/server/dashboard/fleet.py b/src/server/dashboard/fleet.py index 8c5a37f3f..4e7f137b6 100644 --- a/src/server/dashboard/fleet.py +++ b/src/server/dashboard/fleet.py @@ -1,27 +1,20 @@ +"""The one view every page and fact reads: runs joined to their pods and workloads, worker and placement records, and nodes.""" + import asyncio import json import time -from dataclasses import dataclass -from datetime import UTC, datetime -from typing import Literal +from collections import OrderedDict from server import openrl_status +from server.dashboard import sources from server.model_metadata import decode_model_metadata from server.store import get_state_store +from server.telemetry import key +from server.worker_manager import STOPPED, WAITING_FOR_PLACEMENT, owner_id -CACHE_SECONDS, STALE_SECONDS, READ_TIMEOUT_SECONDS = 5, 60, 6 -Lifecycle = Literal["ended", "active"] - - -@dataclass -class Entry: - at: float = 0.0 - task: asyncio.Future | None = None - last_good: asyncio.Future | None = None - last_good_at: float = 0.0 - - -cache: dict[str, Entry] = {} +CACHE_SECONDS, OPS_SECONDS, CACHE_ENTRIES = 5, 10, 32 +cache: OrderedDict[str, dict] = OrderedDict() # least recently used first +STALE_SECONDS = 60 def succeeded(task: asyncio.Future | None) -> bool: @@ -29,52 +22,177 @@ def succeeded(task: asyncio.Future | None) -> bool: async def cached(name: str, seconds: float, read) -> object: - entry = cache.setdefault(name, Entry()) - task = entry.task + """One read serves every caller. Once it is older than `seconds`, or failed, a fresh read starts and callers get the last + good result until it lands, unless that is far older than the window. Only the CACHE_ENTRIES most recently used are kept.""" + entry = cache.setdefault(name, {"at": 0.0, "task": None, "good": None, "good_at": 0.0}) + cache.move_to_end(name) + while len(cache) > CACHE_ENTRIES: + cache.popitem(last=False) + task = entry["task"] if succeeded(task): - entry.last_good, entry.last_good_at = task, entry.at - if task is None or (task.done() and (not succeeded(task) or time.monotonic() - entry.at > seconds)): - entry.at, entry.task = time.monotonic(), asyncio.ensure_future(read()) - entry.task.add_done_callback(lambda t: t.cancelled() or t.exception()) # retrieved, so asyncio does not log it - if succeeded(entry.task): - return entry.task.result() - if entry.last_good is not None and time.monotonic() - entry.last_good_at <= STALE_SECONDS: - return entry.last_good.result() - return await asyncio.shield(entry.task) + entry["good"], entry["good_at"] = task, entry["at"] + if task is None or (task.done() and (not succeeded(task) or time.monotonic() - entry["at"] > seconds)): + entry["at"], entry["task"] = time.monotonic(), asyncio.ensure_future(read()) + entry["task"].add_done_callback(lambda t: t.cancelled() or t.exception()) # retrieved, so asyncio does not log it + if succeeded(entry["task"]): + return entry["task"].result() + # A result much older than its window is no answer: after an idle spell the caller waits for the fresh read. + if entry["good"] is not None and time.monotonic() - entry["good_at"] <= max(STALE_SECONDS, 6 * seconds): + return entry["good"].result() + return await asyncio.shield(entry["task"]) async def snapshot() -> dict: - return await cached("inventory", CACHE_SECONDS, read) + return await cached("fleet", CACHE_SECONDS, read) + + +async def operations(run_id: str) -> list[dict]: + """A run's command records, trainer ops and failed samples, oldest first.""" + + async def read(): + store = get_state_store() + ops, samples = await asyncio.gather(store.read_samples(key("operations", run_id)), store.read_samples(key("operations", run_id, "samples"))) + return sorted(ops + samples, key=lambda r: r["at"]) + + return await cached(run_id, OPS_SECONDS, read) + + +async def node_gpus() -> dict[str, list[dict]]: + """Each node's GPUs now, from DCGM: its UUID and slot, in slot order.""" + found: dict[str, list[dict]] = {} + now = time.time() + for row in await sources.prom(f'max by (Hostname, UUID, gpu) (DCGM_FI_DEV_GPU_UTIL{{UUID!=""{sources.scope()}}})', now, now): + found.setdefault(row["metric"].get("Hostname"), []).append({"id": row["metric"]["UUID"], "name": f"gpu-{row['metric'].get('gpu')}"}) + return {node: sorted(gpus, key=lambda d: (len(d["name"]), d["name"])) for node, gpus in found.items()} -def lifecycle(status: str, live: bool) -> Lifecycle: - return "active" if status == "active" and live else "ended" +def lifecycle(run: dict, live: bool, workloads: list | None) -> str: + """ended: finished, or nothing serves it and no session is live. waiting: a worker is not placed yet. + placed: every worker is placed. active: the scheduler cannot be read, so placement is unknown.""" + states = {w["state"] for w in run["workloads"]} + if run["status"] != "active" or (states and states <= {STOPPED}) or not (states or live): + return "ended" + return "active" if workloads is None else "waiting" if not states or WAITING_FOR_PLACEMENT in states else "placed" -def join(run_id: str, raw: str, live: set[str]) -> dict: +def join(run_id: str, raw: str, steps: dict, live: set, cluster: dict) -> dict: + """A run is its model. LoRA runs may share a base-model runtime; an FFT or exclusive run's runtime is the run itself.""" row, meta = json.loads(raw), decode_model_metadata(raw) - name = row.get("name") or row.get("run_name") - run = {"run_id": run_id, "name": name or f"{(meta.base_model or 'Run').split('/')[-1]} · {run_id[:8]}", "display_name": name} - run.update(model=meta.base_model, recipe_name=row.get("recipe_name"), fine_tuning_type=meta.fine_tuning_type, client_status=meta.status) - run["lifecycle"] = lifecycle(meta.status, any(s in live for s in meta.session_ids)) - run.update(created_at=meta.created_at, completed_at=meta.completed_at, session_ids=meta.session_ids, settings=openrl_status.settings_of(meta)) - return run | {"shared_runtime": meta.runtime(run_id) != run_id} + runtime, kind = meta.runtime(run_id), "lora" if meta.fine_tuning_type == "lora" else "fft" + shared = runtime != run_id + # A finished run on a shared runtime lets go of it; the runtime's pods and workloads now serve others. + keep = meta.status == "active" or not shared + owner = owner_id(runtime).rstrip("-") + pods = [p for p in cluster["pods"] if keep and p["owner"] == owner] + workloads = [w for w in cluster["workloads"] or [] if keep and w["model_id"] == runtime and w["training_kind"] == kind] + run = {"run_id": run_id, "name": row.get("name") or row.get("run_name") or f"{(meta.base_model or 'Run').split('/')[-1]} · {run_id[:8]}"} + run.update(model=meta.base_model, display_name=row.get("name") or row.get("run_name"), recipe_name=row.get("recipe_name")) + run.update(fine_tuning_type=meta.fine_tuning_type, status=meta.status, created_at=meta.created_at, completed_at=meta.completed_at) + run.update(steps=int(steps.get(key("steps", run_id), 0)), settings=openrl_status.settings_of(meta), session_ids=meta.session_ids) + run.update(runtime_id=runtime, shared_runtime=shared, pods=pods, workloads=workloads, nodes=sorted({p["node"] for p in pods if p["node"]})) + run["lifecycle"] = lifecycle(run, any(s in live for s in meta.session_ids), cluster["workloads"]) + return run async def read() -> dict: - store, observed_at = get_state_store(), datetime.now(UTC).isoformat(timespec="seconds") - scans = [asyncio.wait_for(store.scan_values(prefix), READ_TIMEOUT_SECONDS) for prefix in ("open_rl:model_meta:", "open_rl:session:")] - metas, sessions = await asyncio.gather(*scans, return_exceptions=True) - errors, runs = [], [] - if failed := next((r for r in (metas, sessions) if isinstance(r, Exception)), None): - errors.append(f"Redis read failed: {f'no reply in {READ_TIMEOUT_SECONDS} s' if isinstance(failed, TimeoutError) else failed}") - metas, sessions = {}, {} - live = {name.removeprefix("open_rl:session:") for name in sessions} - for name, raw in metas.items(): - run_id = name.removeprefix("open_rl:model_meta:") - try: - runs.append(join(run_id, raw, live)) - except Exception as exc: - errors.append(f"model record {run_id} unreadable: {exc}") - runs.sort(key=lambda r: -r["created_at"]) - return {"observed_at": observed_at, "errors": errors, "runs": runs} + store, now = get_state_store(), time.time() + cluster, gpus = await asyncio.gather(sources.kubernetes(), node_gpus(), return_exceptions=True) + if isinstance(gpus, Exception): + gpus, cluster["errors"] = {}, [*cluster["errors"], f"GPU inventory unavailable: {gpus}"] + metas, steps, workers, live = await asyncio.gather(*(store.scan_values(key(kind, "")) for kind in ("model_meta", "steps", "worker", "session"))) + live = {name.removeprefix(key("session", "")) for name in live} + runs = sorted( + (join(name.removeprefix(key("model_meta", "")), raw, steps, live, cluster) for name, raw in metas.items()), key=lambda r: -r["created_at"] + ) + placements = await asyncio.gather(*(store.read_samples(key("placements", runtime)) for runtime in {r["runtime_id"] for r in runs})) + + async def last_op(run): # when a run that never said it finished last did anything + run["last_completed_at"] = max((s["at"] for s in await store.read_samples(key("operations", run["run_id"]), last=1)), default=None) + + await asyncio.gather(*(last_op(run) for run in runs if run["status"] == "active")) + nodes = [{**n, "devices": gpus.get(n["name"], []), "gpu_capacity": max(n["gpu_capacity"], len(gpus.get(n["name"], [])))} for n in cluster["nodes"]] + fleet = {"observed_at": now, "runs": runs, "workers": [json.loads(w) for w in workers.values()], "placements": [p for ps in placements for p in ps]} + return fleet | {"nodes": nodes, "pods": cluster["pods"], "workloads": cluster["workloads"], "errors": cluster["errors"]} + + +def lanes(fleet: dict, run_id: str | None = None, node_id: str | None = None) -> list[dict]: + """One lane per placement record, with its worker process's phase and GPU turns. A lane on a shared runtime + becomes one lane per run, which shows only that run's ops.""" + found = [] + for record in fleet["placements"]: + start, end = record["placed_at"], record["released_at"] or fleet["observed_at"] + members = [r for r in fleet["runs"] if r["runtime_id"] == record["model_id"] and r["created_at"] <= end and (r["completed_at"] or end) >= start] + # Pod names repeat across placements of one Workload; the process ran inside this placement's span. + procs = [p for p in fleet["workers"] if record["pod"] and p.get("pod") == record["pod"] and p["started_at"] <= end and p["heartbeat_at"] >= start] + ids = [r["run_id"] for r in members] + lane = {"id": f"{record['workload']}@{record['placed_at']:.0f}", "name": record["workload"], "run_ids": ids, "runtime_id": record["model_id"]} + lane.update(label=members[0]["name"] if len(members) == 1 else record["model_id"], shared_runtime=record["model_id"] not in ids if ids else None) + lane.update(role=record["role"], node=record["node"] or "", pod=record["pod"], devices=record["gpu_uuids"], device_count=len(record["gpu_uuids"])) + lane.update(phase=max(procs, key=lambda p: p["started_at"])["phase"] if procs else None, turns=[t for p in procs for t in p.get("turns") or []]) + lane.update(waits=[w for p in procs for w in p.get("waits") or []], terminations=record.get("terminations") or []) + lane.update(start=start, end=end, ended=record["released_at"] is not None) + for member in members if lane["shared_runtime"] else [None]: + if member is None: + found.append(lane) + else: + label = f"{(member['model'] or lane['label']).split('/')[-1]} · {member['run_id'][:8]}" + found.append({**lane, "id": f"{lane['id']}:{member['run_id']}", "allocation_id": lane["id"], "run_ids": [member["run_id"]], "label": label}) + return [p for p in found if (not run_id or run_id in p["run_ids"]) and (not node_id or p["node"] == node_id)] + + +def merge(spans: list) -> list: + result = [] + for start, end in sorted(spans): + if result and start <= result[-1][1]: + result[-1][1] = max(end, result[-1][1]) + else: + result.append([start, end]) + return result + + +async def activity(fleet: dict, low: float, high: float, run_id: str | None = None, node_id: str | None = None) -> list[dict]: + """Each lane in the window with its GPU turns and waits for them (none on a shared lane) and the ops it ran, clipped to it. A sampler lane's + samples are the groups it served and its failed sends. Never utilization.""" + shown = [p for p in lanes(fleet, run_id, node_id) if p["start"] <= high and p["end"] >= low] + wanted = sorted({n for lane in shown for n in lane["run_ids"]}) + records = dict(zip(wanted, await asyncio.gather(*(operations(n) for n in wanted)))) + for lane in shown: + lo, hi, turns, waits = max(low, lane["start"]), min(high, lane["end"]), lane.pop("turns"), lane.pop("waits") + turns = [] if lane["shared_runtime"] else [[max(lo, a), min(hi, b)] for a, b in turns if a < hi and b > lo] + # A grant inside half a second is the slicer's own round trip, not a wait for another holder. + waits = [] if lane["shared_runtime"] else [[max(lo, a), min(hi, b)] for a, b in waits if a < hi and b > lo and b - a >= 0.5] + ops: dict[str, list] = {} + for r in (r for n in lane["run_ids"] for r in records[n]): + start = max(lo, r.get("started_at") or r["at"] - r.get("elapsed_seconds", 0)) + # A sample lands on the replica pod that answered it. + if r.get("role") == lane["role"] and r.get("pod") in (None, lane["pod"]) and start < min(hi, r["at"]): + ops.setdefault(r["operation"], []).append([start, min(hi, r["at"])]) + # A lane with turns was held only during them: an offloaded trainer saves off the GPU, so its ops alone do not mean occupancy. + occupied = turns or [s for group in ops.values() for s in group] + lane.update(ops={n: merge(s) for n, s in ops.items()}, exact=bool(turns), intervals=merge(occupied)) + lane["waits"] = merge(waits) + return shown + + +def steps(ops: list[dict]) -> list[dict]: + """Trainer ops grouped by the step they belong to: the step's window, and per op its count, time and metric spread.""" + found: dict[int, dict] = {} + completed = 0 # records from before ops carried their step count optim_steps instead + for s in (s for s in ops if s["operation"] != "sample"): + number = s["step"] if isinstance(s.get("step"), int) else completed + completed += s["operation"] == "optim_step" and s.get("status") == "succeeded" + step = found.setdefault(number, {"step": number, "started_at": s["started_at"], "ended_at": s["at"], "ops": {}, "nonfinite": set()}) + step["started_at"], step["ended_at"] = min(step["started_at"], s["started_at"]), max(step["ended_at"], s["at"]) + op = step["ops"].setdefault(s["operation"], {"count": 0, "failed": 0, "seconds": 0.0, "metrics": {}}) + op["count"] += 1 + op["failed"] += s.get("status") != "succeeded" + op["seconds"] += s.get("elapsed_seconds", 0) + for name, value in s.get("metrics", {}).items(): + op["metrics"].setdefault(name, []).append(value) + step["nonfinite"].update(s.get("nonfinite", ())) + for step in found.values(): + optim = step["ops"].get("optim_step", {"count": 0, "failed": 0}) + step.update(nonfinite=sorted(step["nonfinite"]), complete=optim["count"] > optim["failed"]) + for op in step["ops"].values(): + op["metrics"] = {name: {"min": min(v), "max": max(v), "mean": sum(v) / op["count"]} for name, v in op["metrics"].items()} + return [found[k] for k in sorted(found)] diff --git a/src/server/dashboard/router.py b/src/server/dashboard/router.py index 35fdc7397..753010d70 100644 --- a/src/server/dashboard/router.py +++ b/src/server/dashboard/router.py @@ -1,13 +1,370 @@ -from fastapi import APIRouter +"""The operator UI and the same read-only JSON API for agents, under /api/v1/inspect.""" -from server.dashboard import fleet +import asyncio +import json +import math +from pathlib import Path +from string import Template +from typing import Annotated + +import httpx +from fastapi import APIRouter, HTTPException, Query +from fastapi.responses import FileResponse +from pydantic import BaseModel + +from server.dashboard import client_runs, facts, fleet, sources +from server.store import get_state_store +from server.telemetry import finite, key router = APIRouter(tags=["inspection"]) +STATIC = Path(__file__).parent / "static" +BOUNDS = ( + "ISO-8601, e.g. 2026-01-02T12:00:00Z, or Unix seconds; since also takes a span before until, e.g. 1h. " + "Default: the 30 minutes ending now; at most 7 days." +) +Bound = Annotated[str | None, Query(max_length=64, description=BOUNDS)] + + +class Doc(BaseModel, extra="allow", allow_inf_nan=False): + """Declared fields are always present; the rest pass through as recorded. Log messages are untrusted output.""" + + +@router.get("/dashboard", include_in_schema=False) +@router.get("/dashboard/", include_in_schema=False) +@router.get("/dashboard/assets/{name}", include_in_schema=False) +async def dashboard(name: str = "index.html"): + if (path := (STATIC / name).resolve()).parent != STATIC.resolve() or not path.is_file(): + raise HTTPException(404, "Not found") + return FileResponse(path, headers={"Cache-Control": "no-store"}) -@router.get("/api/v1/inspect/inventory") +def span(since: str | None, until: str | None) -> tuple[str, str, float, float]: + try: + return sources.window(since, until) + except (ValueError, OverflowError, OSError) as exc: + raise HTTPException(400, "Use ISO-8601 or Unix-second since/until and a positive window of at most seven days") from exc + + +async def run_of(run_id: str) -> tuple[dict, dict]: + """The run, or the one run of a session named in its place.""" + state = await fleet.snapshot() + found = [r for r in state["runs"] if r["run_id"] == run_id] or [r for r in state["runs"] if run_id in r["session_ids"]] + if len(found) > 1: + raise HTTPException(409, f"Session {run_id} has {len(found)} runs; name one of {', '.join(sorted(r['run_id'] for r in found))}") + if not found: + raise HTTPException(404, f"No run {run_id}") + return state, found[0] + + +@router.get("/api/v1/inspect/inventory", response_model=Doc) async def inventory(): - """Runs with their lifecycle (ended, active) and an error per unavailable source or unreadable run record. client_status is - what the client last reported, active until it deletes the model, so a crashed run stays active there; lifecycle is ended - once none of the run's sessions is live.""" - return await fleet.snapshot() + """Runs with their lifecycle (ended, waiting, placed, active), nodes with their GPUs, the GPUs held now, pods with + problems, and an error per unavailable source. A run's facts are /runs/{id}/summary.""" + await sources.discover() + state = await fleet.snapshot() + hidden = ("workloads", "runtime_id", "pods") + # status is what the client last reported (active until it deletes the model), so a crashed run stays active there; + # lifecycle is what the cluster shows. + runs = [ + {("client_status" if k == "status" else k): v for k, v in r.items() if k not in hidden} + | {"pods": [{"name": p["name"], "problem": p["problem"]} for p in r["pods"]]} + for r in state["runs"] + ] + fields = ("id", "label", "run_ids", "role", "node", "pod", "devices", "device_count") + held = [{k: lane[k] for k in fields} for lane in fleet.lanes(state) if not lane["ended"]] + pods = [{"name": p["name"], "problem": p["problem"], "restarts": p["restarts"]} for p in state["pods"] if p["problem"]] + cluster = {"available": not any(e.startswith(("Kubernetes", "pods")) for e in state["errors"]), "nodes": state["nodes"], "pods": pods} + return {"observed_at": facts.iso(state["observed_at"]), "errors": state["errors"], "runs": runs, "cluster": cluster, "placements": held} + + +@router.get("/api/v1/inspect/activity", response_model=Doc) +async def activity(since: Bound = None, until: Bound = None, run_id: str | None = None, node_id: str | None = None): + """Each placement's GPU turns, waits (from asking the time slicer for the GPU to the grant, half a second or more), recorded + ops and its pod's container terminations in the window (Unix seconds), fleet-wide, a node's or a run's. + A sampler's `sample` spans are the groups of samples it served, one record per run each. Gaps do not prove idleness.""" + since, until, low, high = span(since, until) + if run_id: + state, run = await run_of(run_id) + run_id = run["run_id"] + else: + state = await fleet.snapshot() + placements = await fleet.activity(state, low, high, run_id, node_id) + return {"placements": placements, "since": since, "until": until, "observed_at": facts.iso(state["observed_at"])} + + +VLLM = "{pod=~$pods$scope}" + + +# The DCGM exporter's collection interval in k8s/deploy. +DCGM_SECONDS = 10 + + +def utilization(form: str, by: str = "UUID, gpu") -> str: + """A GPU's busy percent. GR_ENGINE_ACTIVE averages DCGM's whole collection interval. GPU_UTIL, the fallback where it is + missing, covers about the last second of each interval, so a load with that period aliases.""" + active, util = "DCGM_FI_PROF_GR_ENGINE_ACTIVE{UUID=~$gpus$scope}", "DCGM_FI_DEV_GPU_UTIL{UUID=~$gpus$scope}" + return f"max by ({by}) (100 * {form.format(active)}) or max by ({by}) ({form.format(util)})" + + +QUERIES = { + # By UUID and gpu: DCGM's slot index names a GPU after its node is gone. + "gpu_util": utilization("{}"), + "gpu_memory_mib": "max by (UUID, gpu) (DCGM_FI_DEV_FB_USED{UUID=~$gpus$scope})", + "gpu_memory_peak_mib": "max by (UUID, gpu) (max_over_time(DCGM_FI_DEV_FB_USED{UUID=~$gpus$scope}[${window}s]))", + "gpu_memory_total_mib": "max by (UUID) (DCGM_FI_DEV_FB_TOTAL{UUID=~$gpus$scope})", + # Utilization comes in bursts; the last two minutes say whether a GPU is working now better than one sample. + "gpu_util_now": utilization("avg_over_time({}[2m])", "Hostname, UUID, gpu, modelName"), + # Duty cycle over the window: the share of time a kernel ran, one number per GPU with points=1. + "gpu_busy": utilization("avg_over_time({}[${{window}}s])", "Hostname, UUID, gpu, modelName"), + # From vLLM's and llm-d's published Grafana dashboards. + "vllm_generation_tokens_per_second": f"sum(rate(vllm:generation_tokens_total{VLLM}[2m]))", + "vllm_prompt_tokens_per_second": f"sum(rate(vllm:prompt_tokens_total{VLLM}[2m]))", + "vllm_requests_running": f"sum(vllm:num_requests_running{VLLM})", + "vllm_requests_waiting": f"sum(vllm:num_requests_waiting{VLLM})", + "vllm_kv_cache_usage": f"max(vllm:kv_cache_usage_perc{VLLM})", + "vllm_preemptions_per_second": f"sum(rate(vllm:num_preemptions_total{VLLM}[2m]))", + "vllm_time_to_first_token_p99_seconds": f"histogram_quantile(0.99, sum by (le) (rate(vllm:time_to_first_token_seconds_bucket{VLLM}[2m])))", + "vllm_request_latency_p99_seconds": f"histogram_quantile(0.99, sum by (le) (rate(vllm:e2e_request_latency_seconds_bucket{VLLM}[2m])))", +} + + +def subject(state: dict, run_id: str | None, node_id: str | None, low: float, high: float) -> tuple[list[str] | None, list[str]]: + """The GPUs and pods a run's or a node's placements held in the window, plus a node's GPUs now; None GPUs is the fleet.""" + if not (run_id or node_id): + return None, [] + held = [p for p in fleet.lanes(state, run_id, node_id) if p["start"] <= high and p["end"] >= low] + now = [d["id"] for n in state["nodes"] if not run_id and n["name"] == node_id for d in n["devices"]] + return sorted({uuid for p in held for uuid in p["devices"]} | set(now)), sorted({p["pod"] for p in held if p["pod"]}) + + +@router.get("/api/v1/inspect/query", response_model=Doc) +async def query( + name: Annotated[str | None, Query(description="One of the named queries; leave it out to list them")] = None, + run_id: str | None = None, + node_id: str | None = None, + since: Bound = None, + until: Bound = None, + points: int = Query(300, ge=1, le=1000), +): + """A named PromQL query over a run's GPUs and pods, a node's, or (GPU queries only) the fleet's, at `points` resolution, + `step` seconds apart. points=1 evaluates once over the whole window, which is how a duty cycle is one number per GPU. + A gpu_util point is a DCGM reading placed at the start of the reading_seconds it averaged, so it lines up with turns.""" + if name not in QUERIES: + raise HTTPException(400, f"{'Unknown query' if name else 'Name a query'}; use one of {sorted(QUERIES)}") + since, until, low, high = span(since, until) + await sources.discover() + gpus, pods = subject(await fleet.snapshot(), run_id, node_id, low, high) + step = sources.step_seconds(low, high, points) + result = {"name": name, "since": since, "until": until, "step": step, "series": [], "source": sources.prometheus_source()} + if gpus == [] and "$gpus" in QUERIES[name] or not pods and "$pods" in QUERIES[name]: + return {**result, "error": "No GPUs or pods recorded for this subject in this window"} + values = {"gpus": '".+"' if gpus is None else sources.any_of(gpus), "pods": sources.any_of(pods), "window": max(1, int(high - low))} + result["query"] = text = Template(QUERIES[name]).substitute(values, scope=sources.scope()) + try: + rows = await sources.prom(text, high if points == 1 else low, high, points) + except Exception as exc: + return {**result, "error": str(exc) if isinstance(exc, LookupError) else f"Query failed: {type(exc).__name__}"} + # A reading averages the interval before it was collected, and a range query returns it up to one reading later. + shift = 2 * DCGM_SECONDS if name == "gpu_util" else 0 + series = [{"labels": r["metric"], "points": [[float(t) - shift, float(v)] for t, v in r["values"] if math.isfinite(float(v))]} for r in rows] + return result | {"series": series} | ({"reading_seconds": DCGM_SECONDS} if shift else {}) + + +@router.get("/api/v1/inspect/accelerators", response_model=Doc) +async def accelerators(run_id: str | None = None, node_id: str | None = None, since: Bound = None, until: Bound = None): + """Every GPU in the fleet, a node's, or a run's in the window: node, slot, model, holders (the newest placement on it + and any time-sliced with it, with their pods' container terminations), busy (percent of the window DCGM saw the GPU + engine active), and peak and total memory in MiB. The window defaults to the last hour, or an ended run's last hour.""" + if run_id and since is None and until is None: + state, run = await run_of(run_id) + run_id, ends = run["run_id"], [p["end"] for p in fleet.lanes(state, run["run_id"])] + if run["lifecycle"] == "ended" and ends: + since, until = "1h", facts.iso(max(ends)) + since, until, low, high = span(since or ("1h" if until is None else None), until) + names = {"busy": "gpu_busy", "memory_peak_mib": "gpu_memory_peak_mib", "memory_total_mib": "gpu_memory_total_mib"} + found = dict(zip(names, await asyncio.gather(*(query(name, run_id, node_id, since, until, 1) for name in names.values())))) + result = {"since": since, "until": until, "gpus": [], "source": found["busy"]["source"]} + if error := next((r["error"] for r in found.values() if r.get("error")), None): + return {**result, "error": error} + value = {k: {s["labels"]["UUID"]: round(s["points"][-1][1], 1) for s in r["series"] if s["points"]} for k, r in found.items()} + held = sorted((p for p in fleet.lanes(await fleet.snapshot(), run_id) if p["start"] <= high and p["end"] >= low), key=lambda p: p["start"]) + placed: dict[str, list[dict]] = {} + for lane in held: + for uuid in lane["devices"]: + placed.setdefault(uuid, []).append(lane) + # A time-sliced GPU has several holders at once, every placement that overlaps the newest one. + holders = {uuid: [p for p in lanes if p["end"] >= lanes[-1]["start"]] for uuid, lanes in placed.items()} + labels_of = {series["labels"]["UUID"]: series["labels"] for series in found["busy"]["series"]} + for uuid, labels in labels_of.items(): + node = labels.get("Hostname") or next((lane["node"] for lane in reversed(holders.get(uuid, []))), None) + gpu = {"uuid": uuid, "node": node, "slot": labels.get("gpu"), "model": labels.get("modelName")} + gpu |= {name: value[name][uuid] for name in names if uuid in value[name]} + if uuid in holders: + gpu["holders"] = [{k: lane[k] for k in ("run_ids", "role", "pod", "start", "end", "ended", "terminations")} for lane in holders[uuid]] + result["gpus"].append(gpu) + result["gpus"].sort(key=lambda g: (g["node"] or "", int(g["slot"] or 0))) + return result + + +# Prefixes of the cookbook's metrics an agent reads first; the rest is a fields= away. +KEY_METRICS = ( + *("env/all/reward", "test/env/all/reward", "env/all/lab/criteria_pass_fraction", "test/env/all/lab/criteria_pass_fraction"), + *("optim/kl", "optim/entropy", "time/total", "progress/"), +) + + +async def client_directory(run: dict) -> str | None: + """The run's client directory under OPEN_RL_CLIENT_RUNS_ROOT, if exactly one matches it.""" + candidates = await fleet.cached("client directories", 30, lambda: asyncio.to_thread(client_runs.directories)) + return await asyncio.to_thread(client_runs.match, run, candidates) + + +@router.get("/api/v1/inspect/runs/{run_id}/steps", response_model=Doc) +async def steps( + run_id: str, + last: int = Query(20, ge=1, le=1000), + fields: Annotated[str | None, Query(description="Comma-separated prefixes of client metrics to keep, e.g. env/all/reward; * keeps all")] = None, +): + """The run's last steps, oldest first: window, completion, trainer ops (count, failures, seconds, metric spread), and the + `client` metrics its client wrote to metrics.jsonl for the step: by default the key ones (rewards, KL, entropy, step time, + progress), with every name the client logged in client_metric_names.""" + _, run = await run_of(run_id) + run_id = run["run_id"] + keep = () if fields == "*" else tuple(p for p in fields.split(",") if p) if fields else KEY_METRICS + every, result = fleet.steps(await fleet.operations(run_id)), {} + # Some runs keep their client metrics in Redis; metrics.jsonl wins a step both have. + client = {row["step"]: finite(row["metrics"]) for row in await get_state_store().read_samples(f"open_rl:run_metrics:{run_id}")} + if directory := await client_directory(run): + try: + client |= await asyncio.to_thread(client_runs.metrics, directory) + except (OSError, ValueError) as exc: # steps stand without the client's metrics + result["client_error"] = f"{directory}/metrics.jsonl unreadable: {type(exc).__name__}" + result["client_directory"] = directory + for step in every[-last:]: + step["client"] = {k: v for k, v in client.get(step["step"], {}).items() if not keep or k.startswith(keep)} + result["client_metric_names"] = sorted({name for metrics in client.values() for name in metrics}) + return result | {"run_id": run_id, "total": len(every), "steps": every[-last:]} + + +@router.get("/api/v1/inspect/runs/{run_id}/commands", response_model=Doc) +async def commands(run_id: str, since: Bound = None, until: Bound = None, role: str | None = None, limit: int = Query(100, ge=1, le=5000)): + """The run's raw command records (trainer ops, the groups of samples a sampler served, and failed samples), + newest first; without bounds, all of them. A group's record has its count of samples.""" + run_id = (await run_of(run_id))[1]["run_id"] + low, high = span(since, until)[2:] if since or until else (0, math.inf) + wanted = (lambda r: True) if role is None else (lambda r: (r.get("role") == "sampler") == (role == "sampler")) + found = [r for r in reversed(await fleet.operations(run_id)) if low <= r["at"] <= high and wanted(r)] + return {"run_id": run_id, "total": len(found), "commands": found[:limit]} + + +@router.get("/api/v1/inspect/runs", response_model=Doc) +async def runs(ended: bool = False): + """Every live run in one line, longest since its last op first, so a stuck run is one read away. An op includes a sample. + ended=true adds ended runs.""" + state = await fleet.snapshot() + chosen = [r for r in state["runs"] if ended or r["lifecycle"] != "ended"] + # A few runs at a time, since each read in flight holds a store connection. + limit = asyncio.Semaphore(128) + + async def line(run): + async with limit: + return await facts.fleet_line(state, run) + + found = await asyncio.gather(*(line(r) for r in chosen)) + lines = [{"run_id": r["run_id"], "name": r["name"], **line} for r, line in zip(chosen, found)] + return {"runs": sorted(lines, key=lambda r: -(r["since_last_op_s"] if r["since_last_op_s"] is not None else math.inf))} + + +@router.get("/api/v1/inspect/runs/{run_id}/summary", response_model=Doc) +async def summary(run_id: str): + """The run's facts, each block only when it has content: pace, goodput, unanswered requests and their holders, the ledger + (lost = enqueued - popped - queued), failures and relaunches with log reads, saves, optimizer, samplers, config, + placement, incidents, and GPU duty by role over the run's span (gpu_busy, percent). No verdict; reason from the facts. + + A vLLM sampler keeps one record per group of samples it served and one per failed sample. In samplers, sends is the + run's sample count, failed and cancelled count its records, and median_s is the median time of the groups it served. + In goodput, rollout is trainer idle time while one of those groups was being served.""" + state, run = await run_of(run_id) + run_id = run["run_id"] + ctx = await facts.context(state, run) + if relaunched := facts.relaunches(ctx): + # The scheduler names a Workload's pod orw-, so the Workload's name matches both; older records hold the pod's. + names = sorted({lane["name"].removeprefix("orw-") for lane in ctx["lanes"]}) + try: + ctx["changes"] = await sources.audit_events(names, relaunched[0]["stopped_at"] - 300, relaunched[-1]["started_at"] + 60) + except Exception as exc: + ctx["changes"] = {"error": f"Audit log unavailable: {exc if isinstance(exc, LookupError) else type(exc).__name__}"} + found = {} + if ops := ctx["ops"]: + # Duty over the run's own span: averaging in the idle days after it ended would understate it. + end = min(state["observed_at"], facts.last_op_at(ctx) + 60) + duty = await query("gpu_busy", run_id, None, facts.iso(max(ops[0]["at"] - 60, end - 7 * 86400)), facts.iso(end), 1) + roles: dict[str, set[str]] = {} + for lane in ctx["lanes"]: + for uuid in lane["devices"]: + roles.setdefault(uuid, set()).add(lane["role"]) + busy: dict[str, list[float]] = {} + for series in (s for s in duty["series"] if s["points"]): + # A GPU that time-sliced roles shared counts under the roles together, e.g. sampler+trainer. + busy.setdefault("+".join(sorted(roles.get(series["labels"].get("UUID")) or {"other"})), []).append(series["points"][-1][1]) + if busy: + found["gpu_busy"] = {role: round(sum(v) / len(v), 1) for role, v in busy.items()} + # On a shared runtime these are the runtime's GPUs, busy with every run it serves. + found["gpu_busy_window"] = {k: duty[k] for k in ("since", "until")} | {"scope": "runtime" if run["shared_runtime"] else "run"} + return {"run_id": run_id, **facts.summary(ctx), **found} + + +@router.get("/api/v1/inspect/sessions/{session_id}/summary", response_model=Doc) +async def session_summary(session_id: str): + """The session's record, whether its registry key is live, its SDK errors, and a line per model over the session's span.""" + store = get_state_store() + if (record := json.loads(await store.get_value(key("session_record", session_id)) or "null")) is None: + raise HTTPException(404, f"No session {session_id}") + live = await store.get_value(key("session", session_id)) is not None + # A model's ops belong to this session while it was alive, which ends when its live key expires after last_seen_at. + low, high = record["created_at"], math.inf if live else record["last_seen_at"] + 120 + state, models = await fleet.snapshot(), [] + for run in (r for r in state["runs"] if r["run_id"] in record["model_ids"]): + models.append({"model_id": run["run_id"], **facts.line(await facts.context(state, run, low, high))}) + errors = await store.read_samples(key("sdk_errors", session_id)) + return {**record, "live": live, "models": models} | ({"sdk_errors": {"count": len(errors), "last": errors[-1].get("message")}} if errors else {}) + + +@router.get("/api/v1/inspect/logs", response_model=Doc) +async def logs( + run_id: str, + pod: Annotated[str | None, Query(description="One of the run's pods, or 'client' for its client's lines")] = None, + q: str = Query("", max_length=1024), + severity: str | None = Query(None, pattern="^(DEBUG|INFO|WARNING|ERROR|CRITICAL|UNKNOWN)$"), + since: Bound = None, + until: Bound = None, + limit: int = Query(200, ge=1, le=1000), + cursor: Annotated[str | None, Query(max_length=16384, description="next_cursor of a page with the same filters and window")] = None, +): + """Newest first: the run's worker pods by owner label from Cloud Logging, and with pod=client its client's logs.log. That + file's lines carry no time, so since and until do not apply to it and its cursor is a byte offset.""" + since, until, low, high = span(since, until) + await sources.discover() + state, run = await run_of(run_id) + run_id = run["run_id"] + pods = {lane["pod"]: {"pod": lane["pod"], "node": lane["node"], "role": lane["role"]} for lane in fleet.lanes(state, run_id) if lane["pod"]} + result = {"run_id": run_id, "source": "gke", "since": since, "until": until, "sources": list(pods.values()), "records": [], "next_cursor": None} + if directory := await client_directory(run): + result["sources"].append({"pod": sources.CLIENT, "node": None, "role": sources.CLIENT}) + if directory and pod == sources.CLIENT: + if cursor and not cursor.isdigit(): + raise HTTPException(400, "Invalid or expired log cursor; repeat the query without it") + result["source"] = path = f"{directory}/logs.log" + try: + page = await asyncio.to_thread(client_runs.log_page, directory, int(cursor) if cursor else None, limit, q, severity) + return {**result, **page, "available": True} + except OSError as exc: + return {**result, "available": False, "error": f"{path} unreadable: {type(exc).__name__}"} + try: + return {**result, **await sources.log_page(run, since, until, cursor=cursor, limit=limit, q=q, pod=pod, severity=severity), "available": True} + except httpx.HTTPStatusError as exc: + if exc.response.status_code == 400 and cursor: + raise HTTPException(400, "Invalid or expired log cursor; repeat the query without it") from None + return {**result, "available": False, "error": f"Cloud Logging refused the read (HTTP {exc.response.status_code}); check credentials and IAM"} + except Exception as exc: + return {**result, "available": False, "error": str(exc) if isinstance(exc, LookupError) else "Cloud Logging unavailable"} diff --git a/src/server/dashboard/sources.py b/src/server/dashboard/sources.py new file mode 100644 index 000000000..32a041a4e --- /dev/null +++ b/src/server/dashboard/sources.py @@ -0,0 +1,327 @@ +"""What the dashboard reads outside Redis: Kubernetes, Prometheus (OPEN_RL_PROMETHEUS_URL, else GKE's) and Cloud Logging.""" + +import asyncio +import json +import os +import re +import threading +import time +from datetime import UTC, datetime, timedelta +from functools import cache + +import google.auth +import google.oauth2.credentials +import httpx +from google.auth.transport.requests import Request + +from server.telemetry import epoch +from server.worker_manager import owner_id + +client: httpx.AsyncClient | None = None +METADATA = {"project": "project/project-id", "cluster": "instance/attributes/cluster-name", "location": "instance/attributes/cluster-location"} +detected: dict = {} +discovered_at = -60.0 +credentials = None +credentials_lock = threading.Lock() + + +async def send(method: str, url: str, **kwargs) -> httpx.Response: + global client + client = client or httpx.AsyncClient(timeout=10) + return (await client.request(method, url, **kwargs)).raise_for_status() + + +def namespace() -> str: + return os.getenv("OPEN_RL_WORKER_NAMESPACE") or os.getenv("POD_NAMESPACE") or "openrl-system" + + +def configuration() -> dict: + """OPEN_RL_GKE_PROJECT, _CLUSTER and _LOCATION, else what the metadata server said; OPEN_RL_GKE_TELEMETRY=0 turns GKE off.""" + fields = {key: os.getenv(f"OPEN_RL_GKE_{key.upper()}") or detected.get(key, "") for key in METADATA} + off = os.getenv("OPEN_RL_GKE_TELEMETRY", "auto").strip().lower() not in ("auto", "1", "true") + return {**fields, "configured": not off and all(fields.values())} + + +async def discover() -> None: + """In a cluster, read the GKE identity from the metadata server; retry a failed read after a minute.""" + global detected, discovered_at + if configuration()["configured"] or not os.getenv("KUBERNETES_SERVICE_HOST") or time.monotonic() - discovered_at < 60: + return + discovered_at = time.monotonic() + async with httpx.AsyncClient(timeout=1, trust_env=False, follow_redirects=False) as metadata: + + async def read(path: str) -> str: + response = await metadata.get(f"http://metadata.google.internal/computeMetadata/v1/{path}", headers={"Metadata-Flavor": "Google"}) + if response.status_code != 200 or response.headers.get("Metadata-Flavor") != "Google" or not 0 < len(response.text.strip()) <= 256: + raise ValueError("Invalid metadata response") + return response.text.strip() + + values = await asyncio.gather(*(read(path) for path in METADATA.values()), return_exceptions=True) + if not any(isinstance(value, Exception) for value in values): # never partial: plain GCE also exposes a project + detected = dict(zip(METADATA, values)) + + +def access_token() -> str: + global credentials + with credentials_lock: + if credentials is None: + credentials, _ = google.auth.default(scopes=["https://www.googleapis.com/auth/cloud-platform.read-only"]) + if not credentials.valid: + credentials.refresh(Request()) + return credentials.token + + +async def gcp_headers() -> dict[str, str]: + # A hung token refresh holds the lock; bound it rather than start a thread per call. + headers = {"Authorization": f"Bearer {await asyncio.wait_for(asyncio.to_thread(access_token), timeout=10)}"} + # A person's gcloud login draws on gcloud's shared quota and fails with 429 unless it bills this project. + if isinstance(credentials, google.oauth2.credentials.Credentials) and not credentials.quota_project_id: + headers["x-goog-user-project"] = configuration()["project"] + return headers + + +def problem(status: dict) -> str | None: + """The one line that explains a pod that is not simply running. A container running again after a restart is not one.""" + if status.get("phase") == "Failed": + return f"Failed: {': '.join(p for p in (status.get('reason'), status.get('message')) if p) or 'see logs'}" + for cs in status.get("containerStatuses") or []: + waiting, ended = cs["state"].get("waiting"), cs["state"].get("terminated") + if waiting and waiting.get("reason") not in (None, "ContainerCreating", "PodInitializing"): + return f"{waiting['reason']}: {waiting.get('message') or ''}".strip(": ") + if ended and (ended.get("exitCode") or ended.get("reason") not in (None, "Completed")): + return f"{ended.get('reason') or 'Terminated'}: exit code {ended.get('exitCode')}" + (f" — {ended['message']}" if ended.get("message") else "") + if status.get("phase") == "Pending": + unscheduled = [c for c in status.get("conditions") or [] if c["type"] == "PodScheduled" and c["status"] != "True"] + return f"Unschedulable: {unscheduled[0].get('message') or 'no node available'}" if unscheduled else "Pending" + return None + + +def incidents(pod: str, statuses: list[dict]) -> list[dict]: + """The last termination of each container that restarted: when, why, and the exit code.""" + ended = [(cs["name"], t) for cs in statuses if (t := (cs.get("lastState") or {}).get("terminated")) and t.get("finishedAt")] + return [ + {"at": epoch(t["finishedAt"]), "pod": pod, "reason": t.get("reason") or "Terminated", "message": f"{name} exited with code {t.get('exitCode')}"} + for name, t in ended + ] + + +@cache +def kubernetes_api() -> tuple: + """The Kubernetes clients, built once per process from the in-cluster service account, else the kubeconfig. A failed + build is not cached, so the next read tries again.""" + from kubernetes import client, config # the cluster extra installs it + + try: + config.load_incluster_config() + except Exception: + config.load_kube_config() + return client.CoreV1Api(), client.CustomObjectsApi() + + +async def kubernetes() -> dict: + """Pods, nodes and scheduler Workloads, one plain list each; workloads is None when the scheduler cannot be read or is + not installed.""" + try: + from server.scheduler_worker_manager import age_seconds, worker_state + + core, custom = kubernetes_api() + except Exception as exc: + return {"pods": [], "nodes": [], "workloads": None, "errors": [f"Kubernetes unavailable: {exc}"]} + ns, errors = namespace(), [] + + async def items(name: str, call, *args) -> list[dict] | None: + try: + # Read as raw JSON, because the client's own decoding of a large pod list costs more than the request. + found = await asyncio.to_thread(call, *args, _preload_content=False, _request_timeout=6) + return json.loads(found.data)["items"] + except Exception as exc: + if getattr(exc, "status", None) != 404: # no scheduler CRD on a local stack + errors.append(f"{name} read failed: {exc}") + return None + + pods, nodes, workloads = await asyncio.gather( + items("pods", core.list_namespaced_pod, ns), + items("nodes", core.list_node), + items("workloads", custom.list_namespaced_custom_object, "openrl.io", "v1alpha1", ns, "workloads"), + ) + result, now = {"pods": [], "nodes": [], "workloads": None if workloads is None else [], "errors": errors}, time.time() + for pod in pods or []: + name, labels, statuses = pod["metadata"]["name"], pod["metadata"].get("labels") or {}, pod["status"].get("containerStatuses") or [] + row = {"name": name, "node": pod["spec"].get("nodeName"), "owner": labels.get("timeslice.io/owner"), "problem": problem(pod["status"])} + result["pods"].append(row | {"incidents": incidents(name, statuses), "restarts": sum(cs.get("restartCount") or 0 for cs in statuses)}) + for node in nodes or []: + labels, status = node["metadata"].get("labels") or {}, node["status"] + row = {"name": node["metadata"]["name"], "ready": any(c["type"] == "Ready" and c["status"] == "True" for c in status.get("conditions") or [])} + row.update(accelerator=labels.get("cloud.google.com/gke-accelerator") or labels.get("nvidia.com/gpu.product")) + row["gpu_capacity"] = int((status.get("capacity") or {}).get("nvidia.com/gpu", 0)) + result["nodes"].append(row | {"openrl_enabled": labels.get("openrl.io/enabled") == "true"}) # the scheduler places only on enabled nodes + for item in workloads or []: + spec, status, meta = item["spec"], item.get("status") or {}, item["metadata"] + deleting = bool(meta.get("deletionTimestamp")) + state, _ = worker_state(phase=status.get("phase") or "Pending", reason=status.get("reason") or "", node=status.get("nodeName"), deleting=deleting) + placed = next((c for c in status.get("conditions") or [] if c.get("type") == "Placed"), {}) + row = {"name": meta["name"], "role": spec["role"], "model_id": spec["modelID"], "training_kind": spec.get("trainingKind"), "state": state} + row.update(node=status.get("nodeName"), age_seconds=age_seconds(meta.get("creationTimestamp"), now)) + result["workloads"].append(row | {"message": placed.get("message") or status.get("reason")}) + return result + + +def scope() -> str: + return f",cluster={json.dumps(configuration()['cluster'])}" if configuration()["cluster"] else "" + + +def any_of(values: list[str]) -> str: + """A PromQL regex matcher for exactly these label values.""" + return json.dumps("|".join(re.escape(v) for v in values)) + + +def prometheus_source() -> str | None: + return "prometheus" if os.getenv("OPEN_RL_PROMETHEUS_URL") else "gke" if configuration()["configured"] else None + + +def step_seconds(start: float, end: float, points: int) -> int: + """The seconds between the points prom returns for [start, end]. DCGM is read and scraped every 10 s; a finer step only + repeats samples.""" + return max(10, int((end - start) / points)) + + +async def prom(query: str, start: float, end: float, points: int = 1) -> list[dict]: + """Prometheus matrix rows; start == end evaluates once, at end.""" + url, headers = os.getenv("OPEN_RL_PROMETHEUS_URL", "").rstrip("/"), {} + if not url: + if not configuration()["configured"]: + raise LookupError("No GPU metrics source: not on GKE and OPEN_RL_PROMETHEUS_URL is unset") + url, headers = f"https://monitoring.googleapis.com/v1/projects/{configuration()['project']}/location/global/prometheus", await gcp_headers() + step = step_seconds(start, end, points) + # Samples land on multiples of step, so queries made at different moments share timestamps. + start -= start % step if start < end else 0 + payload = ( + await send("GET", f"{url}/api/v1/query_range", params={"query": query, "start": start, "end": end, "step": step}, headers=headers) + ).json() + if payload.get("status") != "success": + raise ValueError(payload.get("error") or "query failed") + return payload["data"]["result"] + + +# GKE copies each pod label onto its log entries as k8s-pod/, with the dots in the key as underscores. +OWNER = 'labels."k8s-pod/timeslice_io/owner"' +# An in-cluster client labels its own lines with its run and this source, so they read with the run's worker logs. +RUN_LABEL = "openrl_run_id" +SOURCE_LABEL = "openrl_source" +CLIENT = "client" +TELEMETRY = "telemetry" # telemetry's own event lines, kept as history and never read here +# GKE files every stderr line as ERROR, though most are progress bars, INFO logs and warnings; such a line takes the level it prints. +RELABELED = ("DEBUG", "INFO", "WARNING") +LEVEL, PROGRESS = re.compile(r"\b(DEBUG|INFO|WARNING|WARN|ERROR|CRITICAL)\b"), re.compile(r"\r|\d+%(\||\s+Completed)") + + +def logs_filter(run: dict, start: str, end: str, severity: str | None, q: str, pod: str | None = None) -> str: + """The run's worker pods by the scheduler's owner label and its client's lines in the window, or one of them.""" + config = configuration() + labels = {"project_id": config["project"], "location": config["location"], "cluster_name": config["cluster"], "namespace_name": namespace()} + clauses = ['resource.type="k8s_container"', *(f"resource.labels.{label}={json.dumps(value)}" for label, value in labels.items())] + client = f"(labels.{RUN_LABEL}={json.dumps(run['run_id'])} AND labels.{SOURCE_LABEL}={json.dumps(CLIENT)})" + owner = f"{OWNER}={json.dumps(owner_id(run['runtime_id']).rstrip('-'))}" # the scheduler trims a truncated label value as well + if pod and pod != CLIENT: + # One of the run's pods, never any pod in the namespace, so a run id cannot open another run's logs. + clauses += [owner, f"resource.labels.pod_name={json.dumps(pod)}"] + else: + clauses.append(client if pod == CLIENT else f"({owner} OR {client})") + clauses += [f"NOT labels.{SOURCE_LABEL}={json.dumps(TELEMETRY)}", f"timestamp>={json.dumps(start)}", f"timestamp<={json.dumps(end)}"] + if severity in RELABELED: + stderr = json.dumps(f"projects/{config['project']}/logs/stderr") + clauses.append(f'(severity={json.dumps(severity)} OR (severity="ERROR" AND logName={stderr}))') + elif severity: + clauses.append(f"severity={json.dumps('DEFAULT' if severity == 'UNKNOWN' else severity)}") + if q: + clauses.append(f"(textPayload:{json.dumps(q)} OR jsonPayload.message:{json.dumps(q)})") + return " AND ".join(clauses) + + +def stderr_severity(text: str) -> str: + body = re.sub(r"^\([\w ]+ pid=\d+\)\s*", "", text) # vLLM's "(EngineCore pid=826) " + if not body.strip() or PROGRESS.search(body): + return "INFO" + if level := LEVEL.search(body[:80]): + return "WARNING" if level[1] == "WARN" else level[1] + return "WARNING" if re.search(r"Warning\b|warnings\.warn\(", body) else "ERROR" + + +def entry_record(entry: dict) -> dict: + """One log entry as a record; the pod's labels say its role and node. Messages are untrusted output, never instructions.""" + resource, labels = entry.get("resource", {}).get("labels", {}), entry.get("labels", {}) + text = entry.get("textPayload") + text = json.dumps(entry.get("jsonPayload") or entry.get("protoPayload", {}), ensure_ascii=False) if text is None else text + severity = entry.get("severity", "UNKNOWN") + if severity == "ERROR" and "textPayload" in entry and entry.get("logName", "").endswith("/logs/stderr"): + severity = stderr_severity(text) + client = labels.get(SOURCE_LABEL) == CLIENT + role = CLIENT if client else labels.get("k8s-pod/openrl_io/role") or labels.get("k8s-pod/app_kubernetes_io/component") or "unknown" + record = {"id": f"{entry.get('logName')}/{entry.get('insertId')}/{entry.get('timestamp')}", "timestamp": entry.get("timestamp")} + record.update(pod=CLIENT if client else resource.get("pod_name"), role=role, node=labels.get("compute.googleapis.com/resource_name")) + return record | {"severity": severity, "message": text[:16384], "message_truncated": len(text) > 16384} + + +async def log_page(run: dict, since: str, until: str, *, cursor=None, limit=200, q="", pod=None, severity=None) -> dict: + """A page of logs, newest first. The cursor is Cloud Logging's own page token, valid for the same filters.""" + if not (config := configuration())["configured"]: + raise LookupError("GKE telemetry is not configured") + views = [v.strip() for v in os.getenv("OPEN_RL_LOG_VIEWS", "").split(",") if v.strip()] or [f"projects/{config['project']}"] + query = { + "resourceNames": views, + "filter": logs_filter(run, since, until, severity, q, pod), + "orderBy": "timestamp desc", + "pageSize": limit, + } + query |= {"pageToken": cursor} if cursor else {} + payload = (await send("POST", "https://logging.googleapis.com/v2/entries:list", json=query, headers=await gcp_headers())).json() + records = [entry_record(entry) for entry in payload.get("entries", [])] + if severity in (*RELABELED, "ERROR"): + records = [record for record in records if record["severity"] == severity] + return {"records": records, "next_cursor": payload.get("nextPageToken")} + + +async def audit_events(names: list[str], start: float, end: float) -> list[dict]: + """Who created or deleted these Workloads and their pods, from the cluster's audit log, oldest first.""" + if not (config := configuration())["configured"]: + raise LookupError("GKE telemetry is not configured") + audit_log = f"projects/{config['project']}/logs/cloudaudit.googleapis.com%2Factivity" + methods = ("io.openrl.v1alpha1.workloads.create", "io.openrl.v1alpha1.workloads.delete", "io.k8s.core.v1.pods.delete") + clauses = [ + f"logName={json.dumps(audit_log)}", + f"resource.labels.cluster_name={json.dumps(config['cluster'])}", + "(" + " OR ".join(f"protoPayload.resourceName:{json.dumps(name)}" for name in names) + ")", + "(" + " OR ".join(f"protoPayload.methodName={json.dumps(m)}" for m in methods) + ")", + f"timestamp>={json.dumps(datetime.fromtimestamp(start, UTC).isoformat())}", + f"timestamp<={json.dumps(datetime.fromtimestamp(end, UTC).isoformat())}", + ] + query = {"resourceNames": [f"projects/{config['project']}"], "filter": " AND ".join(clauses), "orderBy": "timestamp asc", "pageSize": 50} + payload = (await send("POST", "https://logging.googleapis.com/v2/entries:list", json=query, headers=await gcp_headers())).json() + return [ + { + "at": e.get("timestamp"), + "action": e["protoPayload"]["methodName"].rsplit(".", 2)[-2] + " " + e["protoPayload"]["methodName"].rsplit(".", 1)[-1], + "resource": e["protoPayload"].get("resourceName", "").rsplit("/", 1)[-1], + "by": e["protoPayload"].get("authenticationInfo", {}).get("principalEmail"), + } + for e in payload.get("entries", []) + ] + + +def window(since: str | None = None, until: str | None = None) -> tuple[str, str, float, float]: + """(since, until) in ISO-8601 UTC, then in Unix seconds; either form is accepted. until defaults to now, since to 30 minutes before it or a + span before it such as 1h; at most seven days.""" + + def parse(value: str) -> datetime: + if re.fullmatch(r"\d+(\.\d+)?", value): + return datetime.fromtimestamp(float(value), UTC) + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + return (parsed if parsed.tzinfo else parsed.replace(tzinfo=UTC)).astimezone(UTC) + + end = parse(until) if until else datetime.now(UTC) + if ago := re.fullmatch(r"(\d+)([mhd])", since or ""): + since = (end - timedelta(**{{"m": "minutes", "h": "hours", "d": "days"}[ago[2]]: int(ago[1])})).isoformat() + start = parse(since) if since else end - timedelta(minutes=30) + if start >= end or end - start > timedelta(days=7): + raise ValueError("Time range must be positive and no longer than seven days") + return start.isoformat(timespec="microseconds"), end.isoformat(timespec="microseconds"), start.timestamp(), end.timestamp() diff --git a/src/server/dashboard/static/index.html b/src/server/dashboard/static/index.html new file mode 100644 index 000000000..e5b5b63bf --- /dev/null +++ b/src/server/dashboard/static/index.html @@ -0,0 +1,12 @@ + + + + + + + OpenRL · Cluster + + + +
+ diff --git a/src/server/dashboard/static/style.css b/src/server/dashboard/static/style.css new file mode 100644 index 000000000..99a0ee4ca --- /dev/null +++ b/src/server/dashboard/static/style.css @@ -0,0 +1,351 @@ +:root { color-scheme: light dark; /* A scrollbar appearing as content loads would otherwise narrow and reflow the page. */ scrollbar-gutter: stable; } +body { margin: 0; background: light-dark(#ffffff, #131519); } +#app { --ink: light-dark(#202328, #e6e8ed); --dim: light-dark(#626874, #a0a6b1); --ground: light-dark(#ffffff, #131519); --paper: light-dark(#f7f8fa, #191c21); --line: light-dark(#e3e6eb, #2a2e35); --hover: light-dark(#eff1f4, #20242b); --selected: light-dark(#e8ebf0, #282d35); --control-border: light-dark(#c9ced7, #424954); --accent: light-dark(#416bc3, #83a9ff); --hue-0: var(--accent); --hue-1: light-dark(#357e70, #76b3a2); --hue-2: light-dark(#7a60bb, #ad98dc); --hue-3: light-dark(#a87434, #c3a072); --hue-4: light-dark(#a8496b, #cd8ca5); --hue-5: light-dark(#2f8a9a, #7cb3bd); --hue-6: light-dark(#858533, #b5b579); --hue-7: light-dark(#b85538, #e39a7f); --status-success: light-dark(#27725c, #83baa5); --status-warning: light-dark(#895e1e, #ccb079); --status-error: light-dark(#ab4249, #da949a); --status-complete: light-dark(#496a97, #9bb0ce); --radius: 5px; --gutter: 22px; --node-label-width: 200px; --gpu-index-width: 38px; --gpu-index-gap: 6px; --sans: Inter, ui-sans-serif, system-ui, sans-serif; --mono: "SFMono-Regular", Consolas, monospace; color: var(--ink); background: var(--ground); width: 100%; min-height: 100dvh; font: 13px/1.5 var(--sans); -webkit-font-smoothing: antialiased; } +#app * { box-sizing: border-box; } +#app :is(h1, h2, p, figure) { margin: 0; } +#app a { color: inherit; text-decoration: none; } +#app a:hover { text-decoration: underline; text-underline-offset: 3px; } +#app button { cursor: pointer; font: inherit; color: inherit; } +#app :where(input, select, .chip, .time-shift) { min-height: 32px; max-width: 100%; padding: 5px 9px; border: 1px solid var(--line); border-radius: var(--radius); background-color: var(--paper); color: var(--ink); font: inherit; line-height: 20px; box-shadow: none; } +#app select { appearance: none; padding-right: 30px; background-image: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='12' height='8' viewBox='0 0 12 8'%3E%3Cpath d='m1 1.5 5 5 5-5' fill='none' stroke='%23878d98' stroke-width='1.4' stroke-linecap='round' stroke-linejoin='round'/%3E%3C/svg%3E"); background-repeat: no-repeat; background-position: right 10px center; } +#app input::placeholder { color: var(--dim); opacity: 1; } +#app :is(input, select):hover, #app .chip:hover:not(:disabled) { border-color: var(--control-border); } +#app .chip:hover:not(:disabled) { background-color: var(--hover); } +#app .chip[aria-pressed="true"] { background: var(--selected); border-color: var(--control-border); } +#app button:disabled { cursor: default; opacity: 0.5; } +#app :is(a, button, input, select):focus-visible { outline: 1px solid var(--accent); outline-offset: 3px; } +#app :is(input, select):focus { border-color: var(--control-border); box-shadow: none; } +#app :is(input, select):focus:not(:focus-visible) { outline: none; } +#app .appbar { display: flex; flex-wrap: wrap; align-items: center; gap: 12px 28px; padding: 14px var(--gutter); border-bottom: 1px solid var(--line); } +#app nav { display: flex; flex-wrap: wrap; gap: 20px; } +#app nav a { display: inline-flex; align-items: center; min-height: 32px; color: var(--dim); } +#app nav a:hover { color: var(--ink); text-decoration: none; } +#app nav [aria-current="page"] { color: var(--ink); text-decoration: underline; text-decoration-thickness: 2px; text-underline-offset: 8px; } +#app .brand { font-size: 16px; font-weight: 500; } +#app #connection { margin-left: auto; color: var(--dim); font-size: 12px; } +#app .pad { padding: 22px var(--gutter); } +#app .heading { margin-bottom: 16px; font-size: 22px; font-weight: 500; line-height: 1.3; overflow-wrap: anywhere; } +#app .mono { font-family: var(--mono); } +#app .muted { color: var(--dim); } +#app .micro { font-size: 11px; } +#app .empty-state { padding: 16px 0; color: var(--dim); } +/* Jobs, processes and health share the same table rhythm. */ +#app :is(.job-list-head, .job-list-row) { display: grid; grid-template-columns: minmax(0, 1fr) 112px 100px 112px 100px; align-items: center; gap: 16px; padding: 12px 0; } +#app .job-list-head { color: var(--dim); font-size: 12px; } +#app .job-list-row { border-top: 1px solid var(--line); } +#app .job-list-row > * { min-width: 0; overflow-wrap: anywhere; } +#app .job-list-row:hover { text-decoration: none; } +#app .job-list-row:hover .job-identity > span:first-child { text-decoration: underline; text-underline-offset: 3px; } +#app .job-identity { display: grid; gap: 3px; min-width: 0; overflow-wrap: anywhere; } +#app .job-identity .micro { font-size: 12px; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } +#app :is(.job-list-head, .job-list-row) > :nth-child(n + 4) { text-align: right; font-variant-numeric: tabular-nums; } +/* Overview: search and status chips above the run list. */ +#app .overview-toolbar { display: flex; flex-wrap: wrap; align-items: center; gap: 10px 16px; margin-bottom: 14px; } +#app #run-search { flex: 1 1 280px; max-width: 420px; } +#app .run-filters { display: flex; flex-wrap: wrap; gap: 6px; } +#app .metric-filters { display: flex; flex-wrap: wrap; gap: 8px; align-items: center; margin-bottom: 12px; } #app .metric-filters input { width: 28ch; } +#app .run-filter-count { margin-left: 4px; color: var(--dim); font-variant-numeric: tabular-nums; } +#app :is(.job-list-head, .job-list-row).run-row { position: relative; grid-template-columns: minmax(0, 1fr) 120px 72px 88px 160px; gap: 24px; } +/* The run link covers its row. */ +#app .run-row .job-identity a::after { content: ""; position: absolute; inset: 0; } +#app .run-row:hover .job-identity a { text-decoration: underline; text-underline-offset: 3px; } +#app :is(.job-list-head, .job-list-row).run-row > :is(:nth-child(3), :nth-child(4)) { text-align: right; font-variant-numeric: tabular-nums; } +#app .run-row > .run-session, #app .job-list-head.run-row > :nth-child(5) { text-align: left; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } +/* Sessions group the run rows; a session's runs are indented under its header. */ +#app .state { display: inline-flex; align-items: center; gap: 7px; white-space: nowrap; } +#app :is(.state-dot, .health-dot) { width: 6px; height: 6px; border-radius: 50%; background: currentColor; flex: none; } +#app .state-running { color: var(--status-success); } +#app .state-pending { color: var(--status-warning); } +#app .state-failed { color: var(--status-error); } +#app .state-completed { color: var(--status-complete); } +#app .state-neutral { color: var(--dim); } +#app :is(.run-table, .scheduler-table) { width: 100%; border-collapse: collapse; } +#app :is(.run-table, .scheduler-table) :is(th, td) { padding: 12px 12px 12px 0; border-bottom: 1px solid var(--line); text-align: left; vertical-align: top; overflow-wrap: anywhere; } +#app :is(.run-table, .scheduler-table) :is(th, td):last-child { padding-right: 0; } +#app :is(.run-table, .scheduler-table) th { color: var(--dim); font-weight: 500; } +#app .run-table { table-layout: fixed; min-width: 720px; } +#app .run-table th:first-child { width: 38%; } +#app .run-table :is(th, td):last-child { text-align: right; } +#app .health-status { --health-tone: var(--status-warning); display: inline-flex; align-items: center; gap: 8px; white-space: nowrap; padding: 4px 8px; border-radius: var(--radius); color: var(--health-tone); background: color-mix(in srgb, var(--health-tone) 9%, var(--ground)); } +#app .health-error { --health-tone: var(--status-error); } +#app .health-success { --health-tone: var(--status-success); } +#app .health-message { display: flex; align-items: baseline; flex-wrap: wrap; gap: 8px 12px; margin: 12px 0; overflow-wrap: anywhere; } +/* Nodes: the header, GPU lanes and duty values use one column grid. */ +#app .nodes-heading { display: flex; justify-content: space-between; align-items: center; flex-wrap: wrap; gap: 12px 16px; margin-bottom: 18px; } +#app :is(.nodes-heading, .run-heading) .heading { margin: 0; } +#app .time-control { position: relative; display: grid; grid-template-columns: 32px minmax(0, 1fr) 32px 80px; width: 424px; max-width: 100%; align-items: center; gap: 8px; color: var(--dim); font-size: 12px; } +#app .time-picker { min-width: 0; color: var(--ink); } +#app .time-summary { display: flex; align-items: center; justify-content: space-between; gap: 8px; height: 32px; padding: 5px 10px; border: 1px solid var(--line); border-radius: var(--radius); background: var(--paper); list-style: none; cursor: pointer; line-height: 20px; white-space: nowrap; } +#app .time-summary::-webkit-details-marker { display: none; } +#app .time-summary > span { overflow: hidden; text-overflow: ellipsis; } +#app .time-chevron { display: block; flex: none; width: 16px; height: 16px; fill: none; stroke: var(--dim); stroke-width: 1.5; stroke-linecap: round; stroke-linejoin: round; } +#app .time-summary:hover, #app .time-picker[open] .time-summary { border-color: var(--control-border); } +#app .time-summary:focus-visible { outline: 1px solid var(--accent); outline-offset: 3px; } +#app .time-shift { display: grid; place-items: center; width: 32px; height: 32px; padding: 0; line-height: 20px; } +#app .time-popover { position: absolute; z-index: 5; top: 38px; left: 0; right: 88px; display: grid; gap: 12px; padding: 14px; border: 1px solid var(--control-border); border-radius: var(--radius); background: var(--ground); } +#app .time-popover label { display: grid; align-items: center; grid-template-columns: 70px minmax(0, 1fr); gap: 8px; } +#app .time-popover :is(input, select) { width: 100%; height: 32px; } +#app .time-control input { min-width: 0; } +#app :is(.node-lane, .node-time-header) { display: grid; grid-template-columns: var(--node-label-width) minmax(0, 1fr) 48px; align-items: start; gap: 12px; } +#app .node-lane { padding: 10px 0; border-bottom: 1px solid var(--line); } +#app .node-time-header { padding-bottom: 8px; color: var(--dim); font-size: 11px; } +#app .node-axis { position: relative; display: flex; justify-content: space-between; margin-left: calc(var(--gpu-index-width) + var(--gpu-index-gap)); font-variant-numeric: tabular-nums; } +#app .node-axis span:nth-child(2) { position: absolute; left: 33.333333%; } +#app .node-axis span:nth-child(3) { position: absolute; left: 66.666667%; } +#app .node-axis span:not(:first-child):not(:last-child) { transform: translateX(-50%); } +#app .node-duty { text-align: right; font-variant-numeric: tabular-nums; line-height: 26px; } +#app .node-time-header .node-duty { line-height: inherit; } +#app .node-lane-label { display: flex; align-items: center; gap: 7px; min-width: 0; min-height: 26px; padding: 0; border: 0; background: transparent; text-align: left; white-space: nowrap; } +#app .node-lane-label:hover .node-accelerator { text-decoration: underline; text-underline-offset: 3px; } +#app .node-accelerator { min-width: 0; overflow: hidden; text-overflow: ellipsis; } +#app .unavailable-selection { display: flex; justify-content: space-between; align-items: center; gap: 12px; margin-bottom: 12px; color: var(--status-warning); } +#app .claim-label { flex: none; padding: 1px 5px; border-radius: 3px; background: var(--paper); color: var(--dim); font-size: 11px; line-height: 18px; white-space: nowrap; } +#app .gpu-capacity { display: grid; grid-template-columns: var(--gpu-index-width) minmax(0, 1fr); gap: var(--gpu-index-gap); min-width: 0; } +#app .gpu-lane-ids { display: grid; align-items: center; color: var(--dim); font: 11px var(--mono); } +#app .gpu-lane-ids span { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } +#app .capacity-track { position: relative; min-width: 0; background: repeating-linear-gradient( to bottom, transparent 0 calc(var(--gpu-lane-height) - 1px), color-mix(in srgb, var(--line) 70%, transparent) calc(var(--gpu-lane-height) - 1px) var(--gpu-lane-height) ); } +#app :is(.capacity-track, .node-axis, .activity-track, .activity-axis, .allocation-expansion .chart-plot svg, .allocation-expansion .chart-empty) { touch-action: pan-y; cursor: grab; } +#app.node-navigating :is(.capacity-track, .node-axis, .activity-track, .activity-axis, .allocation-expansion .chart-plot svg, .allocation-expansion .chart-empty) { cursor: grabbing; user-select: none; } +#app .capacity-track::before { content: ""; position: absolute; inset: 0; pointer-events: none; background: repeating-linear-gradient( to right, transparent 0 calc(33.333333% - 1px), color-mix(in srgb, var(--line) 45%, transparent) calc(33.333333% - 1px) 33.333333% ); } +#app .hue-0 { --allocation-color: var(--hue-0); } +#app .hue-1 { --allocation-color: var(--hue-1); } +#app .hue-2 { --allocation-color: var(--hue-2); } +#app .hue-3 { --allocation-color: var(--hue-3); } +#app .hue-4 { --allocation-color: var(--hue-4); } +#app .hue-5 { --allocation-color: var(--hue-5); } +#app .hue-6 { --allocation-color: var(--hue-6); } +#app .hue-7 { --allocation-color: var(--hue-7); } +#app .capacity-allocation { container: allocation / inline-size; position: absolute; overflow: hidden; display: flex; align-items: center; justify-content: space-between; gap: 8px; min-width: 0; padding: 0; border: 0; border-radius: 2px; background: color-mix( in srgb, var(--allocation-color, var(--accent)) 19%, var(--ground) ); text-align: left; } +#app .capacity-allocation:hover { background: color-mix( in srgb, var(--allocation-color, var(--accent)) 27%, var(--ground) ); } +#app .capacity-allocation.selected:not(.held) { z-index: 1; background: color-mix( in srgb, var(--allocation-color, var(--accent)) 27%, var(--ground) ); outline: 1px solid color-mix(in srgb, var(--allocation-color, var(--accent)) 70%, var(--line)); } +#app .capacity-allocation.ended { --allocation-color: var(--dim); color: var(--dim); } +#app .capacity-allocation.held { overflow: hidden; background: color-mix(in srgb, var(--line) 45%, var(--ground)); } +#app :is(.allocation-name, .allocation-count) { position: relative; z-index: 1; } +#app .capacity-allocation.unknown-mapping { position: relative; width: 100%; height: 26px; margin-block: 3px; } +#app .allocation-name { margin-left: 7px; min-width: 0; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } +#app .allocation-count { margin-right: 7px; flex: none; margin-left: auto; color: var(--dim); font-size: 11px; white-space: nowrap; } +@container allocation (max-width: 160px) { + #app :is(.allocation-name, .allocation-count) { display: none; } +} +#app .hold { position: absolute; top: 0; bottom: 0; background: color-mix(in srgb, var(--allocation-color) 35%, var(--ground)); pointer-events: auto; } +#app .hold:is(:hover, .selected) { background: color-mix(in srgb, var(--allocation-color) 65%, var(--ground)); } +#app .allocation-expansion { --activity-label-width: clamp(160px, 25vw, 240px); display: grid; grid-template-columns: minmax(0, 1fr); gap: 12px; margin: 0 calc(-1 * var(--gutter)); padding: 16px var(--gutter); border-bottom: 1px solid var(--line); } +#app .allocation-expansion > div { min-width: 0; } +#app .node-gpu-detail { display: grid; grid-template-columns: minmax(0, 1fr); gap: 12px; border-top: 1px solid var(--line); padding-top: 16px; } +#app .allocation-detail-head { display: flex; flex-wrap: wrap; align-items: center; justify-content: space-between; gap: 12px; } +#app .allocation-detail-head h2 { font-size: 15px; font-weight: 500; line-height: 1.4; overflow-wrap: anywhere; } +#app .detail-close { display: grid; place-items: center; width: 28px; height: 28px; padding: 0; border: 1px solid transparent; border-radius: var(--radius); color: var(--dim); background: transparent; font-size: 22px; line-height: 1; } +#app .detail-close:hover { border-color: var(--line); background: var(--paper); color: var(--ink); } +#app .allocation-device-picker { display: flex; flex-wrap: wrap; gap: 4px; } +#app .allocation-device-picker .chip { border-color: transparent; background: transparent; color: var(--dim); } +#app .allocation-device-picker .chip:not([data-device="all"]) { min-width: 10ch; /* Fits "7 · 100%", so arriving and updating values never resize the chips. */ font-variant-numeric: tabular-nums; /* Equal-width digits: a new value of the same length doesn't move the text. */ text-align: center; } #app .allocation-device-picker .chip[aria-pressed="true"], #app .allocation-device-picker .chip:hover { border-color: var(--line); background: var(--paper); color: var(--ink); } #app .allocation-memory { color: var(--dim); font-size: 12px; } #app .allocation-memory strong { display: inline-block; min-width: 10ch; /* Fits "1536.0 GiB", so the label doesn't slide when the value arrives. */ margin-left: 8px; color: var(--ink); font-size: 15px; font-weight: 500; text-align: right; } +#app .allocation-detail-footer { display: flex; flex-wrap: wrap; justify-content: space-between; gap: 12px; color: var(--dim); font-size: 11px; } +#app :is(.activity-header, .activity-row) { display: grid; grid-template-columns: var(--activity-label-width) minmax(0, 1fr); column-gap: 12px; align-items: center; } +#app .activity-header { padding-block: 4px 10px; } +#app .activity-header h3 { margin: 0; font-size: 13px; font-weight: 500; } +#app .activity-axis { display: flex; justify-content: space-between; color: var(--dim); font: 11px var(--mono); } +#app .activity-row { min-height: 46px; border-top: 1px solid var(--line); } +#app .activity-label { display: flex; align-items: center; gap: 8px; min-width: 0; padding: 6px 8px; border: 1px solid transparent; border-radius: var(--radius); background: none; text-align: left; font-size: 12px; } +#app a.activity-label:hover { background: var(--hover); } +#app .activity-row[data-selected="true"] .activity-label { border-color: var(--line); background: var(--paper); } +#app .activity-swatch { flex: none; width: 9px; height: 9px; border-radius: 2px; background: color-mix(in srgb, var(--allocation-color) 65%, var(--ground)); } +#app .activity-label-text { min-width: 0; } +#app .activity-name { display: block; min-width: 0; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } +#app .activity-meta { display: block; margin-top: 2px; color: var(--dim); font-size: 11px; } +#app .activity-track { position: relative; min-width: 0; height: 46px; display: flex; align-items: center; background: repeating-linear-gradient(to right, var(--line) 0 1px, transparent 1px 25%); box-shadow: inset -1px 0 var(--line); } +#app .activity-track svg { display: block; width: 100%; height: 24px; } +#app .activity-block { fill: color-mix(in srgb, var(--allocation-color) 35%, var(--ground)); cursor: default; } +#app .activity-block[data-placement] { cursor: pointer; } +#app .activity-block[data-selected="true"], #app .activity-block:hover { fill: color-mix(in srgb, var(--allocation-color) 65%, var(--ground)); } +#app .activity-block[data-selected="true"] { stroke: var(--allocation-color); stroke-width: 1px; } +/* A native SVG outline spans the gaps between intervals; stroke each interval instead. */ +#app .activity-block:focus { outline: none; } +#app .activity-block:focus-visible { stroke: var(--accent); stroke-width: 2px; } +/* Worker-recorded ops inside a GPU turn. The turn underneath stays visible + where no op was recorded. */ +#app [data-op] { --op-color: #2a9d8f; } +#app [data-op="forward_backward"] { --op-color: #3366cc; } +#app [data-op="forward"] { --op-color: #6fa8dc; } +#app [data-op="optim_step"] { --op-color: #8e44ad; } +#app [data-op="sample"] { --op-color: #2e9e5b; } +#app [data-op="weight_sync"] { --op-color: #e67e22; } +#app [data-op="wake_up"], #app [data-op="sleep"] { --op-color: #95a5a6; } +#app [data-op="init"] { --op-color: #5d6d7e; } +#app .activity-block.op { fill: var(--op-color); } +#app .activity-block.op:hover { fill: color-mix(in srgb, var(--op-color) 75%, var(--ink, #000)); } +#app .op-legend { display: flex; flex-wrap: wrap; gap: 4px 14px; margin: 8px 0 0 calc(var(--activity-label-width, 0px) + 12px); font-size: 12px; color: var(--muted, inherit); } +#app .op-legend > span { display: inline-flex; align-items: center; gap: 5px; } +#app .op-swatch { width: 10px; height: 10px; border-radius: 2px; background: var(--op-color); } +#app .op-swatch.turn { background: color-mix(in srgb, currentColor 25%, transparent); } +#app .cross-node-activity { --activity-label-width: clamp(160px, 25vw, 240px); border-top: 1px solid var(--line); padding-top: 16px; } +#app .cross-node-activity .activity-meta { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } +/* The Run timeline has no pan gesture; a click opens the logs at that time. */ +#app section.cross-node-activity .activity-track { cursor: pointer; } +#app [data-crosshair] :is(.activity-track, .capacity-track)::after { content: ""; position: absolute; pointer-events: none; inset-block: 0; left: var(--crosshair); border-left: 1px dashed var(--dim); } +#app .activity-tooltip { position: fixed; z-index: 10; pointer-events: none; max-width: min(360px, calc(100vw - 16px)); max-height: calc(100dvh - 16px); overflow: hidden; overflow-wrap: anywhere; padding: 9px 12px; border: 1px solid var(--control-border); border-radius: var(--radius); background: var(--paper); font-size: 12px; } +#app .activity-tooltip-label { display: block; margin-bottom: 4px; } +#app :is(.activity-tooltip-time, .activity-tooltip-detail) { color: var(--dim); } +#app .activity-status { position: absolute; left: 8px; max-width: calc(100% - 16px); padding: 2px 4px; color: var(--dim); background: var(--ground); font-size: 11px; } +#app .activity-status.unavailable { color: var(--status-warning); } +/* Chart layout stays separate from the SVG gridlines. */ +#app div.chart-grid { display: grid; grid-template-columns: repeat(auto-fit, minmax(min(100%, 340px), 1fr)); gap: 24px 20px; margin-top: 16px; } +#app .chart { --metric-stroke: var(--dim); --plot-height: 160px; min-width: 0; min-height: calc(var(--plot-height) + 82px); } +#app .chart > .chart-empty { display: grid; place-items: center; height: var(--plot-height); margin: 0; } +#app .chart[data-tone="accent"] { --metric-stroke: color-mix(in srgb, var(--accent) 75%, var(--dim)); } +#app .allocation-expansion .chart { --plot-height: 100px; min-height: calc(var(--plot-height) + 82px); } +#app .allocation-expansion .chart-plot { grid-template-columns: var(--activity-label-width) minmax(0, 1fr); column-gap: 12px; } +#app .allocation-expansion .chart-empty { margin-left: calc(var(--activity-label-width) + 12px); } +#app .allocation-expansion .chart-line { stroke-width: 1.5; } +#app .chart-head { display: flex; flex-wrap: wrap; align-items: baseline; justify-content: space-between; gap: 5px 16px; margin-bottom: 12px; } +#app .chart-head h2 { font-size: 13px; font-weight: 500; overflow-wrap: anywhere; } +#app .chart-latest { display: flex; align-items: baseline; gap: 7px; color: var(--dim); font-size: 11px; white-space: nowrap; } +#app .chart-latest strong { color: var(--ink); font: 500 15px var(--mono); } +#app .chart-plot { position: relative; display: grid; grid-template-columns: 44px minmax(0, 1fr); grid-template-rows: var(--plot-height) auto; column-gap: 8px; min-width: 0; } +#app .chart-y { margin-block: -6px; display: flex; flex-direction: column; justify-content: space-between; text-align: right; color: var(--dim); font: 11px var(--mono); } +#app .chart-plot:focus-visible { outline: 1px solid var(--control-border); outline-offset: 4px; } +#app .chart-plot svg { display: block; width: 100%; height: 100%; overflow: visible; border-bottom: 1px solid var(--line); } +#app .chart-x { grid-column: 2; position: relative; display: flex; justify-content: space-between; padding-top: 8px; color: var(--dim); font: 11px var(--mono); } +#app svg .chart-grid { stroke: var(--line); stroke-width: 1; } +#app .chart-line { fill: none; stroke: var(--metric-stroke); stroke-width: 1.6; stroke-linejoin: round; stroke-linecap: round; } +#app .chart-area { fill: var(--metric-stroke); fill-opacity: 0.07; } +#app .chart-band { fill: var(--allocation-color); fill-opacity: 0.55; } +#app .activity-block.wait { fill: var(--dim); fill-opacity: 0.16; } +#app .op-swatch.wait { background: var(--dim); opacity: 0.3; } +#app .chart-band.sampler { fill-opacity: 0.28; } +#app .chart-band-swatch { display: inline-block; width: 9px; height: 9px; margin-right: 5px; border-radius: 2px; background: var(--allocation-color); opacity: 0.55; vertical-align: -1px; } +#app .chart-band-key.sampler .chart-band-swatch { opacity: 0.28; } +#app .chart-dot { stroke: var(--metric-stroke); stroke-width: 5; stroke-linecap: round; } +#app .chart-hover { width: 0; position: absolute; top: 0; height: var(--plot-height); pointer-events: none; z-index: 2; } +#app .chart-hover[hidden] { display: none; } +#app .chart-cursor { position: absolute; inset: 0 auto 0 0; border-left: 1px dashed var(--control-border); } +#app .chart-crosshair { stroke: var(--control-border); stroke-dasharray: 3 3; } +#app .chart-tip { position: absolute; top: 6px; left: 8px; padding: 5px 8px; border: 1px solid var(--control-border); border-radius: var(--radius); background: var(--paper); color: var(--ink); font: 11px/1.5 var(--mono); white-space: nowrap; } +#app .chart-point { position: absolute; width: 6px; height: 6px; border-radius: 50%; background: var(--metric-stroke); transform: translate(-50%, -50%); } +#app .chart-summary { display: flex; flex-wrap: wrap; gap: 5px 18px; margin-top: 10px; color: var(--dim); font-size: 11px; } +#app .chart-summary strong { margin-left: 4px; color: var(--ink); font-family: var(--mono); font-weight: 400; } +#app .chart-note { color: var(--status-warning); } +#app .chart-empty { min-height: var(--plot-height); display: flex; align-items: center; color: var(--dim); } +/* Run details and logs. */ +#app .overview-back { margin-bottom: 16px; color: var(--dim); } +#app .run-heading { display: flex; align-items: center; flex-wrap: wrap; gap: 8px 16px; } +#app .run-description { margin-top: 4px; color: var(--dim); overflow-wrap: anywhere; } +#app .run-state { margin-top: 12px; overflow-wrap: anywhere; } +#app .run-toolbar { display: flex; align-items: center; justify-content: space-between; flex-wrap: wrap; gap: 8px 16px; margin: 16px 0; padding-bottom: 12px; border-bottom: 1px solid var(--line); } +#app .run-time-controls label { display: flex; align-items: center; flex-wrap: wrap; gap: 8px; } +#app .run-time-controls :is(label, a) { color: var(--dim); font-size: 12px; } +#app .run-incidents { display: grid; gap: 8px; margin-bottom: 16px; } +#app .run-incident { display: flex; align-items: baseline; flex-wrap: wrap; gap: 4px 12px; font-size: 12px; } +#app .incident-time { color: var(--dim); font-family: var(--mono); } +#app .incident-error .incident-label { color: var(--status-error); } +#app .incident-warning .incident-label { color: var(--status-warning); } +#app .incident-context { color: var(--dim); overflow-wrap: anywhere; } +#app .run-incident a { color: var(--dim); white-space: nowrap; } +#app .run-metric-summary { display: flex; flex-wrap: wrap; gap: 8px 16px; margin-bottom: 16px; } +#app .run-activity { border-top: 0; padding-top: 0; } +#app .run-activity .activity-title { margin-top: 8px; margin-bottom: 4px; } +#app .run-activity .activity-intro { margin: 0 0 12px; font-size: 12px; } +#app .run-activity :is(.activity-header, .activity-row) { grid-template-columns: var(--activity-label-width) 88px minmax(0, 1fr) 48px; } +#app .run-activity .activity-gpu { color: var(--dim); font: 11px var(--mono); overflow-wrap: anywhere; } +#app .activity-gpu-toggle { padding: 0; border: 0; background: none; text-align: left; cursor: pointer; } +#app .activity-gpu-toggle:hover { color: var(--ink); } +#app .activity-chevron { display: inline-block; width: 12px; } +#app .activity-duty { padding: 10px 0 14px; border-top: 1px solid var(--line); } +#app :is(.activity-mode, .chart-mode) { display: inline-flex; gap: 4px; } +#app .activity-mode { margin: 0 0 12px; } +#app .chart-mode { margin-left: 12px; margin-right: auto; } +#app .activity-duty .chart { --plot-height: 100px; min-height: calc(var(--plot-height) + 82px); } +/* The plot lines up with the lane tracks: past the label and GPU columns, short of the duty column. */ +#app .activity-duty .chart-plot { grid-template-columns: calc(var(--activity-label-width) + 100px) minmax(0, 1fr); column-gap: 12px; margin-right: 60px; } +#app .run-activity .activity-row[data-continued] { min-height: 30px; border-top-color: transparent; } +#app .run-activity .activity-row[data-continued] .activity-track { height: 30px; } +#app .run-activity .activity-row[data-lead] { min-height: 30px; } +#app .run-activity .activity-row[data-lead] .activity-track { height: 30px; } +/* A group's label overhangs the empty label cells of its later rows, so every GPU row keeps one pitch. */ +#app .run-activity .activity-row[data-lead] .activity-label { align-self: start; align-items: flex-start; height: 30px; overflow: visible; } +#app .activity-source { margin-top: 12px; color: var(--dim); font-size: 11px; } +#app .run-metrics { margin-top: 8px; } +#app .time-control.run-time-controls { grid-template-columns: 32px minmax(0, 1fr) 32px auto auto; width: 520px; } +#app .log-toolbar { display: flex; flex-wrap: wrap; gap: 8px; margin-bottom: 12px; } +#app #run-panel > p + .log-toolbar { margin-top: 8px; } +#app .log-toolbar input { flex: 1; min-width: 100px; } +#app .log-toolbar select { max-width: min(100%, 340px); } +#app .log-toolbar .log-scope { display: flex; align-items: center; gap: 6px; color: var(--dim); font-size: 12px; white-space: nowrap; } +#app .log-toolbar .log-scope input { flex: none; min-width: 0; margin: 0; } +/* Fits "Jump to latest", so pausing by scrolling doesn't shift the toolbar. */ +#app .log-toolbar button { min-width: 8.75em; } +#app #log-status { display: grid; gap: 6px; color: var(--dim); font-size: 12px; } +#app #log-status:not(:empty) { margin-bottom: 12px; } +#app .source-error { margin: 8px 0; color: var(--status-warning); font-size: 12px; overflow-wrap: anywhere; } +#app .snapshot-error:not([hidden]) { display: flex; align-items: center; justify-content: space-between; gap: 8px 16px; margin: 16px var(--gutter) 0; padding: 12px; border: 1px solid color-mix(in srgb, var(--status-warning) 35%, var(--line)); border-radius: var(--radius); color: var(--status-warning); background: var(--paper); overflow-wrap: anywhere; } +#app .snapshot-error button { flex: none; } +#app pre, #app .workspace-logrow { white-space: pre-wrap; overflow-wrap: anywhere; font: 12px/1.65 var(--mono); } +/* At the bottom of the page the tabs, toolbar and lines fill the window; at the box's ends the wheel scrolls the page. */ +#app #log-lines { max-height: calc(100svh - 170px); overflow: auto; overflow-anchor: none; scrollbar-gutter: stable; } +#app .workspace-logrow { display: grid; grid-template-columns: 12ch var(--tag-width) minmax(0, 1fr); gap: 10px; padding: 1px 6px; border-left: 2px solid transparent; line-height: 1.5; } +#app .workspace-logrow[data-severity="error"] { border-left-color: var(--status-error); background: color-mix(in srgb, var(--status-error) 7%, transparent); } +#app .workspace-logrow[data-severity="warning"] { border-left-color: var(--status-warning); background: color-mix(in srgb, var(--status-warning) 7%, transparent); } +#app .log-time { color: var(--dim); white-space: nowrap; font-variant-numeric: tabular-nums; } +#app .log-pod { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; color: var(--allocation-color, var(--dim)); } +#app .log-message { min-width: 0; margin: 0; font: inherit; } +#app .log-message mark { border-radius: 2px; background: color-mix(in srgb, var(--hue-3) 30%, transparent); color: inherit; } +#app .log-older { margin: 0 0 8px; } +#app .log-error { color: var(--status-error); } +#app .log-time-scope { display: flex; align-items: baseline; flex-wrap: wrap; gap: 8px 16px; margin-bottom: 12px; color: var(--dim); font-size: 12px; } +#app .log-time-scope a { color: var(--ink); } +#app .run-json-link { margin-top: 20px; color: var(--dim); font-size: 12px; } +/* Section titles and wide tables on the Run and Health pages. */ +#app .scheduler-title { margin: 24px 0 12px; font-size: 15px; font-weight: 500; line-height: 1.4; } +#app :is(.scheduler-table-wrap, .table-scroll) { overflow-x: auto; } +#app .scheduler-table { min-width: 650px; } +/* Health: only the evidence wraps, so pod names and links stay whole. */ +#app .scheduler-table :is(th, td):not(:nth-child(3)) { white-space: nowrap; } +/* Keep the identity readable before switching to compact rows. */ +@media (min-width: 761px) and (max-width: 1100px) { + #app :is(.job-list-head, .job-list-row).run-row { grid-template-columns: minmax(0, 1fr) 112px 56px 72px 140px; gap: 16px; } +} +@media (max-width: 760px) { + #app :is(.job-list-head, .job-list-row).run-row { grid-template-columns: minmax(0, 1fr) minmax(100px, auto); gap: 6px 12px; } + #app .job-list-head.run-row > :nth-child(n + 3) { display: none; } + #app .run-row > :nth-child(2) { justify-self: end; } + #app .run-row > :nth-child(n + 3) { color: var(--dim); font-size: 12px; } + #app .run-row > :nth-child(3) { grid-column: 1; text-align: left; } + #app .run-row > :nth-child(4) { grid-column: 2; text-align: right; } + #app .job-list-row.run-row > :nth-child(3)::after { content: " steps"; } + #app .job-list-row.run-row > :nth-child(4)::before { content: "Elapsed "; } + #app .job-list-row.run-row > .run-session { grid-column: 1 / -1; text-align: left; } +} +@media (max-width: 580px) { + #app .time-control { width: 100%; } + #app { --gutter: 14px; --gpu-index-width: 24px; --gpu-index-gap: 4px; } + #app .appbar { padding-block: 12px; gap: 8px 16px; } + #app .appbar > nav { order: 3; width: 100%; flex-wrap: nowrap; gap: 16px; overflow-x: auto; white-space: nowrap; font-size: 12px; } + #app #run-search { max-width: none; } + #app .pad { padding-block: 16px; } + #app .heading { font-size: 20px; } + #app :is(.node-lane, .node-time-header) { grid-template-columns: minmax(0, 1fr) 36px; gap: 6px 8px; } + #app .node-time-header > span:first-child { display: none; } + #app .node-lane-label { grid-column: 1 / -1; min-height: 20px; gap: 6px; font-size: 12px; } + #app .time-control { grid-template-columns: 32px minmax(0, 1fr) 32px; } + #app .time-control > [data-copy-view] { grid-column: 1 / -1; justify-self: end; } + #app .time-popover { right: 0; } + #app .claim-label { padding-inline: 3px; font-size: 10px; } + #app .node-axis span:not(:first-child):not(:last-child), #app .chart-x span:not(:first-child):not(:last-child) { display: none; } + #app .capacity-track::before { background: none; border-right: 1px solid var(--line); } + #app :is(.allocation-expansion, .cross-node-activity) { --activity-label-width: 40px; } + #app :is(.activity-header, .activity-row) { grid-template-columns: minmax(0, 1fr); } + #app .activity-header h3 { display: none; } + #app .activity-label { margin-top: 4px; } + #app .activity-track { height: 34px; } + #app .op-legend { margin-left: 0; } + #app .activity-axis span:not(:first-child):not(:last-child) { display: none; } + #app .run-activity :is(.activity-header, .activity-row) { grid-template-columns: 64px minmax(0, 1fr) 36px; } + #app .run-activity .activity-label { grid-column: 1 / -1; } + #app .run-activity .activity-axis { grid-column: 2; } + #app .run-activity .activity-row > span:empty { display: none; } + #app .chart-tip { max-width: 230px; white-space: normal; width: max-content; } + #app .run-toolbar { align-items: flex-start; } + #app .run-time-controls { width: 100%; justify-content: space-between; } + #app .log-toolbar > :is(input, select) { flex: 1 1 100%; width: 100%; min-width: 0; max-width: 100%; } + #app .workspace-logrow { grid-template-columns: auto minmax(0, 1fr); gap: 0 10px; padding-block: 4px; } + #app .log-message { grid-column: 1 / -1; } +} diff --git a/src/server/dashboard/ui/Activity.svelte b/src/server/dashboard/ui/Activity.svelte new file mode 100644 index 000000000..62f4a686c --- /dev/null +++ b/src/server/dashboard/ui/Activity.svelte @@ -0,0 +1,86 @@ + + tooltip = crosshair.at = null} onresize={() => tooltip = crosshair.at = null} /> + +
tooltip = crosshair.at = null} onpointerdown={() => tooltip = crosshair.at = null}> + {#if acrossNodes}

GPU activity

One row per accelerator. Colored bars are ops the process on that GPU ran; grey is GPU time held between ops. Duty is the share of time a kernel ran on the GPU, measured by DCGM.

+
{/if} + {#if !shown.length}

{empty}

+ {:else} +
+

{acrossNodes ? "Process" : "Run"}

{#if acrossNodes}

GPU

{/if}
{#each [0, 1, 2, 3, 4] as tick}{axisTime(range.start + (range.now - range.start) * tick / 4, range)}{/each}
{#if acrossNodes}Duty{/if}
+ {#each rows as { key, lane: p, device, first, lead } (key)} + {@const name = acrossNodes ? processLabel(p, inventory.cluster.nodes, { gpus: false }) : p.label} + {@const meta = acrossNodes ? p.node ? shortNodeName(p.node, inventory.cluster.nodes) : p.ended && p.pod ? p.pod : "Node unassigned" : p.role || "process"} + {@const link = href(p)} + {@const status = p.error ? "Ops unavailable" : !p.intervals.length ? "No recorded activity" : ""} +
+ {#if first} + {name}{meta}{p.ended ? " · Process ended" : p.phase && p.phase !== "ready" ? ` · ${p.phase.replace("_", " ")}` : ""}{!link ? acrossNodes ? p.node ? " · Node no longer reported" : "" : p.run_ids?.length ? " · Run record unavailable" : " · No linked run" : ""} + {:else}{/if} + {#if acrossNodes}{#if device && p.node}{:else}{device?.label ?? "GPUs not recorded"}{/if}{/if} + +
onpick(Math.round(timeAt(event, range))))}> + {#if mode === "lanes"} + {#if p.waits?.length} inspect(event, p.waits, `Waiting for the GPU · ${rowLabel(p)}`, "From asking the time slicer to the grant")} />{/if} + + select(event, p)} onkeydown={(event) => select(event, p)} onpointermove={(event) => inspect(event, p.intervals, rowLabel(p), p.exact ? "GPU turns" : "Recorded operations")} /> + {#each Object.entries(p.ops).sort(([a], [b]) => opRank(a) - opRank(b)) as [op, spans] (op)} + + select(event, p)} onkeydown={(event) => select(event, p)} onpointermove={(event) => inspect(event, spans, `${opLabel(op)} · ${rowLabel(p)}`, "Worker-recorded op")} /> + {/each} + {/if} + {#if status && first}{status}{/if} +
+ {#if acrossNodes}{dutyText(device)}{/if} +
+ {#if (open[key] || mode === "duty") && device && p.node}
{/if} + {/each} + {#if ops.length && mode === "lanes"}
{#each ops as op}{opLabel(op)}{/each}held, unlabelled{#if shown.some((p) => p.waits?.length)}waiting for the GPU{/if}
{/if} +
+ {/if} + {#if error && shown.length}

Activity unavailable: {error}

{/if} + {#if warnings.length}

{warnings.join(" · ")}

{/if} + {#if showSource && activities.length}

{activitySource(activities)}

{/if} + +
diff --git a/src/server/dashboard/ui/App.svelte b/src/server/dashboard/ui/App.svelte new file mode 100644 index 000000000..35f902942 --- /dev/null +++ b/src/server/dashboard/ui/App.svelte @@ -0,0 +1,54 @@ + + + + { if (!document.hidden) inventory.retry(); }} /> +
+
+ OpenRL + + {connection} +
+ +
+ {#if inventory.data} + {#if view.page === "run"} + {#key view.id}{/key} + {:else if view.page === "nodes"} + {:else}{/if} + {:else if !inventory.error}Loading cluster…{/if} +
+
diff --git a/src/server/dashboard/ui/Chart.svelte b/src/server/dashboard/ui/Chart.svelte new file mode 100644 index 000000000..62a8b61e6 --- /dev/null +++ b/src/server/dashboard/ui/Chart.svelte @@ -0,0 +1,128 @@ + + + selectedAt = null} /> +
+

{title}

{#if readingSeconds}{#if bands.length}{/if}{/if}{#if data.length && mode !== "running"}Latest {valueText(data.at(-1)[1], unit)}{/if}
+ {#if !data.length}

{empty}

+ {:else} + +
selectedAt = crosshair.at = null} onfocusout={() => selectedAt = null} onkeydown={keyboard}> +
{#each scale.ticks as tick}{chartNumber(tick)}{/each}
+ + {#each scale.ticks as tick}{/each} + {#each paths as path}{#if path.area}{/if}{/each} + {#each bands as band, i (band.key)}{#each band.spans as [a, b]}{/each} + {#each paths as path}{#if path.area}{/if}{/each}{/each} + {#each paths as path}{/each} + {#if !hover && crosshairX !== null}{/if} + +
{#each Array.from({ length: tickCount + 1 }, (_, i) => start + (end - start) * i / (tickCount || 1)) as at}{label(at)}{/each}
+ +
+
{#if mode === "running"}Something ran {valueText(100 * running.reduce((sum, [a, b]) => sum + b - a, 0) / (end - start || 1), "%")} of the range{:else}Average {valueText(values.reduce((sum, value) => sum + value, 0) / values.length, unit)}Peak {valueText(Math.max(...values), unit)}{#if readingSeconds}{mode === "rolling" ? `Mean of the last minute's ${readingSeconds} s readings` : `One reading per ${readingSeconds} s`}{/if}{/if}{#each bands as band (band.key)}{band.label}{/each}{#if note}{note}{/if}
+ {/if} +
diff --git a/src/server/dashboard/ui/CopyLink.svelte b/src/server/dashboard/ui/CopyLink.svelte new file mode 100644 index 000000000..9ad745479 --- /dev/null +++ b/src/server/dashboard/ui/CopyLink.svelte @@ -0,0 +1,12 @@ + + diff --git a/src/server/dashboard/ui/GpuDuty.svelte b/src/server/dashboard/ui/GpuDuty.svelte new file mode 100644 index 000000000..7d6bfef66 --- /dev/null +++ b/src/server/dashboard/ui/GpuDuty.svelte @@ -0,0 +1,14 @@ + + + diff --git a/src/server/dashboard/ui/Logs.svelte b/src/server/dashboard/ui/Logs.svelte new file mode 100644 index 000000000..88fccce1f --- /dev/null +++ b/src/server/dashboard/ui/Logs.svelte @@ -0,0 +1,127 @@ + + +{#if view.event !== null} +
{logRange.since.slice(0, 10)} · {logRange.since.slice(11, 19)}–{logRange.until.slice(11, 19)}All logs
+{/if} +
+ + + {#if run.shared_runtime}{/if} + +
+
{status} + {#if error}{error}{records.length ? " · Showing previously fetched records" : ""}{/if} +
+ +
{ const box = event.currentTarget; if (view.end === null && box.scrollHeight - box.clientHeight - box.scrollTop > 24) pause(); }}> + {#if cursor} + {:else if records.length === 2000}

2,000 records shown. Narrow the time range or search to inspect more.

{/if} + {#each shown as record (record.id)} + {@const at = timestamp(record.timestamp)} +
+ {tag(record.pod)}
{#each match ? record.message.split(match) : [record.message] as part, i}{#if i % 2}{part}{:else}{part}{/if}{/each}{#if record.message_truncated}{"\n[Message truncated by source]"}{/if}
+
+ {:else}{#if !pending && !error}

{data?.note || "No logs match this time range and filter"}

{/if}{/each} +
diff --git a/src/server/dashboard/ui/Metrics.svelte b/src/server/dashboard/ui/Metrics.svelte new file mode 100644 index 000000000..65a3b5b8f --- /dev/null +++ b/src/server/dashboard/ui/Metrics.svelte @@ -0,0 +1,73 @@ + + +{#snippet placeholders(count)}
{#each Array.from({ length: count }) as _, i (i)}{/each}
{/snippet} + +
+ {#if steps.error}

{steps.error}{rows.length ? " · Showing previously fetched steps" : ""}

{/if} +

Client metrics

+ {#if allClient.length}
+ +
+ {#each ["key", ...groups, "all"] as key (key)}{/each} +
+
{/if} + {#if clientCharts.length}
+ {#each clientCharts as [key, title, unit = ""] (key)} row.client?.[key])} start={stepRange.start} end={stepRange.end} empty="No values" />{/each} +
{:else if allClient.length}

No client metric matches

{:else if loading}{@render placeholders(FIRST.length)}{:else if steps.data}

0 metrics

{/if} +

Trainer metrics

+ {#if trainerCharts.length}
+ {#each trainerCharts as [op, name] (`${op} ${name}`)} row.ops[op]?.metrics[name]?.mean)} start={stepRange.start} end={stepRange.end} tone="accent" empty="No values" />{/each} +
{:else if loading}{@render placeholders(3)}{:else if steps.data}

0 metrics

{/if} +

Operations

+
Completed steps {run.steps ?? "—"}
+ {#if loading}{@render placeholders(3)}{/if} +
+ {#each opNames as op (op)} row.ops[op] && row.ops[op].seconds / row.ops[op].count)} start={stepRange.start} end={stepRange.end} min={0} tone="accent" empty="No values" />{/each} + {#if steps.data && !rows.length}

No operations recorded

{/if} +
+

Samplers

+ + {#if run.shared_runtime}

The pool's samplers serve every run on this base model; see their load on Nodes.

+ {:else}
+ {#each SAMPLER_CHARTS as [key, title, unit, scale = 1] (key)}{/each} +
{/if} +
diff --git a/src/server/dashboard/ui/Nodes.svelte b/src/server/dashboard/ui/Nodes.svelte new file mode 100644 index 000000000..4baf52c54 --- /dev/null +++ b/src/server/dashboard/ui/Nodes.svelte @@ -0,0 +1,143 @@ + + scrollSelection = true} onkeydown={escapeKey} /> +
range, getNow: () => observed, onGesture: (value) => navigating = !!value }} data-crosshair={cursor !== null ? "" : undefined} style:--crosshair={`${cursor ?? 0}%`}> +

Accelerator nodes

+
+ +
+
+ {#each errors as error}

{error}

{/each} + {#if history.error}{/if} + {#if !inventory.cluster.available && !errors.length}

Kubernetes unavailable

{/if} + {#if unavailable}
{view.node ? "The selected node is no longer reported." : "The selected placement is not in retained history."}
{/if} +
Node
{#each [0, 1, 2, 3] as step}{axisTime(range.start + (range.now - range.start) * step / 3, range)}{/each}
Duty
+ {#each loading ? [] : nodes as item (item.name)} + {@const devices = item.devices || []} + {@const height = devices.length === 1 ? 30 : 22} + {@const placements = all.filter((p) => p.node === item.name && p.start <= range.now && p.end >= range.start)} + {@const current = range.live ? inventory.placements.filter((p) => p.node === item.name) : placements.filter((p) => p.start <= range.now && p.end >= range.now)} + {@const unmapped = placements.filter((p) => !p.devices.length || p.devices.some((id) => !devices.some((d) => d.id === id)))} +
+
+
{#if devices.length}
{#each devices as device (device.id)}{deviceLabel(device.name)}{/each}
crosshair.at = event.buttons ? null : timeAt(event, range)} onpointerleave={() => crosshair.at = null}> + {#each capacityBars(devices, placements, range, view.placement) as bar (bar.key)} + + {/each} +
{/if} + {#each unmapped as p (p.id)}{/each} + {#if unmapped.length}Device mapping unavailable{:else if !devices.length && item.gpu_capacity}

GPUs without DRA devices

{/if}
+ {dutyCycle(item)} +
+ {#if item.name === view.node} + {@const label = `${acceleratorLabel(item)} · ${shortNodeName(item.name, inventory.cluster.nodes)}`} +

{label}

+ {#if devices.length > 1}
+ + + {#each devices as device (device.id)}{@const value = lastValue(deviceSeries(metrics.devices.filter((d) => d.id === device.id), "utilization"), range)}{/each} +
{/if} +
+ {#if item.gpu_capacity}
+ {#if queries.utilization.error}{/if} +
{activitySource(visible)}GPU memory {Number.isFinite(gpuMemory) ? `${(gpuMemory / 1024).toFixed(1)} GiB` : "—"}{#if gpuMemoryError} · {Number.isFinite(gpuMemory) ? "Stale" : "Unavailable"}: {gpuMemoryError}{/if}
{/if} +
+ {/if} +
+ {:else}{#if loading}

Loading activity…

{:else if !errors.length && inventory.cluster.available}

No accelerator nodes reported by Kubernetes

{/if}{/each} +
diff --git a/src/server/dashboard/ui/QueryChart.svelte b/src/server/dashboard/ui/QueryChart.svelte new file mode 100644 index 000000000..9ec4a09a7 --- /dev/null +++ b/src/server/dashboard/ui/QueryChart.svelte @@ -0,0 +1,14 @@ + + +
+ [at, value * scale])} start={range.start} end={range.now} min={0} {navigating} empty={result.error || "No samples in this time range"} note={result.error && series.length ? `${result.error} · Showing previously fetched samples` : ""} /> +
diff --git a/src/server/dashboard/ui/Run.svelte b/src/server/dashboard/ui/Run.svelte new file mode 100644 index 000000000..a8927d96e --- /dev/null +++ b/src/server/dashboard/ui/Run.svelte @@ -0,0 +1,94 @@ + + +

← {view.back ? "Nodes" : "Overview"}

+{#if !run} +

Run

+ {#if error}

{error}

{:else}

{view.id ? "Loading run…" : "Choose a run from Overview"}

{/if} + {#if detail.error}{/if} +{:else} +

{title}

+ {#if error}

{error}

{/if} + {#if description}

{description}

{/if} +
+ {#each (run.pods || []).filter((p) => p.problem) as pod}

{pod.name}: {pod.problem}

{/each} + {#if !finished && run.shared_runtime}

Shared runtime: its workers also serve other runs

{/if} +
+
range, getNow: () => observed, onGesture: (value) => navigating = !!value }}> +
+ +
Agent JSON ↗
+
+
+ {#if tab === "logs"} + {#if validEvent}{/if} + {:else if tab === "metrics"} + {:else} +
location.hash = link("logs", { event: at, end: range.now })} />
+ {/if} +
+
+{/if} diff --git a/src/server/dashboard/ui/Status.svelte b/src/server/dashboard/ui/Status.svelte new file mode 100644 index 000000000..96827b198 --- /dev/null +++ b/src/server/dashboard/ui/Status.svelte @@ -0,0 +1,7 @@ + +{label} diff --git a/src/server/dashboard/ui/TimeControl.svelte b/src/server/dashboard/ui/TimeControl.svelte new file mode 100644 index 000000000..a65921e1a --- /dev/null +++ b/src/server/dashboard/ui/TimeControl.svelte @@ -0,0 +1,33 @@ + + { if (picker?.open && !picker.contains(event.target)) picker.open = false; }} /> + +
{timeLabel} +
+ + {#if range.live}Following current time{:else}{/if} + W/S zoom · A/D pan +
+
+ diff --git a/src/server/dashboard/ui/Views.svelte b/src/server/dashboard/ui/Views.svelte new file mode 100644 index 000000000..8f61b0e2a --- /dev/null +++ b/src/server/dashboard/ui/Views.svelte @@ -0,0 +1,81 @@ + + +{#snippet healthStatus(label, tone)} + {label} +{/snippet} + +{#if view.page === "overview"} +

Overview

+
+ +
+ {#each FILTERS as [key, label, title]} + + {/each} +
+
+
+ {#if shown.length} +
+ JobStatusStepsElapsedSession +
+ {#each shown as run (run.run_id)} + {@const label = runDescription(run, null)} + {@const model = (run.model || "Run").split("/").at(-1)} + {@const session = sessionOf(run)} +
+ {model} · {run.run_id.slice(0, 8)}{#if KINDS[run.fine_tuning_type] || run.fine_tuning_type}{` · ${KINDS[run.fine_tuning_type] || run.fine_tuning_type}`}{/if}{#if label}{label}{/if} + {run.steps ?? "—"}{elapsedTime(run, inventory.observed_at)} + {session ?? "—"} +
+ {/each} + {/if} +
+ {#if !shown.length}
{inventory.runs.length ? "No runs match the search and status filter" : "No runs recorded"}
{/if} +{:else if view.page === "health"} +

Health

+ {#each errors as error}

{@render healthStatus("Source unavailable", "warning")}{error}

{/each} + {#if issues.length} +
+ {#each issues as issue}{/each} +
IssueResourceEvidence
{@render healthStatus(issue.issue, issue.tone)}{issue.resource}{issue.evidence}{#if issue.link}{issue.label} ↗{/if}
+ {:else if inventory.cluster.available === true && !errors.length}

{@render healthStatus("Healthy", "success")}No pod problems, node problems or source errors.

{/if} +

Diagnostic JSON ↗ · API reference ↗

+{/if} diff --git a/src/server/dashboard/ui/format.js b/src/server/dashboard/ui/format.js new file mode 100644 index 000000000..f5c525432 --- /dev/null +++ b/src/server/dashboard/ui/format.js @@ -0,0 +1,27 @@ +export const duration = (total) => total < 90 ? `${Math.round(total)}s` : total < 5400 ? `${Math.round(total / 60)}m` : total < 172800 ? `${(total / 3600).toFixed(1)}h` : `${(total / 86400).toFixed(1)}d`; +const seconds = (value) => value === null || value === undefined || value === "" ? NaN : typeof value === "number" ? value : Date.parse(value) / 1000; +// A run is finished once the gateway marks it so, or once nothing serves it any more; a client +// that never deletes its model leaves the status active. +export const isFinished = (run) => run?.lifecycle === "ended"; +// When a finished run stopped: when it was marked complete, else its last completed work. +export const finishedAt = (run) => [seconds(run.completed_at), run.last_completed_at].find(Number.isFinite) ?? NaN; +// One label: the gateway's status once it is final, else the lifecycle. +export const runLabel = (run) => run.client_status !== "active" ? run.client_status : { ended: "ended", waiting: "queued", placed: "running" }[run.lifecycle] || "active"; +export function elapsedTime(run, observedAt) { + const stop = isFinished(run) ? finishedAt(run) : NaN; + const start = seconds(run.created_at), end = Number.isFinite(stop) ? stop : seconds(observedAt); + return Number.isFinite(start) && Number.isFinite(end) ? duration(Math.max(0, end - start)) : "—"; +} +export function shortNodeName(name, nodes = []) { + const full = String(name || "—"), parts = full.split("-"); + const others = nodes.map((n) => typeof n === "string" ? n : n.name).filter((n) => n && n !== full); + for (let i = parts.length - 1; i >= 0; i--) { + const suffix = parts.slice(i).join("-"); + if (suffix && !others.some((other) => other === suffix || other.endsWith(`-${suffix}`))) return suffix; + } + return full; +} +export const KINDS = { lora: "LoRA", full: "FFT", fft: "FFT" }; +export const runDescription = (run, title = run.name) => [run.display_name, run.recipe_name].filter((v, i, values) => v && v !== title && values.indexOf(v) === i).join(" · "); +export const escapeRegExp = (text) => text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); +export const family = (id) => `hue-${Array.from(id || "").reduce((sum, c) => sum + c.charCodeAt(0), 0) % 8}`; diff --git a/src/server/dashboard/ui/index.js b/src/server/dashboard/ui/index.js new file mode 100644 index 000000000..2383efde6 --- /dev/null +++ b/src/server/dashboard/ui/index.js @@ -0,0 +1,3 @@ +import { mount } from "svelte"; +import App from "./App.svelte"; +mount(App, { target: document.getElementById("app") }); diff --git a/src/server/dashboard/ui/node-time.js b/src/server/dashboard/ui/node-time.js new file mode 100644 index 000000000..cdc264211 --- /dev/null +++ b/src/server/dashboard/ui/node-time.js @@ -0,0 +1,124 @@ +import { crosshair, MAX_WINDOW } from "./state.svelte.js"; + +export const WINDOWS = [[60, "1 minute"], [600, "10 minutes"], [1800, "30 minutes"], [3600, "1 hour"], [10800, "3 hours"], [21600, "6 hours"], [86400, "24 hours"], [259200, "3 days"], [MAX_WINDOW, "7 days"]]; +const clamp = (value, low, high) => Math.max(low, Math.min(high, value)); +const SURFACES = ".capacity-track, .node-axis, .activity-track, .activity-axis, #placement-detail .chart-plot svg, #placement-detail .chart-empty"; + +// Native gesture listeners are scoped to the mounted Nodes component. Svelte +// reacts to view changes; the query window stays frozen until a gesture ends. +export function nodeTime(element, { view, getRange, getNow, onGesture }) { + let gesture = null, drag = null, pinch = null, settle, ignoreClickUntil = 0; + const events = new AbortController(), root = element.closest("#app"); + const listen = (target, name, handler, options = {}) => target.addEventListener(name, handler, { ...options, signal: events.signal }); + const fraction = (x, box) => clamp((x - box.left) / (box.width || 1), 0, 1); + function begin(kind) { + if (gesture) return; + gesture = { kind, duration: view.duration, end: view.end }; + onGesture(getRange()); + root?.classList.add("node-navigating"); + } + function select(duration, end) { + duration = clamp(Math.round(duration), 60, MAX_WINDOW); + view.duration = duration; + view.end = clamp(end, getNow() - MAX_WINDOW + duration, getNow()); + } + function finish(cancel = false) { + clearTimeout(settle); + if (drag?.pointerId !== undefined && element.hasPointerCapture(drag.pointerId)) element.releasePointerCapture(drag.pointerId); + drag = pinch = null; + if (!gesture) return; + if (cancel) { view.duration = gesture.duration; view.end = gesture.end; } + if (gesture.kind === "drag") ignoreClickUntil = performance.now() + 400; + gesture = null; + onGesture(null); + root?.classList.remove("node-navigating"); + } + function moveDrag(x, y, touch = false) { + if (!drag) return false; + const dx = x - drag.x, dy = y - drag.y; + if (!gesture) { + if (touch && Math.abs(dy) > 6 && Math.abs(dy) > Math.abs(dx)) { drag = null; return false; } + if (Math.abs(dx) < 6) return false; + begin("drag"); + if (drag.pointerId !== undefined) element.setPointerCapture(drag.pointerId); + } + select(drag.range.now - drag.range.start, drag.range.now - dx / (drag.box.width || 1) * (drag.range.now - drag.range.start)); + return true; + } + listen(element, "wheel", (event) => { + const track = event.target.closest(SURFACES); + if (!track || !(event.ctrlKey || event.metaKey) || (gesture && gesture.kind !== "wheel")) return; + event.preventDefault(); + begin("wheel"); + const range = getRange(), at = fraction(event.clientX, track.getBoundingClientRect()); + const delta = event.deltaY * (event.deltaMode === 1 ? 16 : event.deltaMode === 2 ? track.clientHeight : 1); + const duration = clamp(Math.round((range.now - range.start) * Math.exp(clamp(delta * 0.002, -1, 1))), 60, MAX_WINDOW); + select(duration, range.start + at * (range.now - range.start) + (1 - at) * duration); + clearTimeout(settle); + settle = setTimeout(() => finish(), 200); + }, { passive: false }); + listen(element, "pointerdown", (event) => { + const track = event.target.closest(SURFACES); + if (!track || event.pointerType === "touch" || event.button !== 0 || event.target.closest("[data-placement], a")) return; + finish(); + drag = { x: event.clientX, y: event.clientY, box: track.getBoundingClientRect(), range: getRange(), pointerId: event.pointerId }; + }); + listen(window, "pointermove", (event) => { if (drag?.pointerId === event.pointerId && moveDrag(event.clientX, event.clientY)) event.preventDefault(); }); + listen(window, "pointerup", (event) => { if (drag?.pointerId === event.pointerId) finish(); }); + listen(window, "pointercancel", (event) => { if (drag?.pointerId === event.pointerId) finish(true); }); + listen(element, "touchstart", (event) => { + const track = event.target.closest(SURFACES); + if (!track) return; + if (event.touches.length === 2) { + event.preventDefault(); + if (gesture?.kind === "wheel") finish(); + begin("pinch"); + gesture.kind = "pinch"; + drag = null; + const [a, b] = event.touches; + pinch = { box: track.getBoundingClientRect(), range: getRange(), x: (a.clientX + b.clientX) / 2, distance: Math.hypot(a.clientX - b.clientX, a.clientY - b.clientY) || 1 }; + } else if (event.touches.length === 1 && !event.target.closest("[data-placement], a")) { + const touch = event.touches[0]; + drag = { x: touch.clientX, y: touch.clientY, box: track.getBoundingClientRect(), range: getRange() }; + } + }, { passive: false }); + listen(element, "touchmove", (event) => { + if (pinch && event.touches.length === 2) { + event.preventDefault(); + const [a, b] = event.touches, x = (a.clientX + b.clientX) / 2; + const duration = clamp(Math.round((pinch.range.now - pinch.range.start) * pinch.distance / (Math.hypot(a.clientX - b.clientX, a.clientY - b.clientY) || 1)), 60, MAX_WINDOW); + const anchor = pinch.range.start + fraction(pinch.x, pinch.box) * (pinch.range.now - pinch.range.start); + select(duration, anchor + (1 - fraction(x, pinch.box)) * duration); + } else if (drag && event.touches.length === 1 && moveDrag(event.touches[0].clientX, event.touches[0].clientY, true)) event.preventDefault(); + }, { passive: false }); + listen(element, "touchend", () => { if (drag || pinch) finish(); }); + listen(element, "touchcancel", () => { if (drag || pinch) finish(true); }); + listen(element, "click", (event) => { + if (event.detail && performance.now() < ignoreClickUntil) { ignoreClickUntil = 0; event.preventDefault(); event.stopImmediatePropagation(); } + }, { capture: true }); + listen(window, "keydown", (event) => { + if (event.key === "Escape" && (gesture || drag || pinch)) { event.preventDefault(); event.stopImmediatePropagation(); finish(true); } + }, { capture: true }); + // Perfetto keys: W/S zoom around the crosshair (else the middle), A/D pan by a fifth of the window. + listen(window, "keydown", (event) => { + const key = event.key.toLowerCase(); + if (!["w", "a", "s", "d"].includes(key) || event.ctrlKey || event.metaKey || event.altKey || event.target.closest("input, select, textarea") || (gesture && gesture.kind !== "keys")) return; + event.preventDefault(); + begin("keys"); + const range = getRange(), span = range.now - range.start; + if (key === "a" || key === "d") { + select(span, range.now + (key === "d" ? 0.2 : -0.2) * span); + // Keep the crosshair under the pointer, which did not move. + if (crosshair.at !== null) crosshair.at += view.end - range.now; + } else { + const duration = clamp(Math.round(span * (key === "w" ? 0.8 : 1.25)), 60, MAX_WINDOW); + const at = crosshair.at === null ? 0.5 : clamp((crosshair.at - range.start) / span, 0, 1); + select(duration, range.start + at * span + (1 - at) * duration); + } + clearTimeout(settle); + settle = setTimeout(() => finish(), 300); + }); + listen(window, "blur", () => finish(true)); + listen(window, "hashchange", () => finish()); + return { destroy() { finish(); events.abort(); } }; +} diff --git a/src/server/dashboard/ui/state.svelte.js b/src/server/dashboard/ui/state.svelte.js new file mode 100644 index 000000000..56c67e43e --- /dev/null +++ b/src/server/dashboard/ui/state.svelte.js @@ -0,0 +1,156 @@ +import { untrack } from "svelte"; + +const queryString = (params) => { + const query = new URLSearchParams(Object.entries(params).filter(([, v]) => v !== null && v !== undefined && v !== "")).toString(); + return query ? `?${query}` : ""; +}; +export const viewLink = (path, params = {}) => `#${path}${queryString(params)}`; +export const inspectUrl = (resource, params = {}) => `/api/v1/inspect/${resource}${queryString(params)}`; +export const nodeLink = (placement, range) => viewLink("nodes", { placement, duration: range.now - range.start, end: range.now }); +export const now = (inventory) => Date.parse(inventory?.observed_at) / 1000 || Date.now() / 1000; +const cache = new Map(); +// The longest window the pages show, matching the inspection API's seven-day limit. +export const MAX_WINDOW = 7 * 86400; +export const saved = $state({ overview: "#overview", nodes: "#nodes" }); +const number = (params, key, fallback) => params.get(key)?.trim() && Number.isFinite(Number(params.get(key))) ? Number(params.get(key)) : fallback; +export function parseView(hash = location.hash) { + const [path, search = ""] = hash.slice(1).split("?"); + const [page, id = "", tab = "activity"] = path.split("/").map((part) => { try { return decodeURIComponent(part); } catch { return part; } }); + const params = new URLSearchParams(search), duration = Math.max(60, Math.min(MAX_WINDOW, Math.round(number(params, "duration", 1800)))); + const end = number(params, "end", null), event = number(params, "event", null), back = params.get("back"); + return { page: ["nodes", "run", "health"].includes(page) ? page : "overview", id, tab, explicitWindow: params.has("duration") || params.has("end"), + duration, end: end !== null && end >= duration && end <= 8.64e12 ? end : null, + event: event !== null && (end === null || (event >= end - duration && event <= end)) ? event : null, + back: back?.startsWith("#nodes?") ? back : null, + node: params.get("node"), placement: params.get("placement"), gpu: params.get("gpu") || "all", + q: params.get("q") || "", pod: params.get("pod") || "", + status: ["active", "queued", "finished"].includes(params.get("status")) ? params.get("status") : "all" }; +} +export const view = $state(parseView()); +export const rangeOf = (inventory, selection = view) => { + const observed = now(inventory), duration = Math.max(60, Math.min(MAX_WINDOW, Math.round(selection.duration || 1800))); + const end = Math.max(observed - MAX_WINDOW + duration, Math.min(observed, selection.end ?? observed)); + return { start: end - duration, now: end, live: selection.end === null }; +}; +// Grafana's shared crosshair: hovering any chart or timeline marks the same instant on the others. +export const crosshair = $state({ at: null }); +export const timeAt = (event, range) => { + const box = event.currentTarget.getBoundingClientRect(); + return range.start + Math.max(0, Math.min(1, (event.clientX - box.left) / (box.width || 1))) * (range.now - range.start); +}; +export const crosshairOffset = (range) => crosshair.at === null || crosshair.at < range.start || crosshair.at > range.now ? null : 100 * (crosshair.at - range.start) / (range.now - range.start || 1); +export function currentView(freeze = false, tab = view.tab) { + const { page, id, duration, end, back, event, q, pod, node, placement, gpu, status } = view; + if (page === "overview") return viewLink(page, { q, status: status === "all" ? null : status }); + if (page === "run") return viewLink(`run/${encodeURIComponent(id)}/${tab || "activity"}`, { duration, end: freeze ? end ?? now(cache.get(inspectUrl("inventory"))?.data) : end, back, event, q, pod }); + if (page === "nodes") return viewLink(page, { duration, end: freeze ? end ?? now(cache.get(inspectUrl("inventory"))?.data) : end, node, placement, gpu: node && gpu !== "all" ? gpu : null }); + return viewLink(page + (id ? `/${encodeURIComponent(id)}` : "")); +} +export async function get(url, signal, previous) { + const timeout = AbortSignal.timeout(15000); + let response; + try { response = await fetch(url, { cache: "no-store", headers: previous?.etag ? { "If-None-Match": previous.etag } : undefined, signal: signal ? AbortSignal.any([signal, timeout]) : timeout }); } + catch (error) { + if (signal?.aborted) throw error; + throw new Error(timeout.aborted ? "Dashboard request timed out. Try again." : "Cannot reach the dashboard. Check the connection and try again."); + } + if (response.status === 304 && previous?.etag) return previous; + if (!response.ok) { + const data = await response.json().catch(() => null); + const detail = Array.isArray(data?.detail) ? data.detail.map((item) => `${(item.loc || []).filter((part) => part !== "body").join(".")}: ${item.msg}`).join("; ") : data?.detail; + const error = new Error([detail, data?.error].find((v) => typeof v === "string") || ({ 401: "Authentication required", 403: "Access denied", 404: "Requested data not found", 429: "Too many requests. Try again shortly." }[response.status] || `Dashboard request failed (${response.status})`)); + error.status = response.status; + throw error; + } + const data = await response.json().catch(() => { throw new Error("The dashboard returned an invalid response. Try again."); }); + return { data, etag: response.headers.get("ETag") }; +} + +// Reuse recent responses across navigation; changing the key or unmounting aborts old work. +// A live view keys on the user's selection, so its moving window is read at each poll instead of restarting it. +class QueryState { + // Responses are immutable snapshots. Only replacing the tree needs to be reactive. + data = $state.raw(null); + error = $state(null); + status = $state(null); + pending = $state(false); + // A brief failure keeps the last response and retries; this says it may be out of date. + stale = $state(false); + retry = $state(() => {}); +} +// remember: whether responses go in the shared cache. A live view's window moves every poll, so its +// responses are never revisited and would only push out the ones that are. +export function createQuery(url, { poll = 0, pollWhen = () => true, scope = () => true, key = url, merge = (next) => next, remember = () => true } = {}) { + let previousScope; + const result = new QueryState(); + const selectedKey = $derived.by(key), selectedScope = $derived.by(scope); + $effect(() => { + const requestedKey = selectedKey, requestedScope = selectedScope; + return untrack(() => { + result.retry = () => {}; + if (requestedScope !== previousScope) result.data = result.error = result.status = null; + previousScope = requestedScope; + const initialUrl = requestedKey && url(); + if (!initialUrl) return; + result.data = cache.get(initialUrl)?.data ?? result.data; + const controller = new AbortController(); + let failures = 0, timer; + const schedule = (delay, retry) => { + if (delay) timer = setTimeout(() => { + if (!document.hidden && (retry || pollWhen())) load(); + else schedule(delay, retry); + }, delay); + }; + const load = result.retry = async () => { + const requested = url(); + if (!requested || result.pending || controller.signal.aborted) return; + clearTimeout(timer); + result.pending = true; + let retry = false; + try { + const { data, etag } = await get(requested, controller.signal, cache.get(requested)); + if (controller.signal.aborted) return; + failures = 0; + Object.assign(result, { error: data.error || null, status: null, stale: false }); + if (data !== result.data && (!result.error || !result.data)) result.data = merge(data, result.data); + if (!result.error && remember()) { + cache.set(requested, { data: result.data, etag }); + if (cache.size > 24) cache.delete([...cache.keys()].find((key) => key !== inspectUrl("inventory"))); + } + } catch (error) { + if (controller.signal.aborted) return; + // A dropped tunnel or a busy server is usually brief: keep what is shown, retry on the one + // schedule, and report the failure only if it lasts. A rejected request is reported at once. + failures += 1; + retry = (!error.status || error.status >= 500 || error.status === 429) && failures < 4; + if (retry) result.stale = Boolean(result.data); + else Object.assign(result, { error: error.message, status: error.status || null, stale: false }); + } + finally { + if (!controller.signal.aborted) { + result.pending = false; + schedule(retry ? 2000 * 2 ** (failures - 1) : poll, retry); + } + } + }; + load(); + return () => { controller.abort(); clearTimeout(timer); result.pending = false; }; + }); + }); + return result; +} + +export async function copyLink(url, success = "Link copied") { + try { await navigator.clipboard.writeText(url); return success; } + catch { + const focused = document.activeElement, input = Object.assign(document.createElement("textarea"), { value: url }); + input.style.cssText = "position:fixed;opacity:0"; + document.body.append(input); + input.select(); + let copied = false; + try { copied = document.execCommand("copy"); } catch {} + input.remove(); + focused?.focus({ preventScroll: true }); + return copied ? success : "Copy failed"; + } +} diff --git a/src/server/dashboard/ui/visualization.js b/src/server/dashboard/ui/visualization.js new file mode 100644 index 000000000..84feaf4e6 --- /dev/null +++ b/src/server/dashboard/ui/visualization.js @@ -0,0 +1,72 @@ +// Presentation geometry only: identity and attribution come from the inspection API. +import { family } from "./format.js"; +export { shortNodeName } from "./format.js"; +const numbers = ["standard", "compact"].map((notation) => new Intl.NumberFormat(undefined, { maximumSignificantDigits: 4, notation })); +const percent = new Intl.NumberFormat(undefined, { maximumFractionDigits: 1 }); +export const chartNumber = (value) => numbers[Number(Math.abs(value) >= 10000)].format(value); +export const chartTime = (at, seconds = false) => new Date(at * 1000).toISOString().slice(11, seconds ? 19 : 16); +export const valueText = (value, unit) => unit === "%" ? `${percent.format(value)}%` : `${chartNumber(value)}${unit ? ` ${unit}` : ""}`; +export const axisTime = (at, range) => `${range.now - range.start >= 86400 ? `${new Date(at * 1000).toISOString().slice(5, 10)} ` : ""}${chartTime(at, range.now - range.start <= 120)}`; +export const placementColor = (p) => family(p.run_ids?.[0] || p.runtime_id); +// The exporter's collection interval in k8s/deploy; each engine-activity reading averages it. +export const DCGM_READING_SECONDS = 10; +export const dutyBands = (lanes, device = null) => lanes.filter((p) => p.intervals.length && (!device || p.devices.includes(device))).map((p) => ({ key: p.id, label: `${p.label} · ${p.role || "process"}`, hue: placementColor(p), role: p.role, spans: p.intervals })); +export const deviceLabel = (name) => /(\d+)$/.exec(String(name ?? ""))?.[1] ?? String(name ?? ""); +export const acceleratorLabel = (node) => node.accelerator ? node.accelerator.replace(/^nvidia[- ]/i, "").replace(/-\d+gb$/i, "").replace(/-/g, " ").toUpperCase() : node.gpu_capacity ? "GPU" : "CPU"; +export function clipPlacements(placements = [], range, error = null, status = null) { + const clip = (spans = []) => spans.map(([a, b]) => [Math.max(range.start, a), Math.min(range.now, b)]).filter(([a, b]) => b > a); + // A request failure belongs to every lane; a lane's own error is only about its ops. + return placements.map((p) => ({ ...p, intervals: clip(p.intervals), ops: Object.fromEntries(Object.entries(p.ops || {}).map(([name, spans]) => [name, clip(spans)])), error: status ? error : p.error, errorStatus: status || p.errorStatus })); +} +export function activitySource(placements) { + if (!placements.length) return ""; + const source = placements.every((p) => p.exact) ? "GPU turns" : placements.some((p) => p.exact) ? "GPU turns and recorded operations" : "Recorded operations; may include GPU wait time"; + return `${source}. Gaps may include unrecorded activity.`; +} +export function processLabel(p, nodes, { gpus: named = true } = {}) { + const node = nodes.find((n) => n.name === p.node), devices = p.devices.map((id) => node?.devices.find((d) => d.id === id)); + // A lane rebuilt from recorded ops has no known GPUs, which is not zero GPUs. + const gpus = devices.length && devices.every(Boolean) ? `GPU${devices.length === 1 ? "" : "s"} ${devices.map((d) => deviceLabel(d.name)).join(", ")}` : p.device_count ? `${p.device_count} GPU${p.device_count === 1 ? "" : "s"}` : null; + const role = p.role || "Process", accelerator = node && acceleratorLabel(node); + return [role[0].toUpperCase() + role.slice(1), accelerator === "GPU" ? null : accelerator, named && gpus].filter(Boolean).join(" · "); +} +export const OP_ORDER = ["forward_backward", "forward", "optim_step", "sample", "init", "wake_up", "sleep", "weight_sync"]; +export const opRank = (name) => OP_ORDER.includes(name) ? OP_ORDER.indexOf(name) : OP_ORDER.length; +export const opLabel = (name) => name.replace(/_/g, " "); +export function claimLabel(node, placements, live) { + if (live && !node.ready) return "Offline"; + if (!node.gpu_capacity) return "CPU"; + if (node.openrl_enabled === false) return "Not enabled"; + if (!placements.every((p) => p.devices.length === p.device_count && p.devices.every((id) => node.devices.some((d) => d.id === id)))) return "— claimed"; + return `${new Set(placements.flatMap((p) => p.devices)).size}/${node.gpu_capacity} claimed`; +} +export function capacityBars(devices, placements, range, selected) { + // Every held GPU draws the turns and ops its holders ran, never one block naming the run; the name is a click away. + const height = devices.length === 1 ? 30 : 22; + const span = (from, to) => `left:${100 * Math.max(0, Math.max(range.start, from) - range.start) / (range.now - range.start)}%;width:${100 * Math.max(0, Math.min(range.now, to) - Math.max(range.start, from)) / (range.now - range.start)}%`; + return devices.flatMap((device, i) => { + const rows = placements.filter((p) => p.devices.includes(device.id)).sort((a, b) => a.start - b.start || a.id.localeCompare(b.id)); + if (!rows.length) return []; + const from = Math.max(range.start, Math.min(...rows.map((p) => p.start))), to = Math.min(range.now, Math.max(...rows.map((p) => p.end))); + return [{ key: `gpu:${device.id}`, placement: rows.find((p) => p.id === selected) || rows.at(-1), selected: rows.some((p) => p.id === selected), title: `GPU activity: ${rows.map((p) => `${p.label} ${p.role || ""}`.trim()).join(", ")}`, + style: `${span(from, to)};top:${i * height + 2}px;height:${height - 4}px`, + holds: rows.flatMap((p) => p.intervals.map(([a, b]) => ({ placement: p, style: `left:${100 * (Math.max(a, from) - from) / (to - from || 1)}%;width:${100 * (Math.min(b, to) - Math.max(a, from)) / (to - from || 1)}%` }))) }]; + }); +} +// sortedPoints must be in time order; unsorted input gives wrong gaps without an error. +export function gapFor(sortedPoints, floor) { + const spacings = sortedPoints.slice(1).map(([at], i) => at - sortedPoints[i][0]).filter((d) => d > 0).sort((a, b) => a - b); + return Math.max(floor, 3 * (spacings[Math.floor(spacings.length / 2)] || 0)); +} +// Aggregate one physical generation per slot and timestamp; incomplete or ambiguous samples stay gaps. +export function deviceSeries(devices, field) { + const slots = new Set(devices.map((d) => d.id)).size, byTime = new Map(); + for (const device of devices) for (const [at, value] of device[field] || []) { + if (!byTime.has(at)) byTime.set(at, new Map()); + const values = byTime.get(at), samples = values.get(device.id) || []; + if (Number.isFinite(value)) samples.push(value); + values.set(device.id, samples); + } + return [...byTime].sort((a, b) => a[0] - b[0]).map(([at, values]) => [at, values.size === slots && [...values.values()].every((samples) => samples.length === 1) ? [...values.values()].reduce((sum, samples) => sum + samples[0], 0) / (field === "utilization" ? slots : 1) : null]); +} +export const lastValue = (points, range) => points.findLast(([at]) => at >= range.start && at <= range.now)?.[1]; diff --git a/src/server/model_metadata.py b/src/server/model_metadata.py index 8ec44e568..c9cc45330 100644 --- a/src/server/model_metadata.py +++ b/src/server/model_metadata.py @@ -15,6 +15,13 @@ SPARSE_DELTA_VERSION = 2 +def display_metadata(value: dict[str, Any]) -> dict[str, str]: + """Only bounded, explicitly supplied labels become run display metadata.""" + return { + key: value[key].strip()[:1024] for key in ("name", "run_name", "recipe_name", "git_rev") if isinstance(value.get(key), str) and value[key].strip() + } + + @dataclass class WeightSyncConfig: """How the trainer publishes weights each step: sparse deltas or full checkpoints. diff --git a/src/server/sampling_dispatcher.py b/src/server/sampling_dispatcher.py index a3eebe3d4..60c97be00 100644 --- a/src/server/sampling_dispatcher.py +++ b/src/server/sampling_dispatcher.py @@ -19,6 +19,7 @@ import redis.asyncio as redis from server.sampling_store import SamplingStore, job_of +from server.telemetry import op_finished, sampled_group from server.vllm_options import sample_params, sample_result logger = logging.getLogger(__name__) @@ -110,9 +111,14 @@ async def dispatch(self) -> None: await self.read() if await self.ready(): # Read after the await above, so a request whose result was saved meanwhile is not sent again. - for r in self.to_send()[: self.concurrency - len(self.in_flight)]: + batch, started, tasks = self.to_send()[: self.concurrency - len(self.in_flight)], time.time(), [] + for r in batch: self.in_flight[r["request_id"]] = task = asyncio.create_task(self.send(r)) task.add_done_callback(lambda _, request_id=r["request_id"]: self.in_flight.pop(request_id)) + tasks.append(task) + if tasks: + # the group record once every send of this dispatch has answered + asyncio.gather(*tasks, return_exceptions=True).add_done_callback(lambda _, b=batch, t=started: sampled_group(b, t)) async def ready(self) -> bool: """Whether the sampler serves the current revision, loading it if it can. A revision it can never load fails its @@ -177,14 +183,17 @@ async def set_result(self, r: dict[str, Any], result: dict[str, Any]) -> None: async def send(self, r: dict[str, Any]) -> None: """Saves the sampler's answer. None, or an error, leaves the record pending to be sent again.""" self.served[job_of(r)] = next(self.sends) + started = time.time() try: # A sampler that never answers must not hold the send past its record's deadline. timeout = max(0.0, accepted_at(r) + SAMPLE_DEADLINE_SECONDS - time.time()) result = await asyncio.wait_for(self.sampler.generate(r), timeout) except Exception as exc: + op_finished(r, "sampler", r.get("pool"), started, type(exc).__name__) logger.info("Retrying %s after %r", r["request_id"], exc) return if result is not None: + op_finished(r, "sampler", r.get("pool"), started, result=result) await self.set_result(r, result) diff --git a/src/server/scheduler_worker_manager.py b/src/server/scheduler_worker_manager.py index cbead33e1..f7d5851a1 100644 --- a/src/server/scheduler_worker_manager.py +++ b/src/server/scheduler_worker_manager.py @@ -32,6 +32,7 @@ from server.estimator import Footprint, footprint from server.lora_snapshots import snapshot_root from server.sampling_dispatcher import revision_file, tmp_root +from server.telemetry import SAMPLER_METRICS_PORT from server.vllm_options import ENGINE_ENV from server.worker_manager import ( FAILED, @@ -52,6 +53,7 @@ GROUP = "openrl.io" VERSION = "v1alpha1" PLURAL = "workloads" +OPS_SETTINGS = ("OPEN_RL_OPS_TTL_DAYS", "OPEN_RL_OPS_SAMPLE_LIMIT") EPP_IMAGE = "ghcr.io/llm-d/llm-d-router-endpoint-picker:v0.11.0" ENVOY_IMAGE = "docker.io/envoyproxy/envoy:distroless-v1.33.2" @@ -148,7 +150,7 @@ def pod_env(worker: Worker) -> list[dict[str, Any]]: # MAX_JOBS caps FlashInfer's JIT build, which otherwise runs one ~3GB # compiler per core and blows through the pod's host memory limit. passed = (*ENGINE_ENV, "OPEN_RL_TRAIN_TOKEN_BUDGET", "MAX_JOBS") - for name in (*passed, "OPEN_RL_SAMPLE_DEADLINE_SECONDS"): + for name in (*passed, "OPEN_RL_SAMPLE_DEADLINE_SECONDS", *OPS_SETTINGS): if os.getenv(name): values[name] = os.environ[name] # vLLM keys its torch.compile artifacts by model, GPU type and engine config. @@ -166,6 +168,9 @@ def pod_env(worker: Worker) -> list[dict[str, Any]]: env.append(downward("OPEN_RL_ACCEL_TIMESLICER_HOST", "status.hostIP")) if worker.pool: env.append(downward("POD_IP", "status.podIP")) + # The pod identity a worker stamps on the ops it records, for the dashboard. + for name, field in (("POD_NAME", "metadata.name"), ("POD_UID", "metadata.uid"), ("NODE_NAME", "spec.nodeName")): + env.append(downward(name, field)) return env @@ -241,6 +246,8 @@ def pod_template(worker: Worker) -> dict[str, Any]: if pull_policy := os.getenv("OPEN_RL_WORKER_IMAGE_PULL_POLICY"): template["spec"]["containers"][0]["imagePullPolicy"] = pull_policy + if worker.role == "sampler": + template["spec"]["containers"][0]["ports"] = [{"name": "http", "containerPort": SAMPLER_METRICS_PORT}] return template @@ -565,6 +572,22 @@ def load_workers(self) -> list[dict[str, Any]]: ) return workers + def workloads(self) -> list[dict[str, Any]]: + """Our Workload objects as the API returns them, read fresh, for the placement records.""" + selector = "app.kubernetes.io/managed-by=open-rl-api-server" + return self.custom_api.list_namespaced_custom_object(GROUP, VERSION, self.namespace, PLURAL, label_selector=selector)["items"] + + def pod_terminations(self) -> dict[str, list[dict[str, Any]]]: + """Each worker pod's container terminations, the current one and the one before a restart, for the placement records.""" + found: dict[str, list[dict[str, Any]]] = {} + for pod in client.CoreV1Api().list_namespaced_pod(self.namespace, label_selector="openrl.io/worker").items: + for cs in pod.status.container_statuses or []: + for ended in (cs.state and cs.state.terminated, cs.last_state and cs.last_state.terminated): + if ended and ended.finished_at: + entry = {"at": ended.finished_at.timestamp(), "container": cs.name, "reason": ended.reason, "exit_code": ended.exit_code} + found.setdefault(pod.metadata.name, []).append(entry | {"signal": ended.signal}) + return found + def dispatcher_available(self, pool: str) -> bool: try: deployment = self.apps_api.read_namespaced_deployment(pool, self.namespace) diff --git a/src/server/session_registry.py b/src/server/session_registry.py index ac2c40167..1adf27d0e 100644 --- a/src/server/session_registry.py +++ b/src/server/session_registry.py @@ -8,36 +8,75 @@ Everything lives in the store, so an API server restart keeps it: - open_rl:session: present while the session is live. Each heartbeat - resets its expiry, so a silent session vanishes - on its own. Its value is the user_metadata the - client opened the session with, plus its openrl. - tags, the defaults for every model it creates. - open_rl:owner: the sessions using this owner's workers. - open_rl:owners every owner that has workers. + open_rl:session: present while the session is live. Each heartbeat + resets its expiry, so a silent session vanishes + on its own. Its value is the user_metadata the + client opened the session with, plus its openrl. + tags, the defaults for every model it creates. + open_rl:session_record: when the session started and was last heard + from, its redacted metadata and the models it + attached. It outlives the live key, for the + inspection API. It is telemetry, so it is + written in the background and a failed write + is dropped. + open_rl:owner: the sessions using this owner's workers. + open_rl:owners every owner that has workers. Nothing here is atomic. The API server holds a lock per owner around attach and around the in_use check and the teardown that follows it, so a session cannot attach to an owner between the check and the delete. """ +import asyncio import json +import time +from collections import defaultdict from typing import Any from server.store import StateStore +from server.telemetry import TTL_SECONDS, best_effort, key, log_event, redact, write class SessionRegistry: def __init__(self, state: StateStore, ttl_seconds: float = 120.0): self.state = state self.ttl_seconds = ttl_seconds + self.record_locks: defaultdict[str, asyncio.Lock] = defaultdict(asyncio.Lock) # the API server runs as one replica async def heartbeat(self, session_id: str) -> None: - key = f"open_rl:session:{session_id}" - await self.state.set_value(key, await self.state.get_value(key) or "{}", ttl_seconds=self.ttl_seconds) + await self.keep_alive(session_id) + self.note(session_id) + + async def keep_alive(self, session_id: str) -> None: + live_key = f"open_rl:session:{session_id}" + await self.state.set_value(live_key, await self.state.get_value(live_key) or "{}", ttl_seconds=self.ttl_seconds) async def update_metadata(self, session_id: str, user_metadata: dict[str, Any]) -> None: await self.state.set_value(f"open_rl:session:{session_id}", json.dumps(user_metadata), ttl_seconds=self.ttl_seconds) + self.note(session_id, metadata=user_metadata) + + @best_effort + def note(self, session_id: str, *, model_id: str | None = None, metadata: dict | None = None) -> None: + """A session's creation, heartbeat or model attach, kept on its record. Creation and attach also log a line; a + heartbeat, every ten seconds, only moves last_seen_at.""" + write(self.update_record(session_id, model_id, metadata)) + if metadata is not None: + log_event("session_created", {"session_id": session_id, "metadata": metadata}, session_id=session_id) + if model_id: + log_event("model_attached", {"session_id": session_id, "model_id": model_id}, run_id=model_id, session_id=session_id) + + async def update_record(self, session_id: str, model_id: str | None, metadata: dict | None) -> None: + record_key, now = key("session_record", session_id), time.time() + # Heartbeats and attaches rewrite the same record. One at a time, or a heartbeat drops a model attached meanwhile. + async with self.record_locks[session_id]: + saved = await self.state.get_value(record_key) + record = json.loads(saved) if saved else {"session_id": session_id, "created_at": now, "model_ids": []} + record["last_seen_at"] = now + if metadata is not None: + record["metadata"] = redact(metadata) + if model_id and model_id not in record["model_ids"]: + record["model_ids"].append(model_id) + await self.state.set_value(record_key, json.dumps(record), ttl_seconds=TTL_SECONDS) async def user_metadata(self, session_id: str | None) -> dict[str, Any]: raw = await self.state.get_value(f"open_rl:session:{session_id}") if session_id else None @@ -50,7 +89,7 @@ async def live(self, session_id: str) -> bool: async def attach(self, session_id: str, owner: str) -> None: """The session is using this owner's workers from now on.""" - await self.heartbeat(session_id) + await self.keep_alive(session_id) await self.state.add_to_set("open_rl:owners", owner) await self.state.add_to_set(f"open_rl:owner:{owner}", session_id) diff --git a/src/server/store.py b/src/server/store.py index a37b8ed86..21182f8d3 100644 --- a/src/server/store.py +++ b/src/server/store.py @@ -269,17 +269,64 @@ async def remove_from_set(self, key: str, member: str) -> None: async def set_members(self, key: str) -> set[str]: pass + @abstractmethod + def set_value_sync(self, key: str, value: str, ttl_seconds: float | None = None) -> None: + """set_value without the event loop, for a thread.""" + pass + + @abstractmethod + async def append_sample(self, key: str, sample: dict[str, Any], limit: int, ttl_seconds: float | None) -> None: + """Append to a list that keeps the newest `limit` samples and expires ttl_seconds after the last append, or never.""" + pass + + @abstractmethod + async def read_samples(self, key: str, last: int | None = None) -> list[dict[str, Any]]: + """The samples in order, or only the newest `last`.""" + pass + + @abstractmethod + async def replace_samples(self, key: str, samples: list[dict[str, Any]], ttl_seconds: float | None) -> None: + """Replace a list's samples in one step; it expires ttl_seconds later, or never.""" + pass + + @abstractmethod + async def expire(self, key: str, ttl_seconds: float) -> None: + """Expire a key ttl_seconds from now, whatever it holds.""" + pass + + @abstractmethod + async def increment(self, key: str, ttl_seconds: float | None) -> int: + """Atomically count up from zero; the counter expires ttl_seconds after the last increment, or never.""" + pass + + @abstractmethod + async def increment_field(self, key: str, field: str, ttl_seconds: float | None) -> int: + """increment for one of several counts kept under one key, which all expire together.""" + pass + + @abstractmethod + async def read_counts(self, key: str) -> dict[str, int]: + """The counts increment_field keeps under key.""" + pass + @abstractmethod async def scan_values(self, prefix: str) -> dict[str, str]: """Every live value whose key starts with prefix.""" pass + @abstractmethod + async def list_length(self, key: str) -> int | None: + """Items in a list, such as a request queue; None where this store holds no queues.""" + pass + class InMemoryStateStore(StateStore): def __init__(self): self.kv_store: dict[str, str] = {} self.expiries: dict[str, float] = {} self.sets: dict[str, set[str]] = {} + self.samples: dict[str, list[str]] = {} + self.counts: dict[str, dict[str, int]] = {} async def set_value(self, key: str, value: str, ttl_seconds: float | None = None) -> None: self.kv_store[key] = value @@ -311,9 +358,46 @@ async def remove_from_set(self, key: str, member: str) -> None: async def set_members(self, key: str) -> set[str]: return set(self.sets.get(key, set())) + def set_value_sync(self, key: str, value: str, ttl_seconds: float | None = None) -> None: + self.kv_store[key] = value + self.expiries.pop(key, None) + if ttl_seconds is not None: + self.expiries[key] = time.monotonic() + ttl_seconds + + async def append_sample(self, key: str, sample: dict[str, Any], limit: int, ttl_seconds: float | None) -> None: + samples = self.samples.setdefault(key, []) + samples.append(json.dumps(sample, allow_nan=False)) + del samples[:-limit] + + async def read_samples(self, key: str, last: int | None = None) -> list[dict[str, Any]]: + return [json.loads(sample) for sample in self.samples.get(key, [])[-(last or 0) :]] + + async def replace_samples(self, key: str, samples: list[dict[str, Any]], ttl_seconds: float | None) -> None: + self.samples[key] = [json.dumps(sample, allow_nan=False) for sample in samples] + + async def expire(self, key: str, ttl_seconds: float) -> None: + if key in self.kv_store: + self.expiries[key] = time.monotonic() + ttl_seconds # lists, sets and counts here never expire + + async def increment(self, key: str, ttl_seconds: float | None) -> int: + value = int(self.get_value_sync(key) or 0) + 1 + await self.set_value(key, str(value), ttl_seconds) + return value + + async def increment_field(self, key: str, field: str, ttl_seconds: float | None) -> int: + counts = self.counts.setdefault(key, {}) + counts[field] = counts.get(field, 0) + 1 + return counts[field] + + async def read_counts(self, key: str) -> dict[str, int]: + return dict(self.counts.get(key, {})) + async def scan_values(self, prefix: str) -> dict[str, str]: return {key: value for key in list(self.kv_store) if key.startswith(prefix) and (value := self.get_value_sync(key)) is not None} + async def list_length(self, key: str) -> int | None: + return None # the in-memory queues live in InMemoryStore + class RedisStateStore(StateStore): def __init__(self, client: redis.Redis, sync_client: sync_redis.Redis): @@ -342,10 +426,66 @@ async def remove_from_set(self, key: str, member: str) -> None: async def set_members(self, key: str) -> set[str]: return set(await self.redis.smembers(key)) + def set_value_sync(self, key: str, value: str, ttl_seconds: float | None = None) -> None: + self.sync_redis.set(key, value, px=None if ttl_seconds is None else int(ttl_seconds * 1000)) + + async def append_sample(self, key: str, sample: dict[str, Any], limit: int, ttl_seconds: float | None) -> None: + async with self.redis.pipeline(transaction=True) as pipeline: + pipeline.rpush(key, json.dumps(sample, allow_nan=False)) + pipeline.ltrim(key, -limit, -1) + if ttl_seconds: + pipeline.expire(key, int(ttl_seconds)) + else: + pipeline.persist(key) + await pipeline.execute() + + async def read_samples(self, key: str, last: int | None = None) -> list[dict[str, Any]]: + raw = await self.redis.lrange(key, -(last or 0), -1) + # Decoded off the event loop: a dashboard read can be thousands of samples. + return await asyncio.to_thread(lambda: [json.loads(sample) for sample in raw]) + + async def replace_samples(self, key: str, samples: list[dict[str, Any]], ttl_seconds: float | None) -> None: + async with self.redis.pipeline(transaction=True) as pipeline: + pipeline.delete(key) + if samples: + pipeline.rpush(key, *(json.dumps(sample, allow_nan=False) for sample in samples)) + if ttl_seconds: + pipeline.expire(key, int(ttl_seconds)) + await pipeline.execute() + + async def expire(self, key: str, ttl_seconds: float) -> None: + await self.redis.expire(key, int(ttl_seconds)) + + async def increment(self, key: str, ttl_seconds: float | None) -> int: + async with self.redis.pipeline(transaction=True) as pipeline: + pipeline.incr(key) + if ttl_seconds: + pipeline.expire(key, int(ttl_seconds)) + else: + pipeline.persist(key) + value, _ = await pipeline.execute() + return value + + async def increment_field(self, key: str, field: str, ttl_seconds: float | None) -> int: + async with self.redis.pipeline(transaction=True) as pipeline: + pipeline.hincrby(key, field, 1) + if ttl_seconds: + pipeline.expire(key, int(ttl_seconds)) + else: + pipeline.persist(key) + value, _ = await pipeline.execute() + return value + + async def read_counts(self, key: str) -> dict[str, int]: + return {field: int(value) for field, value in (await self.redis.hgetall(key)).items()} + async def scan_values(self, prefix: str) -> dict[str, str]: keys = [key async for key in self.redis.scan_iter(match=prefix + "*", count=1000)] return {key: value for key, value in zip(keys, await self.redis.mget(keys)) if value is not None} if keys else {} + async def list_length(self, key: str) -> int | None: + return await self.redis.llen(key) + @cache def redis_client(redis_url: str) -> redis.Redis: diff --git a/src/server/telemetry.py b/src/server/telemetry.py new file mode 100644 index 000000000..9e08ea629 --- /dev/null +++ b/src/server/telemetry.py @@ -0,0 +1,551 @@ +"""The records OpenRL keeps about its own runs, written where things happen, for the inspection API to read. + +A run's events go through record(), which keeps them in the store and also prints each as one JSON line labelled with +the run, for history. Sessions, placements and SDK events print the same lines through log_event(). + +Telemetry never changes what it observes. Every hook swallows its own errors, every store write runs in the background +and is dropped after WRITE_SECONDS, and no hook changes what an op returns or raises. +""" + +import asyncio +import functools +import json +import math +import os +import re +import socket +import threading +import time +import uuid +from collections import defaultdict +from collections.abc import Awaitable, Callable, Coroutine +from contextlib import asynccontextmanager, suppress +from dataclasses import dataclass +from datetime import datetime +from numbers import Real + +from server.store import get_state_store + +TTL_SECONDS = float(os.getenv("OPEN_RL_OPS_TTL_DAYS", "7")) * 86400 or None # 0 keeps records forever +SAMPLE_LIMIT = int(os.getenv("OPEN_RL_OPS_SAMPLE_LIMIT", "2000")) # command records kept per list +STEP_TIMES = 64 +SDK_ERRORS = 200 +MAX_TURNS = 200 +HEARTBEAT_SECONDS = 5 +STALE_SECONDS = 30 # a worker record not rewritten for this long belongs to a gone process +WRITE_SECONDS = 5 +FINISH_SECONDS = 5 # how long an exiting process waits for its last writes +OPEN_PLACEMENTS = "open_rl:placements:open" # models holding a placement not yet released +FAILED = "RequestFailedResponse" + +SECRET = re.compile(r"authorization|cookie|api[-_]?key|access[-_]?token|secret|password", re.I) +CONFIG_ENV = ("OPEN_RL_", "OPENRL_", "VLLM_", "NCCL_", "CUDA_", "PYTORCH_", "TORCH_") + + +def key(*parts: str) -> str: + return ":".join(("open_rl", *parts)) + + +def epoch(stamp: str | None) -> float | None: + return datetime.fromisoformat(stamp.replace("Z", "+00:00")).timestamp() if stamp else None + + +def rank_zero() -> bool: + # Every torchrun rank runs the same ops; rank 0 records them. + return os.getenv("RANK", "0") == "0" + + +def identity() -> dict[str, str]: + """The pod this process runs in, as the scheduler passes it.""" + return {field: value for field, env in (("pod", "POD_NAME"), ("pod_uid", "POD_UID"), ("node", "NODE_NAME")) if (value := os.getenv(env))} + + +def run_of(request: dict) -> str | None: + # a pool's sampling record names its run as job; trainer requests carry model_id + return request.get("logical_run_id") or request.get("job") or request.get("model_id") + + +def operation_name(request: dict, role: str) -> str: + # A forward-only call travels as forward_backward, and its loss must not mix with training losses. + return "forward" if request.get("forward_only") else str(request.get("op") or ("sample" if role == "sampler" else "unknown")) + + +def redact(value): + # The SDK attaches full request and response headers to fatal errors, and the request carries the API key. + if isinstance(value, dict): + return {k: "[redacted]" if SECRET.search(str(k)) else redact(v) for k, v in value.items() if k not in ("request_headers", "response_headers")} + return [redact(v) for v in value] if isinstance(value, list) else value + + +writes: set[asyncio.Task] = set() # held so a write in flight is not garbage collected + + +def best_effort(hook): + @functools.wraps(hook) + def quiet(*args, **kwargs): + with suppress(Exception): + hook(*args, **kwargs) + + return quiet + + +def write(coroutine: Coroutine, timeout: float | None = None) -> None: + """Run a store write in the background. One that fails or takes longer than timeout (WRITE_SECONDS) is dropped.""" + + async def bounded(): + with suppress(Exception): + await asyncio.wait_for(coroutine, timeout or WRITE_SECONDS) + + try: + task = asyncio.get_running_loop().create_task(bounded()) + except RuntimeError: # no event loop to run it on + coroutine.close() + return + writes.add(task) + task.add_done_callback(writes.discard) + + +# Worker records + + +def gpu_uuids() -> list[str] | None: + """The GPUs this process can see, in CUDA order, as the UUIDs DCGM labels its series with. None when NVML cannot be + read. pynvml comes with vLLM, so a trainer may lack it, which says nothing about how many GPUs it has.""" + try: + import pynvml + + pynvml.nvmlInit() + uuids = [pynvml.nvmlDeviceGetUUID(pynvml.nvmlDeviceGetHandleByIndex(i)) for i in range(pynvml.nvmlDeviceGetCount())] + except Exception: + return None + visible = os.getenv("CUDA_VISIBLE_DEVICES") + return [gpu for i, gpu in enumerate(uuids) if visible is None or str(i) in visible.split(",") or gpu in visible] + + +def alive(record: dict, now: float) -> bool: + return record.get("phase") != "exited" and now - record.get("heartbeat_at", 0) < STALE_SECONDS + + +@dataclass +class Outstanding: + """A request this worker popped and has not answered.""" + + op: str + run_id: str | None + since: float + observed: bool = False # observe_operation records the op, so answered need not + + +class Worker: + """This process's worker record. A thread rewrites it, so it stays fresh while loading a model holds the event loop.""" + + def __init__(self, role: str, runtime_id: str | None) -> None: + now = time.time() + self.record = { + # One record per process, not per pod, so a container restarted in place does not overwrite its predecessor's. + "id": uuid.uuid4().hex[:16], + "node": socket.gethostname(), + **identity(), + "role": role, + "runtime_id": runtime_id, + "gpus": gpu_uuids(), + "started_at": now, + "phase": "starting", + "phase_since": now, + "turns": [], + "waits": [], + # The settings the scheduler passed as env, which is what this process runs with. + "config": {"env": {k: v for k, v in sorted(os.environ.items()) if k.startswith(CONFIG_ENV) and not SECRET.search(k)}}, + } + self.outstanding: dict[str, Outstanding] = {} + self.stopped = threading.Event() + self.heartbeat = threading.Thread(target=self.beat, name="worker-heartbeat", daemon=True) + + def snapshot(self) -> str: + """The record as JSON, with outstanding requests counted by op and run, each group showing its oldest.""" + groups: dict[tuple, dict] = {} + for entry in list(self.outstanding.values()): # a copy, since the event loop changes it while the heartbeat reads + group = groups.setdefault((entry.op, entry.run_id), {"op": entry.op, "run_id": entry.run_id, "since": entry.since, "count": 0}) + group["count"] += 1 + return json.dumps({**self.record, "heartbeat_at": time.time(), "outstanding": list(groups.values())}) + + @best_effort + def save(self) -> None: + write(get_state_store().set_value(key("worker", self.record["id"]), self.snapshot(), ttl_seconds=TTL_SECONDS)) + + def beat(self) -> None: + while not self.stopped.wait(HEARTBEAT_SECONDS): + with suppress(Exception): + get_state_store().set_value_sync(key("worker", self.record["id"]), self.snapshot(), ttl_seconds=TTL_SECONDS) + + +current: Worker | None = None # this process's record while it serves + +# The sampler pod's port named http, which the sampler PodMonitoring scrapes. +SAMPLER_METRICS_PORT = 9095 + + +@best_effort +def serve_metrics() -> None: + """Serve this process's Prometheus registry, where an in-process vLLM engine records its own metrics. Always in the + 0.0.4 text format, which keeps vLLM's vllm: names; offered OpenMetrics, the scraper asks for them escaped to vllm_.""" + from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + + from prometheus_client.exposition import CONTENT_TYPE_PLAIN_0_0_4, generate_latest # in the sampler image through vLLM + + class Metrics(BaseHTTPRequestHandler): + def do_GET(self) -> None: + body = generate_latest() + self.send_response(200) + self.send_header("Content-Type", CONTENT_TYPE_PLAIN_0_0_4) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, *args) -> None: # one line per scrape would flood the sampler's log + pass + + server = ThreadingHTTPServer(("", SAMPLER_METRICS_PORT), Metrics) + threading.Thread(target=server.serve_forever, name="sampler-metrics", daemon=True).start() + + +@asynccontextmanager +async def serving(role: str, runtime_id: str | None): + """Keep this process's worker record while the body runs, and mark it exited after.""" + global current + if rank_zero(): + with suppress(Exception): + current = Worker(role, runtime_id) + current.heartbeat.start() + current.save() + try: + yield current + finally: + await finish() + + +async def finish() -> None: + """Mark this process exited and land the writes in flight, within FINISH_SECONDS. Also called before an os._exit.""" + global current + worker, current = current, None + with suppress(Exception): + async with asyncio.timeout(FINISH_SECONDS): + if worker is not None: + worker.stopped.set() # stop the heartbeat first, so no late beat overwrites the exit + await asyncio.to_thread(worker.heartbeat.join, WRITE_SECONDS) + worker.record.update(phase="exited", phase_since=time.time()) + worker.save() + await asyncio.gather(*writes, return_exceptions=True) + + +@best_effort +def set_phase(phase: str) -> None: + if current is not None and current.record["phase"] != phase: + current.record.update(phase=phase, phase_since=time.time()) + + +@asynccontextmanager +async def gpu_turn(slot): + """Take the time slicer's slot for the body and show it on the worker record: the wait from asking to the grant, then + the turn, which is the hold.""" + asked = time.time() + async with slot: + started = time.time() + if current is not None: + current.record["waits"] = [*current.record["waits"][1 - MAX_TURNS :], [asked, started]] + set_phase("on_gpu") + try: + yield + finally: + if current is not None: + current.record["turns"] = [*current.record["turns"][1 - MAX_TURNS :], [started, time.time()]] + set_phase("ready") + + +@best_effort +def popped(requests: list[dict]) -> None: + """Requests this worker took from its queue, each outstanding until answered. A shared trainer's delete_model is queued + without being counted as enqueued, so it is not counted here either.""" + if current is None: + return + for request in requests: + if request.get("request_id") and request.get("op") != "delete_model": + run_id, op = run_of(request), operation_name(request, current.record["role"]) + current.outstanding[request["request_id"]] = Outstanding(op, run_id, time.time()) + record(run_id, "popped", request_id=request["request_id"], op=op) + + +@best_effort +def answered(request_id: str, result: dict | None = None) -> None: + """The request's answer is written. A failure answered without its op running, such as a failed wake or parse, gets its + command record here, so every failed answer has one.""" + if current is None or (entry := current.outstanding.pop(request_id, None)) is None: + return + record(entry.run_id, "answered", request_id=request_id, op=entry.op) + if not entry.observed and (result or {}).get("type") == FAILED: + request = {"op": entry.op, "request_id": request_id, "logical_run_id": entry.run_id} + op_finished(request, current.record["role"], current.record["runtime_id"], entry.since, FAILED) + + +# Command records + + +@dataclass +class RunSteps: + """A run's optimizer steps as this process counts them. start is the run's count when this process first ran one of + its ops, read in the background, and done_here the steps finished here since.""" + + start: asyncio.Task + done_here: int = 0 + + +run_steps: dict[str, RunSteps] = {} + + +async def read_step_count(run_id: str) -> int | None: + with suppress(Exception): + return int(await asyncio.wait_for(get_state_store().get_value(key("steps", run_id)), WRITE_SECONDS) or 0) + return None + + +async def observe_operation(request: dict, role: str, runtime_id: str | None, call: Callable[[], Awaitable[dict]]) -> dict: + """Run one op and record how it went. Returns or raises exactly as the op does.""" + if not rank_zero(): + return await call() + started_at = time.time() + with suppress(Exception): + if current is not None and (entry := current.outstanding.get(request.get("request_id"))): + entry.observed = True + # Read the run's step count before this op can change it. + if (run_id := run_of(request)) and run_id not in run_steps and operation_name(request, role) != "sample": + run_steps[run_id] = RunSteps(asyncio.ensure_future(read_step_count(run_id))) + try: + result = await call() + except Exception as error: + op_finished(request, role, runtime_id, started_at, type(error).__name__) + raise + op_finished(request, role, runtime_id, started_at, result=result) + return result + + +def numbers(metrics: dict | None) -> dict[str, float]: + """The metric values that are numbers, as floats. A bool is not one.""" + return {k: float(v) for k, v in (metrics or {}).items() if isinstance(v, Real) and not isinstance(v, bool)} + + +def finite(metrics: dict | None) -> dict[str, float]: + return {k: v for k, v in numbers(metrics).items() if math.isfinite(v)} + + +def command_record(request: dict, role: str, runtime_id: str | None, started_at: float, error_type: str | None, result) -> dict: + """How one op went. It failed with error_type, or as its result says. No payload, output or error message is kept.""" + result = result if isinstance(result, dict) else {} + if error_type is None and result.get("type") == FAILED: + error_type = FAILED + now, values = time.time(), numbers(result.get("metrics")) + record = { + "at": now, + "started_at": started_at, + "operation": operation_name(request, role), + "request_id": request.get("request_id"), + "run_id": run_of(request), + "runtime_id": runtime_id, + "role": role, + "status": "failed" if error_type else "succeeded", + "error_type": error_type, + "elapsed_seconds": now - started_at, + **identity(), + "metrics": finite(values), + } + # JSON has no NaN or infinity, so name the metrics that were, rather than leave them missing. + if nonfinite := sorted(k for k, v in values.items() if not math.isfinite(v)): + record["nonfinite"] = nonfinite + return record + + +@best_effort +def op_finished(request: dict, role: str, runtime_id: str | None, started_at: float, error_type: str | None = None, result=None) -> None: + """Record how one op went. A sample a vLLM sampler answered is only counted, with the time of the last, since vLLM's own + metrics time what it answers.""" + op = command_record(request, role, runtime_id, started_at, error_type, result) + if op["operation"] == "sample" and role == "sampler" and op["status"] == "succeeded": + count_sample(op["run_id"], op["at"]) + else: + record(op["run_id"], "op", **op) + + +@best_effort +def sampled_group(requests: list[dict], started_at: float) -> None: + """One record per run for the samples a vLLM sampler served together, so its lane shows when it sampled without a + record per sample. op_finished counted each sample, and a failed one keeps its own record too.""" + store, groups = get_state_store(), defaultdict(list) + for request in requests: + groups[run_of(request)].append(request) + for run_id, group in groups.items(): + if run_id: + op = command_record(group[0], "sampler", group[0].get("model_id"), started_at, None, None) | {"request_id": None, "count": len(group)} + write(store.append_sample(key("operations", run_id, "samples"), op, SAMPLE_LIMIT, TTL_SECONDS)) + log_event("op", op, run_id=run_id) + + +def count_sample(run_id: str | None, at: float) -> None: + if run_id: + store = get_state_store() + write(store.increment(key("sampled", run_id), TTL_SECONDS)) + write(store.set_value(key("sampled_at", run_id), str(at), ttl_seconds=TTL_SECONDS)) + + +def keep_op(run_id: str, op: dict) -> None: + """Keep one op's command record, and for a trainer op the run's failure and step counts.""" + store = get_state_store() + if op["operation"] == "sample": + count_sample(run_id, op["at"]) + # Samples keep their own list, so a run's sampling never evicts its training ops. + write(store.append_sample(key("operations", run_id, "samples"), op, SAMPLE_LIMIT, TTL_SECONDS)) + return + steps = run_steps.get(run_id) + # done_here is taken before this op's own step is counted below. The write may wait WRITE_SECONDS on the step read. + write(append_trainer_op(op, steps, steps.done_here if steps else 0), 2 * WRITE_SECONDS) + # The few records a run's one-line summary reads instead of its command history. + if op["status"] == "failed": + count_request(run_id, "failed") + elif op["operation"] == "optim_step": + if steps is not None: + steps.done_here += 1 + write(store.increment(key("steps", run_id), TTL_SECONDS)) + write(store.append_sample(key("step_times", run_id), op["at"], STEP_TIMES, TTL_SECONDS)) + + +async def append_trainer_op(record: dict, steps: RunSteps | None, done_here: int) -> None: + """Keep a trainer op, stamped with the step it belongs to once the run's count is read. An unread count leaves it off.""" + if steps is not None and (start := await steps.start) is not None: + record["step"] = start + done_here + await get_state_store().append_sample(key("operations", record["run_id"]), record, SAMPLE_LIMIT, TTL_SECONDS) + + +def count_request(run_id: str, event: str) -> None: + """Count one more of the run's requests reaching event, which is enqueued, popped, answered or failed. A request + enqueued but never popped shows up as a gap between the counts.""" + write(get_state_store().increment_field(key("ledger", run_id), event, TTL_SECONDS)) + + +# Run events + + +@best_effort +def record(run_id: str | None, kind: str, /, **fields) -> None: + """Keep one event about a run and log it. kind is enqueued, popped or answered, which count the run's requests, or op, + whose fields are a command record from command_record. The store write runs in the background.""" + if not run_id: + return + with suppress(Exception): + if kind == "op": + keep_op(run_id, fields) + else: + count_request(run_id, kind) + log_event(kind, fields, run_id=run_id) + + +@best_effort +def log_event(kind: str, fields: dict, *, run_id: str | None = None, session_id: str | None = None, severity: str | None = None) -> None: + """Print one event as a JSON line, labelled with its run or session, for the year of history Cloud Logging keeps. The + inspection API never reads these lines. A failed op logs as an error.""" + labels = {"openrl_source": "telemetry"} | ({"openrl_run_id": run_id} if run_id else {}) | ({"openrl_session_id": session_id} if session_id else {}) + line = {"severity": severity or ("ERROR" if fields.get("status") == "failed" else "INFO"), "message": f"telemetry {kind}"} + print(json.dumps(line | redact(fields) | {"logging.googleapis.com/labels": labels}, default=str), flush=True) + + +# Placement records + +placements_lock = asyncio.Lock() # sweeps run in the background and may overlap + + +def placement(workload: dict, terminations: dict[str, list[dict]]) -> dict: + """A Workload's placement, with the container terminations its pod has shown since, so they outlive the pod.""" + spec, status = workload["spec"], workload["status"] + placed_at = epoch(status["placedAt"]) + return { + "workload": workload["metadata"]["name"], + "role": spec["role"], + "model_id": spec["modelID"], + "owner": spec.get("ownerID"), + "pod": status.get("podName"), + "node": status.get("nodeName"), + "gpu_uuids": status.get("gpuUUIDs") or [], + "placed_at": placed_at, + "released_at": epoch(status.get("releasedAt")), + "terminations": [t for t in terminations.get(status.get("podName"), []) if t["at"] >= placed_at], + } + + +def merge_placements(records: list[dict], live: dict[str, dict], now: float) -> list[dict]: + """A model's placement records after a sweep. A record whose Workload still holds that placement takes its current + state, an open one whose Workload is gone or placed again is released now, and a new placement is added.""" + fresh, merged = dict(live), [] + for old in records: + if (new := fresh.get(old["workload"])) and new["placed_at"] == old["placed_at"]: + # A restarted container's earlier termination drops out of its pod status; the record keeps it. + kept = old.get("terminations", []) + merged.append(fresh.pop(old["workload"]) | {"terminations": kept + [t for t in new["terminations"] if t not in kept]}) + elif old["released_at"] is None: + merged.append({**old, "released_at": now}) + else: + merged.append(old) + return merged + list(fresh.values()) + + +async def record_placements(store, workloads: list[dict], terminations: dict[str, list[dict]]) -> None: + """Keep one record per Workload placement, which outlives the Workload, and the set of models still holding one.""" + now = time.time() + live: defaultdict[str, dict[str, dict]] = defaultdict(dict) # model -> Workload name -> placement + for workload in workloads: + if (workload.get("status") or {}).get("placedAt"): # the scheduler stamps it on bind + p = placement(workload, terminations) + live[p["model_id"]][p["workload"]] = p + async with placements_lock: + opened = await store.set_members(OPEN_PLACEMENTS) + for model_id in opened | live.keys(): + records_key = key("placements", model_id) + records = await store.read_samples(records_key) + merged = merge_placements(records, live[model_id], now) + held = any(r["released_at"] is None for r in merged) + if merged != records: + await store.replace_samples(records_key, merged, TTL_SECONDS) + for changed in (p for p in merged if p not in records): # placed, moved or released + log_event("placement", changed, run_id=model_id) + elif held and TTL_SECONDS: # a placement held longer than the TTL must outlive it + await store.expire(records_key, TTL_SECONDS) + if held and model_id not in opened: + await store.add_to_set(OPEN_PLACEMENTS, model_id) + elif not held and model_id in opened: + await store.remove_from_set(OPEN_PLACEMENTS, model_id) + + +# SDK events + + +@best_effort +def ingest_sdk_events(store, body: dict) -> None: + """Log the tinker SDK's own telemetry, one redacted JSON line per event, and keep a session's error events.""" + context = {k: body.get(k) for k in ("session_id", "process_uuid", "sdk_version", "platform")} + session_id, events = body.get("session_id"), body.get("events") + if not isinstance(events, list): + return + session_id = session_id if isinstance(session_id, str) and session_id else None + for event in [redact(e) for e in events if isinstance(e, dict)]: + severity = str(event.get("severity") or "INFO").upper() + log_event("sdk_event", {**context, "event": event}, session_id=session_id, severity=severity) + if severity in ("ERROR", "CRITICAL") and session_id: + write(store.append_sample(key("sdk_errors", session_id), sdk_error(event, severity), SDK_ERRORS, TTL_SECONDS)) + + +def sdk_error(event: dict, severity: str) -> dict: + data = event.get("event_data") if isinstance(event.get("event_data"), dict) else {} + name = str(event.get("error_type") or event.get("event_name") or event.get("event") or "event").rsplit(".", 1)[-1] + status = f"HTTP {data['status_code']}: " if data.get("status_code") else "" + message = status + str(event.get("error_message") or data.get("detail") or data.get("exception") or data.get("message") or "") + try: + at = datetime.fromisoformat(str(event.get("timestamp"))).timestamp() + except ValueError: + at = time.time() + return {"at": at, "severity": severity, "reason": f"SDK {name}", "message": message[:2000]} diff --git a/src/server/training_requests_processor.py b/src/server/training_requests_processor.py index 53c586f39..2fedabe40 100644 --- a/src/server/training_requests_processor.py +++ b/src/server/training_requests_processor.py @@ -22,6 +22,7 @@ from server.model_metadata import get_model_metadata from server.sampling_store import SamplingStore from server.store import RedisStore, RequestStore, StateStore, get_state_store, get_store +from server.telemetry import answered, finish, gpu_turn, observe_operation, popped, serving, set_phase from training import commands from training.commands import parse_command from training.distributed import broadcast_object, is_distributed, is_primary, local_rank @@ -126,6 +127,7 @@ async def run(self) -> None: if self.time_slicer is not None: await self.time_slicer.register(self.workload) self.snapshot_registered = True + set_phase("ready") while True: try: await self.announce_readiness() @@ -183,6 +185,7 @@ async def exit_gracefully(self, unregister: bool = True) -> None: await self.time_slicer.close() except Exception: pass + await finish() os._exit(0) async def next_batch(self) -> list[dict[str, Any]]: @@ -205,6 +208,7 @@ async def publish_result(self, request_id: str | None, result: dict[str, Any]) - # Only rank 0 writes to the store; the other ranks computed the same result. if request_id is not None and is_primary(): await self.store.set_result(request_id, result) + answered(request_id, result) async def run_once(self) -> None: batch = await self.next_batch() @@ -214,6 +218,7 @@ async def run_once(self) -> None: shutdown = any(req.get("op") == "shutdown_workers" for req in batch) work = [req for req in batch if req.get("op") != "shutdown_workers"] + popped(work) model_id = batch[0].get("model_id", "default") results: list[tuple[str | None, dict[str, Any]]] = [] @@ -268,7 +273,7 @@ async def handle_batch(self, requests: list[dict[str, Any]], results: list[tuple save_reqs = [r for r in requests if r.get("op") in save_ops] if gpu_reqs: - async with self.time_slicer.acquire(self.workload): + async with gpu_turn(self.time_slicer.acquire(self.workload)): await asyncio.to_thread(self.worker.wake_up) try: for request in gpu_reqs: @@ -282,7 +287,7 @@ async def handle_batch(self, requests: list[dict[str, Any]], results: list[tuple for request in save_reqs: results.append(await self.handle_request(request)) else: - async with self.time_slicer.acquire(self.workload): + async with gpu_turn(self.time_slicer.acquire(self.workload)): for request in save_reqs: results.append(await self.handle_request(request)) @@ -301,7 +306,7 @@ async def handle_request(self, raw_request: dict[str, Any]) -> tuple[str | None, ctx = propagate.extract(command.trace_context) if command.trace_context else None token = otel_context.attach(ctx) if ctx else None - result = await self.dispatch_operation(command) + result = await observe_operation(raw_request, "trainer", self.model_id, lambda: self.dispatch_operation(command)) return request_id, result except Exception as exc: traceback.print_exc() @@ -312,6 +317,12 @@ async def handle_request(self, raw_request: dict[str, Any]) -> tuple[str | None, if token: otel_context.detach(token) + def save_metrics(self) -> dict[str, Any]: + """The last save's metrics (a sparse delta's density and size) for the op record. The API server drops them from + the client's response, which the SDK's save types do not carry.""" + metrics = self.worker.last_save_metrics + return {"metrics": dict(metrics)} if metrics else {} + async def dispatch_operation(self, command: commands.TrainingCommand) -> dict[str, Any]: match command: case commands.CreateModel(): @@ -364,7 +375,7 @@ async def dispatch_operation(self, command: commands.TrainingCommand) -> dict[st return result case commands.SaveState(): result = await asyncio.to_thread(self.worker.save_state, command.model_id, command.state_path, command.include_optimizer, command.kind) - return {"path": result.get("path", command.state_path), "type": "state_saved"} + return {"path": result.get("path", command.state_path), "type": "state_saved", **self.save_metrics()} case commands.LoadWeights(): await asyncio.to_thread(self.worker.load_from_state, command.model_id, command.state_path, command.restore_optimizer) return {"path": command.state_path, "type": "weights_loaded"} @@ -376,7 +387,7 @@ async def dispatch_operation(self, command: commands.TrainingCommand) -> dict[st elif self.sampling is not None and is_primary() and ref: # A sampler serves a LoRA save by its reference, so the adapter is frozen under that name. await asyncio.to_thread(freeze_adapter, command.model_id, self.worker.base_model_name, command.path, command.sampling_session_id) - return {"path": command.path, "sampling_session_id": command.sampling_session_id, "type": "sampler_weights_saved"} + return {"path": command.path, "sampling_session_id": command.sampling_session_id, "type": "sampler_weights_saved", **self.save_metrics()} case commands.DeleteModel(): await asyncio.to_thread(self.worker.delete_model, command.model_id) return {"status": "ok", "type": "model_deleted"} @@ -472,6 +483,11 @@ def run_probe_server(): ) +async def serve_trainer(args: argparse.Namespace) -> None: + async with serving("trainer", os.getenv("OPEN_RL_MODEL_ID") or args.model_id): + await main_async(args) + + def start_request_processing_loop() -> None: parser = argparse.ArgumentParser() parser.add_argument("--model-id", help="Model id whose per-model request queue this dedicated trainer worker drains.") @@ -485,7 +501,7 @@ def start_request_processing_loop() -> None: cuda_devs = os.getenv("CUDA_VISIBLE_DEVICES", "ALL") print(f"-> Hardware : CUDA_VISIBLE_DEVICES={cuda_devs}") - asyncio.run(main_async(args)) + asyncio.run(serve_trainer(args)) if __name__ == "__main__": diff --git a/src/server/vllm_sampler.py b/src/server/vllm_sampler.py index 791f1c3f1..c15832f17 100644 --- a/src/server/vllm_sampler.py +++ b/src/server/vllm_sampler.py @@ -9,6 +9,7 @@ import logging import os import signal +import time import uuid from contextlib import asynccontextmanager from pathlib import Path @@ -19,6 +20,7 @@ from server.lora_snapshots import snapshot_root from server.sampling_dispatcher import Dispatcher, RevisionUnloadable, delta_applies, failed, redis_client from server.sampling_store import SamplingStore +from server.telemetry import gpu_turn, sampled_group, serve_metrics, serving, set_phase from server.vllm_options import engine_args, sample_params, sample_result logger = logging.getLogger(__name__) @@ -133,7 +135,7 @@ async def holding_gpu(time_slicer: TimeSlicerClient | None, workload: WorkloadRe if time_slicer is None: yield return - async with time_slicer.acquire(workload): + async with gpu_turn(time_slicer.acquire(workload)): if sampler is not None: await sampler.wake() try: @@ -159,7 +161,9 @@ async def dispatch(self) -> None: async with holding_gpu(self.time_slicer, self.workload, self.sampler): await self.read() # requests that arrived while the turn was pending join it if await self.ready(): - await asyncio.gather(*(self.send(r) for r in self.to_send()[: self.concurrency])) + batch, started = self.to_send()[: self.concurrency], time.time() + await asyncio.gather(*(self.send(r) for r in batch)) + sampled_group(batch, started) # one record per run for the group, so the lane shows when it sampled def make_engine(sleep_mode: bool, is_lora: bool) -> Any: @@ -181,6 +185,13 @@ async def run_sampling_worker(pool: str) -> None: store = SamplingStore(redis_client(os.environ["REDIS_URL"])) if time_slicer is not None: await time_slicer.register(workload) + serve_metrics() # vLLM's own registry on the http port, for the sampler PodMonitoring + async with serving("sampler", pool): # this process's worker record, rewritten every few seconds + await serve_pool(pool, store, time_slicer, workload) + + +async def serve_pool(pool: str, store: SamplingStore, time_slicer: TimeSlicerClient | None, workload: WorkloadRef | None) -> None: + """The worker's life after registration: boot the engine, serve the pool, and give everything back at the end.""" sampler = None try: async with holding_gpu(time_slicer, workload): @@ -189,6 +200,7 @@ async def run_sampling_worker(pool: str) -> None: if time_slicer is not None: await sampler.sleep() # give the memory back before releasing the turn Path(READY_FILE).touch() + set_phase("ready") concurrency = int(os.getenv("OPEN_RL_DISPATCH_CONCURRENCY", "256")) if time_slicer is not None: dispatcher = TimeSlicedDispatcher(store, pool, sampler, concurrency, time_slicer, workload) diff --git a/src/server/worker_manager.py b/src/server/worker_manager.py index f8cf9316f..f88bc91b4 100644 --- a/src/server/worker_manager.py +++ b/src/server/worker_manager.py @@ -157,6 +157,10 @@ def inspect_owner(self, owner: str) -> list[dict[str, Any]]: `age_seconds` where the launcher knows them.""" ... + def workloads(self) -> list[dict[str, Any]]: + """The scheduler Workload objects this manager created; none for a launcher without a scheduler.""" + ... + class LocalWorkerManager: """Runs each runtime as a subprocess of the API server, for development.""" @@ -265,6 +269,9 @@ def inspect_owner(self, owner: str) -> list[dict[str, Any]]: ) return workers + def workloads(self) -> list[dict[str, Any]]: + return [] + def python_command(extras: list[str], module: str, args: list[str]) -> list[str]: if shutil.which("uv"): diff --git a/src/training/fft_trainer_worker.py b/src/training/fft_trainer_worker.py index c85620268..4c1b18109 100644 --- a/src/training/fft_trainer_worker.py +++ b/src/training/fft_trainer_worker.py @@ -213,6 +213,7 @@ def save_state(self, model_id: str, state_path: str, include_optimizer: bool = F with open(os.path.join(state_path, "metadata.json"), "w") as f: json.dump(metadata, f) + self.last_save_metrics = {} print(f"Saved full fine-tuning state to {state_path}") return {"path": state_path} @@ -287,6 +288,15 @@ def save_state_delta( with open(os.path.join(state_path, "metadata.json"), "w") as f: json.dump(metadata, f) + # The op record keeps these, so the share of the model that moved each step is a step metric. + self.last_save_metrics = { + "delta/density_pct": metadata["density_pct"], + "delta/changed_elements": float(total_changed), + "delta/total_elements": float(total_elements), + "delta/layers": float(len(layer_names_list)), + "time/delta_collect": collect_time, + "time/delta_write": save_file_time, + } print(f"Saved sparse delta ({metadata['density_pct']}% changed elements, {total_changed}/{total_elements}) to {state_path}") return {"path": state_path, "density_pct": metadata["density_pct"]} diff --git a/src/training/trainer_worker.py b/src/training/trainer_worker.py index 0141ab7ee..1f336b4e3 100644 --- a/src/training/trainer_worker.py +++ b/src/training/trainer_worker.py @@ -46,6 +46,8 @@ class BaseTrainerWorker: def __init__(self): self.tokenizer: PreTrainedTokenizerBase | None = None self.last_token_budget: int | None = None + # What the last save moved, for the op record: a sparse delta's density and size, or empty for a full checkpoint. + self.last_save_metrics: dict[str, float] = {} if torch.cuda.is_available(): self.device = torch.device("cuda", local_rank()) diff --git a/tests/test_api_server_paths.py b/tests/test_api_server_paths.py index 06283b117..4161fea24 100644 --- a/tests/test_api_server_paths.py +++ b/tests/test_api_server_paths.py @@ -7,9 +7,10 @@ from fastapi.testclient import TestClient -from server import api_server +from server import api_server, telemetry from server.store import InMemoryStateStore, InMemoryStore from server.worker_manager import LocalWorkerManager +from tests.api_client import asgi_client from tests.test_session_lifecycle import RuntimeManager @@ -20,6 +21,7 @@ class ApiServerTest(unittest.TestCase): def setUp(self) -> None: self.enterContext(patch.object(api_server, "store", InMemoryStore())) self.enterContext(patch.object(api_server, "state", InMemoryStateStore())) + self.enterContext(patch.object(api_server.session_registry, "state", api_server.state)) self.client = TestClient(api_server.app) def post(self, path: str, body: dict, **kwargs): @@ -578,5 +580,39 @@ def test_an_fft_job_sharing_its_trainers_gpu_cannot_ask_for_more_samplers(self) self.assertEqual(self.metadata(exclusive.json()["request_id"])["sampler_replicas"], 4) +class TelemetryRecordsTest(ApiServerTest): + """Records are written in the background, so each test runs its requests in one loop and lets the writes land.""" + + def setUp(self) -> None: + super().setUp() + # The in-memory store has no Redis, which the vLLM pool path needs on delete; these tests are about the records. + self.enterContext(patch.dict(os.environ, {"SAMPLING_BACKEND": "torch"})) + self.enterContext(patch("server.worker_manager.get_state_store", return_value=api_server.state)) + self.enterContext(patch.object(telemetry, "get_state_store", return_value=api_server.state)) + + def call(self, *requests: tuple[str, dict]) -> list[dict]: + async def run() -> list[dict]: + async with asgi_client() as client: + answers = [(await client.post(f"/api/v1/{path}", json=body)).json() for path, body in requests] + await asyncio.gather(*list(telemetry.writes)) + return answers + + return asyncio.run(run()) + + def test_a_shared_trainers_delete_is_counted_neither_enqueued_nor_popped(self) -> None: + [created] = self.call(("create_model", {"base_model": "m"})) + self.assertEqual(self.post("delete_model", {"model_id": created["request_id"]}).status_code, 200) + + async def trainer() -> None: + async with telemetry.serving("trainer", "m"): + requests = await api_server.store.get_requests(active_set_id="m-1") + self.assertEqual([r["op"] for r in requests], ["create_model", "delete_model"]) + telemetry.popped(requests) + + asyncio.run(trainer()) + counts = asyncio.run(api_server.state.read_counts(telemetry.key("ledger", created["request_id"]))) + self.assertEqual(counts, {"enqueued": 1, "popped": 1}) + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_dashboard.py b/tests/test_dashboard.py index 03b4ab655..672e4ea8b 100644 --- a/tests/test_dashboard.py +++ b/tests/test_dashboard.py @@ -1,47 +1,47 @@ +"""The inspection routes as an agent calls them, over a snapshot with one run and its recorded optim_step.""" + import asyncio import unittest from unittest import mock from fastapi.testclient import TestClient -from server import api_server -from server.dashboard import fleet -from server.model_metadata import TrainingModelMetadata +from server.dashboard import fleet, router +from server.dashboard.__main__ import app from server.store import InMemoryStateStore +from server.telemetry import key - -def meta(**fields) -> str: - return TrainingModelMetadata(base_model="Qwen/Qwen3-8B", **fields).model_dump_json() +RUN = {"run_id": "r1", "name": "Run one", "model": "Qwen/base", "runtime_id": "r1", "created_at": 0, "completed_at": None} +RUN |= {"lifecycle": "placed", "status": "active", "workloads": [], "pods": [], "settings": {}, "session_ids": []} -class InventoryTest(unittest.TestCase): - def inventory(self, **records: str) -> dict: - store = InMemoryStateStore() - for key, value in records.items(): - asyncio.run(store.set_value(f"open_rl:{key}", value)) +class ApiTest(unittest.TestCase): + def setUp(self): + self.store = InMemoryStateStore() + op = {"run_id": "r1", "role": "trainer", "pod": "trainer-0", "operation": "optim_step", "status": "succeeded", "started_at": 100, "at": 150} + asyncio.run(self.store.append_sample(key("operations", "r1"), op, 10, None)) + self.snapshot = {"observed_at": 200, "placements": [], "workers": [], "runs": [RUN], "nodes": [], "pods": [], "errors": []} fleet.cache.clear() - self.enterContext(mock.patch.object(fleet, "get_state_store", return_value=store)) - return TestClient(api_server.app).get("/api/v1/inspect/inventory").json() - - def test_a_run_is_active_only_while_the_client_has_a_live_session(self): - body = self.inventory( - **{ - "model_meta:live": meta(session_ids=["s1"], created_at=3), - "model_meta:idle": meta(session_ids=["s2"], created_at=2), - "model_meta:done": meta(status="completed", session_ids=["s1"], created_at=1), - "session:s1": "{}", - } - ) - - runs = [(r["run_id"], r["lifecycle"], r["client_status"]) for r in body["runs"]] - self.assertEqual(runs, [("live", "active", "active"), ("idle", "ended", "active"), ("done", "ended", "completed")]) - self.assertEqual(body["errors"], []) - - def test_a_malformed_model_record_is_skipped_with_an_error(self): - body = self.inventory(**{"model_meta:good": meta(), "model_meta:bad": "not json"}) - - self.assertEqual([r["run_id"] for r in body["runs"]], ["good"]) - self.assertTrue(any("bad" in error for error in body["errors"]), body["errors"]) + self.enterContext(mock.patch.object(fleet, "snapshot", mock.AsyncMock(return_value=self.snapshot))) + self.enterContext(mock.patch("server.dashboard.facts.get_state_store", return_value=self.store)) + self.enterContext(mock.patch.object(fleet, "get_state_store", return_value=self.store)) + self.enterContext(mock.patch.object(router, "query", mock.AsyncMock(return_value={"series": []}))) # Prometheus has no series + self.client = TestClient(app) + + def test_routes_are_read_only_and_a_run_summary_reads_the_store(self): + summary = self.client.get("/api/v1/inspect/runs/r1/summary").json() + self.assertEqual((summary["steps"], summary["last_op"]["operation"]), (1, "optim_step")) + self.assertEqual(self.client.get("/api/v1/inspect/runs/missing/summary").status_code, 404) + self.assertTrue(all(set(methods) == {"get"} for methods in self.client.get("/openapi.json").json()["paths"].values())) + + def test_a_session_id_names_its_one_run_and_a_session_with_several_runs_is_refused(self): + self.snapshot["runs"] = [{**RUN, "session_ids": ["s1", "s2"]}, {**RUN, "run_id": "r2", "runtime_id": "r2", "session_ids": ["s2"]}] + by_session = self.client.get("/api/v1/inspect/runs/s1/summary").json() + self.assertEqual((by_session["run_id"], by_session["steps"]), ("r1", 1)) + refused = self.client.get("/api/v1/inspect/runs/s2/summary") + self.assertEqual(refused.status_code, 409) + self.assertIn("r1, r2", refused.text) + self.assertEqual(self.client.get("/api/v1/inspect/runs/s3/summary").status_code, 404) if __name__ == "__main__": diff --git a/tests/test_dashboard_facts.py b/tests/test_dashboard_facts.py new file mode 100644 index 000000000..b38ef2917 --- /dev/null +++ b/tests/test_dashboard_facts.py @@ -0,0 +1,104 @@ +"""Run facts replay recorded incidents: each incident must show up as a fact an agent can read without a verdict.""" + +import asyncio +import unittest +from unittest import mock + +from server.dashboard import facts, fleet +from server.dashboard.facts import request_queue, sampler_queue +from server.store import InMemoryStateStore +from server.telemetry import key + +NOW = 100_000.0 + + +def op(at, operation, status="succeeded", role="trainer", **fields): + return {"at": at, "started_at": at - 10, "operation": operation, "status": status, "role": role, "elapsed_seconds": 10.0, "metrics": {}, **fields} + + +def healthy_steps(count=5, every=600.0): + return [op(NOW - (count - i) * every, "optim_step", metrics={"grad_norm:mean": 1.0 + i}) for i in range(count)] + + +def worker(started_at, heartbeat_at, **fields): + return {"id": str(started_at), "pod": "t-0", "role": "trainer", "runtime_id": "r", "started_at": started_at, "heartbeat_at": heartbeat_at, **fields} + + +class RunFactsTest(unittest.TestCase): + """Records go into a real in-memory store, and the facts are read back through the same context the routes use.""" + + def setUp(self): + self.enterContext(mock.patch.object(facts, "get_state_store", side_effect=lambda: self.store)) + self.enterContext(mock.patch.object(fleet, "get_state_store", side_effect=lambda: self.store)) + + def facts_of(self, ops=(), workers=(), ledger=None, queued=0, lifecycle="placed", pods=(), samples_queued=0, completed_at=None): + self.store = InMemoryStateStore() + fleet.cache.clear() + + async def read(): + for record in ops: + await self.store.append_sample(key("operations", "r", *(["samples"] if record["operation"] == "sample" else [])), record, 10_000, None) + for event, value in (ledger or {}).items(): + for _ in range(value): + await self.store.increment_field(key("ledger", "r"), event, None) + queues = {request_queue("r"): queued, sampler_queue("r"): samples_queued} # the in-memory store keeps no queues + self.store.list_length = mock.AsyncMock(side_effect=queues.get) + run = {"run_id": "r", "runtime_id": "r", "lifecycle": lifecycle, "created_at": 0.0, "completed_at": completed_at, "workloads": []} + run.update(pods=list(pods), settings={"openrl.fine_tuning_type": "lora", "openrl.trainer_gpus": 1}) + ctx = await facts.context({"observed_at": NOW, "workers": list(workers), "placements": [], "runs": [run]}, run) + return ctx, facts.summary(ctx), facts.line(ctx) + + return asyncio.run(read()) + + def test_requests_held_by_a_trainer_that_stopped_reporting_name_the_dead_holder(self): # I1 + held = {"op": "forward_backward", "run_id": "r", "count": 3, "since": NOW - 207360} + _, summary, line = self.facts_of(healthy_steps(), workers=[worker(0, NOW - 600, pod="harvey-trainer-0", outstanding=[held])]) + (request,) = summary["requests"]["unanswered"] + self.assertEqual((request["pod"], request["holder_alive"], request["age_s"]), ("harvey-trainer-0", False, 207360)) + self.assertEqual(line["held_by_dead_worker"], 3) + + def test_a_request_lost_between_push_and_pop_shows_in_the_ledger_and_queued_samples_show_too(self): # I4 + counts = {"enqueued": 11, "popped": 10, "answered": 10} + _, lost, _ = self.facts_of(healthy_steps(), ledger=counts) + self.assertEqual(lost["requests"]["ledger"]["lost"], 1) + _, queued, line = self.facts_of(healthy_steps(), ledger=counts, queued=1, samples_queued=64) + self.assertEqual((queued["requests"]["ledger"]["lost"], queued["requests"]["samples_queued"], line["samples_queued"]), (0, 64, 64)) + + def test_a_nan_gradient_names_the_op_step_and_metric(self): # I30 + _, summary, _ = self.facts_of([*healthy_steps(), op(NOW - 5, "optim_step", nonfinite=["grad_norm:mean"], step=5)]) + first = summary["optimizer"]["nonfinite"]["first"] + self.assertEqual((first["operation"], first["step"], first["metrics"]), ("optim_step", 5, ["grad_norm:mean"])) + + def test_a_run_that_ended_on_a_failed_create_model_names_the_error_and_where_to_read_it(self): + _, summary, _ = self.facts_of([op(NOW - 600, "create_model", status="failed", error_type="ValueError", pod="t-0")], lifecycle="ended") + self.assertEqual( + (summary["steps"], summary["last_op"]["operation"], summary["failures"]["ops"][0]["error_type"]), (0, "create_model", "ValueError") + ) + logs = summary["failures"]["logs"] + self.assertEqual((logs["read"], logs["args"]["pod"], logs["args"]["q"]), ("logs", "t-0", "ValueError")) + + def test_relaunch_rules(self): + first, after = worker(NOW - 1000, NOW - 905, pod_uid="u1"), {"started_at": NOW - 880, "heartbeat_at": NOW} + holding = [{"op": "create_model", "run_id": "r", "count": 1, "since": NOW - 950}] + cases = { + "container restarted in place": ([first, {**first, **after}], None, 1), + "new pod after one died holding work": ([{**first, "outstanding": holding}, {**first, **after, "pod_uid": "u2"}], None, 1), + "clean exit": ([{**first, "phase": "exited"}, {**first, **after}], None, 0), + "idle pod recreated, as a shared runtime's teardown does": ([first, {**first, **after, "pod_uid": "u2"}], None, 0), + "next run on the shared runtime, same pod name": ([first, {**first, "started_at": NOW - 300, "heartbeat_at": NOW}], NOW - 500, 0), + } + for name, (workers, completed_at, expected) in cases.items(): + with self.subTest(name): + _, summary, line = self.facts_of(workers=workers, completed_at=completed_at) + self.assertEqual((line["relaunches"], "relaunches" in summary), (expected, bool(expected))) + + def test_an_oom_killed_container_is_an_incident_and_a_healthy_run_has_no_failure_blocks(self): + oom = {"at": NOW - 60, "pod": "t-0", "reason": "OOMKilled", "message": "worker exited with code 137"} + _, summary, _ = self.facts_of(healthy_steps(), pods=[{"name": "t-0", "incidents": [oom]}]) + self.assertEqual(summary["incidents"][0]["reason"], "OOMKilled") + _, healthy, _ = self.facts_of(healthy_steps(), ledger={"enqueued": 5, "popped": 5, "answered": 5}) + self.assertFalse({"failures", "relaunches", "incidents", "samplers"} & set(healthy)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_dashboard_lanes.py b/tests/test_dashboard_lanes.py new file mode 100644 index 000000000..9a3f72d7e --- /dev/null +++ b/tests/test_dashboard_lanes.py @@ -0,0 +1,41 @@ +"""A lane's occupancy on a time-sliced GPU is its turns; an op recorded outside every turn ran off the GPU (an offloaded save).""" + +import asyncio +import unittest +from unittest import mock + +from server.dashboard import fleet +from server.store import InMemoryStateStore +from server.telemetry import key + +PLACEMENT = {"workload": "fft-r-trainer", "placed_at": 50.0, "released_at": None, "model_id": "r", "pod": "t-0", "role": "trainer"} +PLACEMENT |= {"node": "n1", "gpu_uuids": ["GPU-1"], "terminations": []} +RUN = {"run_id": "r", "runtime_id": "r", "name": "Run", "model": "Qwen/base", "created_at": 50.0, "completed_at": None} + + +def worker(turns): + return {"id": "w", "pod": "t-0", "role": "trainer", "started_at": 50.0, "heartbeat_at": 190.0, "phase": "ready", "turns": turns, "waits": []} + + +class LaneOccupancyTest(unittest.TestCase): + def activity(self, turns): + store = InMemoryStateStore() + save = {"at": 125.0, "started_at": 124.0, "operation": "save_weights_for_sampler", "status": "succeeded", "role": "trainer", "metrics": {}} + asyncio.run(store.append_sample(key("operations", "r"), save, 100, None)) + fleet.cache.clear() + with mock.patch.object(fleet, "get_state_store", return_value=store): + state = {"observed_at": 200.0, "placements": [PLACEMENT], "runs": [RUN], "workers": [worker(turns)]} + return asyncio.run(fleet.activity(state, 0.0, 200.0))[0] + + def test_an_op_outside_every_turn_is_shown_but_does_not_occupy_the_gpu(self): + lane = self.activity(turns=[[100.0, 110.0]]) + self.assertEqual((lane["exact"], lane["intervals"]), (True, [[100.0, 110.0]])) + self.assertEqual(lane["ops"], {"save_weights_for_sampler": [[124.0, 125.0]]}) + + def test_without_turns_the_ops_stand_in_for_occupancy(self): + lane = self.activity(turns=[]) + self.assertEqual((lane["exact"], lane["intervals"]), (False, [[124.0, 125.0]])) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_fft_batch_failure.py b/tests/test_fft_batch_failure.py index 962393168..ccaf32133 100644 --- a/tests/test_fft_batch_failure.py +++ b/tests/test_fft_batch_failure.py @@ -1,10 +1,12 @@ import asyncio import contextlib +import json import os import unittest from unittest.mock import patch -from server.store import InMemoryStore +from server import telemetry +from server.store import InMemoryStateStore, InMemoryStore from server.training_requests_processor import TrainingRequestsProcessor @@ -85,5 +87,46 @@ async def handled(request): self.assertEqual(exits, [False]) +class FailedAnswerRecordTest(unittest.TestCase): + """Every failed answer a client gets has one failed record, including failures before the op runs.""" + + def setUp(self): + self.state = InMemoryStateStore() + self.enterContext(patch.object(telemetry, "get_state_store", return_value=self.state)) + + def test_a_wake_or_parse_failure_records_every_request_it_failed(self) -> None: + wake = processor(BatchStore([{"request_id": "fb-1", "op": "forward_backward", "model_id": "run-a"}]), SlicerStub()) + parse = processor(BatchStore([{"request_id": "bad-1", "op": "optim_step", "model_id": "run-a", "adam_params": "not a dict"}]), None) + + async def serve(): + async with telemetry.serving("trainer", "run-a"): + for proc in (wake, parse): + with contextlib.suppress(RuntimeError): + await proc.run_once() + return await self.state.read_samples(telemetry.key("operations", "run-a")) + + records = [(r["request_id"], r["operation"], r["status"]) for r in asyncio.run(serve())] + self.assertEqual(records, [("fb-1", "forward_backward", "failed"), ("bad-1", "optim_step", "failed")]) + + def test_an_exit_after_shutdown_says_the_process_exited_and_lands_its_writes(self) -> None: + proc = processor(BatchStore([{"request_id": "s-1", "op": "shutdown_workers", "model_id": "run-a"}]), SlicerStub()) + increment = self.state.increment_field + + async def slow_increment(*args): + await asyncio.sleep(0.05) + return await increment(*args) + + async def serve(): + async with telemetry.serving("trainer", "run-a") as worker: + telemetry.record("run-a", "popped") # still in flight when the process exits + with patch("os._exit", side_effect=SystemExit), self.assertRaises(SystemExit): + await proc.run_once() + record = json.loads(await self.state.get_value(telemetry.key("worker", worker.record["id"])) or "{}") + return record.get("phase"), (await self.state.read_counts(telemetry.key("ledger", "run-a"))).get("popped") + + with patch.object(self.state, "increment_field", slow_increment): + self.assertEqual(asyncio.run(serve()), ("exited", 1)) + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_session_lifecycle.py b/tests/test_session_lifecycle.py index dd84f8b6d..6fbd096be 100644 --- a/tests/test_session_lifecycle.py +++ b/tests/test_session_lifecycle.py @@ -23,6 +23,9 @@ def ensure(self, model_id, role): def release(self, model_id): self.released_models.append(model_id) + def workloads(self): + return [] + def release_owner(self, owner): self.released.append(owner) return {"test-base"} if owner == "test-base" else set() diff --git a/tests/test_telemetry_faults.py b/tests/test_telemetry_faults.py new file mode 100644 index 000000000..5fb65e592 --- /dev/null +++ b/tests/test_telemetry_faults.py @@ -0,0 +1,100 @@ +"""Telemetry must never change what a worker does. Each test serves the same work with a healthy telemetry store and with +one that fails, and expects the same answers, in about the same time.""" + +import asyncio +import os +import threading +import time +import unittest +from unittest.mock import AsyncMock, Mock, patch + +from server import telemetry +from server.store import InMemoryStateStore +from server.training_requests_processor import TrainingRequestsProcessor +from tests.test_fft_batch_failure import BatchStore, SlicerStub + +try: + from server import vllm_sampler + from tests.test_sampler_patch import make_engine +except ImportError: # vLLM is a GPU extra + vllm_sampler = None + +BOUND = 1.0 # a hung write is dropped after WRITE_SECONDS; the serve must finish well inside it + + +class FaultyStore: + """Every call raises, as with Redis down, or waits forever, as with a Redis that never answers.""" + + def __init__(self, hangs: bool): + self.hangs = hangs + + def __getattr__(self, name): + if not self.hangs: + raise ConnectionError(f"telemetry store {name} failed") + if name.endswith("_sync"): + return lambda *args, **kwargs: threading.Event().wait() + return lambda *args, **kwargs: asyncio.Event().wait() + + +def no_store(): + raise RuntimeError("no telemetry store configured") + + +FAULTS = {"raises": lambda: FaultyStore(False), "hangs": lambda: FaultyStore(True), "is missing": no_store} + + +class TelemetryFaultTest(unittest.TestCase): + def setUp(self) -> None: + self.enterContext(patch.object(telemetry, "WRITE_SECONDS", 5 * BOUND)) + self.enterContext(patch.object(telemetry, "FINISH_SECONDS", BOUND)) + self.enterContext(patch.dict(telemetry.run_steps, clear=True)) + + def assert_unchanged(self, work) -> None: + async def serve(): + async with telemetry.serving("trainer", "run-a"): + started = time.monotonic() + result = await asyncio.wait_for(work(), 2 * BOUND) + exiting = time.monotonic() + return result, exiting - started, time.monotonic() - exiting + + healthy = InMemoryStateStore() + with patch.object(telemetry, "get_state_store", lambda: healthy): + expected, _, _ = asyncio.run(serve()) + self.assertTrue(expected) + self.assertTrue(healthy.samples or healthy.get_value_sync(telemetry.key("sampled", "run-a")), "the healthy serve recorded its ops") + for fault, store in FAULTS.items(): + with self.subTest(store=fault), patch.object(telemetry, "get_state_store", store): + result, took, exiting = asyncio.run(serve()) + self.assertEqual(result, expected) + self.assertLess(took, BOUND) + self.assertLess(exiting, 2 * BOUND) + + def test_a_time_sliced_training_request_is_answered_the_same(self) -> None: + trainer = Mock(cpu_offload=True, optim_step=lambda adam_params, model_id: {"metrics": {"grad_norm:mean": 0.5}}) + + async def train(): + store = BatchStore([{"request_id": f"os-{i}", "op": "optim_step", "model_id": "run-a", "adam_params": {}} for i in range(2)]) + with patch.dict(os.environ, {"REDIS_URL": "redis://test"}): + processor = TrainingRequestsProcessor(store, trainer, "run-a", time_slicer=SlicerStub()) + await processor.run_once() + return store.futures + + self.assert_unchanged(train) + + @unittest.skipIf(vllm_sampler is None, "needs vLLM") + def test_a_sample_request_is_answered_the_same(self) -> None: + async def sample(): + store = AsyncMock() + store.get_sampling_requests_for_model.side_effect = [ + [{"request_id": "s-1", "model_id": "run-a", "prompt_token_ids": [1, 2], "max_tokens": 1}], + [{"request_id": vllm_sampler.SHUTDOWN_SENTINEL}], + ] + with patch.object(vllm_sampler, "get_state_store", return_value=AsyncMock()): + await vllm_sampler.serve("run-a", store, make_engine, SlicerStub(), vllm_sampler.WorkloadRef("sampler-run-a")) + return {call.args[0]: call.args[1] for call in store.set_result.call_args_list} + + self.assert_unchanged(sample) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_trainer_optimizer_correctness.py b/tests/test_trainer_optimizer_correctness.py index 2a9891441..dbc5cc464 100644 --- a/tests/test_trainer_optimizer_correctness.py +++ b/tests/test_trainer_optimizer_correctness.py @@ -510,6 +510,32 @@ async def test_full_processor_saves_sampler_checkpoint_as_full_state(self) -> No }, ) + async def test_full_processor_sampler_save_records_delta_metrics(self) -> None: + """A delta save's density and size travel with the op result, where the command record keeps them.""" + + class _DeltaWorker(_RecordingFullWorker): + def save_state(self, model_id, state_path, include_optimizer=False, kind="state"): + self.last_save_metrics = {"delta/density_pct": 6.97, "delta/changed_elements": 34439901.0, "delta/total_elements": 494032768.0} + return super().save_state(model_id, state_path, include_optimizer, kind) + + worker = _DeltaWorker() + store = _FutureStoreStub() + time_slicer = _TimeSlicerStub() + with patch.dict(os.environ, {"OPEN_RL_TMP_DIR": "/tmp/open-rl-test", "REDIS_URL": "redis://localhost:6379"}): + processor = training_requests_processor_module.TrainingRequestsProcessor(store, worker, "model-a", time_slicer=time_slicer) + await processor.process_request( + { + "request_id": "req-a", + "model_id": "model-a", + "op": "save_weights_for_sampler", + "payload": {"path": "tinker://model-a/sampler_weights/final", "sampling_session_id": "tinker://model-a/sampler_weights/sampler-7"}, + }, + ) + result = store.results["req-a"] + self.assertEqual(result["type"], "sampler_weights_saved") + self.assertEqual(result["metrics"]["delta/density_pct"], 6.97) + self.assertEqual(result["metrics"]["delta/total_elements"], 494032768.0) + async def test_full_processor_requires_redis(self) -> None: with patch.dict(os.environ, {"OPEN_RL_ENABLE_FFT": "true"}, clear=True), self.assertRaisesRegex(RuntimeError, "REDIS_URL"): await training_requests_processor_module.run_training_requests_processor(_RecordingFullWorker(), "model-a") diff --git a/tests/test_worker_manager.py b/tests/test_worker_manager.py index c0916d964..12bf803d6 100644 --- a/tests/test_worker_manager.py +++ b/tests/test_worker_manager.py @@ -390,6 +390,17 @@ def test_internal_future_result_types_translate_to_tinker_types(self) -> None: {"type": public_type, "path": "/tmp/x"}, ) + def test_save_results_keep_delta_metrics_out_of_the_client_response(self) -> None: + """The worker's delta metrics ride on the result for the op record; the SDK's save responses do not carry them.""" + metrics = {"delta/density_pct": 6.97, "delta/total_elements": 494032768.0} + for internal_type in ("state_saved", "sampler_weights_saved", "weights_saved"): + with self.subTest(internal_type=internal_type): + translated = api_server.translate_future_result({"type": internal_type, "path": "/tmp/x", "metrics": metrics}) + self.assertNotIn("metrics", translated) + # forward_backward keeps its metrics: the client reads them + translated = api_server.translate_future_result({"type": "forward_backward_completed", "metrics": {"loss:mean": 1.0}}) + self.assertEqual(translated["metrics"], {"loss:mean": 1.0}) + class LocalWorkerManagerSamplerLaunchTest(unittest.TestCase): def setUp(self) -> None: diff --git a/tests/test_worker_record.py b/tests/test_worker_record.py new file mode 100644 index 000000000..f010499bb --- /dev/null +++ b/tests/test_worker_record.py @@ -0,0 +1,108 @@ +import asyncio +import json +import time +import unittest +from unittest import mock + +from server import telemetry +from server.store import InMemoryStateStore + + +def call(result=None, error=None): + return mock.AsyncMock(return_value=result or {}, side_effect=error) + + +class WorkerRecordTest(unittest.IsolatedAsyncioTestCase): + def setUp(self) -> None: + self.store = InMemoryStateStore() + self.enterContext(mock.patch.object(telemetry, "get_state_store", return_value=self.store)) + self.enterContext(mock.patch.dict("os.environ", {"POD_NAME": "pod", "POD_UID": "uid", "NODE_NAME": "node"})) + self.enterContext(mock.patch.dict(telemetry.run_steps, clear=True)) + + async def saved(self, worker_id: str) -> dict: + await asyncio.gather(*list(telemetry.writes)) + return json.loads(await self.store.get_value(telemetry.key("worker", worker_id))) + + async def recorded(self, *parts: str) -> list[dict]: + await asyncio.gather(*list(telemetry.writes)) # writes are fire-and-forget + return await self.store.read_samples(telemetry.key("operations", *parts)) + + async def test_the_record_shows_settings_and_a_clean_exit(self) -> None: + env = {"OPEN_RL_FINE_TUNING_TYPE": "lora", "OPEN_RL_API_KEY": "sk-private", "HOME": "/root"} + with mock.patch.dict("os.environ", env): + async with telemetry.serving("trainer", "run-a") as worker: + record = await self.saved(worker.record["id"]) + self.assertTrue(telemetry.alive(record, record["heartbeat_at"])) + self.assertEqual(record["config"]["env"], {"OPEN_RL_FINE_TUNING_TYPE": "lora"}) + record = await self.saved(worker.record["id"]) + self.assertEqual(record["phase"], "exited") + self.assertFalse(telemetry.alive(record, record["heartbeat_at"])) + self.assertNotIn("sk-private", json.dumps(record)) + + async def test_the_record_stays_fresh_while_model_loading_blocks_the_event_loop(self) -> None: + with mock.patch.object(telemetry, "HEARTBEAT_SECONDS", 0.02): + async with telemetry.serving("trainer", "run-a") as worker: + blocked_at = time.time() + time.sleep(0.3) # loading weights holds the loop, and the heartbeat must not need it + record = json.loads(self.store.get_value_sync(telemetry.key("worker", worker.record["id"]))) + self.assertGreater(record["heartbeat_at"] - blocked_at, 0.2) + + async def test_popped_requests_stay_outstanding_and_are_counted_until_answered(self) -> None: + async with telemetry.serving("trainer", "base") as worker: + telemetry.popped([{"request_id": f"f{i}", "op": "forward_backward", "model_id": "run-a"} for i in range(3)] + [{"op": "no id"}]) + telemetry.popped([{"request_id": "s1", "model_id": "base", "logical_run_id": "run-b"}]) + # A shared trainer's delete_model was never counted as enqueued, so it is not outstanding either. + telemetry.popped([{"request_id": "d1", "op": "delete_model", "model_id": "run-a"}]) + telemetry.answered("f0") + groups = {(g["op"], g["run_id"]): g["count"] for g in json.loads(worker.snapshot())["outstanding"]} + self.assertEqual(groups, {("forward_backward", "run-a"): 2, ("unknown", "run-b"): 1}) + self.assertEqual(await self.store.read_counts(telemetry.key("ledger", "run-a")), {"popped": 3, "answered": 1}) + + async def test_an_op_raises_as_it_would_and_only_rank_zero_records_its_failure(self) -> None: + request = {"model_id": "run-a", "op": "train", "payload": "private input"} + with self.assertRaisesRegex(ValueError, "private error"): + await telemetry.observe_operation(request, "trainer", "run-a", call(error=ValueError("private error"))) + with self.assertRaises(asyncio.CancelledError): + await telemetry.observe_operation(request, "trainer", "run-a", call(error=asyncio.CancelledError())) + with mock.patch.dict("os.environ", {"RANK": "1"}): + await telemetry.observe_operation(request, "trainer", "run-a", call()) + (sample,) = await self.recorded("run-a") + self.assertEqual((sample["status"], sample["error_type"]), ("failed", "ValueError")) + self.assertNotIn("private", repr(sample)) + + async def test_trainer_ops_carry_their_step_and_nonfinite_metrics_and_sampling_never_evicts_them(self) -> None: + await self.store.set_value(telemetry.key("steps", "r"), "5") # a restarted trainer resumes the run's count + failed, nan = {"type": "RequestFailedResponse"}, {"metrics": {"loss": 1.5, "grad_norm": float("nan"), "kl": float("inf")}} + for op, result in (("forward_backward", nan), ("optim_step", None), ("optim_step", failed), ("forward_backward", None)): + await telemetry.observe_operation({"model_id": "r", "op": op}, "trainer", "r", call(result)) + await telemetry.observe_operation({"model_id": "r", "op": "forward_backward", "forward_only": True}, "trainer", "r", call()) + with mock.patch.object(telemetry, "SAMPLE_LIMIT", 3): + for i in range(6): # a shared sampler's requests name the base model's queue and the run they sample for + request = {"op": "sample", "model_id": "base", "logical_run_id": "r", "request_id": f"s{i}"} + telemetry.op_finished(request, "sampler", "base", 0.0, "EngineDied" if i % 2 else None) + trainer = await self.recorded("r") + self.assertEqual( + [(s["operation"], s["status"], s["step"]) for s in trainer], + [("forward_backward", "succeeded", 5), ("optim_step", "succeeded", 5), ("optim_step", "failed", 6)] + + [("forward_backward", "succeeded", 6), ("forward", "succeeded", 6)], + ) + self.assertEqual((trainer[0]["metrics"], trainer[0]["nonfinite"]), ({"loss": 1.5}, ["grad_norm", "kl"])) # JSON has no NaN + self.assertEqual(await self.store.get_value(telemetry.key("steps", "r")), "6") + + +if __name__ == "__main__": + unittest.main() + + +class PlacementRecordTest(unittest.TestCase): + def test_a_placement_keeps_its_pods_termination_after_the_pod_status_drops_it(self) -> None: + workload = { + "metadata": {"name": "w"}, + "spec": {"role": "trainer", "modelID": "m"}, + "status": {"placedAt": "2026-10-21T06:00:00Z", "podName": "p"}, + } + oom = {"at": telemetry.epoch("2026-10-21T06:30:00Z"), "container": "worker", "reason": "OOMKilled", "exit_code": 137, "signal": None} + first = telemetry.merge_placements([], {"w": telemetry.placement(workload, {"p": [oom]})}, time.time()) + later = telemetry.merge_placements(first, {"w": telemetry.placement(workload, {})}, time.time()) + self.assertEqual(later[0]["terminations"], [oom]) + self.assertIsNone(later[0]["released_at"])