diff --git a/.github/workflows/amd-exporter-image.yml b/.github/workflows/amd-exporter-image.yml new file mode 100644 index 0000000000..75e0a4551e --- /dev/null +++ b/.github/workflows/amd-exporter-image.yml @@ -0,0 +1,146 @@ +name: AMD exporter image + +on: + push: + branches: [chore/powerx-amd-exporter-image] + paths: [.github/workflows/amd-exporter-image.yml, tools/amd-exporter/**] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: amd-exporter-image-${{ github.ref }} + cancel-in-progress: false + +jobs: + image: + name: Build and publish the AMD power exporter on CPU + runs-on: ubuntu-24.04 + permissions: + contents: read + actions: read # Reuse the pinned prepared runtime artifact. + packages: write # Publish the verified image to the repository-linked GHCR package. + timeout-minutes: 30 + env: + SOURCE_SHA: 9d0eb8c88af99f1dbe2914d803382092671456d9 + PATCH_SHA256: b0bb7d3477be7ff5b3b455d17f83c454d3aae96ac8bf2e289306d3fd3df0e7a5 + GPU_AGENT_SHA: bcae57f9b395363f3340ad0b7393659b7df13f37 + ARCHIVE_NAME: device-metrics-exporter-v1.5.4-10.2.0a20261001.tar.gz + ARCHIVE_SHA256: 0ba397bc31abc1c538e24f8d7264fb421cc5fb8dd59b2e1c493a8efec404c2ba + BASE_IMAGE: powerx-amd-recovered:runtime-36985488002 + BASE_CONFIG_ID: sha256:aa151c70573fd5d3f94538c73adac6b235f052bb7dc9a8681ad2bfd0a8417929 + RECOVERY_SQUASH_SHA256: 0f2cbb4e2c41c1506db74c6653ea09ec5e7d42236d9532e2bc38fb9e0be5879e + IMAGE_REF: ghcr.io/semianalysisai/amd-device-metrics-exporter:powerx-9d0eb8c-cache-${{ github.sha }} + ENROOT_PACKAGE: enroot_4.2.0-1_amd64.deb + ENROOT_PACKAGE_SHA256: 50bfcf8336bfe1a5d19481081d7800ae80300a27795595757f7d20957e079a62 + steps: + - name: Checkout build recipe + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Checkout exact upstream source + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: ROCm/device-metrics-exporter + ref: 9d0eb8c88af99f1dbe2914d803382092671456d9 + path: exporter-source + persist-credentials: false + - name: Install matching Go toolchain + uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6 + with: + go-version: 1.25.13 + cache: false + - name: Download the retained prepared runtime + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + repository: SemiAnalysisAI/InferenceX + run-id: 36985488002 + artifact-ids: 11217112734 + github-token: ${{ github.token }} + path: ${{ runner.temp }}/amd-exporter-image + - name: Apply cache patch and test the exporter + run: | + set -eo pipefail + source inferencex-e2e/benchmarks/benchmark_lib.sh --validation-only + check_env_vars SOURCE_SHA PATCH_SHA256 GPU_AGENT_SHA GITHUB_WORKSPACE + printf '%s %s\n' "$PATCH_SHA256" tools/amd-exporter/cache.patch | sha256sum --check - + test "$(git -C exporter-source rev-parse HEAD)" = "$SOURCE_SHA" + git -C exporter-source apply "$GITHUB_WORKSPACE/tools/amd-exporter/cache.patch" + python3 -m unittest discover -s tools/amd-exporter -p 'test_*.py' -v + cd exporter-source + go test -race ./pkg/amdgpu/gpuagent -run 'Test(ParseGPUGetCacheTTL|CacheRead(HonorsGPUGetCacheTTL|ErrorClearsSlot|ConcurrentTTL)|NewAgentGPUGetCacheTTL)' -count=1 + go test ./pkg/exporter -run TestExporterOptions -count=1 + go test ./pkg/exporter/utils + CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -C cmd/exporter -trimpath \ + -ldflags "-X main.Version=1.5.4-powerx-cache -X main.GitCommit=$SOURCE_SHA+patch.$PATCH_SHA256 -X main.ROCmCommit=unknown" \ + -o "$GITHUB_WORKSPACE/tools/amd-exporter/amd-metrics-exporter" + - name: Build and check the image without GPU allocation + env: + IMAGE_DIR: ${{ runner.temp }}/amd-exporter-image + OUTPUT_DIR: ${{ runner.temp }}/amd-exporter-output + run: | + set -eo pipefail + source inferencex-e2e/benchmarks/benchmark_lib.sh --validation-only + check_env_vars IMAGE_DIR OUTPUT_DIR RECOVERY_SQUASH_SHA256 BASE_IMAGE IMAGE_REF \ + ENROOT_PACKAGE ENROOT_PACKAGE_SHA256 SOURCE_SHA PATCH_SHA256 GITHUB_REPOSITORY GITHUB_SHA + printf '%s %s\n' "$RECOVERY_SQUASH_SHA256" "$IMAGE_DIR/amd-exporter.sqsh" | sha256sum --check - + curl --fail --location --output "$IMAGE_DIR/$ENROOT_PACKAGE" \ + "https://github.com/NVIDIA/enroot/releases/download/v4.2.0/$ENROOT_PACKAGE" + printf '%s %s\n' "$ENROOT_PACKAGE_SHA256" "$IMAGE_DIR/$ENROOT_PACKAGE" | sha256sum --check - + sudo apt-get update + sudo apt-get install --yes squashfs-tools "$IMAGE_DIR/$ENROOT_PACKAGE" + sudo unsquashfs -d "$IMAGE_DIR/rootfs" "$IMAGE_DIR/amd-exporter.sqsh" + mkdir -p "$OUTPUT_DIR" + sudo python3 tools/amd-exporter/recover-base.py "$IMAGE_DIR/rootfs" \ + "$IMAGE_DIR/provenance.json" tools/amd-exporter/base-runtime.json \ + "$BASE_IMAGE" "$OUTPUT_DIR/base-recovery.json" + docker build --pull=false --build-arg "BASE_IMAGE=$BASE_IMAGE" \ + --label "org.opencontainers.image.source=https://github.com/$GITHUB_REPOSITORY" \ + --label "org.opencontainers.image.revision=$GITHUB_SHA" \ + --label "io.semianalysis.exporter.upstream=$SOURCE_SHA" \ + --label "io.semianalysis.exporter.patch-sha256=$PATCH_SHA256" \ + --tag "$IMAGE_REF" tools/amd-exporter + mkdir -p "$OUTPUT_DIR" + docker run --rm --entrypoint /home/amd/bin/server "$IMAGE_REF" --version | tee "$OUTPUT_DIR/version.txt" + if docker run --rm -e AMD_GPU_GET_CACHE_TTL=invalid --entrypoint /home/amd/bin/server "$IMAGE_REF" > "$OUTPUT_DIR/invalid-ttl.log" 2>&1; then + echo 'Invalid cache TTL unexpectedly started the exporter' >&2 + exit 1 + fi + grep -F 'invalid GPUGet cache configuration:' "$OUTPUT_DIR/invalid-ttl.log" + python3 tools/amd-exporter/verify-image.py "$BASE_IMAGE" "$IMAGE_REF" + - name: Publish immutable image + env: + GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }} + OUTPUT_DIR: ${{ runner.temp }}/amd-exporter-output + run: | + set -eo pipefail + source inferencex-e2e/benchmarks/benchmark_lib.sh --validation-only + check_env_vars GHCR_TOKEN GITHUB_ACTOR IMAGE_REF OUTPUT_DIR + printf '%s' "$GHCR_TOKEN" | docker login ghcr.io --username "$GITHUB_ACTOR" --password-stdin + docker push "$IMAGE_REF" + docker image inspect "$IMAGE_REF" > "$OUTPUT_DIR/image-inspect.json" + - name: Prepare runner squash and provenance + env: + IMAGE_DIR: ${{ runner.temp }}/amd-exporter-image + OUTPUT_DIR: ${{ runner.temp }}/amd-exporter-output + run: | + set -eo pipefail + source inferencex-e2e/benchmarks/benchmark_lib.sh --validation-only + check_env_vars IMAGE_DIR OUTPUT_DIR IMAGE_REF ENROOT_PACKAGE ENROOT_PACKAGE_SHA256 SOURCE_SHA \ + PATCH_SHA256 GPU_AGENT_SHA ARCHIVE_NAME ARCHIVE_SHA256 BASE_CONFIG_ID GITHUB_REPOSITORY \ + GITHUB_RUN_ID GITHUB_RUN_ATTEMPT GITHUB_SHA + enroot import --output "$OUTPUT_DIR/amd-exporter.sqsh" "dockerd://$IMAGE_REF" + unsquashfs -s "$OUTPUT_DIR/amd-exporter.sqsh" + cp tools/amd-exporter/cache.patch "$OUTPUT_DIR/cache.patch" + cp tools/amd-exporter/amd-metrics-exporter "$OUTPUT_DIR/amd-metrics-exporter" + python3 tools/amd-exporter/provenance.py "$OUTPUT_DIR" + cd "$OUTPUT_DIR" + sha256sum amd-exporter.sqsh amd-metrics-exporter cache.patch > SHA256SUMS + - name: Upload image evidence and runner squash + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: amd-dme-cache-squash + path: ${{ runner.temp }}/amd-exporter-output/ + if-no-files-found: error + compression-level: 0 diff --git a/tools/amd-exporter/Dockerfile b/tools/amd-exporter/Dockerfile new file mode 100644 index 0000000000..8808220458 --- /dev/null +++ b/tools/amd-exporter/Dockerfile @@ -0,0 +1,4 @@ +ARG BASE_IMAGE +FROM ${BASE_IMAGE} +COPY --chmod=0755 amd-metrics-exporter /home/amd/bin/server +ENV AMD_GPU_GET_CACHE_TTL=0s diff --git a/tools/amd-exporter/README.md b/tools/amd-exporter/README.md new file mode 100644 index 0000000000..09e951335e --- /dev/null +++ b/tools/amd-exporter/README.md @@ -0,0 +1,13 @@ +# AMD power exporter image + +**English** | [中文](README_zh.md) + +The CPU-only `AMD exporter image` workflow builds a fresh-sample exporter and publishes a unique tag to `ghcr.io/semianalysisai/amd-device-metrics-exporter`. It does not start a GPU benchmark or change production data. + +The official ROCm nightly artifact from run `36867107435` has been deleted upstream. Reuse the retained Enroot conversion from InferenceX run `36985488002`, artifact `11217112734`, verified against squash SHA256 `0f2cbb4e2c41c1506db74c6653ea09ec5e7d42236d9532e2bc38fb9e0be5879e` and its original provenance. It contains source `9d0eb8c88af99f1dbe2914d803382092671456d9`. The runtime includes the upstream 255 W sentinel correction. `cache.patch` makes the exporter's GPUGet response-cache TTL a startup setting, `AMD_GPU_GET_CACHE_TTL`: `cmd/exporter/main.go` parses it once and fails fast on a bad value, the exporter and gpuagent options carry it the same way `exit-on-agent-down` travels, and `cacheRead` applies it. Unset keeps the upstream 15 s cache, so the patch changes nothing unless the variable is set; `0s` makes every scrape issue a new GPUGet. The locking, query timeout and error handling around the cache are untouched. Its tests cover the parser, the cache decision at 15 s, 1 s and 0 s, the error path, sixteen concurrent readers at `0s`, and the exporter and agent defaults. `0s` is the only value that guarantees a fresh read per scrape: the cache timestamp is taken after GPUGet returns, so any positive TTL serves cached data on some scrapes. Each read holds the cache lock for the duration of its RPC, which the one-hour MI355X run tolerated at a 1.01 s worst gap. It replaces the earlier copy of [functionstackx/device-metrics-exporter#1](https://github.com/functionstackx/device-metrics-exporter/pull/1), which read the variable twice and changed the default to 1 s. + +`base-runtime.json` records the original Docker configuration and hashes/symlink targets for 61 critical paths, extracted from the retained official OCI archive. Recovery verifies GPUAgent, entrypoint, AMD-SMI, ROCm sysdeps and companion binaries before importing the root filesystem with the original environment, entrypoint, labels and working directory. Enroot flattened layers, normalized ownership/permissions and generated its `/etc` startup files; the reconstructed base has a new identity and is not the original OCI image. Only `/home/amd/bin/server` is replaced in the subsequent overlay. `AMD_GPU_GET_CACHE_TTL=0s` forces each exporter request to issue a new GPUGet RPC. This removes exporter response reuse; actual hardware freshness, overhead and one-second benchmark collection still require live validation. + +The workflow verifies the patch checksum, runs focused cache race tests, builds Linux/amd64, checks the image's version and invalid-setting rejection, and verifies the unchanged reconstructed base layers and runtime configuration. It then publishes a commit-specific tag and saves an Enroot squash with `SHA256SUMS` and `provenance.json` in `amd-dme-cache-squash`. Consumers must record the published registry digest and the squash checksum. `base-recovery.json` records the recovery run/artifact, source receipt, runtime manifest hash, verified-file count and reconstructed image identity. The source commit, patch hash, GPUAgent commit and workflow revision are separate provenance fields; the embedded ROCm commit is explicitly unknown. + +Dispatch this workflow manually on the reviewed branch, or push the dedicated `chore/powerx-amd-exporter-image` branch. The job has a 30-minute CPU limit and needs `packages: write`. It does not move a `latest` tag. A successful image build is preparation, not AMD measurement acceptance. diff --git a/tools/amd-exporter/README_zh.md b/tools/amd-exporter/README_zh.md new file mode 100644 index 0000000000..0799280587 --- /dev/null +++ b/tools/amd-exporter/README_zh.md @@ -0,0 +1,13 @@ +# AMD 功耗 exporter 镜像 + +[English](README.md) | **中文** + +仅使用 CPU 的 `AMD exporter image` 工作流构建可获取新读数的 exporter，并向 `ghcr.io/semianalysisai/amd-device-metrics-exporter` 发布唯一标签。它不启动 GPU 基准测试，也不修改生产数据。 + +官方 ROCm nightly 工作流 `36867107435` 的原始产物已被上游删除。现在复用 InferenceX 工作流 `36985488002` 保留的 Enroot 转换产物 `11217112734`，校验 squash SHA256 `0f2cbb4e2c41c1506db74c6653ea09ec5e7d42236d9532e2bc38fb9e0be5879e` 及原始来源回执。其源码为 `9d0eb8c88af99f1dbe2914d803382092671456d9`。该运行时包含上游 255 W 哨兵值修复。`cache.patch` 把 exporter 的 GPUGet 响应缓存 TTL 变成启动配置 `AMD_GPU_GET_CACHE_TTL`：`cmd/exporter/main.go` 解析一次并在取值非法时立即退出，exporter 与 gpuagent 的 options 像 `exit-on-agent-down` 一样把它传下去，最终由 `cacheRead` 应用。未设置时保持上游 15 s 缓存，因此不设该变量时补丁不改变任何行为；`0s` 让每次 scrape 都发起新的 GPUGet。缓存周围的加锁、查询超时和错误处理保持不变。测试覆盖解析器、15 s / 1 s / 0 s 三种缓存判定、错误路径、`0s` 下 16 个并发读取，以及 exporter 与 agent 的默认值。只有 `0s` 能保证每次 scrape 都是新读数：缓存时间戳在 GPUGet 返回后才记录，任何正的 TTL 都会让部分 scrape 拿到缓存数据。每次读取在 RPC 期间持有缓存锁，一小时的 MI355X 运行在最差 1.01 s 间隔下未受影响。它取代此前复制自 [functionstackx/device-metrics-exporter#1](https://github.com/functionstackx/device-metrics-exporter/pull/1) 的版本；旧版本读取该变量两次，并把默认值改成了 1 s。 + +`base-runtime.json` 从保留的官方 OCI 归档提取原始 Docker 配置，以及 61 个关键路径的文件哈希和符号链接目标。恢复过程先验证 GPUAgent、入口脚本、AMD-SMI、ROCm sysdeps 和配套二进制，再使用原始环境变量、入口、标签及工作目录导入根文件系统。Enroot 已合并镜像层、规范化所有权和权限，并生成 `/etc` 启动文件，因此重建基础镜像使用新标识，不能称为原始 OCI 镜像。之后仅在新增镜像层替换 `/home/amd/bin/server`。`AMD_GPU_GET_CACHE_TTL=0s` 强制每次 exporter 请求发起新的 GPUGet RPC。这消除了 exporter 响应复用；硬件实际刷新率、采集开销和基准测试中的一秒采集仍需实机验证。 + +工作流校验补丁摘要，运行缓存竞态测试，构建 Linux/amd64 二进制，检查版本与非法配置拒绝行为，并验证重建基础镜像层及运行时配置未变。随后发布提交专属标签，生成 Enroot squash，并将 `SHA256SUMS`、`provenance.json` 保存为 `amd-dme-cache-squash` 产物。使用方应记录发布的注册表摘要及 squash 校验值。`base-recovery.json` 记录恢复工作流和产物、原始来源回执、运行时清单哈希、已验证文件数及重建镜像标识。源码提交、补丁哈希、GPUAgent 提交和工作流版本分别记录；二进制中的 ROCm 提交明确标记为未知。 + +在已审阅分支手动触发，或推送专用分支 `chore/powerx-amd-exporter-image`。CPU 作业上限为 30 分钟，需要 `packages: write` 权限，不移动 `latest` 标签。镜像构建通过只代表准备完成，不代表 AMD 功耗测量已验收。 diff --git a/tools/amd-exporter/base-runtime.json b/tools/amd-exporter/base-runtime.json new file mode 100644 index 0000000000..b80774a263 --- /dev/null +++ b/tools/amd-exporter/base-runtime.json @@ -0,0 +1,349 @@ +{ + "official_image_config_digest": "sha256:aa151c70573fd5d3f94538c73adac6b235f052bb7dc9a8681ad2bfd0a8417929", + "official_archive_sha256": "0ba397bc31abc1c538e24f8d7264fb421cc5fb8dd59b2e1c493a8efec404c2ba", + "recovery": { + "repository": "SemiAnalysisAI/InferenceX", + "run_id": 36985488002, + "artifact_id": 11217112734, + "artifact_name": "amd-dme-nightly-squash", + "squash_sha256": "0f2cbb4e2c41c1506db74c6653ea09ec5e7d42236d9532e2bc38fb9e0be5879e" + }, + "config": { + "Env": [ + "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/amd/bin/:/opt/rocm/bin/:/opt/nic/bin:/opt/nic/sbin", + "container=oci", + "LD_LIBRARY_PATH=/home/amd/lib:/opt/rocm/lib:/opt/rocm-10.1.0rc3/lib/rocm_sysdeps/lib:", + "GPUAGENT_EVENTS_DISABLE=1", + "AMD_METRICS_EXPORTER_ENABLE_CPER=1" + ], + "Entrypoint": [ + "/home/amd/tools/entrypoint.sh" + ], + "WorkingDir": "/", + "Labels": { + "HOURLY_TAG": "", + "OS": "registry.access.redhat.com/ubi9/ubi-minimal:9.8", + "ROCM_COMMIT": "", + "ROCM_VERSION": "10.1.0rc3", + "architecture": "x86_64", + "build-date": "2026-09-30T07:43:22Z", + "com.redhat.component": "ubi9-minimal-container", + "com.redhat.license_terms": "https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI", + "cpe": "cpe:/a:redhat:enterprise_linux:9::appstream", + "description": "AMD Device Metrics Exporter enables Prometheus-format metrics collection for AMD GPUs in HPC and AI environments. It provides detailed telemetry including temperature, utilization, memory usage, and power consumption.", + "distribution-scope": "public", + "io.buildah.version": "1.44.0", + "io.k8s.description": "The Universal Base Image Minimal is a stripped down image that uses microdnf as a package manager. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.", + "io.k8s.display-name": "Red Hat Universal Base Image 9 Minimal", + "io.openshift.expose-services": "", + "io.openshift.tags": "minimal rhel9", + "maintainer": "praveenkumar.shanmugam@amd.com,nitish.bhat@amd.com,yan.sun3@amd.com,shrey.ajmera@amd.com", + "name": "amd-device-metrics-exporter", + "org.opencontainers.image.created": "2026-09-30T07:43:22Z", + "org.opencontainers.image.revision": "f354e3d450169972cb1f8df64acae506d96d4c49", + "release": "v1.5.4", + "summary": "AMD Device Metrics Exporter enables out-of-the-box metrics collection for AMD GPUs on Kubernetes.", + "url": "https://catalog.redhat.com/en/search?searchType=containers", + "vcs-ref": "f354e3d450169972cb1f8df64acae506d96d4c49", + "vcs-type": "git", + "vendor": "Advanced Micro Devices, Inc.", + "version": "v1.5.4" + } + }, + "files": { + "home/amd/bin/amdgpuhealth": { + "type": "file", + "sha256": "793f66e2c3f9dc86105dddbf4d22140dfcad7c1eb2693694cd93f455d5d63744", + "mode": 493 + }, + "home/amd/bin/gpuagent": { + "type": "file", + "sha256": "8c2b013b98a29448804cac534d62d0c96a45d564669aa7d5ee2435d7b1784cc0", + "mode": 493 + }, + "home/amd/bin/gpuctl": { + "type": "file", + "sha256": "b4df351009b557233241fe0dd0cd4c671014a628a8e27e1110ab8c3509178f4c", + "mode": 493 + }, + "home/amd/bin/metrics-exporter-ts.sh": { + "type": "file", + "sha256": "81dd25273d8b457224f3c62bf3bdfb5da5ba8f8f27e09ef807919dc2930653cd", + "mode": 493 + }, + "home/amd/bin/metricsclient": { + "type": "file", + "sha256": "f2d6168da03e938ac72fbcc59a4e07d0705c5eac34ba516fb2f97b513432009b", + "mode": 493 + }, + "home/amd/bin/rocpctl": { + "type": "file", + "sha256": "cb86c07456f8d54e91b12ecba704a60aeb5ccf81c46c314668f8ba2934d840c2", + "mode": 493 + }, + "home/amd/bin/server": { + "type": "file", + "sha256": "ffafbaffb534c44b2d352dfc5604a0daf1c396b7a149b0a7a4f18762732b5f36", + "mode": 493 + }, + "home/amd/lib/libamd_smi.so": { + "type": "symlink", + "target": "libamd_smi.so.27" + }, + "home/amd/lib/libamd_smi.so.27": { + "type": "file", + "sha256": "77859323dce97cda578e9d2e332fded749622a52dadb804c7ace09de23e8f329", + "mode": 493 + }, + "home/amd/lib/librocm_sysdeps_drm.so.2": { + "type": "file", + "sha256": "cf203bcf5f42df87c5e292b254ea6cd801c3612027c04e5f29bfc71607877775", + "mode": 493 + }, + "home/amd/lib/librocm_sysdeps_drm_amdgpu.so.1": { + "type": "file", + "sha256": "5485dbf004f79fe2ca51a11ff89625299a8602bb46ebc4f2a25471e81825faf5", + "mode": 493 + }, + "home/amd/lib/librocm_sysdeps_mnl.so.0": { + "type": "file", + "sha256": "d73ed5e00af232ae518ff738218d3795f7b66c60208c2b11f78183f27f373175", + "mode": 493 + }, + "home/amd/lib/librocm_sysdeps_nl_3.so.200": { + "type": "file", + "sha256": "f83b147afcb0b69336ac0cea582766168b11c0187857198d588fdd10c1f9bac8", + "mode": 493 + }, + "home/amd/lib/librocm_sysdeps_nl_genl_3.so.200": { + "type": "file", + "sha256": "ab2dc13a18577ce7d313088ab90d5dcf945ac3f49a061a65381799b4a64fc0c0", + "mode": 493 + }, + "home/amd/tools/entrypoint.sh": { + "type": "file", + "sha256": "70e04a4ba3c46e36b341db851683ca5597ab2806802e1cf81b0c0fd47361a0fe", + "mode": 493 + }, + "opt/rocm": { + "type": "symlink", + "target": "/etc/alternatives/rocm" + }, + "opt/rocm-10.1.0rc3/lib/libamd_smi.so": { + "type": "symlink", + "target": "libamd_smi.so.27" + }, + "opt/rocm-10.1.0rc3/lib/libamd_smi.so.27": { + "type": "symlink", + "target": "libamd_smi.so.27.1.0" + }, + "opt/rocm-10.1.0rc3/lib/libamd_smi.so.27.1.0": { + "type": "file", + "sha256": "77859323dce97cda578e9d2e332fded749622a52dadb804c7ace09de23e8f329", + "mode": 493 + }, + "opt/rocm-10.1.0rc3/lib/librocpclient.so": { + "type": "file", + "sha256": "a020125a4477056cb64552986eee95a3c7d162b91a3d1eb67ee92d2f00b38c89", + "mode": 493 + }, + "opt/rocm-10.1.0rc3/lib/rocm_sysdeps/lib/librocm_sysdeps_asm.so.1": { + "type": "file", + "sha256": "747121912a67621171749399ca510a62d0b4b2ee9044ebd24c9dd31c8e5e0371", + "mode": 493 + }, + "opt/rocm-10.1.0rc3/lib/rocm_sysdeps/lib/librocm_sysdeps_blkid.so.1": { + "type": "file", + "sha256": "b08a7263000676fb1d31d81a9581707bd836ef1ac756bf2228c92ef91548d083", + "mode": 493 + }, + "opt/rocm-10.1.0rc3/lib/rocm_sysdeps/lib/librocm_sysdeps_bz2.so": { + "type": "file", + "sha256": "fe4102b002321cea17405cf9c6aa04cf46e726bea5666c40ac8639684314de02", + "mode": 493 + }, + "opt/rocm-10.1.0rc3/lib/rocm_sysdeps/lib/librocm_sysdeps_cap.so.2": { + "type": "file", + "sha256": "988916874bd84ecd9e4755160a193073b5336242e387fb0e40257b83edc678dc", + "mode": 493 + }, + "opt/rocm-10.1.0rc3/lib/rocm_sysdeps/lib/librocm_sysdeps_drm.so.2": { + "type": "file", + "sha256": "cf203bcf5f42df87c5e292b254ea6cd801c3612027c04e5f29bfc71607877775", + "mode": 493 + }, + "opt/rocm-10.1.0rc3/lib/rocm_sysdeps/lib/librocm_sysdeps_drm_amdgpu.so.1": { + "type": "file", + "sha256": "6e6ac3431944eba947506faf56386c02934893543121246e5cc96ea2ea8d2b6a", + "mode": 493 + }, + "opt/rocm-10.1.0rc3/lib/rocm_sysdeps/lib/librocm_sysdeps_dw.so.1": { + "type": "file", + "sha256": "c15ba3964365ad35ed1b072347163e8dcd8e8d3b0c19646537126f629275b9fd", + "mode": 493 + }, + "opt/rocm-10.1.0rc3/lib/rocm_sysdeps/lib/librocm_sysdeps_elf.so.1": { + "type": "file", + "sha256": "cc7c1d8faa3f067072c6496cdaba16ed792d94130c41b2b0d5f01bc6367631f5", + "mode": 493 + }, + "opt/rocm-10.1.0rc3/lib/rocm_sysdeps/lib/librocm_sysdeps_expat.so.1": { + "type": "file", + "sha256": "b85cdd6fe2004ea5e7a1eea1c4c4522e606ecaaedfbc22d6c9f09a2b77bd91a3", + "mode": 493 + }, + "opt/rocm-10.1.0rc3/lib/rocm_sysdeps/lib/librocm_sysdeps_formw.so.6": { + "type": "file", + "sha256": "eb0ca92e4ca0d46b69366c736ba40ee409e8f64a9249e591999a873c5838e8fd", + "mode": 493 + }, + "opt/rocm-10.1.0rc3/lib/rocm_sysdeps/lib/librocm_sysdeps_gallium_drv_video.so": { + "type": "file", + "sha256": "c36a5a5049f6bd1a6bb4180cedb8b5f0c3f2f0d8720b783704254398882309a7", + "mode": 493 + }, + "opt/rocm-10.1.0rc3/lib/rocm_sysdeps/lib/librocm_sysdeps_gmp.so.10": { + "type": "file", + "sha256": "f009167d6003de15b5a4bfb9c670433d1da1df1ea8d67ea41c55875386fa8d01", + "mode": 493 + }, + "opt/rocm-10.1.0rc3/lib/rocm_sysdeps/lib/librocm_sysdeps_hwloc.so.5": { + "type": "file", + "sha256": "52d240623482edb7b23cf9bb28888839575e331be228c7b8e22a9640e220fe81", + "mode": 493 + }, + "opt/rocm-10.1.0rc3/lib/rocm_sysdeps/lib/librocm_sysdeps_liblzma.so.5": { + "type": "symlink", + "target": "librocm_sysdeps_liblzma.so.5.8.1" + }, + "opt/rocm-10.1.0rc3/lib/rocm_sysdeps/lib/librocm_sysdeps_liblzma.so.5.8.1": { + "type": "file", + "sha256": "15402b36e2fdb0762e0b6590c3470fccec5e80cc8010c84ac4835573d592c02d", + "mode": 493 + }, + "opt/rocm-10.1.0rc3/lib/rocm_sysdeps/lib/librocm_sysdeps_menuw.so.6": { + "type": "file", + "sha256": "f1052d214a022a4d2ff2068a08995b63f2a06df54f581cea6aafa7b1a11ff54d", + "mode": 493 + }, + "opt/rocm-10.1.0rc3/lib/rocm_sysdeps/lib/librocm_sysdeps_mnl.so.0": { + "type": "file", + "sha256": "d73ed5e00af232ae518ff738218d3795f7b66c60208c2b11f78183f27f373175", + "mode": 493 + }, + "opt/rocm-10.1.0rc3/lib/rocm_sysdeps/lib/librocm_sysdeps_mount.so.1": { + "type": "file", + "sha256": "0f06b3b8383061360f68805f0e6aeca6a209301c233fcb29caec13c96f326987", + "mode": 493 + }, + "opt/rocm-10.1.0rc3/lib/rocm_sysdeps/lib/librocm_sysdeps_mpfr.so.6": { + "type": "file", + "sha256": "7c8fb735972b878ddce57964f52a716bf72886f364c103f2df953e95495650a0", + "mode": 493 + }, + "opt/rocm-10.1.0rc3/lib/rocm_sysdeps/lib/librocm_sysdeps_ncursesw.so.6": { + "type": "file", + "sha256": "1c1396486d718fc64695f40194d989832b06ec9ab25747649ff4b0b313c941fe", + "mode": 493 + }, + "opt/rocm-10.1.0rc3/lib/rocm_sysdeps/lib/librocm_sysdeps_nl_3.so.200": { + "type": "file", + "sha256": "f83b147afcb0b69336ac0cea582766168b11c0187857198d588fdd10c1f9bac8", + "mode": 493 + }, + "opt/rocm-10.1.0rc3/lib/rocm_sysdeps/lib/librocm_sysdeps_nl_genl_3.so.200": { + "type": "file", + "sha256": "ab2dc13a18577ce7d313088ab90d5dcf945ac3f49a061a65381799b4a64fc0c0", + "mode": 493 + }, + "opt/rocm-10.1.0rc3/lib/rocm_sysdeps/lib/librocm_sysdeps_nl_idiag_3.so.200": { + "type": "file", + "sha256": "62b8b23b149b061e2a95117710eef354ff95de185452ce05af6aaf8b29ee60ad", + "mode": 493 + }, + "opt/rocm-10.1.0rc3/lib/rocm_sysdeps/lib/librocm_sysdeps_nl_nf_3.so.200": { + "type": "file", + "sha256": "f21015e3a6894857cd5d4dfa27d6de52bbfadb4247aac2edc0114c1f8af7da25", + "mode": 493 + }, + "opt/rocm-10.1.0rc3/lib/rocm_sysdeps/lib/librocm_sysdeps_nl_route_3.so.200": { + "type": "file", + "sha256": "3709ce228b7a82bcb3a4b93a54ce26c2b6fa38b6f0b49c849d5caea02c8ca7bc", + "mode": 493 + }, + "opt/rocm-10.1.0rc3/lib/rocm_sysdeps/lib/librocm_sysdeps_nl_xfrm_3.so.200": { + "type": "file", + "sha256": "d892573aaf11ce964c86a548345a61b9c32218711b7b4c3aac07d245a56bbc7f", + "mode": 493 + }, + "opt/rocm-10.1.0rc3/lib/rocm_sysdeps/lib/librocm_sysdeps_numa.so.1": { + "type": "file", + "sha256": "333b980e3f706f7196c73a33967c7979e9fd47b6d5f5f51ed3990142ef4142cb", + "mode": 493 + }, + "opt/rocm-10.1.0rc3/lib/rocm_sysdeps/lib/librocm_sysdeps_panelw.so.6": { + "type": "file", + "sha256": "470e0b21fd509b0e48edb8631045a85c760e00d272607497fb9b2cade058b34e", + "mode": 493 + }, + "opt/rocm-10.1.0rc3/lib/rocm_sysdeps/lib/librocm_sysdeps_pciaccess.so.0": { + "type": "file", + "sha256": "e344afed823a3b2d0f149c218ed5ee2545d637ae9e03e5a5e4e9d54e2bdb9410", + "mode": 493 + }, + "opt/rocm-10.1.0rc3/lib/rocm_sysdeps/lib/librocm_sysdeps_psx.so.2": { + "type": "file", + "sha256": "045c9fcb71a9bf6b781033118f842870aca75201b52e991365e1b38f2ba709a6", + "mode": 493 + }, + "opt/rocm-10.1.0rc3/lib/rocm_sysdeps/lib/librocm_sysdeps_sqlite3.so": { + "type": "file", + "sha256": "44234a62c1ca345fd1a7886b4eeb82b6396ff7737967446c1627d4154c69a69a", + "mode": 493 + }, + "opt/rocm-10.1.0rc3/lib/rocm_sysdeps/lib/librocm_sysdeps_va-drm.so": { + "type": "symlink", + "target": "librocm_sysdeps_va-drm.so.2" + }, + "opt/rocm-10.1.0rc3/lib/rocm_sysdeps/lib/librocm_sysdeps_va-drm.so.2": { + "type": "symlink", + "target": "librocm_sysdeps_va-drm.so.2.2200.0" + }, + "opt/rocm-10.1.0rc3/lib/rocm_sysdeps/lib/librocm_sysdeps_va-drm.so.2.2200.0": { + "type": "file", + "sha256": "f787d8dfdd09b1bdd2d33f4e5a6f53402edf789c5a2d2bb70e759e284c702257", + "mode": 493 + }, + "opt/rocm-10.1.0rc3/lib/rocm_sysdeps/lib/librocm_sysdeps_va.so": { + "type": "symlink", + "target": "librocm_sysdeps_va.so.2" + }, + "opt/rocm-10.1.0rc3/lib/rocm_sysdeps/lib/librocm_sysdeps_va.so.2": { + "type": "symlink", + "target": "librocm_sysdeps_va.so.2.2200.0" + }, + "opt/rocm-10.1.0rc3/lib/rocm_sysdeps/lib/librocm_sysdeps_va.so.2.2200.0": { + "type": "file", + "sha256": "dc4e1cac5d99b98af82638538c004dc770d7a2a14363e6eff32d552107fc2702", + "mode": 493 + }, + "opt/rocm-10.1.0rc3/lib/rocm_sysdeps/lib/librocm_sysdeps_z.so.1": { + "type": "symlink", + "target": "librocm_sysdeps_z.so.1.3.2" + }, + "opt/rocm-10.1.0rc3/lib/rocm_sysdeps/lib/librocm_sysdeps_z.so.1.3.2": { + "type": "file", + "sha256": "27361c0c552d2afd5ecf9079f356309ed827e9a27d35f2fc68c7b5e6b5c83199", + "mode": 493 + }, + "opt/rocm-10.1.0rc3/lib/rocm_sysdeps/lib/librocm_sysdeps_zstd.so.1": { + "type": "symlink", + "target": "librocm_sysdeps_zstd.so.1.5.7" + }, + "opt/rocm-10.1.0rc3/lib/rocm_sysdeps/lib/librocm_sysdeps_zstd.so.1.5.7": { + "type": "file", + "sha256": "5fc4ab0e6899318ece9bea43d3d397d20053418a5e40cb6bc86efe19a801c75e", + "mode": 493 + } + } +} diff --git a/tools/amd-exporter/cache.patch b/tools/amd-exporter/cache.patch new file mode 100644 index 0000000000..a88dd92ad0 --- /dev/null +++ b/tools/amd-exporter/cache.patch @@ -0,0 +1,435 @@ +From b49c383de9177c1773720d71719c501c62bea8ad Mon Sep 17 00:00:00 2001 +From: Wenyao Gao +Date: Wed, 7 Oct 2026 17:12:40 -0700 +Subject: [PATCH] feat: make the GPUGet response cache TTL configurable + +AMD_GPU_GET_CACHE_TTL is parsed once in cmd/exporter and carried through the existing exporter and gpuagent options; unset keeps the 15 s cache and 0s issues a new GPUGet on every read. A zero-value GPUAgentClient literal now gets no cache instead of 15 s; no upstream test relies on that. +--- + cmd/exporter/main.go | 10 + + pkg/amdgpu/gpuagent/gpuagent.go | 32 +++ + .../gpuagent/gpuagent_cache_ttl_test.go | 185 ++++++++++++++++++ + pkg/amdgpu/gpuagent/gpuagent_gpu.go | 8 +- + pkg/exporter/exporter.go | 10 + + pkg/exporter/exporter_test.go | 16 ++ + 6 files changed, 258 insertions(+), 3 deletions(-) + create mode 100644 pkg/amdgpu/gpuagent/gpuagent_cache_ttl_test.go + +diff --git a/cmd/exporter/main.go b/cmd/exporter/main.go +index edfbf05..28ee4b4 100644 +--- a/cmd/exporter/main.go ++++ b/cmd/exporter/main.go +@@ -25,6 +25,7 @@ import ( + "syscall" + "time" + ++ "github.com/ROCm/device-metrics-exporter/pkg/amdgpu/gpuagent" + "github.com/ROCm/device-metrics-exporter/pkg/exporter" + "github.com/ROCm/device-metrics-exporter/pkg/exporter/globals" + "github.com/ROCm/device-metrics-exporter/pkg/exporter/logger" +@@ -38,6 +39,8 @@ var ( + ROCmCommit string + ) + ++const gpuGetCacheTTLEnv = "AMD_GPU_GET_CACHE_TTL" ++ + func main() { + // Check environment variable to determine error handling behavior + relaxedMode := os.Getenv("AMD_EXPORTER_RELAXED_FLAGS_PARSING") != "" +@@ -106,6 +109,12 @@ func main() { + os.Exit(1) + } + ++ gpuGetCacheTTL, err := gpuagent.ParseGPUGetCacheTTL(os.Getenv(gpuGetCacheTTLEnv)) ++ if err != nil { ++ fmt.Printf("invalid GPUGet cache configuration: %s: %v, exiting\n", gpuGetCacheTTLEnv, err) ++ os.Exit(1) ++ } ++ + if !*enableNICMonitoring && !*enableGPUMonitoring && !*enableIFOEMonitoring { + fmt.Printf("NIC Agent, GPU Agent and IFOE Agent are all disabled, exiting") + os.Exit(1) +@@ -128,6 +137,7 @@ func main() { + exporter.WithSRIOV(*sriov), + exporter.WithExitOnAgentDown(*exitOnAgentDown), + exporter.WithExitOnRocpctlError(*exitOnRocpctlError), ++ exporter.WithGPUGetCacheTTL(gpuGetCacheTTL), + exporter.WithBindAddr(*bindAddr), + exporter.WithSlurmClient(*enableSlumrScl), + exporter.WithenableIFOEMonitoring(*enableIFOEMonitoring), +diff --git a/pkg/amdgpu/gpuagent/gpuagent.go b/pkg/amdgpu/gpuagent/gpuagent.go +index a408de3..c9ff711 100644 +--- a/pkg/amdgpu/gpuagent/gpuagent.go ++++ b/pkg/amdgpu/gpuagent/gpuagent.go +@@ -21,6 +21,7 @@ import ( + "errors" + "fmt" + "strconv" ++ "strings" + "sync" + "time" + +@@ -50,6 +51,7 @@ type GPUAgentClient struct { + enableSriov bool + exitOnAgentDown bool // exit DME process when agent is unreachable + exitOnRocpctlError bool // exit DME process when rocpctl auto-disables on failure ++ gpuGetCacheTTL time.Duration + useSocket bool // use socket connection instead of IP:port + socketPath string + +@@ -144,6 +146,35 @@ func WithExitOnRocpctlError(exit bool) GPUAgentClientOptions { + } + } + ++// DefaultGPUGetCacheTTL is the GPUGet response reuse window when no option is passed. ++const DefaultGPUGetCacheTTL = cacheTimer ++ ++// WithGPUGetCacheTTL sets how long a GPUGet response is reused; a non-positive ++// value issues a new RPC on every read. ++func WithGPUGetCacheTTL(ttl time.Duration) GPUAgentClientOptions { ++ return func(ga *GPUAgentClient) { ++ logger.Log.Printf("GPUGet cache TTL set to %v", ttl) ++ ga.gpuGetCacheTTL = ttl ++ } ++} ++ ++// ParseGPUGetCacheTTL parses a cache TTL; empty means DefaultGPUGetCacheTTL and ++// negative values are rejected. ++func ParseGPUGetCacheTTL(value string) (time.Duration, error) { ++ value = strings.TrimSpace(value) ++ if value == "" { ++ return DefaultGPUGetCacheTTL, nil ++ } ++ ttl, err := time.ParseDuration(value) ++ if err != nil { ++ return 0, err ++ } ++ if ttl < 0 { ++ return 0, fmt.Errorf("%v must not be negative", ttl) ++ } ++ return ttl, nil ++} ++ + func (ga *GPUAgentClient) GetGRPCConnection() *grpc.ClientConn { + return ga.conn + } +@@ -174,6 +205,7 @@ func NewAgent(mh *metricsutil.MetricsHandler, opts ...GPUAgentClientOptions) *GP + computeNodeHealthState: true, + enableGPUMonitoring: true, + enableIFOEMonitoring: false, ++ gpuGetCacheTTL: DefaultGPUGetCacheTTL, + } + for _, o := range opts { + o(ga) +diff --git a/pkg/amdgpu/gpuagent/gpuagent_cache_ttl_test.go b/pkg/amdgpu/gpuagent/gpuagent_cache_ttl_test.go +new file mode 100644 +index 0000000..776b330 +--- /dev/null ++++ b/pkg/amdgpu/gpuagent/gpuagent_cache_ttl_test.go +@@ -0,0 +1,185 @@ ++/* ++* ++# Copyright (c) Advanced Micro Devices, Inc. All rights reserved. ++# ++# Licensed under the Apache License, Version 2.0 (the \"License\"); ++# you may not use this file except in compliance with the License. ++# You may obtain a copy of the License at ++# ++# http://www.apache.org/licenses/LICENSE-2.0 ++# ++# Unless required by applicable law or agreed to in writing, software ++# distributed under the License is distributed on an \"AS IS\" BASIS, ++# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++# See the License for the specific language governing permissions and ++# limitations under the License. ++* ++*/ ++package gpuagent ++ ++import ( ++ "context" ++ "errors" ++ "sync" ++ "testing" ++ "time" ++ ++ "github.com/ROCm/device-metrics-exporter/pkg/amdgpu/gen/amdgpu" ++ "github.com/ROCm/device-metrics-exporter/pkg/amdgpu/mock_gen" ++ "github.com/ROCm/device-metrics-exporter/pkg/exporter/config" ++ "github.com/ROCm/device-metrics-exporter/pkg/exporter/logger" ++ "github.com/ROCm/device-metrics-exporter/pkg/exporter/metricsutil" ++ "go.uber.org/mock/gomock" ++ "google.golang.org/grpc" ++) ++ ++func TestParseGPUGetCacheTTL(t *testing.T) { ++ for _, tc := range []struct { ++ value string ++ want time.Duration ++ bad bool ++ }{ ++ {"", DefaultGPUGetCacheTTL, false}, ++ {"15s", 15 * time.Second, false}, ++ {"500ms", 500 * time.Millisecond, false}, ++ {" 0s ", 0, false}, ++ {"0", 0, false}, ++ {"-1s", 0, true}, ++ {"1", 0, true}, ++ {"abc", 0, true}, ++ {"999999999999999999999s", 0, true}, ++ } { ++ t.Run(tc.value, func(t *testing.T) { ++ got, err := ParseGPUGetCacheTTL(tc.value) ++ if (err != nil) != tc.bad || got != tc.want { ++ t.Fatalf("%q: got %v, %v; want %v, error=%v", tc.value, got, err, tc.want, tc.bad) ++ } ++ }) ++ } ++} ++ ++func newCacheTTLClient(t *testing.T, ttl time.Duration) (*GPUAgentGPUClient, *mock_gen.MockGPUSvcClient) { ++ t.Helper() ++ logger.Init(true) ++ ga, err := NewGPUAgentGPUClient(&GPUAgentClient{ctx: context.Background(), gpuGetCacheTTL: ttl}) ++ if err != nil { ++ t.Fatal(err) ++ } ++ t.Cleanup(ga.Close) ++ client := mock_gen.NewMockGPUSvcClient(gomock.NewController(t)) ++ ga.gpuclient = client ++ return ga, client ++} ++ ++func TestCacheReadHonorsGPUGetCacheTTL(t *testing.T) { ++ for _, tc := range []struct { ++ name string ++ ttl time.Duration ++ age time.Duration ++ refresh bool ++ }{ ++ {"default keeps a 2s-old response", DefaultGPUGetCacheTTL, 2 * time.Second, false}, ++ {"1s TTL refreshes a 2s-old response", time.Second, 2 * time.Second, true}, ++ {"0 refreshes even a fresh response", 0, 0, true}, ++ } { ++ t.Run(tc.name, func(t *testing.T) { ++ ga, client := newCacheTTLClient(t, tc.ttl) ++ cached, fresh := &amdgpu.GPUGetResponse{}, &amdgpu.GPUGetResponse{} ++ slot := &ga.gCache.metricsCache ++ slot.resp, slot.ts = cached, time.Now().Add(-tc.age) ++ filter := &amdgpu.GPUGetFilter{SkipProcessStatus: true} ++ want := cached ++ if tc.refresh { ++ client.EXPECT().GPUGet(gomock.Any(), &amdgpu.GPUGetRequest{Filter: filter}).Return(fresh, nil).Times(1) ++ want = fresh ++ } ++ got, err := ga.cacheRead(filter, slot) ++ if err != nil || got != want || slot.resp != want { ++ t.Fatalf("cacheRead = %p, %v; want %p", got, err, want) ++ } ++ }) ++ } ++} ++ ++func TestCacheReadErrorClearsSlot(t *testing.T) { ++ ga, client := newCacheTTLClient(t, time.Second) ++ slot := &ga.gCache.metricsCache ++ slot.resp, slot.ts = &amdgpu.GPUGetResponse{}, time.Now().Add(-2*time.Second) ++ readErr := errors.New("GPUGet failed") ++ fresh := &amdgpu.GPUGetResponse{} ++ gomock.InOrder( ++ client.EXPECT().GPUGet(gomock.Any(), gomock.Any()).Return(nil, readErr), ++ client.EXPECT().GPUGet(gomock.Any(), gomock.Any()).Return(fresh, nil), ++ ) ++ if got, err := ga.cacheRead(nil, slot); !errors.Is(err, readErr) || got != nil || slot.resp != nil { ++ t.Fatalf("failed read retained a response: %v, %v", got, err) ++ } ++ if got, err := ga.cacheRead(nil, slot); err != nil || got != fresh { ++ t.Fatalf("retry = %v, %v", got, err) ++ } ++} ++ ++func TestCacheReadConcurrentTTL(t *testing.T) { ++ for _, tc := range []struct { ++ name string ++ ttl time.Duration ++ calls int ++ }{ ++ {"default reuses one read", DefaultGPUGetCacheTTL, 1}, ++ {"0 reads once per caller", 0, 16}, ++ } { ++ t.Run(tc.name, func(t *testing.T) { ++ ga, client := newCacheTTLClient(t, tc.ttl) ++ fresh := &amdgpu.GPUGetResponse{} ++ client.EXPECT().GPUGet(gomock.Any(), gomock.Any()).DoAndReturn(func(context.Context, *amdgpu.GPUGetRequest, ...grpc.CallOption) (*amdgpu.GPUGetResponse, error) { ++ time.Sleep(10 * time.Millisecond) ++ return fresh, nil ++ }).Times(tc.calls) ++ start := make(chan struct{}) ++ var wg sync.WaitGroup ++ for i := 0; i < 16; i++ { ++ wg.Add(1) ++ go func() { ++ defer wg.Done() ++ <-start ++ if got, err := ga.cacheRead(nil, &ga.gCache.metricsCache); err != nil || got != fresh { ++ t.Errorf("concurrent read = %v, %v", got, err) ++ } ++ }() ++ } ++ close(start) ++ wg.Wait() ++ }) ++ } ++} ++ ++func TestNewAgentGPUGetCacheTTL(t *testing.T) { ++ logger.Init(true) ++ handler, err := metricsutil.NewMetrics(config.NewConfigHandler("config.json", config.GPUAgentConfig{})) ++ if err != nil { ++ t.Fatal(err) ++ } ++ for _, tc := range []struct { ++ name string ++ opts []GPUAgentClientOptions ++ want time.Duration ++ }{ ++ {"no option keeps the upstream default", nil, DefaultGPUGetCacheTTL}, ++ {"option disables reuse", []GPUAgentClientOptions{WithGPUGetCacheTTL(0)}, 0}, ++ } { ++ t.Run(tc.name, func(t *testing.T) { ++ ga := NewAgent(handler, tc.opts...) ++ if ga == nil || len(ga.clients) == 0 { ++ t.Fatal("NewAgent returned no GPU client") ++ } ++ gpu, ok := ga.clients[0].(*GPUAgentGPUClient) ++ if !ok { ++ t.Fatalf("first client is %T, want *GPUAgentGPUClient", ga.clients[0]) ++ } ++ defer gpu.Close() ++ if gpu.gpuGetCacheTTL != tc.want { ++ t.Fatalf("GPU client TTL = %v, want %v", gpu.gpuGetCacheTTL, tc.want) ++ } ++ }) ++ } ++} +diff --git a/pkg/amdgpu/gpuagent/gpuagent_gpu.go b/pkg/amdgpu/gpuagent/gpuagent_gpu.go +index 3e5d0f3..a19d644 100644 +--- a/pkg/amdgpu/gpuagent/gpuagent_gpu.go ++++ b/pkg/amdgpu/gpuagent/gpuagent_gpu.go +@@ -154,6 +154,7 @@ type GPUAgentGPUClient struct { + healthState map[string]*metricssvc.GPUState + mockEccField map[string]map[string]uint32 // gpuid->fields->count + gCache *gpuCache ++ gpuGetCacheTTL time.Duration + exportLabels map[string]bool + exportFieldMap map[string]bool // all upper case keys + fieldMetricsMap map[string]FieldMeta +@@ -183,6 +184,7 @@ func NewGPUAgentGPUClient(gpuHandler *GPUAgentClient) (*GPUAgentGPUClient, error + fieldMetricsMap: make(map[string]FieldMeta), + gpuSelectorMap: make(map[int]bool), + gCache: &gpuCache{}, ++ gpuGetCacheTTL: gpuHandler.gpuGetCacheTTL, + gpuHandler: gpuHandler, + fl: gpuHandler.fl, + gpuIDMap: make(map[string]GPUIDMeta), +@@ -418,14 +420,14 @@ func (ga *GPUAgentGPUClient) GetContext() context.Context { + return ctx + } + +-// cacheRead serves the given slot within cacheTimer, else issues a GPUGet with ++// cacheRead serves the given slot within gpuGetCacheTTL, else issues a GPUGet with + // the given filter and refreshes that slot. + func (ga *GPUAgentGPUClient) cacheRead(filter *amdgpu.GPUGetFilter, c *responseCache) (*amdgpu.GPUGetResponse, error) { + now := time.Now() + + // First try fast path with RLock + ga.gCache.RLock() +- if c.resp != nil && now.Sub(c.ts) < cacheTimer { ++ if ga.gpuGetCacheTTL > 0 && c.resp != nil && now.Sub(c.ts) < ga.gpuGetCacheTTL { + res := c.resp + ga.gCache.RUnlock() + logger.Debug("returning metrics from cache") +@@ -438,7 +440,7 @@ func (ga *GPUAgentGPUClient) cacheRead(filter *amdgpu.GPUGetFilter, c *responseC + defer ga.gCache.Unlock() + + // Check again after acquiring Lock to handle the case where another goroutine has already updated the cache +- if c.resp != nil && time.Since(c.ts) < cacheTimer { ++ if ga.gpuGetCacheTTL > 0 && c.resp != nil && time.Since(c.ts) < ga.gpuGetCacheTTL { + logger.Debug("returning metrics from cache (after double-check)") + return c.resp, nil + } +diff --git a/pkg/exporter/exporter.go b/pkg/exporter/exporter.go +index f4261bf..0d5b57d 100644 +--- a/pkg/exporter/exporter.go ++++ b/pkg/exporter/exporter.go +@@ -73,6 +73,7 @@ type Exporter struct { + enableCRI bool + exitOnAgentDown bool + exitOnRocpctlError bool ++ gpuGetTTL time.Duration + bindAddr string + k8sApiClient *k8sclient.K8sClient + svcHandler *metricsserver.SvcHandler +@@ -300,6 +301,7 @@ func NewExporter(agentGrpcport int, configFile string, opts ...ExporterOption) * + disableK8sApi: false, // by default k8s api is enabled + disableK8sScl: false, // by default k8s scheduler client is enabled + enableCRI: true, // by default CRI client is enabled ++ gpuGetTTL: gpuagent.DefaultGPUGetCacheTTL, + } + + for _, o := range opts { +@@ -477,6 +479,13 @@ func WithExitOnRocpctlError(exit bool) ExporterOption { + } + } + ++func WithGPUGetCacheTTL(ttl time.Duration) ExporterOption { ++ return func(e *Exporter) { ++ logger.Log.Printf("GPUGet cache TTL set to %v", ttl) ++ e.gpuGetTTL = ttl ++ } ++} ++ + // StartMain - doesn't return it exits only on failure + func (e *Exporter) StartMain() { + defer e.Close() +@@ -514,6 +523,7 @@ func (e *Exporter) StartMain() { + gpuagent.WithIFOEMonitoring(e.enableIFOEMonitoring), + gpuagent.WithExitOnAgentDown(e.exitOnAgentDown), + gpuagent.WithExitOnRocpctlError(e.exitOnRocpctlError), ++ gpuagent.WithGPUGetCacheTTL(e.gpuGetTTL), + } + if e.agentConfig.UseSocket { + opts = append(opts, gpuagent.WithSocketConnection(e.agentConfig.SocketPath)) +diff --git a/pkg/exporter/exporter_test.go b/pkg/exporter/exporter_test.go +index d02b6c0..6a6e46a 100644 +--- a/pkg/exporter/exporter_test.go ++++ b/pkg/exporter/exporter_test.go +@@ -29,6 +29,7 @@ import ( + "testing" + "time" + ++ "github.com/ROCm/device-metrics-exporter/pkg/amdgpu/gpuagent" + "github.com/ROCm/device-metrics-exporter/pkg/exporter/config" + "github.com/ROCm/device-metrics-exporter/pkg/exporter/globals" + "github.com/ROCm/device-metrics-exporter/pkg/exporter/logger" +@@ -523,6 +524,21 @@ func TestExporterOptions(t *testing.T) { + assert.Assert(t, e.exitOnAgentDown, "exitOnAgentDown should be enabled") + }, + }, ++ { ++ name: "GPUGet cache TTL defaults to the gpuagent value", ++ verify: func(e *Exporter) { ++ assert.Equal(t, e.gpuGetTTL, gpuagent.DefaultGPUGetCacheTTL, "default GPUGet cache TTL should match gpuagent") ++ }, ++ }, ++ { ++ name: "GPUGet cache TTL disabled", ++ options: []ExporterOption{ ++ WithGPUGetCacheTTL(0), ++ }, ++ verify: func(e *Exporter) { ++ assert.Equal(t, e.gpuGetTTL, time.Duration(0), "GPUGet cache TTL should be zero") ++ }, ++ }, + } + + for _, tc := range tests { diff --git a/tools/amd-exporter/provenance.py b/tools/amd-exporter/provenance.py new file mode 100644 index 0000000000..14f9052cd6 --- /dev/null +++ b/tools/amd-exporter/provenance.py @@ -0,0 +1,60 @@ +import hashlib +import json +import os +import sys +from pathlib import Path + + +def sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as source: + for block in iter(lambda: source.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +output = Path(sys.argv[1]) +image = json.loads((output / "image-inspect.json").read_text())[0] +repository = os.environ["IMAGE_REF"].rsplit(":", 1)[0] +digests = [ + value for value in image["RepoDigests"] if value.startswith(repository + "@sha256:") +] +if len(digests) != 1: + raise ValueError("Expected exactly one published digest for the target image") +receipt = { + "source_repository": "ROCm/device-metrics-exporter", + "source_sha": os.environ["SOURCE_SHA"], + "source_run_id": 36867107435, + "source_artifact_id": 11169689175, + "source_artifact_name": "build-dme-10.2.0a20261001", + "gpu_agent_sha": os.environ["GPU_AGENT_SHA"], + "cache_patch": {"sha256": sha256(output / "cache.patch"), "ttl": "0s"}, + "binary": { + "name": "amd-metrics-exporter", + "sha256": sha256(output / "amd-metrics-exporter"), + }, + "archive": { + "name": os.environ["ARCHIVE_NAME"], + "sha256": os.environ["ARCHIVE_SHA256"], + }, + "base_recovery": json.loads((output / "base-recovery.json").read_text()), + "image": { + "loaded_tag": os.environ["IMAGE_REF"], + "config_digest": image["Id"], + "reference": digests[0], + }, + "squash": { + "name": "amd-exporter.sqsh", + "sha256": sha256(output / "amd-exporter.sqsh"), + }, + "preparation": { + "repository": os.environ["GITHUB_REPOSITORY"], + "run_id": os.environ["GITHUB_RUN_ID"], + "run_attempt": os.environ["GITHUB_RUN_ATTEMPT"], + "workflow_sha": os.environ["GITHUB_SHA"], + }, +} +if receipt["cache_patch"]["sha256"] != os.environ["PATCH_SHA256"]: + raise ValueError("Cache patch digest changed during preparation") +(output / "provenance.json").write_text(json.dumps(receipt, indent=2) + "\n") +print(digests[0]) diff --git a/tools/amd-exporter/recover-base.py b/tools/amd-exporter/recover-base.py new file mode 100644 index 0000000000..9fd8689c86 --- /dev/null +++ b/tools/amd-exporter/recover-base.py @@ -0,0 +1,139 @@ +import hashlib +import json +import os +import stat +import subprocess +import sys +import tempfile +from pathlib import Path + + +def sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as source: + for block in iter(lambda: source.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def container_path(root: Path, path: str) -> Path: + pending = path.split("/") + parts = [] + links = 0 + while pending: + part = pending.pop(0) + if part in ("", "."): + continue + if part == "..": + if not parts: + raise ValueError("Path escapes the container root") + parts.pop() + continue + candidate = root.joinpath(*parts, part) + if candidate.is_symlink() and pending: + links += 1 + if links > 40: + raise ValueError("Too many symlinks in container path") + target = os.readlink(candidate) + if target.startswith("/"): + parts = [] + pending = target.split("/") + pending + else: + parts.append(part) + return root.joinpath(*parts) + + +def verify_rootfs(root: Path, files: dict) -> None: + for name, expected in files.items(): + path = container_path(root, name) + mode = path.lstat().st_mode + if expected["type"] == "symlink": + if not stat.S_ISLNK(mode) or os.readlink(path) != expected["target"]: + raise ValueError(f"Runtime symlink changed: {name}") + elif not stat.S_ISREG(mode) or sha256(path) != expected["sha256"]: + raise ValueError(f"Runtime bytes changed: {name}") + elif mode & 0o111 != expected["mode"] & 0o111: + raise ValueError(f"Runtime executable bits changed: {name}") + + +def config_changes(config: dict) -> list[str]: + changes = [] + for entry in config.get("Env", []): + key, value = entry.split("=", 1) + changes += ["--change", f"ENV {key}={json.dumps(value)}"] + for key, value in config.get("Labels", {}).items(): + changes += ["--change", f"LABEL {key}={json.dumps(value)}"] + for key in ("Entrypoint", "Cmd", "WorkingDir", "User"): + if config.get(key): + instruction = "WORKDIR" if key == "WorkingDir" else key.upper() + changes += ["--change", f"{instruction} {json.dumps(config[key])}"] + return changes + + +def verify_config(expected: dict, actual: dict) -> None: + for key in ("Env", "Entrypoint", "Cmd", "WorkingDir", "User", "Labels"): + if expected.get(key) != actual.get(key) and not ( + key in ("Cmd", "User") and not expected.get(key) and not actual.get(key) + ): + raise ValueError(f"Recovered Docker configuration changed: {key}") + + +def verify_provenance(source: dict, fixture: dict) -> None: + checks = ( + (source["squash"]["sha256"], fixture["recovery"]["squash_sha256"]), + (source["archive"]["sha256"], fixture["official_archive_sha256"]), + (source["image"]["config_digest"], fixture["official_image_config_digest"]), + (str(source["preparation"]["run_id"]), str(fixture["recovery"]["run_id"])), + (source["preparation"]["repository"], fixture["recovery"]["repository"]), + ) + if any(actual != expected for actual, expected in checks): + raise ValueError("Prepared runtime provenance does not match the pinned base") + + +def main() -> None: + root, source_path, fixture_path, image_ref, output_path = sys.argv[1:] + root = Path(root) + fixture = json.loads(Path(fixture_path).read_text()) + source = json.loads(Path(source_path).read_text()) + verify_provenance(source, fixture) + verify_rootfs(root, fixture["files"]) + with tempfile.TemporaryFile() as archive: + subprocess.run( + ["tar", "--numeric-owner", "-C", str(root), "-cpf", "-", "."], + stdout=archive, + check=True, + ) + archive.seek(0) + subprocess.run( + [ + "docker", + "import", + "--platform=linux/amd64", + *config_changes(fixture["config"]), + "-", + image_ref, + ], + stdin=archive, + check=True, + ) + image = json.loads( + subprocess.check_output(["docker", "image", "inspect", image_ref]) + )[0] + verify_config(fixture["config"], image["Config"]) + receipt = { + "kind": "reconstructed_from_enroot_squash", + "recovery": fixture["recovery"], + "official_image_config_digest": fixture["official_image_config_digest"], + "official_archive_sha256": fixture["official_archive_sha256"], + "reconstructed_image_config_digest": image["Id"], + "reconstructed_layers": image["RootFS"]["Layers"], + "critical_files_verified": len(fixture["files"]), + "runtime_manifest_sha256": sha256(Path(fixture_path)), + "conversion": "Enroot 4.2.0: flattened layers, root ownership, normalized permissions, generated /etc/{environment,fstab,rc,rc.local}; original Docker config restored", + "source_provenance": source, + } + Path(output_path).write_text(json.dumps(receipt, indent=2) + "\n") + + +if __name__ == "__main__": + main() diff --git a/tools/amd-exporter/test_recover_base.py b/tools/amd-exporter/test_recover_base.py new file mode 100644 index 0000000000..7620dfdfd8 --- /dev/null +++ b/tools/amd-exporter/test_recover_base.py @@ -0,0 +1,97 @@ +import copy +import hashlib +import importlib.util +import json +import tempfile +import unittest +from pathlib import Path + +spec = importlib.util.spec_from_file_location( + "recover_base", Path(__file__).with_name("recover-base.py") +) +module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(module) + + +class RuntimeRecoveryTest(unittest.TestCase): + def setUp(self): + self.directory = tempfile.TemporaryDirectory() + self.addCleanup(self.directory.cleanup) + self.root = Path(self.directory.name) + (self.root / "opt/runtime/lib").mkdir(parents=True) + self.library = self.root / "opt/runtime/lib/amdsmi.so" + self.library.write_bytes(b"retained runtime library") + self.library.chmod(0o755) + (self.root / "opt/rocm").symlink_to("/opt/runtime") + (self.root / "opt/runtime/lib/amdsmi").symlink_to("amdsmi.so") + self.files = { + "opt/rocm": {"type": "symlink", "target": "/opt/runtime"}, + "opt/rocm/lib/amdsmi": {"type": "symlink", "target": "amdsmi.so"}, + "opt/rocm/lib/amdsmi.so": { + "type": "file", + "mode": 0o755, + "sha256": hashlib.sha256(self.library.read_bytes()).hexdigest(), + }, + } + + def test_accepts_retained_bytes_and_container_absolute_symlink(self): + module.verify_rootfs(self.root, self.files) + + def test_rejects_changed_runtime_library(self): + self.library.write_bytes(b"different version") + with self.assertRaisesRegex(ValueError, "Runtime bytes changed"): + module.verify_rootfs(self.root, self.files) + + def test_rejects_changed_symlink(self): + link = self.root / "opt/runtime/lib/amdsmi" + link.unlink() + link.symlink_to("other.so") + with self.assertRaisesRegex(ValueError, "Runtime symlink changed"): + module.verify_rootfs(self.root, self.files) + + def test_rejects_lost_executable_bit(self): + self.library.chmod(0o644) + with self.assertRaisesRegex(ValueError, "executable bits changed"): + module.verify_rootfs(self.root, self.files) + + def test_rejects_path_outside_container(self): + with self.assertRaisesRegex(ValueError, "escapes"): + module.container_path(self.root, "../outside") + + def test_original_runtime_config_can_be_restored(self): + fixture = json.loads(Path(__file__).with_name("base-runtime.json").read_text()) + config = fixture["config"] + actual = copy.deepcopy(config) + actual.update({"Cmd": None, "User": ""}) + module.verify_config(config, actual) + changes = module.config_changes(config) + self.assertIn('ENTRYPOINT ["/home/amd/tools/entrypoint.sh"]', changes) + self.assertIn('ENV GPUAGENT_EVENTS_DISABLE="1"', changes) + actual["Env"] = ["PATH=/bin"] + with self.assertRaisesRegex(ValueError, "Env"): + module.verify_config(config, actual) + + def test_rejects_another_squash_in_source_receipt(self): + fixture = { + "official_archive_sha256": "archive", + "official_image_config_digest": "config", + "recovery": { + "squash_sha256": "expected", + "run_id": 10, + "repository": "owner/repo", + }, + } + source = { + "squash": {"sha256": "expected"}, + "archive": {"sha256": "archive"}, + "image": {"config_digest": "config"}, + "preparation": {"run_id": "10", "repository": "owner/repo"}, + } + module.verify_provenance(source, fixture) + source["squash"]["sha256"] = "other" + with self.assertRaisesRegex(ValueError, "provenance"): + module.verify_provenance(source, fixture) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/amd-exporter/test_verify_image.py b/tools/amd-exporter/test_verify_image.py new file mode 100644 index 0000000000..94367c769c --- /dev/null +++ b/tools/amd-exporter/test_verify_image.py @@ -0,0 +1,51 @@ +import copy +import importlib.util +import unittest +from pathlib import Path + +spec = importlib.util.spec_from_file_location( + "verify_image", Path(__file__).with_name("verify-image.py") +) +module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(module) + + +class ImageVerificationTest(unittest.TestCase): + def setUp(self): + self.base = { + "RootFS": {"Layers": ["base", "gpu-agent"]}, + "Config": { + "Entrypoint": ["/home/amd/tools/entrypoint.sh"], + "Cmd": None, + "WorkingDir": "/home/amd", + "User": "root", + "Env": ["PATH=/bin"], + }, + "Os": "linux", + "Architecture": "amd64", + } + self.candidate = copy.deepcopy(self.base) + self.candidate["RootFS"]["Layers"].append("new-exporter") + self.candidate["Config"]["Env"].append("AMD_GPU_GET_CACHE_TTL=0s") + + def test_accepts_binary_overlay(self): + module.verify(self.base, self.candidate) + + def test_rejects_changed_base_layer(self): + self.candidate["RootFS"]["Layers"][1] = "different-gpu-agent" + with self.assertRaisesRegex(ValueError, "preserve every base layer"): + module.verify(self.base, self.candidate) + + def test_rejects_changed_entrypoint(self): + self.candidate["Config"]["Entrypoint"] = ["/bin/sh"] + with self.assertRaisesRegex(ValueError, "Entrypoint"): + module.verify(self.base, self.candidate) + + def test_rejects_cache_reuse(self): + self.candidate["Config"]["Env"][-1] = "AMD_GPU_GET_CACHE_TTL=1s" + with self.assertRaisesRegex(ValueError, "cache bypass"): + module.verify(self.base, self.candidate) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/amd-exporter/verify-image.py b/tools/amd-exporter/verify-image.py new file mode 100644 index 0000000000..afc9f8db6d --- /dev/null +++ b/tools/amd-exporter/verify-image.py @@ -0,0 +1,34 @@ +import json +import subprocess +import sys + + +def verify(base: dict, candidate: dict) -> None: + old_layers = base["RootFS"]["Layers"] + new_layers = candidate["RootFS"]["Layers"] + if new_layers[:-1] != old_layers: + raise ValueError( + "Candidate must preserve every base layer and add exactly one layer" + ) + for key in ("Entrypoint", "Cmd", "WorkingDir", "User"): + if base["Config"][key] != candidate["Config"][key]: + raise ValueError(f"Runtime configuration changed: {key}") + if candidate["Os"] != "linux" or candidate["Architecture"] != "amd64": + raise ValueError("Expected a linux/amd64 image") + cache_values = [ + entry + for entry in candidate["Config"]["Env"] + if entry.startswith("AMD_GPU_GET_CACHE_TTL=") + ] + if cache_values != ["AMD_GPU_GET_CACHE_TTL=0s"]: + raise ValueError("Exporter cache bypass must be explicit in the image") + + +if __name__ == "__main__": + images = json.loads( + subprocess.check_output(["docker", "image", "inspect", *sys.argv[1:]]) + ) + if len(images) != 2: + raise ValueError("Expected base and candidate image references") + verify(*images) + print("Base layers, runtime entrypoint and cache bypass verified")