From 8c416737598b78ad8b7c9fd60c75774712377c12 Mon Sep 17 00:00:00 2001 From: chen21019 Date: Mon, 7 Sep 2026 21:59:59 +0800 Subject: [PATCH 1/4] feat: discover host accelerators and preserve safe Docker hardware options --- .github/workflows/codeql-verification.yml | 4 +- .github/workflows/security-release-gate.yml | 20 ++-- .github/workflows/validate.yml | 16 +-- core/compute/compute_unix.go | 94 +++++++++++++--- core/compute/hardware_docker_test.go | 81 ++++++++++++++ core/compute/hardware_test.go | 69 ++++++++++++ core/hostinfo/hardware.go | 48 +++++++++ core/hostinfo/hardware_linux.go | 100 ++++++++++++++++++ core/hostinfo/hardware_linux_test.go | 53 ++++++++++ core/hostinfo/hardware_other.go | 9 ++ docs/releases/node-agent-0.13.24.md | 13 +++ handlers/common.go | 1 + model/instance.go | 17 ++- third_party/go-rancher/v2/device_request.go | 10 ++ .../go-rancher/v2/generated_container.go | 4 + .../go-rancher/v2/generated_launch_config.go | 4 + .../v2/generated_secondary_launch_config.go | 4 + 17 files changed, 509 insertions(+), 38 deletions(-) create mode 100644 core/compute/hardware_docker_test.go create mode 100644 core/compute/hardware_test.go create mode 100644 core/hostinfo/hardware.go create mode 100644 core/hostinfo/hardware_linux.go create mode 100644 core/hostinfo/hardware_linux_test.go create mode 100644 core/hostinfo/hardware_other.go create mode 100644 docs/releases/node-agent-0.13.24.md create mode 100644 third_party/go-rancher/v2/device_request.go diff --git a/.github/workflows/codeql-verification.yml b/.github/workflows/codeql-verification.yml index 008544418..780c3c4e9 100644 --- a/.github/workflows/codeql-verification.yml +++ b/.github/workflows/codeql-verification.yml @@ -38,7 +38,7 @@ jobs: set -euo pipefail test -z "$(git status --porcelain)" git merge-base --is-ancestor c8663d12dd253ef13258750dca056d4b1219fc10 HEAD - grep -Fq 'VERSION_OVERRIDE: v0.13.23' .github/workflows/validate.yml + grep -Fq 'VERSION_OVERRIDE: v0.13.24' .github/workflows/validate.yml - name: Install the release Go toolchain uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 @@ -65,7 +65,7 @@ jobs: export GOFLAGS=-mod=vendor mkdir -p "$GITHUB_WORKSPACE/bin" CGO_ENABLED=0 go build -trimpath -tags 'netgo osusergo' \ - -ldflags='-w -s -X main.VERSION=v0.13.23' \ + -ldflags='-w -s -X main.VERSION=v0.13.24' \ -o "$GITHUB_WORKSPACE/bin/node-agent" ./ - name: Analyze without publishing temporary alerts diff --git a/.github/workflows/security-release-gate.yml b/.github/workflows/security-release-gate.yml index 7589c6ee2..9faf70153 100644 --- a/.github/workflows/security-release-gate.yml +++ b/.github/workflows/security-release-gate.yml @@ -19,9 +19,9 @@ jobs: timeout-minutes: 120 env: DAPPER_IMAGE: pasturestack/node-agent-dapper:${{ github.sha }} - RUNTIME_IMAGE: pasturestack/node-agent:v0.13.23 + RUNTIME_IMAGE: pasturestack/node-agent:v0.13.24 TRIVY_IMAGE: aquasec/trivy:0.74.0@sha256:62b1e65e8869bc4b4c6aa4fa2b21595256c7c2f6018a9d9ad61caf87187c1969 - VERSION_OVERRIDE: v0.13.23 + VERSION_OVERRIDE: v0.13.24 steps: - name: Check out candidate uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -49,19 +49,19 @@ jobs: set -euo pipefail CROSS=1 VERSION_OVERRIDE="$VERSION_OVERRIDE" make DAPPER_IMAGE="$DAPPER_IMAGE" ci make DAPPER_IMAGE="$DAPPER_IMAGE" IMAGE_NAME=pasturestack/node-agent \ - TAG=v0.13.23 VERSION_OVERRIDE="$VERSION_OVERRIDE" package-image - test -s dist/artifacts/node-agent-0.13.23.tar.gz - test -s dist/artifacts/node-agent-0.13.23-windows-amd64.zip + TAG=v0.13.24 VERSION_OVERRIDE="$VERSION_OVERRIDE" package-image + test -s dist/artifacts/node-agent-0.13.24.tar.gz + test -s dist/artifacts/node-agent-0.13.24-windows-amd64.zip test "$(cat dist/image)" = "$RUNTIME_IMAGE" - gzip -t dist/artifacts/node-agent-0.13.23.tar.gz - unzip -t dist/artifacts/node-agent-0.13.23-windows-amd64.zip - sha256sum dist/artifacts/node-agent-0.13.23.tar.gz \ - dist/artifacts/node-agent-0.13.23-windows-amd64.zip \ + gzip -t dist/artifacts/node-agent-0.13.24.tar.gz + unzip -t dist/artifacts/node-agent-0.13.24-windows-amd64.zip + sha256sum dist/artifacts/node-agent-0.13.24.tar.gz \ + dist/artifacts/node-agent-0.13.24-windows-amd64.zip \ bin/node-agent > evidence/product-artifacts.sha256 docker image inspect "$RUNTIME_IMAGE" > evidence/runtime-image-inspect.json docker run --rm --entrypoint /usr/bin/node-agent "$RUNTIME_IMAGE" --version \ > evidence/runtime-version.txt - grep -Fxq 'node-agent version v0.13.23' evidence/runtime-version.txt + grep -Fxq 'node-agent version v0.13.24' evidence/runtime-version.txt - name: Verify reproducible Linux binary shell: bash diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index c7981095c..8b263ada0 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -34,12 +34,12 @@ jobs: id: package env: CROSS: "1" - VERSION_OVERRIDE: v0.13.23 + VERSION_OVERRIDE: v0.13.24 run: | set -euo pipefail make ci - linux_asset="dist/artifacts/node-agent-0.13.23.tar.gz" - windows_asset="dist/artifacts/node-agent-0.13.23-windows-amd64.zip" + linux_asset="dist/artifacts/node-agent-0.13.24.tar.gz" + windows_asset="dist/artifacts/node-agent-0.13.24-windows-amd64.zip" test -s "$linux_asset" test -s "$windows_asset" gzip -t "$linux_asset" @@ -47,8 +47,8 @@ jobs: ( cd dist/artifacts sha256sum \ - node-agent-0.13.23.tar.gz \ - node-agent-0.13.23-windows-amd64.zip >SHA256SUMS + node-agent-0.13.24.tar.gz \ + node-agent-0.13.24-windows-amd64.zip >SHA256SUMS ) printf 'NODE_AGENT_VALIDATION_OK source=%s version=%s artifacts=linux,windows\n' \ "$GITHUB_SHA" "$VERSION_OVERRIDE" @@ -56,10 +56,10 @@ jobs: - name: Retain reviewed release candidate uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: node-agent-0.13.23 + name: node-agent-0.13.24 path: | - dist/artifacts/node-agent-0.13.23.tar.gz - dist/artifacts/node-agent-0.13.23-windows-amd64.zip + dist/artifacts/node-agent-0.13.24.tar.gz + dist/artifacts/node-agent-0.13.24-windows-amd64.zip dist/artifacts/SHA256SUMS if-no-files-found: error retention-days: 30 diff --git a/core/compute/compute_unix.go b/core/compute/compute_unix.go index 89d06f516..be11c7437 100755 --- a/core/compute/compute_unix.go +++ b/core/compute/compute_unix.go @@ -9,6 +9,7 @@ import ( "fmt" "net/netip" "os" + "path" "strconv" "strings" @@ -310,24 +311,10 @@ func setupFieldsHostConfig(fields model.InstanceFields, hostConfig *container.Ho hostConfig.SecurityOpt = fields.SecurityOpt - deviceMappings := []container.DeviceMapping{} - devices := fields.Devices - for _, device := range devices { - parts := strings.Split(device, ":") - permission := "rwm" - if len(parts) == 3 { - permission = parts[2] - } - deviceMappings = append(deviceMappings, - container.DeviceMapping{ - PathOnHost: parts[0], - PathInContainer: parts[1], - CgroupPermissions: permission, - }) + if err := setupHardware(fields, hostConfig); err != nil { + return err } - hostConfig.Devices = deviceMappings - hostConfig.DNS = make([]netip.Addr, 0, len(fields.DNS)) for _, value := range fields.DNS { address, err := netip.ParseAddr(value) @@ -413,6 +400,81 @@ func setupFieldsHostConfig(fields model.InstanceFields, hostConfig *container.Ho return nil } +func setupHardware(fields model.InstanceFields, hostConfig *container.HostConfig) error { + if fields.ShmSize < 0 { + return errors.New("shmSize must be non-negative bytes") + } + if fields.ShmSize > 0 && fields.IpcMode != "" && fields.IpcMode != "private" && fields.IpcMode != "shareable" { + return errors.New("shmSize requires private or shareable IPC") + } + if strings.Trim(fields.Runtime, "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_.-") != "" { + return errors.New("runtime must be a registered runtime name") + } + hostConfig.Runtime = fields.Runtime + hostConfig.DeviceRequests = nil + for _, request := range fields.DeviceRequests { + if request.Count < -1 || (request.Count != 0 && len(request.DeviceIDs) > 0) || (request.Count == 0 && len(request.DeviceIDs) == 0) { + return errors.New("deviceRequests requires count (-1 for all or a positive integer) OR deviceIds") + } + if len(request.Capabilities) == 0 { + return errors.New("deviceRequests requires capabilities") + } + for _, capabilities := range request.Capabilities { + if len(capabilities) == 0 { + return errors.New("deviceRequests capability groups cannot be empty") + } + for _, capability := range capabilities { + if strings.TrimSpace(capability) == "" { + return errors.New("deviceRequests capabilities cannot be blank") + } + } + } + seen := map[string]bool{} + for _, id := range request.DeviceIDs { + if strings.TrimSpace(id) == "" || seen[id] { + return errors.New("deviceRequests deviceIds must be non-empty and unique") + } + seen[id] = true + } + hostConfig.DeviceRequests = append(hostConfig.DeviceRequests, container.DeviceRequest{ + Driver: request.Driver, Count: request.Count, DeviceIDs: request.DeviceIDs, + Capabilities: request.Capabilities, Options: request.Options, + }) + } + hostConfig.Devices = nil + for _, device := range fields.Devices { + parts := strings.Split(device, ":") + if len(parts) > 3 || !path.IsAbs(parts[0]) || path.Clean(parts[0]) != parts[0] { + return errors.New("devices requires an absolute host path, optionally :container-path[:rwm]; CDI identifiers are not supported") + } + target, permissions := parts[0], "rwm" + if len(parts) >= 2 { + if path.IsAbs(parts[1]) { + target = parts[1] + } else if len(parts) == 2 { + permissions = parts[1] + } else { + return errors.New("devices container path must be absolute") + } + } + if len(parts) == 3 { + permissions = parts[2] + } + if path.Clean(target) != target || permissions == "" || strings.Trim(permissions, "rwm") != "" { + return errors.New("invalid devices path or permissions") + } + for _, permission := range "rwm" { + if strings.Count(permissions, string(permission)) > 1 { + return errors.New("duplicate device permission") + } + } + hostConfig.Devices = append(hostConfig.Devices, container.DeviceMapping{ + PathOnHost: parts[0], PathInContainer: target, CgroupPermissions: permissions, + }) + } + return nil +} + func setupComputeResourceFields(hostConfig *container.HostConfig, instance model.Instance) { const dockerMinMemoryReservation = 6 * 1024 * 1024 diff --git a/core/compute/hardware_docker_test.go b/core/compute/hardware_docker_test.go new file mode 100644 index 000000000..7b7ec397d --- /dev/null +++ b/core/compute/hardware_docker_test.go @@ -0,0 +1,81 @@ +//go:build linux + +package compute + +import ( + "context" + "encoding/json" + "fmt" + "os" + "testing" + "time" + + "github.com/PastureStack/node-agent/internal/dockerapi/client" + "github.com/PastureStack/node-agent/internal/dockerapi/types" + "github.com/PastureStack/node-agent/model" + "github.com/moby/moby/api/types/container" +) + +// Explicit opt-in: uses an already present immutable image, an isolated network +// namespace, no host mounts/ports, and removes only the exact container it made. +// Do not invoke the legacy whole-daemon test harness on a shared development VM. +func TestHardwareDockerRoundTrip(t *testing.T) { + image := os.Getenv("PASTURESTACK_HARDWARE_TEST_IMAGE") + if image == "" { + t.Skip("set PASTURESTACK_HARDWARE_TEST_IMAGE to an existing approved sha256 image") + } + if len(image) != 71 || image[:7] != "sha256:" { + t.Fatal("test image must be an immutable local image ID") + } + cli, err := client.NewEnvClient() + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second) + defer cancel() + var fields model.InstanceFields + // Decode through the real agent wire contract, including signed limits. + if err = json.Unmarshal([]byte(`{"shmSize":2147483648,"ipcMode":"private","runtime":"runc","pidsLimit":64,"cpuQuota":50000,"cpuPeriod":100000,"runInit":true,"devices":["/dev/null:rw"],"groupAdd":["993"],"tmpfs":{"/tmp":"rw,noexec,nosuid,size=64m"},"sysctls":{"net.core.somaxconn":"1024"},"ulimits":[{"name":"nofile","soft":1024,"hard":4096}]}`), &fields); err != nil { + t.Fatal(err) + } + var hc container.HostConfig + if err = setupFieldsHostConfig(fields, &hc); err != nil { + t.Fatal(err) + } + hc.NetworkMode = "none" + created, err := cli.ContainerCreate(ctx, &container.Config{Image: image, Entrypoint: []string{"/bin/sh"}, + Cmd: []string{"-c", "sleep 60"}, Labels: map[string]string{"io.pasturestack.test": "hardware-roundtrip"}}, &hc, nil, + fmt.Sprintf("pasturestack-hardware-test-%d", time.Now().UnixNano())) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + cleanup, done := context.WithTimeout(context.Background(), 20*time.Second) + defer done() + if err := cli.ContainerRemove(cleanup, created.ID, types.ContainerRemoveOptions{Force: true}); err != nil { + t.Errorf("exact test container cleanup: %v", err) + } + }) + if err = cli.ContainerStart(ctx, created.ID, types.ContainerStartOptions{}); err != nil { + t.Fatal(err) + } + actual, err := cli.ContainerInspect(ctx, created.ID) + if err != nil { + t.Fatal(err) + } + if !actual.State.Running { + t.Fatal("fixture did not start") + } + c := actual.HostConfig + if c.ShmSize != 2147483648 || c.Runtime != "runc" || c.IpcMode != "private" || c.PidsLimit == nil || *c.PidsLimit != 64 || + c.CPUQuota != 50000 || c.CPUPeriod != 100000 || c.Init == nil || !*c.Init || c.Privileged || len(c.CapAdd) != 0 || len(c.SecurityOpt) != 0 { + t.Fatal("Docker runtime did not preserve resource/isolation settings") + } + if c.Tmpfs["/tmp"] != "rw,noexec,nosuid,size=64m" || c.Sysctls["net.core.somaxconn"] != "1024" || + len(c.Ulimits) != 1 || c.Ulimits[0].Name != "nofile" || c.Ulimits[0].Soft != 1024 || c.Ulimits[0].Hard != 4096 || + len(c.Devices) != 1 || c.Devices[0].PathOnHost != "/dev/null" || c.Devices[0].CgroupPermissions != "rw" || + len(c.GroupAdd) != 1 || c.GroupAdd[0] != "993" { + t.Fatal("Docker runtime lost advanced options") + } + t.Log("real Docker create/start/inspect passed; GPU hardware execution is not covered by this test") +} diff --git a/core/compute/hardware_test.go b/core/compute/hardware_test.go new file mode 100644 index 000000000..320c7bff1 --- /dev/null +++ b/core/compute/hardware_test.go @@ -0,0 +1,69 @@ +//go:build linux + +package compute + +import ( + "encoding/json" + "testing" + + "github.com/PastureStack/node-agent/model" + "github.com/moby/moby/api/types/container" +) + +func TestHardwareLaunchContract(t *testing.T) { + var fields model.InstanceFields + err := json.Unmarshal([]byte(`{"runtime":"nvidia","shmSize":2147483648,"ipcMode":"private","groupAdd":["993"],"devices":["/dev/dri/renderD128"],"deviceRequests":[{"driver":"nvidia","deviceIds":["GPU-test"],"capabilities":[["gpu"]]}]}`), &fields) + if err != nil { + t.Fatal(err) + } + var actual container.HostConfig + if err = setupFieldsHostConfig(fields, &actual); err != nil { + t.Fatal(err) + } + if actual.ShmSize != 2147483648 || actual.Runtime != "nvidia" || actual.IpcMode != "private" || actual.GroupAdd[0] != "993" { + t.Fatalf("lost resource options: %+v", actual) + } + if len(actual.DeviceRequests) != 1 || actual.DeviceRequests[0].DeviceIDs[0] != "GPU-test" || actual.DeviceRequests[0].Capabilities[0][0] != "gpu" { + t.Fatal("lost GPU request") + } + if actual.Devices[0].PathInContainer != "/dev/dri/renderD128" { + t.Fatal("shorthand device not preserved") + } + if actual.Privileged || len(actual.SecurityOpt) > 0 { + t.Fatal("hardware must not escalate privilege") + } + encoded, err := json.Marshal(actual) + if err != nil { + t.Fatal(err) + } + var inspected model.HostConfig + if err := json.Unmarshal(encoded, &inspected); err != nil { + t.Fatal(err) + } + if inspected.Runtime != actual.Runtime || len(inspected.DeviceRequests) != 1 || inspected.DeviceRequests[0].DeviceIDs[0] != "GPU-test" { + t.Fatal("Docker inspect drops GPU/runtime identity") + } +} + +func TestHardwareRejectsInvalidRequests(t *testing.T) { + cases := []model.InstanceFields{ + {ShmSize: -1}, {ShmSize: 1024, IpcMode: "host"}, {Runtime: "runc --evil"}, + {Devices: []string{""}}, {Devices: []string{"relative"}}, {Devices: []string{"/dev/dri:"}}, + {Devices: []string{"/dev/dri:/dev/dri:bad"}}, {Devices: []string{"nvidia.com/gpu=all"}}, + {DeviceRequests: []model.DeviceRequest{{Count: -2, Capabilities: [][]string{{"gpu"}}}}}, + {DeviceRequests: []model.DeviceRequest{{Count: 1, DeviceIDs: []string{"GPU-test"}, Capabilities: [][]string{{"gpu"}}}}}, + {DeviceRequests: []model.DeviceRequest{{Count: 1}}}, + {DeviceRequests: []model.DeviceRequest{{Count: 1, Capabilities: [][]string{{}}}}}, + {DeviceRequests: []model.DeviceRequest{{DeviceIDs: []string{"x", "x"}, Capabilities: [][]string{{"gpu"}}}}}, + } + for i, fields := range cases { + if setupFieldsHostConfig(fields, &container.HostConfig{}) == nil { + t.Errorf("case %d accepted", i) + } + } + for _, spec := range []string{"/dev/dri", "/dev/dri:rw", "/dev/dri:/dev/dri:rw"} { + if err := setupFieldsHostConfig(model.InstanceFields{Devices: []string{spec}}, &container.HostConfig{}); err != nil { + t.Errorf("%s: %v", spec, err) + } + } +} diff --git a/core/hostinfo/hardware.go b/core/hostinfo/hardware.go new file mode 100644 index 000000000..3b3f191b9 --- /dev/null +++ b/core/hostinfo/hardware.go @@ -0,0 +1,48 @@ +package hostinfo + +import ( + "context" + "sort" + "time" + + "github.com/moby/moby/api/types/system" +) + +type hardwareDockerClient interface { + Info(context.Context) (system.Info, error) +} + +// HardwareCollector reports capabilities, not reservations or GPU health. +// It does not execute vendor utilities or modify the host configuration. +type HardwareCollector struct { + DockerClient hardwareDockerClient +} + +func (h HardwareCollector) KeyName() string { return "hardwareInfo" } +func (h HardwareCollector) GetLabels(string) (map[string]string, error) { + return map[string]string{}, nil +} + +func (h HardwareCollector) GetData() (map[string]interface{}, error) { + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + info, err := h.DockerClient.Info(ctx) + if err != nil { + return map[string]interface{}{"status": "unavailable"}, err + } + runtimes := make([]string, 0, len(info.Runtimes)) + for name := range info.Runtimes { + runtimes = append(runtimes, name) + } + sort.Strings(runtimes) + devices, scanErr := discoverHardware("/") + status := "available" + if scanErr != nil { + status = "partial" + } + return map[string]interface{}{ + "status": status, "collectedAt": time.Now().UTC().Format(time.RFC3339), + "runtimes": runtimes, "defaultRuntime": info.DefaultRuntime, + "devices": devices, "deviceRequestsSupported": true, + }, scanErr +} diff --git a/core/hostinfo/hardware_linux.go b/core/hostinfo/hardware_linux.go new file mode 100644 index 000000000..54d94c090 --- /dev/null +++ b/core/hostinfo/hardware_linux.go @@ -0,0 +1,100 @@ +//go:build linux + +package hostinfo + +import ( + "fmt" + "os" + "path/filepath" + "regexp" + "sort" + "strings" + "syscall" +) + +var renderNodeName = regexp.MustCompile(`^(renderD|card)[0-9]+$`) + +func readHardwareText(root string, path ...string) string { + data, err := os.ReadFile(filepath.Join(append([]string{root}, path...)...)) + if err != nil { + return "" + } + return strings.TrimSpace(string(data)) +} + +func hardwareNode(root, path string) (map[string]interface{}, bool) { + info, err := os.Stat(filepath.Join(root, path)) + if err != nil || info.Mode()&os.ModeCharDevice == 0 { + return nil, false + } + stat, ok := info.Sys().(*syscall.Stat_t) + if !ok { + return nil, false + } + return map[string]interface{}{"path": path, "groupId": stat.Gid}, true +} + +func discoverHardware(root string) ([]map[string]interface{}, error) { + devices := []map[string]interface{}{} + entries, err := os.ReadDir(filepath.Join(root, "sys/class/drm")) + if err != nil && !os.IsNotExist(err) { + return devices, err + } + for _, entry := range entries { + if !renderNodeName.MatchString(entry.Name()) { + continue + } + node, ok := hardwareNode(root, "/dev/dri/"+entry.Name()) + if !ok { + continue + } + base := "sys/class/drm/" + entry.Name() + "/device" + vendorID := readHardwareText(root, base, "vendor") + vendor := map[string]string{"0x8086": "Intel", "0x1002": "AMD", "0x10de": "NVIDIA", "0x15ad": "VMware"}[vendorID] + if vendor == "" { + vendor = "GPU" + } + pci, _ := filepath.EvalSymlinks(filepath.Join(root, base)) + driver, _ := os.Readlink(filepath.Join(root, base, "driver")) + node["kind"], node["vendor"], node["vendorId"] = "drm", vendor, vendorID + if pci != "" { + node["pciAddress"] = filepath.Base(pci) + } + if driver != "" { + node["driver"] = filepath.Base(driver) + } + node["name"] = fmt.Sprintf("%s %s ยท %s", vendor, readHardwareText(root, base, "device"), entry.Name()) + devices = append(devices, node) + } + if node, ok := hardwareNode(root, "/dev/kfd"); ok { + node["kind"], node["vendor"], node["name"] = "kfd", "AMD", "AMD ROCm /dev/kfd" + devices = append(devices, node) + } + // NVIDIA's kernel driver provides stable UUIDs without requiring nvidia-smi + // to be installed in the agent image. The runtime is reported separately. + gpus, gpuErr := os.ReadDir(filepath.Join(root, "proc/driver/nvidia/gpus")) + if gpuErr != nil && !os.IsNotExist(gpuErr) { + return devices, gpuErr + } + for _, gpu := range gpus { + if !gpu.IsDir() { + continue + } + values := map[string]string{} + for _, line := range strings.Split(readHardwareText(root, "proc/driver/nvidia/gpus", gpu.Name(), "information"), "\n") { + pair := strings.SplitN(line, ":", 2) + if len(pair) == 2 { + values[strings.TrimSpace(pair[0])] = strings.TrimSpace(pair[1]) + } + } + if !strings.HasPrefix(values["GPU UUID"], "GPU-") { + continue + } + devices = append(devices, map[string]interface{}{ + "kind": "nvidia", "vendor": "NVIDIA", "name": values["Model"], + "id": values["GPU UUID"], "pciAddress": gpu.Name(), + }) + } + sort.Slice(devices, func(i, j int) bool { return fmt.Sprint(devices[i]["name"]) < fmt.Sprint(devices[j]["name"]) }) + return devices, nil +} diff --git a/core/hostinfo/hardware_linux_test.go b/core/hostinfo/hardware_linux_test.go new file mode 100644 index 000000000..6e0ea0667 --- /dev/null +++ b/core/hostinfo/hardware_linux_test.go @@ -0,0 +1,53 @@ +//go:build linux + +package hostinfo + +import ( + "os" + "path/filepath" + "testing" +) + +func TestHardwareDiscoveryUsesIdentityNotDeviceGuess(t *testing.T) { + root := t.TempDir() + write := func(name, text string) { + path := filepath.Join(root, name) + if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(text), 0600); err != nil { + t.Fatal(err) + } + } + write("sys/class/drm/renderD128/device/vendor", "0x15ad") + write("sys/class/drm/renderD128/device/device", "0x0405") + if err := os.MkdirAll(filepath.Join(root, "dev/dri"), 0700); err != nil { + t.Fatal(err) + } + if err := os.Symlink("/dev/null", filepath.Join(root, "dev/dri/renderD128")); err != nil { + t.Fatal(err) + } + write("proc/driver/nvidia/gpus/0000:01:00.0/information", "Model: Test GPU\nGPU UUID: GPU-stable-id\n") + devices, err := discoverHardware(root) + if err != nil { + t.Fatal(err) + } + if len(devices) != 2 { + t.Fatalf("expected DRM and NVIDIA devices: %+v", devices) + } + for _, device := range devices { + if device["kind"] == "drm" && (device["vendor"] != "VMware" || device["path"] != "/dev/dri/renderD128" || device["groupId"] == nil) { + t.Fatalf("wrong DRM identity: %+v", device) + } + if device["kind"] == "nvidia" && device["id"] != "GPU-stable-id" { + t.Fatal("NVIDIA identity lost") + } + } +} + +func TestHardwareDiscoveryMissingDevicesIsNotGPUCapability(t *testing.T) { + devices, err := discoverHardware(t.TempDir()) + if err != nil || len(devices) != 0 { + t.Fatalf("missing hardware: %v %v", devices, err) + } +} diff --git a/core/hostinfo/hardware_other.go b/core/hostinfo/hardware_other.go new file mode 100644 index 000000000..44f78e306 --- /dev/null +++ b/core/hostinfo/hardware_other.go @@ -0,0 +1,9 @@ +//go:build !linux + +package hostinfo + +import "errors" + +func discoverHardware(string) ([]map[string]interface{}, error) { + return []map[string]interface{}{}, errors.New("Linux GPU discovery is not available on this OS") +} diff --git a/docs/releases/node-agent-0.13.24.md b/docs/releases/node-agent-0.13.24.md new file mode 100644 index 000000000..24cfa8336 --- /dev/null +++ b/docs/releases/node-agent-0.13.24.md @@ -0,0 +1,13 @@ +# Node Agent v0.13.24 + +Add Linux host runtime and accelerator inventory through the existing host-info +collector. Report NVIDIA UUIDs and DRM/KFD device nodes with their actual groups; +do not install drivers or invoke vendor utilities. + +Preserve LaunchConfig runtime and GPU DeviceRequests through Docker create and +inspect. Reject malformed device bindings and contradictory shared-memory/IPC +settings. Hardware options never implicitly enable privileged mode or host IPC. + +Windows packaging remains supported; these new resource controls target Linux. +Inventory is not proof of CUDA, ROCm or media-workload compatibility. GPU count +controls visibility, not exclusive reservation or scheduling quotas. diff --git a/handlers/common.go b/handlers/common.go index fadff9837..adb90dfdf 100755 --- a/handlers/common.go +++ b/handlers/common.go @@ -135,6 +135,7 @@ func initializeHandlers() *Handler { storageCache := cache.New(5*time.Minute, 30*time.Second) cache := cache.New(5*time.Minute, 30*time.Second) Collectors := []hostinfo.Collector{ + hostinfo.HardwareCollector{DockerClient: clientWithTimeout}, hostinfo.CPUCollector{}, hostinfo.DiskCollector{ Unit: 1048576, diff --git a/model/instance.go b/model/instance.go index 1fa78e18c..81d96c891 100644 --- a/model/instance.go +++ b/model/instance.go @@ -172,6 +172,7 @@ type Resources struct { CpusetCpus string // CpusetCpus 0-2, 0,1 CpusetMems string // CpusetMems 0-2, 0,1 Devices []container.DeviceMapping // List of devices to map inside the container + DeviceRequests []container.DeviceRequest // Preserve GPU identities in Docker inspect round trips DiskQuota int64 // Disk limit (in bytes) KernelMemory int64 // Kernel memory limit (in bytes) MemoryReservation int64 // Memory soft limit (in bytes) @@ -201,8 +202,10 @@ type InstanceFields struct { LogConfig LogConfig SecurityOpt []string Devices []string - DNS []string `json:"dns"` - DNSSearch []string `json:"dnsSearch"` + Runtime string `json:"runtime"` + DeviceRequests []DeviceRequest `json:"deviceRequests"` + DNS []string `json:"dns"` + DNSSearch []string `json:"dnsSearch"` CapAdd []string CapDrop []string RestartPolicy container.RestartPolicy @@ -264,6 +267,16 @@ type LogConfig struct { Config map[string]string } +// DeviceRequest uses the LaunchConfig API spelling, independently of Docker's +// capitalized wire format. Count and DeviceIDs are mutually exclusive. +type DeviceRequest struct { + Driver string `json:"driver,omitempty"` + Count int `json:"count,omitempty"` + DeviceIDs []string `json:"deviceIds,omitempty"` + Capabilities [][]string `json:"capabilities"` + Options map[string]string `json:"options,omitempty"` +} + type DeviceOptions struct { Weight uint16 ReadIops uint64 diff --git a/third_party/go-rancher/v2/device_request.go b/third_party/go-rancher/v2/device_request.go new file mode 100644 index 000000000..df2a1c663 --- /dev/null +++ b/third_party/go-rancher/v2/device_request.go @@ -0,0 +1,10 @@ +package client + +// DeviceRequest is the LaunchConfig hardware contract, not a scheduler reservation. +type DeviceRequest struct { + Driver string `json:"driver,omitempty" yaml:"driver,omitempty"` + Count int `json:"count,omitempty" yaml:"count,omitempty"` + DeviceIDs []string `json:"deviceIds,omitempty" yaml:"device_ids,omitempty"` + Capabilities [][]string `json:"capabilities" yaml:"capabilities"` + Options map[string]string `json:"options,omitempty" yaml:"options,omitempty"` +} diff --git a/third_party/go-rancher/v2/generated_container.go b/third_party/go-rancher/v2/generated_container.go index 3d58ff7d1..f6df24fd9 100644 --- a/third_party/go-rancher/v2/generated_container.go +++ b/third_party/go-rancher/v2/generated_container.go @@ -63,6 +63,10 @@ type Container struct { Devices []string `json:"devices,omitempty" yaml:"devices,omitempty"` + Runtime string `json:"runtime,omitempty" yaml:"runtime,omitempty"` + + DeviceRequests []DeviceRequest `json:"deviceRequests,omitempty" yaml:"device_requests,omitempty"` + DiskQuota int64 `json:"diskQuota,omitempty" yaml:"disk_quota,omitempty"` Dns []string `json:"dns,omitempty" yaml:"dns,omitempty"` diff --git a/third_party/go-rancher/v2/generated_launch_config.go b/third_party/go-rancher/v2/generated_launch_config.go index 2f3c95aec..60856f984 100644 --- a/third_party/go-rancher/v2/generated_launch_config.go +++ b/third_party/go-rancher/v2/generated_launch_config.go @@ -65,6 +65,10 @@ type LaunchConfig struct { Devices []string `json:"devices,omitempty" yaml:"devices,omitempty"` + Runtime string `json:"runtime,omitempty" yaml:"runtime,omitempty"` + + DeviceRequests []DeviceRequest `json:"deviceRequests,omitempty" yaml:"device_requests,omitempty"` + DiskQuota int64 `json:"diskQuota,omitempty" yaml:"disk_quota,omitempty"` Disks []VirtualMachineDisk `json:"disks,omitempty" yaml:"disks,omitempty"` diff --git a/third_party/go-rancher/v2/generated_secondary_launch_config.go b/third_party/go-rancher/v2/generated_secondary_launch_config.go index f233eb233..c0d53657f 100644 --- a/third_party/go-rancher/v2/generated_secondary_launch_config.go +++ b/third_party/go-rancher/v2/generated_secondary_launch_config.go @@ -65,6 +65,10 @@ type SecondaryLaunchConfig struct { Devices []string `json:"devices,omitempty" yaml:"devices,omitempty"` + Runtime string `json:"runtime,omitempty" yaml:"runtime,omitempty"` + + DeviceRequests []DeviceRequest `json:"deviceRequests,omitempty" yaml:"device_requests,omitempty"` + DiskQuota int64 `json:"diskQuota,omitempty" yaml:"disk_quota,omitempty"` Disks []VirtualMachineDisk `json:"disks,omitempty" yaml:"disks,omitempty"` From d39bd5bbdc524019e5145ea51f2469fcc92c70be Mon Sep 17 00:00:00 2001 From: chen21019 Date: Mon, 7 Sep 2026 22:08:43 +0800 Subject: [PATCH 2/4] test: separate valid private shm and explicit host IPC fixtures --- handlers/compute_unix_test.go | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/handlers/compute_unix_test.go b/handlers/compute_unix_test.go index 0dbe99f00..a73f373b5 100755 --- a/handlers/compute_unix_test.go +++ b/handlers/compute_unix_test.go @@ -88,7 +88,9 @@ func (s *ComputeTestSuite) TestNewFields(c *check.C) { fields["shmSize"] = 67108864 fields["groupAdd"] = []string{"root"} fields["uts"] = "host" - fields["ipcMode"] = "host" + // A dedicated shared-memory limit requires a private IPC namespace. + // Host IPC is covered separately without a contradictory shmSize. + fields["ipcMode"] = "private" fields["stopSignal"] = "SIGTERM" fields["ulimits"] = []map[string]interface{}{ { @@ -127,7 +129,7 @@ func (s *ComputeTestSuite) TestNewFields(c *check.C) { c.Assert(inspect.HostConfig.ShmSize, check.Equals, int64(67108864)) c.Assert(inspect.HostConfig.GroupAdd, check.DeepEquals, []string{"root"}) c.Assert(string(inspect.HostConfig.UTSMode), check.Equals, "host") - c.Assert(string(inspect.HostConfig.IpcMode), check.Equals, "host") + c.Assert(string(inspect.HostConfig.IpcMode), check.Equals, "private") c.Assert(inspect.Config.StopSignal, check.Equals, "SIGTERM") ulimits := []units.Ulimit{ { @@ -139,6 +141,19 @@ func (s *ComputeTestSuite) TestNewFields(c *check.C) { c.Assert(*(inspect.HostConfig.Ulimits[0]), check.DeepEquals, ulimits[0]) } +func (s *ComputeTestSuite) TestHostIPCWithoutShmSize(c *check.C) { + deleteContainer("/c861f990-4472-4fa1-960f-65171b544c28") + event, _, fields := unmarshalEventAndInstanceFields(loadEvent("./test_events/instance_activate_basic", c), c) + fields["ipcMode"] = "host" + delete(fields, "shmSize") + reply := testEvent(marshalEvent(event, c), c) + created, ok := utils.GetFieldsIfExist(reply.Data, "instanceHostMap", "instance", "+data", "dockerContainer") + c.Assert(ok, check.Equals, true) + inspect, err := docker.GetClient(docker.DefaultVersion).ContainerInspect(context.Background(), created.(types.Container).ID) + c.Assert(err, check.IsNil) + c.Assert(string(inspect.HostConfig.IpcMode), check.Equals, "host") +} + func (s *ComputeTestSuite) TestDNSFields(c *check.C) { // this test aims to verify that if the dnsSearch is set to rancher.internal, we should add dnssearch from host to // containers From 5f6c975dba961e10470394313a334527b43d01b5 Mon Sep 17 00:00:00 2001 From: chen21019 Date: Mon, 7 Sep 2026 22:16:47 +0800 Subject: [PATCH 3/4] test: compare capability identity across Docker API spellings --- tests/tests/test_docker.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/tests/tests/test_docker.py b/tests/tests/test_docker.py index 194f40f6e..e875ef911 100644 --- a/tests/tests/test_docker.py +++ b/tests/tests/test_docker.py @@ -1101,10 +1101,18 @@ def post(req, resp, valid_resp): instance_activate_assert_host_config(resp) instance_data = resp['data']['instanceHostMap']['instance']['+data'] docker_inspect = instance_data['dockerInspect'] - set_actual_cap_add = set(docker_inspect['HostConfig']['CapAdd']) + # Docker 29 canonicalizes capability names with the CAP_ prefix. + # Compare the exact capability set, accepting either API spelling. + set_actual_cap_add = { + name.removeprefix('CAP_') + for name in docker_inspect['HostConfig']['CapAdd'] + } set_expected_cap_add = set(["MKNOD", "SYS_ADMIN"]) assert set_actual_cap_add == set_expected_cap_add - set_actual_cap_drop = set(docker_inspect['HostConfig']['CapDrop']) + set_actual_cap_drop = { + name.removeprefix('CAP_') + for name in docker_inspect['HostConfig']['CapDrop'] + } set_expected_cap_drop = set(["MKNOD", "SYS_ADMIN"]) assert set_actual_cap_drop == set_expected_cap_drop container_field_test_boiler_plate(resp) From f168d3d02aff13420c7af3606fb8a66dbf1ec54b Mon Sep 17 00:00:00 2001 From: chen21019 Date: Mon, 7 Sep 2026 22:24:27 +0800 Subject: [PATCH 4/4] style: format hardware HostConfig integration --- core/compute/compute_unix.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/compute/compute_unix.go b/core/compute/compute_unix.go index be11c7437..9b5278b8e 100755 --- a/core/compute/compute_unix.go +++ b/core/compute/compute_unix.go @@ -445,7 +445,7 @@ func setupHardware(fields model.InstanceFields, hostConfig *container.HostConfig for _, device := range fields.Devices { parts := strings.Split(device, ":") if len(parts) > 3 || !path.IsAbs(parts[0]) || path.Clean(parts[0]) != parts[0] { - return errors.New("devices requires an absolute host path, optionally :container-path[:rwm]; CDI identifiers are not supported") + return errors.New("devices requires an absolute host path, optionally :container-path[:rwm]; CDI identifiers are not supported") } target, permissions := parts[0], "rwm" if len(parts) >= 2 {