Skip to content

Commit cc15cff

Browse files
committed
chore: publish sources to opensource
1 parent 2b227b0 commit cc15cff

File tree

20 files changed

+1718
-0
lines changed

20 files changed

+1718
-0
lines changed

Dockerfile

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
FROM golang:1.18 as build
2+
3+
WORKDIR /go/app
4+
COPY . /go/app
5+
6+
RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o ydb-disk-manager cmd/ydb-disk-manager/main.go
7+
8+
FROM ubuntu:20.04
9+
10+
WORKDIR /root
11+
12+
COPY --from=build /go/app/ydb-disk-manager /usr/bin/ydb-disk-manager
13+
14+
ENTRYPOINT ["/usr/bin/ydb-disk-manager"]

Makefile

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
image_name = cr.yandex/crpl7ipeu79oseqhcgn2/ydb-disk-manager
2+
image_version = $(shell grep appVersion helm/ydb-disk-manager/Chart.yaml | cut -d ' ' -f 2)
3+
charts_registry = cr.yandex/crpl7ipeu79oseqhcgn2/charts
4+
helm_version = $(shell grep version helm/ydb-disk-manager/Chart.yaml | cut -d ' ' -f 2)
5+
image = $(image_name):$(image_version)
6+
7+
SHELL = /usr/bin/env bash -o pipefail
8+
.SHELLFLAGS = -ec
9+
10+
.PHONY: all
11+
all: build push clean
12+
13+
##@ General
14+
15+
.PHONY: help
16+
help:
17+
@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m<target>\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)
18+
19+
##@ Development
20+
21+
.PHONY: fmt
22+
fmt:
23+
go fmt ./...
24+
25+
.PHONY: vet
26+
vet:
27+
go vet ./...
28+
29+
##@ Build
30+
31+
.PHONY: build
32+
build: fmt vet
33+
docker build -t $(image) -f Dockerfile .
34+
helm package helm/ydb-disk-manager
35+
36+
##@ Deployment
37+
38+
.PHONY: push
39+
push:
40+
docker push $(image)
41+
helm push ydb-disk-manager-$(helm_version).tgz oci://$(charts_registry)/
42+
43+
.PHONY: clean
44+
clean:
45+
rm ydb-disk-manager-$(helm_version).tgz
46+
docker rmi $(image)

README.md

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
## ydb-disk-manager
2+
3+
This is the helper process that solves a Kubernetes-specific problem.
4+
It is intended to be used as a DaemonSet in Kubernetes installations of YDB to make sure YDB has access to disks instead of EPERM.
5+
6+
The codebase is based upon `smarter-device-manager` with vital modifications.
7+
8+
#### ListAndWatch:
9+
10+
The only device advertised to kubelet is `ydb-disk-manager/hostdev`, which is a metadevice that is considered to be always present
11+
on every node (since `/dev` path is always present on every node)
12+
13+
#### Allocate:
14+
15+
`Allocate` response returns a series of disks instead the metadevice `ydb-disk-manager/hostdev` that was allocated. Here, we abuse kubelet
16+
behaviour - kubelet will silently swallow every device provided, will not check if it is the same device that it requested, and will propagate it
17+
down to the container runtime.

cmd/ydb-disk-manager/main.go

Lines changed: 193 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,193 @@
1+
package main
2+
3+
import (
4+
"flag"
5+
"fmt"
6+
"io/ioutil"
7+
"net/http"
8+
"os"
9+
"os/signal"
10+
"syscall"
11+
"time"
12+
13+
"github.com/fsnotify/fsnotify"
14+
"github.com/ydb-platform/ydb-disk-manager/internal/hostdev"
15+
"github.com/ydb-platform/ydb-disk-manager/pkg/api"
16+
"gopkg.in/yaml.v2"
17+
"k8s.io/klog/v2"
18+
pluginapi "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1"
19+
)
20+
21+
type Config struct {
22+
DiskMatch string `yaml:"diskMatch"`
23+
HostProcPath string `yaml:"hostProcPath"`
24+
UpdateInterval time.Duration `yaml:"updateInterval"`
25+
}
26+
27+
var cfg Config
28+
var confFileName string
29+
30+
func usage() {
31+
fmt.Fprintf(os.Stderr, "usage: ydb-disk-manager\n")
32+
flag.PrintDefaults()
33+
os.Exit(2)
34+
}
35+
36+
func main() {
37+
klog.InitFlags(nil)
38+
flag.StringVar(&confFileName, "config", "config/conf.yaml", "set the configuration file to use")
39+
flag.Usage = usage
40+
flag.Parse()
41+
42+
defer klog.Flush()
43+
klog.V(0).Info("Loading ydb-disk-manager")
44+
45+
// Setting up the disks to check
46+
klog.V(0).Infof("Reading configuration file %s", confFileName)
47+
yamlFile, err := ioutil.ReadFile(confFileName)
48+
if err != nil {
49+
klog.Fatal("Reading configuration file failed with: %v", err)
50+
}
51+
err = yaml.Unmarshal(yamlFile, &cfg)
52+
if err != nil {
53+
klog.Fatal("Unmarshal: %v", err)
54+
os.Exit(-1)
55+
}
56+
klog.V(0).Infof("Applied configuration: %v", cfg)
57+
58+
klog.V(0).Info("Starting FS watcher.")
59+
watcher, err := newFSWatcher(pluginapi.DevicePluginPath)
60+
if err != nil {
61+
klog.Error("Failed to create FS watcher.")
62+
os.Exit(1)
63+
}
64+
defer watcher.Close()
65+
66+
klog.V(0).Info("Starting OS watcher.")
67+
sigs := newOSWatcher(syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT)
68+
69+
klog.V(0).Info("Starting /healthz HTTP handler on port :8080")
70+
http.HandleFunc("/healthz", healthHandler)
71+
go func() {
72+
if err = http.ListenAndServe(":8080", nil); err != nil {
73+
klog.Error("Failed to create HTTP health server")
74+
os.Exit(1)
75+
}
76+
}()
77+
78+
runWatcherLoop(watcher, sigs)
79+
}
80+
81+
func runWatcherLoop(watcher *fsnotify.Watcher, sigs chan os.Signal) {
82+
var diskManager *hostdev.DiskManager
83+
var devicePluginInstance *api.DevicePlugin
84+
ticker := time.NewTicker(cfg.UpdateInterval)
85+
tickerCh := make(chan bool)
86+
defer close(tickerCh)
87+
restart := true
88+
defer ticker.Stop()
89+
for {
90+
if restart {
91+
if devicePluginInstance != nil {
92+
devicePluginInstance.Stop()
93+
}
94+
95+
diskManager = hostdev.NewDiskmanager(tickerCh)
96+
devicePluginInstance = api.NewDevicePlugin(diskManager)
97+
98+
if err := diskManager.UpdateDisks(cfg.DiskMatch, "/dev"); err != nil {
99+
klog.V(0).Infof("Failed to update disks: %v", err)
100+
time.Sleep(cfg.UpdateInterval)
101+
continue
102+
}
103+
104+
if err := devicePluginInstance.Serve(); err != nil {
105+
klog.V(0).Info("Could not contact Kubelet, retrying. Did you enable the device plugin feature gate?")
106+
time.Sleep(cfg.UpdateInterval)
107+
continue
108+
}
109+
110+
if err := diskManager.UpdateLocks(cfg.HostProcPath); err != nil {
111+
klog.V(0).Infof("Failed to update locks: %v", err)
112+
time.Sleep(cfg.UpdateInterval)
113+
continue
114+
}
115+
116+
restart = false
117+
}
118+
119+
select {
120+
case <-tickerCh:
121+
klog.V(2).Infof("Resetting ticker to updateInterval: %s", cfg.UpdateInterval)
122+
ticker.Reset(cfg.UpdateInterval)
123+
124+
case event := <-watcher.Events:
125+
if event.Name == pluginapi.KubeletSocket && event.Op&fsnotify.Create == fsnotify.Create {
126+
klog.V(0).Infof("inotify: %s created, restarting.", pluginapi.KubeletSocket)
127+
restart = true
128+
}
129+
130+
case <-ticker.C:
131+
if err := diskManager.UpdateDisks(cfg.DiskMatch, "/dev"); err != nil {
132+
klog.V(0).Infof("Failed to update disks: %v", err)
133+
restart = true
134+
break
135+
}
136+
if err := diskManager.UpdateLocks(cfg.HostProcPath); err != nil {
137+
klog.V(0).Infof("Failed to update locks: %v", err)
138+
restart = true
139+
break
140+
}
141+
142+
case err := <-watcher.Errors:
143+
klog.V(0).Infof("inotify: %s", err)
144+
145+
case s := <-sigs:
146+
switch s {
147+
case syscall.SIGHUP:
148+
klog.V(0).Info("Received SIGHUP, restarting.")
149+
restart = true
150+
default:
151+
klog.V(0).Infof("Received signal \"%v\", shutting down.", s)
152+
if devicePluginInstance != nil {
153+
devicePluginInstance.Stop()
154+
}
155+
return
156+
}
157+
}
158+
}
159+
}
160+
161+
func newFSWatcher(files ...string) (*fsnotify.Watcher, error) {
162+
watcher, err := fsnotify.NewWatcher()
163+
if err != nil {
164+
return nil, err
165+
}
166+
167+
for _, f := range files {
168+
err = watcher.Add(f)
169+
if err != nil {
170+
watcher.Close()
171+
return nil, err
172+
}
173+
}
174+
175+
return watcher, nil
176+
}
177+
178+
func newOSWatcher(sigs ...os.Signal) chan os.Signal {
179+
sigChan := make(chan os.Signal, 1)
180+
signal.Notify(sigChan, sigs...)
181+
182+
return sigChan
183+
}
184+
185+
func healthHandler(w http.ResponseWriter, r *http.Request) {
186+
if err := api.LivenessProbe(api.SocketPath); err != nil {
187+
w.WriteHeader(http.StatusInternalServerError)
188+
fmt.Fprint(w, err)
189+
return
190+
}
191+
w.WriteHeader(http.StatusOK)
192+
fmt.Fprint(w, "Server is healthy")
193+
}

example.yaml

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
---
2+
# Source: ydb-disk-manager/templates/configmap.yaml
3+
apiVersion: v1
4+
kind: ConfigMap
5+
metadata:
6+
name: ydb-disk-manager-config
7+
data:
8+
conf.yaml: |
9+
diskMatch: "^sd[a-z][0-9]*$"
10+
hostProcPath: "/host/proc"
11+
updateInterval: "30s"
12+
---
13+
# Source: ydb-disk-manager/templates/daemonset.yaml
14+
apiVersion: apps/v1
15+
kind: DaemonSet
16+
metadata:
17+
name: ydb-disk-manager
18+
labels:
19+
helm.sh/chart: ydb-disk-manager-0.2.5
20+
app.kubernetes.io/name: ydb-disk-manager
21+
app.kubernetes.io/instance: ydb-disk-manager
22+
app.kubernetes.io/version: "0.2.5"
23+
app.kubernetes.io/managed-by: Helm
24+
spec:
25+
selector:
26+
matchLabels:
27+
app.kubernetes.io/name: ydb-disk-manager
28+
app.kubernetes.io/instance: ydb-disk-manager
29+
template:
30+
metadata:
31+
labels:
32+
app.kubernetes.io/name: ydb-disk-manager
33+
app.kubernetes.io/instance: ydb-disk-manager
34+
spec:
35+
priorityClassName: system-node-critical
36+
containers:
37+
- name: ydb-disk-manager
38+
securityContext:
39+
privileged: true
40+
image: "cr.yandex/crpl7ipeu79oseqhcgn2/ydb-disk-manager:0.2.5"
41+
imagePullPolicy: IfNotPresent
42+
resources:
43+
limits:
44+
memory: 50Mi
45+
requests:
46+
cpu: 50m
47+
memory: 10Mi
48+
env:
49+
- name: DP_DISABLE_PRE_START_CONTAINER
50+
value: "true"
51+
volumeMounts:
52+
- name: device-plugin
53+
mountPath: /var/lib/kubelet/device-plugins
54+
- name: proc-dir
55+
mountPath: /host/proc
56+
readOnly: true
57+
- name: dev-dir
58+
mountPath: /dev
59+
- name: sys-dir
60+
mountPath: /sys
61+
- name: config
62+
mountPath: /root/config
63+
livenessProbe:
64+
httpGet:
65+
path: /healthz
66+
port: 8080
67+
initialDelaySeconds: 15
68+
volumes:
69+
- name: device-plugin
70+
hostPath:
71+
path: /var/lib/kubelet/device-plugins
72+
- name: proc-dir
73+
hostPath:
74+
path: /proc
75+
- name: dev-dir
76+
hostPath:
77+
path: /dev
78+
- name: sys-dir
79+
hostPath:
80+
path: /sys
81+
- name: config
82+
configMap:
83+
name: ydb-disk-manager-config

go.mod

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
module github.com/ydb-platform/ydb-disk-manager
2+
3+
go 1.18
4+
5+
require (
6+
github.com/fsnotify/fsnotify v1.6.0
7+
golang.org/x/net v0.12.0
8+
google.golang.org/grpc v1.56.2
9+
google.golang.org/protobuf v1.31.0
10+
gopkg.in/yaml.v2 v2.4.0
11+
k8s.io/klog/v2 v2.70.1
12+
k8s.io/kubelet v0.25.3
13+
)
14+
15+
require (
16+
github.com/go-logr/logr v1.2.3 // indirect
17+
github.com/gogo/protobuf v1.3.2 // indirect
18+
github.com/golang/protobuf v1.5.3 // indirect
19+
github.com/kr/pretty v0.3.1 // indirect
20+
github.com/rogpeppe/go-internal v1.11.0 // indirect
21+
golang.org/x/sys v0.10.0 // indirect
22+
golang.org/x/text v0.11.0 // indirect
23+
google.golang.org/genproto/googleapis/rpc v0.0.0-20230724170836-66ad5b6ff146 // indirect
24+
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
25+
)

0 commit comments

Comments
 (0)