diff --git a/charts/sourcegraph-executor/dind/README.md b/charts/sourcegraph-executor/dind/README.md index b6f5f8a87..3aa533dba 100644 --- a/charts/sourcegraph-executor/dind/README.md +++ b/charts/sourcegraph-executor/dind/README.md @@ -63,11 +63,13 @@ In addition to the documented values, the `executor` and `private-docker-registr | executor.image.defaultTag | string | `"6.0.0@sha256:0be94a7c91f8273db10fdf46718c6596340ab2acc570e7b85353806e67a27508"` | | | executor.image.name | string | `"executor"` | | | executor.replicaCount | int | `1` | | +| executor.resources | object | `{}` | Resource requests and limits for the executor container. Each queue can override this with its own resources field. | | privateDockerRegistry.enabled | bool | `true` | Whether to deploy the private registry. Only one registry is needed when deploying multiple executors. More information: https://docs.sourcegraph.com/admin/executors/deploy_executors#using-private-registries | | privateDockerRegistry.image.registry | string | `"index.docker.io"` | | -| privateDockerRegistry.image.repository | string | `"docker/regisry"` | | -| privateDockerRegistry.image.tag | int | `2` | | +| privateDockerRegistry.image.repository | string | `"registry"` | | +| privateDockerRegistry.image.tag | int | `3` | | | privateDockerRegistry.storageSize | string | `"10Gi"` | | +| queues | list | `[]` | Optional list of queues to deploy as standalone Deployments. When set, the single executor Deployment is not rendered. Each entry supports: name (required), replicaCount, resources, env (merged with executor.env, queue overrides). | | sourcegraph.affinity | object | `{}` | Affinity, learn more from the [Kubernetes documentation](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity) | | sourcegraph.image.defaultTag | string | `"{{ .Chart.AppVersion }}"` | Global docker image tag | | sourcegraph.image.pullPolicy | string | `"IfNotPresent"` | Global docker image pull policy | diff --git a/charts/sourcegraph-executor/dind/templates/executor/docker-daemon.ConfigMap.yaml b/charts/sourcegraph-executor/dind/templates/executor/docker-daemon.ConfigMap.yaml index f927fcf8f..b06f81105 100644 --- a/charts/sourcegraph-executor/dind/templates/executor/docker-daemon.ConfigMap.yaml +++ b/charts/sourcegraph-executor/dind/templates/executor/docker-daemon.ConfigMap.yaml @@ -1,4 +1,4 @@ -{{- if .Values.executor.enabled -}} +{{- if or .Values.queues .Values.executor.enabled -}} apiVersion: v1 data: daemon.json: | diff --git a/charts/sourcegraph-executor/dind/templates/executor/executor.Deployment.yaml b/charts/sourcegraph-executor/dind/templates/executor/executor.Deployment.yaml index 069481490..6bbf75a1d 100644 --- a/charts/sourcegraph-executor/dind/templates/executor/executor.Deployment.yaml +++ b/charts/sourcegraph-executor/dind/templates/executor/executor.Deployment.yaml @@ -1,7 +1,179 @@ -{{- if .Values.executor.enabled -}} +{{- if .Values.queues }} +{{- range .Values.queues }} +{{- $queue := . }} +{{- $mergedEnv := mergeOverwrite (deepCopy $.Values.executor.env) ($queue.env | default dict) }} +{{- $replicaCount := ($queue.replicaCount | default $.Values.executor.replicaCount) }} +{{- $resources := ($queue.resources | default $.Values.executor.resources) }} +--- apiVersion: apps/v1 kind: Deployment metadata: + name: executor-{{ $queue.name }} + annotations: + description: Runs sourcegraph executors + kubectl.kubernetes.io/default-container: executor + labels: + {{- include "sourcegraph.labels" $ | nindent 4 }} + {{- if $.Values.executor.labels }} + {{- toYaml $.Values.executor.labels | nindent 4 }} + {{- end }} + app: executor-{{ $queue.name }} + deploy: sourcegraph + sourcegraph-resource-requires: no-cluster-admin + app.kubernetes.io/component: executor +spec: + selector: + matchLabels: + {{- include "sourcegraph.selectorLabels" $ | nindent 6 }} + app: executor-{{ $queue.name }} + minReadySeconds: 10 + replicas: {{ $replicaCount }} + revisionHistoryLimit: 10 + strategy: + rollingUpdate: + maxSurge: 1 + maxUnavailable: 1 + type: RollingUpdate + template: + metadata: + annotations: + kubectl.kubernetes.io/default-container: executor + {{- if $.Values.sourcegraph.podAnnotations }} + {{- toYaml $.Values.sourcegraph.podAnnotations | nindent 8 }} + {{- end }} + {{- if $.Values.executor.podAnnotations }} + {{- toYaml $.Values.executor.podAnnotations | nindent 8 }} + {{- end }} + labels: + {{- include "sourcegraph.selectorLabels" $ | nindent 8 }} + {{- if $.Values.sourcegraph.podLabels }} + {{- toYaml $.Values.sourcegraph.podLabels | nindent 8 }} + {{- end }} + {{- if $.Values.executor.podLabels }} + {{- toYaml $.Values.executor.podLabels | nindent 8 }} + {{- end }} + app: executor-{{ $queue.name }} + deploy: sourcegraph + sourcegraph-resource-requires: no-cluster-admin + app.kubernetes.io/component: executor + spec: + containers: + - name: executor + image: {{ include "sourcegraph.image" (list $ "executor") }} + imagePullPolicy: {{ $.Values.sourcegraph.image.pullPolicy }} + livenessProbe: + httpGet: + path: /healthz + port: http-debug + scheme: HTTP + initialDelaySeconds: 60 + timeoutSeconds: 5 + readinessProbe: + httpGet: + path: /healthz + port: http-debug + scheme: HTTP + periodSeconds: 5 + timeoutSeconds: 5 + ports: + - name: http-debug + containerPort: 8080 + terminationMessagePolicy: FallbackToLogsOnError + env: + {{- range $name, $item := $mergedEnv }} + - name: {{ $name }} + {{- $item | toYaml | nindent 14 }} + {{- end }} + - name: EXECUTOR_USE_FIRECRACKER + value: "false" + - name: EXECUTOR_HEALTH_SERVER_ADDR + value: ":8080" + - name: EXECUTOR_JOB_NUM_CPUS + value: "0" + - name: EXECUTOR_JOB_MEMORY + value: "0" + - name: DOCKER_HOST + value: tcp://localhost:2375 + - name: TMPDIR + value: /scratch + - name: EXECUTOR_USE_KUBERNETES + value: "false" + volumeMounts: + - mountPath: /scratch + name: executor-scratch + {{- with $resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + - name: dind + image: "{{ $.Values.dind.image.registry}}/{{ $.Values.dind.image.repository}}:{{ $.Values.dind.image.tag}}" + imagePullPolicy: {{ $.Values.sourcegraph.image.pullPolicy }} + securityContext: + privileged: true + command: + - 'dockerd' + - '--tls=false' + - '--mtu=1200' + - '--registry-mirror=http://private-docker-registry:5000' + - '--host=tcp://0.0.0.0:2375' + livenessProbe: + tcpSocket: + port: 2375 + initialDelaySeconds: 5 + periodSeconds: 5 + failureThreshold: 5 + readinessProbe: + tcpSocket: + port: 2375 + initialDelaySeconds: 10 + periodSeconds: 5 + failureThreshold: 5 + env: + - name: POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + ports: + - containerPort: 2375 + protocol: TCP + volumeMounts: + - mountPath: /scratch + name: executor-scratch + - mountPath: /etc/docker/daemon.json + subPath: daemon.json + name: docker-config + enableServiceLinks: false + {{- with $.Values.sourcegraph.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with $.Values.sourcegraph.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with include "sourcegraph.priorityClassName" (list $ "executor") | trim }}{{ . | nindent 6 }}{{- end }} + {{- with $.Values.sourcegraph.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with $.Values.sourcegraph.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + - name: executor-scratch + emptyDir: {} + - name: docker-config + configMap: + defaultMode: 420 + name: docker-config +{{- end }} +{{- else if .Values.executor.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "executor.name" . }} annotations: description: Runs sourcegraph executors kubectl.kubernetes.io/default-container: executor @@ -57,14 +229,14 @@ spec: timeoutSeconds: 5 readinessProbe: httpGet: - path: /ready + path: /healthz port: http-debug scheme: HTTP periodSeconds: 5 timeoutSeconds: 5 ports: - name: http-debug - containerPort: 6060 + containerPort: 8080 terminationMessagePolicy: FallbackToLogsOnError env: {{- range $name, $item := .Values.executor.env }} @@ -73,6 +245,8 @@ spec: {{- end }} - name: EXECUTOR_USE_FIRECRACKER value: "false" + - name: EXECUTOR_HEALTH_SERVER_ADDR + value: ":8080" - name: EXECUTOR_JOB_NUM_CPUS value: "0" - name: EXECUTOR_JOB_MEMORY @@ -81,9 +255,15 @@ spec: value: tcp://localhost:2375 - name: TMPDIR value: /scratch + - name: EXECUTOR_USE_KUBERNETES + value: "false" volumeMounts: - mountPath: /scratch name: executor-scratch + {{- with .Values.executor.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} - name: dind image: "{{ .Values.dind.image.registry}}/{{ .Values.dind.image.repository}}:{{ .Values.dind.image.tag}}" imagePullPolicy: {{ .Values.sourcegraph.image.pullPolicy }} @@ -93,7 +273,7 @@ spec: - 'dockerd' - '--tls=false' - '--mtu=1200' - - '--registry-mirror=http://executor:5000' + - '--registry-mirror=http://private-docker-registry:5000' - '--host=tcp://0.0.0.0:2375' livenessProbe: tcpSocket: @@ -122,6 +302,7 @@ spec: - mountPath: /etc/docker/daemon.json subPath: daemon.json name: docker-config + enableServiceLinks: false {{- with .Values.sourcegraph.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/charts/sourcegraph-executor/dind/templates/executor/executor.Service.yaml b/charts/sourcegraph-executor/dind/templates/executor/executor.Service.yaml deleted file mode 100644 index 970de1a9c..000000000 --- a/charts/sourcegraph-executor/dind/templates/executor/executor.Service.yaml +++ /dev/null @@ -1,26 +0,0 @@ -{{- if .Values.executor.enabled -}} -apiVersion: v1 -kind: Service -metadata: - annotations: - prometheus.io/port: "6060" - sourcegraph.prometheus/scrape: "true" - {{- if .Values.executor.serviceAnnotations }} - {{- toYaml .Values.executor.serviceAnnotations | nindent 4 }} - {{- end }} - labels: - {{- include "executor.labels" . | nindent 4 }} - {{- if .Values.executor.serviceLabels }} - {{- toYaml .Values.executor.serviceLabels | nindent 4 }} - {{- end }} - name: executor -spec: - ports: - - name: http-debug - port: 6060 - targetPort: http-debug - selector: - {{- include "sourcegraph.selectorLabels" . | nindent 4 }} - app: {{include "executor.name" . }} - type: {{ .Values.executor.serviceType | default "ClusterIP" }} -{{- end }} diff --git a/charts/sourcegraph-executor/dind/templates/private-docker-registry/private-docker-registry.Deployment.yaml b/charts/sourcegraph-executor/dind/templates/private-docker-registry/private-docker-registry.Deployment.yaml index 32554be69..fff6d3379 100644 --- a/charts/sourcegraph-executor/dind/templates/private-docker-registry/private-docker-registry.Deployment.yaml +++ b/charts/sourcegraph-executor/dind/templates/private-docker-registry/private-docker-registry.Deployment.yaml @@ -1,6 +1,6 @@ {{- if .Values.privateDockerRegistry.enabled -}} apiVersion: apps/v1 -kind: Deployment +kind: StatefulSet metadata: name: private-docker-registry labels: @@ -11,13 +11,14 @@ metadata: deploy: sourcegraph app.kubernetes.io/component: private-docker-registry spec: - replicas: {{ .Values.privateDockerRegistry.replicaCount }} + replicas: 1 + serviceName: private-docker-registry selector: matchLabels: {{- include "sourcegraph.selectorLabels" . | nindent 6 }} app: private-docker-registry - strategy: - type: Recreate + updateStrategy: + type: RollingUpdate template: metadata: annotations: @@ -83,8 +84,17 @@ spec: imagePullSecrets: {{- toYaml . | nindent 8 }} {{- end }} - volumes: - - name: cache - persistentVolumeClaim: - claimName: private-docker-registry + volumeClaimTemplates: + - metadata: + name: cache + labels: + deploy: sourcegraph + app.kubernetes.io/component: private-docker-registry + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: {{ .Values.privateDockerRegistry.storageSize }} + storageClassName: {{ .Values.storageClass.name }} {{- end }} diff --git a/charts/sourcegraph-executor/dind/templates/private-docker-registry/private-docker-registry.PersistentVolumeClaim.yaml b/charts/sourcegraph-executor/dind/templates/private-docker-registry/private-docker-registry.PersistentVolumeClaim.yaml deleted file mode 100644 index 619d5af9b..000000000 --- a/charts/sourcegraph-executor/dind/templates/private-docker-registry/private-docker-registry.PersistentVolumeClaim.yaml +++ /dev/null @@ -1,19 +0,0 @@ -{{- if .Values.privateDockerRegistry.enabled -}} -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - labels: - deploy: sourcegraph - app.kubernetes.io/component: private-docker-registry - name: private-docker-registry -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: {{ .Values.privateDockerRegistry.storageSize }} - storageClassName: {{ .Values.storageClass.name }} - {{- if .Values.privateDockerRegistry.volumeName }} - volumeName: {{ .Values.privateDockerRegistry.volumeName }} - {{- end }} -{{- end }} diff --git a/charts/sourcegraph-executor/dind/values.yaml b/charts/sourcegraph-executor/dind/values.yaml index eec0a03c1..9b2d92fce 100644 --- a/charts/sourcegraph-executor/dind/values.yaml +++ b/charts/sourcegraph-executor/dind/values.yaml @@ -54,8 +54,40 @@ storageClass: # learn more from the [Kubernetes documentation](https://kubernetes.io/docs/concepts/storage/storage-classes/#allowed-topologies) allowedTopologies: {} +# -- Optional list of queues to deploy as standalone Deployments. +# When set, the single executor Deployment is not rendered. +# Each entry supports: name (required), replicaCount, resources, env (merged with executor.env, queue overrides). +queues: [] +# - name: codeintel +# replicaCount: 2 +# resources: +# requests: +# cpu: "2" +# memory: 4Gi +# limits: +# cpu: "4" +# memory: 8Gi +# env: +# EXECUTOR_QUEUE_NAME: +# value: codeintel +# - name: batches +# replicaCount: 1 +# resources: +# requests: +# cpu: "1" +# memory: 2Gi +# limits: +# cpu: "2" +# memory: 4Gi +# env: +# EXECUTOR_QUEUE_NAME: +# value: batches + executor: enabled: true + # -- Resource requests and limits for the executor container. + # Each queue can override this with its own resources field. + resources: {} image: defaultTag: 6.0.0@sha256:0be94a7c91f8273db10fdf46718c6596340ab2acc570e7b85353806e67a27508 name: "executor" @@ -86,6 +118,6 @@ privateDockerRegistry: enabled: true image: registry: index.docker.io - repository: docker/regisry - tag: 2 + repository: registry + tag: 3 storageSize: 10Gi