diff --git a/.github/workflows/bootstrap-argocd.yml b/.github/workflows/bootstrap-argocd.yml index 26b455b..f4d9cbf 100644 --- a/.github/workflows/bootstrap-argocd.yml +++ b/.github/workflows/bootstrap-argocd.yml @@ -1,4 +1,4 @@ -name: ArgoCD +name: ArgoCD Bootstrap on: push: @@ -10,31 +10,45 @@ on: permissions: id-token: write contents: read - pull-requests: write - issues: read - security-events: write - jobs: - bootstrap: - name: Bootstrap ${{ matrix.environment }} + changes: + runs-on: ubuntu-latest + + outputs: + dev: ${{ steps.filter.outputs.dev }} + staging: ${{ steps.filter.outputs.staging }} + prod: ${{ steps.filter.outputs.prod }} + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Detect changes + id: filter + uses: dorny/paths-filter@v3 + with: + filters: | + dev: + - 'argocd/dev/**' + staging: + - 'argocd/staging/**' + prod: + - 'argocd/prod/**' + + bootstrap-dev: + name: Bootstrap dev + needs: changes + if: needs.changes.outputs.dev == 'true' runs-on: ubuntu-latest - environment: ${{ matrix.environment }} - - strategy: - matrix: - include: - - environment: dev - cluster: eks-dev-cluster - - environment: staging - cluster: eks-staging-cluster - - environment: prod - cluster: eks-prod-cluster + environment: dev steps: - - uses: actions/checkout@v4 + - name: Checkout + uses: actions/checkout@v4 - - uses: aws-actions/configure-aws-credentials@v4 + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 with: role-to-assume: ${{ secrets.AWS_ROLE_ARN }} aws-region: us-east-1 @@ -48,24 +62,93 @@ jobs: run: | aws eks update-kubeconfig \ --region us-east-1 \ - --name ${{ matrix.cluster }} + --name eks-dev-cluster - - name: Apply Root Application + - name: Apply Argo CD root application run: | - kubectl apply -f argocd/${{ matrix.environment }}/root-app.yaml + kubectl apply \ + --namespace argocd \ + --filename argocd/dev/root-app.yaml - - name: Wait for Root Application + - name: Show Argo CD applications run: | - kubectl wait \ - --for=condition=Established \ - crd/applications.argoproj.io \ - --timeout=60s || true + kubectl get applications \ + --namespace argocd + + bootstrap-staging: + name: Bootstrap staging + needs: changes + if: needs.changes.outputs.staging == 'true' + runs-on: ubuntu-latest + environment: staging + + steps: + - name: Checkout + uses: actions/checkout@v4 - - name: Show Applications + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ secrets.AWS_ROLE_ARN }} + aws-region: us-east-1 + + - name: Install kubectl + uses: azure/setup-kubectl@v4 + with: + version: latest + + - name: Configure kubeconfig run: | - kubectl get applications -n argocd || true + aws eks update-kubeconfig \ + --region us-east-1 \ + --name eks-staging-cluster - - name: Show ApplicationSets + - name: Apply Argo CD root application run: | - kubectl get applicationsets -n argocd || true + kubectl apply \ + --namespace argocd \ + --filename argocd/staging/root-app.yaml + - name: Show Argo CD applications + run: | + kubectl get applications \ + --namespace argocd + + bootstrap-prod: + name: Bootstrap prod + needs: changes + if: needs.changes.outputs.prod == 'true' + runs-on: ubuntu-latest + environment: prod + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ secrets.AWS_ROLE_ARN }} + aws-region: us-east-1 + + - name: Install kubectl + uses: azure/setup-kubectl@v4 + with: + version: latest + + - name: Configure kubeconfig + run: | + aws eks update-kubeconfig \ + --region us-east-1 \ + --name eks-prod-cluster + + - name: Apply Argo CD root application + run: | + kubectl apply \ + --namespace argocd \ + --filename argocd/prod/root-app.yaml + + - name: Show Argo CD applications + run: | + kubectl get applications \ + --namespace argocd diff --git a/argocd/dev/applicationset-apps.yaml b/argocd/dev/applicationset-apps.yaml index a34c818..210bd02 100644 --- a/argocd/dev/applicationset-apps.yaml +++ b/argocd/dev/applicationset-apps.yaml @@ -30,11 +30,20 @@ spec: server: https://kubernetes.default.svc namespace: dev + revisionHistoryLimit: 5 + syncPolicy: automated: prune: true selfHeal: true + retry: + limit: 5 + backoff: + duration: 5s + factor: 2 + maxDuration: 3m + syncOptions: - CreateNamespace=true - \ No newline at end of file + \ No newline at end of file diff --git a/argocd/dev/applicationset-infra.yaml b/argocd/dev/applicationset-infra.yaml index 92b0a5d..e28122e 100644 --- a/argocd/dev/applicationset-infra.yaml +++ b/argocd/dev/applicationset-infra.yaml @@ -29,10 +29,20 @@ spec: server: https://kubernetes.default.svc namespace: '{{namespace}}' + revisionHistoryLimit: 5 + syncPolicy: automated: prune: true selfHeal: true + + retry: + limit: 5 + backoff: + duration: 5s + factor: 2 + maxDuration: 3m + syncOptions: - CreateNamespace=true - \ No newline at end of file + \ No newline at end of file diff --git a/argocd/dev/applicationset-monitoring.yaml b/argocd/dev/applicationset-monitoring.yaml index f472a17..051626e 100644 --- a/argocd/dev/applicationset-monitoring.yaml +++ b/argocd/dev/applicationset-monitoring.yaml @@ -29,8 +29,20 @@ spec: server: https://kubernetes.default.svc namespace: "{{namespace}}" + revisionHistoryLimit: 5 + syncPolicy: automated: prune: true - selfHeal: true - \ No newline at end of file + selfHeal: true + + retry: + limit: 5 + backoff: + duration: 5s + factor: 2 + maxDuration: 3m + + syncOptions: + - CreateNamespace=true + \ No newline at end of file diff --git a/argocd/dev/kustomization.yaml b/argocd/dev/kustomization.yaml index 7c79629..3a8f883 100644 --- a/argocd/dev/kustomization.yaml +++ b/argocd/dev/kustomization.yaml @@ -4,5 +4,5 @@ kind: Kustomization resources: - applicationset-apps.yaml - applicationset-infra.yaml - - applicationset-monitoring.yaml - \ No newline at end of file + - applicationset-monitoring.yaml + \ No newline at end of file diff --git a/argocd/dev/root-app.yaml b/argocd/dev/root-app.yaml index d315540..d58f6b4 100644 --- a/argocd/dev/root-app.yaml +++ b/argocd/dev/root-app.yaml @@ -20,5 +20,5 @@ spec: syncPolicy: automated: prune: true - selfHeal: true - \ No newline at end of file + selfHeal: true + \ No newline at end of file diff --git a/argocd/prod/applicationset-apps.yaml b/argocd/prod/applicationset-apps.yaml index 350b479..d4bd868 100644 --- a/argocd/prod/applicationset-apps.yaml +++ b/argocd/prod/applicationset-apps.yaml @@ -30,11 +30,20 @@ spec: server: https://kubernetes.default.svc namespace: prod + revisionHistoryLimit: 5 + syncPolicy: automated: prune: true selfHeal: true + retry: + limit: 5 + backoff: + duration: 5s + factor: 2 + maxDuration: 3m + syncOptions: - - CreateNamespace=true + - CreateNamespace=true \ No newline at end of file diff --git a/argocd/prod/applicationset-infra.yaml b/argocd/prod/applicationset-infra.yaml index 0ac0b6e..a99e19a 100644 --- a/argocd/prod/applicationset-infra.yaml +++ b/argocd/prod/applicationset-infra.yaml @@ -13,26 +13,35 @@ spec: path: infrastructure/overlay/prod namespace: prod - template: - metadata: - name: '{{name}}-prod' - - spec: - project: default - - source: - repoURL: https://github.com/Emmy-github-webdev/Kubernetes-argocd.git - targetRevision: main - path: '{{path}}' - - destination: - server: https://kubernetes.default.svc - namespace: '{{namespace}}' - - syncPolicy: - automated: - prune: true - selfHeal: true - syncOptions: - - CreateNamespace=true - \ No newline at end of file +template: + metadata: + name: '{{name}}-prod' + + spec: + project: default + + source: + repoURL: https://github.com/Emmy-github-webdev/Kubernetes-argocd.git + targetRevision: main + path: apps/{{name}}/overlays/prod + + destination: + server: https://kubernetes.default.svc + namespace: prod + + revisionHistoryLimit: 5 + + syncPolicy: + automated: + prune: true + selfHeal: true + + retry: + limit: 5 + backoff: + duration: 5s + factor: 2 + maxDuration: 3m + + syncOptions: + - CreateNamespace=true \ No newline at end of file diff --git a/argocd/prod/applicationset-monitoring.yaml b/argocd/prod/applicationset-monitoring.yaml index 15fc493..d1bcf48 100644 --- a/argocd/prod/applicationset-monitoring.yaml +++ b/argocd/prod/applicationset-monitoring.yaml @@ -29,8 +29,20 @@ spec: server: https://kubernetes.default.svc namespace: "{{namespace}}" + revisionHistoryLimit: 5 + syncPolicy: automated: prune: true - selfHeal: true + selfHeal: true + + retry: + limit: 5 + backoff: + duration: 5s + factor: 2 + maxDuration: 3m + + syncOptions: + - CreateNamespace=true \ No newline at end of file diff --git a/argocd/staging/applicationset-apps.yaml b/argocd/staging/applicationset-apps.yaml index ccf6ef7..f4557fa 100644 --- a/argocd/staging/applicationset-apps.yaml +++ b/argocd/staging/applicationset-apps.yaml @@ -30,10 +30,19 @@ spec: server: https://kubernetes.default.svc namespace: staging + revisionHistoryLimit: 5 + syncPolicy: automated: prune: true selfHeal: true + retry: + limit: 5 + backoff: + duration: 5s + factor: 2 + maxDuration: 3m + syncOptions: - - CreateNamespace=true \ No newline at end of file + - CreateNamespace=true \ No newline at end of file diff --git a/argocd/staging/applicationset-infra.yaml b/argocd/staging/applicationset-infra.yaml index ed61138..34770dd 100644 --- a/argocd/staging/applicationset-infra.yaml +++ b/argocd/staging/applicationset-infra.yaml @@ -29,9 +29,19 @@ spec: server: https://kubernetes.default.svc namespace: '{{namespace}}' + revisionHistoryLimit: 5 + syncPolicy: automated: prune: true selfHeal: true + + retry: + limit: 5 + backoff: + duration: 5s + factor: 2 + maxDuration: 3m + syncOptions: - - CreateNamespace=true \ No newline at end of file + - CreateNamespace=true \ No newline at end of file diff --git a/argocd/staging/applicationset-monitoring.yaml b/argocd/staging/applicationset-monitoring.yaml index 4d851b3..49cd88c 100644 --- a/argocd/staging/applicationset-monitoring.yaml +++ b/argocd/staging/applicationset-monitoring.yaml @@ -29,7 +29,19 @@ spec: server: https://kubernetes.default.svc namespace: "{{namespace}}" + revisionHistoryLimit: 5 + syncPolicy: automated: prune: true - selfHeal: true \ No newline at end of file + selfHeal: true + + retry: + limit: 5 + backoff: + duration: 5s + factor: 2 + maxDuration: 3m + + syncOptions: + - CreateNamespace=true \ No newline at end of file diff --git a/infrastructure/overlay/dev/ingress.yaml b/infrastructure/overlay/dev/ingress.yaml index 0ce04b3..dcea6ba 100644 --- a/infrastructure/overlay/dev/ingress.yaml +++ b/infrastructure/overlay/dev/ingress.yaml @@ -11,7 +11,7 @@ metadata: # HTTPS configuration alb.ingress.kubernetes.io/listen-ports: '[{"HTTP":80},{"HTTPS":443}]' - alb.ingress.kubernetes.io/certificate-arn: arn:aws:acm:us-east-1:579871530627:certificate/1187d24f-f377-4202-9cba-de460088f724 + alb.ingress.kubernetes.io/certificate-arn: arn:aws:acm:us-east-1:579871530627:certificate/5b7e620a-b401-4f5d-af4d-ca0cede863be # Redirect HTTP to HTTPS alb.ingress.kubernetes.io/ssl-redirect: '443' diff --git a/platform/ingress/overlays/dev/aws-load-balancer-controller-patch.yaml b/platform/ingress/overlays/dev/aws-load-balancer-controller-patch.yaml index 00362a6..90ac819 100644 --- a/platform/ingress/overlays/dev/aws-load-balancer-controller-patch.yaml +++ b/platform/ingress/overlays/dev/aws-load-balancer-controller-patch.yaml @@ -14,7 +14,7 @@ spec: values: | clusterName: eks-dev-cluster region: us-east-1 - vpcId: vpc-059e75cfd8dfdb0d8 + vpcId: vpc-0c7d69cfad4f143e6 serviceAccount: create: true diff --git a/platform/ingress/overlays/dev/letsencrypt.yaml b/platform/ingress/overlays/dev/letsencrypt.yaml index f244dd8..555b3bc 100644 --- a/platform/ingress/overlays/dev/letsencrypt.yaml +++ b/platform/ingress/overlays/dev/letsencrypt.yaml @@ -8,7 +8,7 @@ metadata: spec: acme: - email: admin@dev.example.com + email: ogaemmanuel@ymail.com server: https://acme-v02.api.letsencrypt.org/directory diff --git a/platform/ingress/overlays/prod/letsencrypt.yaml b/platform/ingress/overlays/prod/letsencrypt.yaml index 11dc037..555b3bc 100644 --- a/platform/ingress/overlays/prod/letsencrypt.yaml +++ b/platform/ingress/overlays/prod/letsencrypt.yaml @@ -8,7 +8,7 @@ metadata: spec: acme: - email: admin@prod.example.com + email: ogaemmanuel@ymail.com server: https://acme-v02.api.letsencrypt.org/directory diff --git a/platform/ingress/overlays/staging/letsencrypt.yaml b/platform/ingress/overlays/staging/letsencrypt.yaml index 5197e93..555b3bc 100644 --- a/platform/ingress/overlays/staging/letsencrypt.yaml +++ b/platform/ingress/overlays/staging/letsencrypt.yaml @@ -8,7 +8,7 @@ metadata: spec: acme: - email: admin@staging.example.com + email: ogaemmanuel@ymail.com server: https://acme-v02.api.letsencrypt.org/directory diff --git a/platform/monitoring/base/kube-prometheus-stack/grafana/dashboards/applications/kustomization.yaml b/platform/monitoring/base/kube-prometheus-stack/grafana/dashboards/applications/kustomization.yaml index 3f88f61..81c9dbe 100644 --- a/platform/monitoring/base/kube-prometheus-stack/grafana/dashboards/applications/kustomization.yaml +++ b/platform/monitoring/base/kube-prometheus-stack/grafana/dashboards/applications/kustomization.yaml @@ -7,11 +7,6 @@ configMapGenerator: files: - springboot.json - - nginx.json - - redis.json - - postgresql.json - - kafka.json - - ingress-nginx.json generatorOptions: diff --git a/platform/monitoring/base/kube-prometheus-stack/grafana/dashboards/applications/springboot.json b/platform/monitoring/base/kube-prometheus-stack/grafana/dashboards/applications/springboot.json index e69de29..01dfca5 100644 --- a/platform/monitoring/base/kube-prometheus-stack/grafana/dashboards/applications/springboot.json +++ b/platform/monitoring/base/kube-prometheus-stack/grafana/dashboards/applications/springboot.json @@ -0,0 +1,32 @@ +{ + "annotations": {"list": [{"builtIn": 1, "datasource": {"type": "grafana", "uid": "-- Grafana --"}, "enable": true, "hide": true, "iconColor": "#5794F2", "name": "Annotations & Alerts", "type": "dashboard"}]}, + "editable": false, + "graphTooltip": 1, + "id": null, + "links": [], + "panels": [ + {"type": "row", "title": "Application fleet", "gridPos": {"h": 1, "w": 24, "x": 0, "y": 0}, "panels": []}, + {"type": "stat", "title": "Deployments", "gridPos": {"h": 4, "w": 6, "x": 0, "y": 1}, "datasource": {"type": "prometheus", "uid": "prometheus"}, "targets": [{"expr": "count(kube_deployment_status_replicas_available{namespace=~\"$namespace\",deployment=~\"$service\"})", "refId": "A"}], "fieldConfig": {"defaults": {"unit": "short", "decimals": 0, "color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "blue", "value": null}]}}, "overrides": []}, "options": {"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "colorMode": "value", "graphMode": "none", "textMode": "value_and_name"}}, + {"type": "stat", "title": "Ready replicas", "gridPos": {"h": 4, "w": 6, "x": 6, "y": 1}, "datasource": {"type": "prometheus", "uid": "prometheus"}, "targets": [{"expr": "sum(kube_deployment_status_replicas_available{namespace=~\"$namespace\",deployment=~\"$service\"})", "refId": "A"}], "fieldConfig": {"defaults": {"unit": "short", "decimals": 0, "thresholds": {"mode": "absolute", "steps": [{"color": "red", "value": null}, {"color": "green", "value": 1}]}, "color": {"mode": "thresholds"}}, "overrides": []}, "options": {"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "colorMode": "background", "graphMode": "area", "textMode": "value_and_name"}}, + {"type": "stat", "title": "Pending pods", "gridPos": {"h": 4, "w": 6, "x": 12, "y": 1}, "datasource": {"type": "prometheus", "uid": "prometheus"}, "targets": [{"expr": "sum(kube_pod_status_phase{namespace=~\"$namespace\",pod=~\"$service.*\",phase=\"Pending\"})", "refId": "A"}], "fieldConfig": {"defaults": {"unit": "short", "decimals": 0, "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}, {"color": "orange", "value": 1}, {"color": "red", "value": 3}]}, "color": {"mode": "thresholds"}}, "overrides": []}, "options": {"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "colorMode": "background", "graphMode": "area", "textMode": "value_and_name"}}, + {"type": "stat", "title": "Restarts / 1h", "gridPos": {"h": 4, "w": 6, "x": 18, "y": 1}, "datasource": {"type": "prometheus", "uid": "prometheus"}, "targets": [{"expr": "sum(increase(kube_pod_container_status_restarts_total{namespace=~\"$namespace\",pod=~\"$service.*\"}[1h]))", "refId": "A"}], "fieldConfig": {"defaults": {"unit": "short", "decimals": 0, "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}, {"color": "orange", "value": 1}, {"color": "red", "value": 5}]}, "color": {"mode": "thresholds"}}, "overrides": []}, "options": {"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "colorMode": "background", "graphMode": "area", "textMode": "value_and_name"}}, + {"type": "row", "title": "Workload detail", "gridPos": {"h": 1, "w": 24, "x": 0, "y": 5}, "panels": []}, + {"type": "timeseries", "title": "CPU usage by service", "gridPos": {"h": 8, "w": 12, "x": 0, "y": 6}, "datasource": {"type": "prometheus", "uid": "prometheus"}, "targets": [{"expr": "sum by (namespace, pod) (rate(container_cpu_usage_seconds_total{namespace=~\"$namespace\",pod=~\"$service.*\",container!=\"\"}[5m]))", "refId": "A", "legendFormat": "{{namespace}} / {{pod}}"}], "fieldConfig": {"defaults": {"unit": "cores", "decimals": 3, "custom": {"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2, "fillOpacity": 10, "showPoints": "never", "spanNulls": true}}, "overrides": []}, "options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["lastNotNull", "max"]}, "tooltip": {"mode": "multi", "sort": "desc"}}}, + {"type": "timeseries", "title": "Memory working set by service", "gridPos": {"h": 8, "w": 12, "x": 12, "y": 6}, "datasource": {"type": "prometheus", "uid": "prometheus"}, "targets": [{"expr": "sum by (namespace, pod) (container_memory_working_set_bytes{namespace=~\"$namespace\",pod=~\"$service.*\",container!=\"\"})", "refId": "A", "legendFormat": "{{namespace}} / {{pod}}"}], "fieldConfig": {"defaults": {"unit": "bytes", "custom": {"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2, "fillOpacity": 10, "showPoints": "never", "spanNulls": true}}, "overrides": []}, "options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["lastNotNull", "max"]}, "tooltip": {"mode": "multi", "sort": "desc"}}}, + {"type": "table", "title": "Replica status", "gridPos": {"h": 8, "w": 24, "x": 0, "y": 14}, "datasource": {"type": "prometheus", "uid": "prometheus"}, "targets": [{"expr": "kube_deployment_spec_replicas{namespace=~\"$namespace\",deployment=~\"$service\"}", "refId": "A", "format": "table", "instant": true, "legendFormat": "{{namespace}} / {{deployment}}"}, {"expr": "kube_deployment_status_replicas_available{namespace=~\"$namespace\",deployment=~\"$service\"}", "refId": "B", "format": "table", "instant": true, "legendFormat": "{{namespace}} / {{deployment}} ready"}], "transformations": [{"id": "joinByField", "options": {"byField": "deployment", "mode": "outer"}}], "fieldConfig": {"defaults": {"custom": {"align": "auto", "displayMode": "auto"}}, "overrides": []}, "options": {"showHeader": true, "footer": {"show": false}}} + ], + "refresh": "30s", + "schemaVersion": 39, + "style": "dark", + "tags": ["applications", "spring-boot", "kubernetes"], + "templating": {"list": [ + {"name": "namespace", "label": "Environment", "type": "query", "datasource": {"type": "prometheus", "uid": "prometheus"}, "query": "label_values(kube_namespace_labels, namespace)", "includeAll": true, "allValue": ".*", "multi": true, "refresh": 1, "sort": 1}, + {"name": "service", "label": "Service", "type": "query", "datasource": {"type": "prometheus", "uid": "prometheus"}, "query": "label_values(kube_deployment_labels{namespace=~\"$namespace\"}, deployment)", "includeAll": true, "allValue": ".*", "multi": true, "refresh": 1, "sort": 1} + ]}, + "time": {"from": "now-6h", "to": "now"}, + "timepicker": {"refresh_intervals": ["30s", "1m", "5m", "15m"], "time_options": ["5m", "15m", "1h", "6h", "12h", "24h", "7d"]}, + "timezone": "browser", + "title": "Applications | Workload Operations", + "uid": "applications-workload-operations", + "version": 1 +} diff --git a/platform/monitoring/base/kube-prometheus-stack/grafana/dashboards/sre/kustomization.yaml b/platform/monitoring/base/kube-prometheus-stack/grafana/dashboards/sre/kustomization.yaml index 5b60396..51a291e 100644 --- a/platform/monitoring/base/kube-prometheus-stack/grafana/dashboards/sre/kustomization.yaml +++ b/platform/monitoring/base/kube-prometheus-stack/grafana/dashboards/sre/kustomization.yaml @@ -7,11 +7,6 @@ configMapGenerator: files: - slo.json - - api-latency.json - - alerts.json - - availability.json - - error-budget.json - - capacity-planning.json generatorOptions: disableNameSuffixHash: true diff --git a/platform/monitoring/base/kube-prometheus-stack/grafana/dashboards/sre/slo.json b/platform/monitoring/base/kube-prometheus-stack/grafana/dashboards/sre/slo.json index e69de29..2b89619 100644 --- a/platform/monitoring/base/kube-prometheus-stack/grafana/dashboards/sre/slo.json +++ b/platform/monitoring/base/kube-prometheus-stack/grafana/dashboards/sre/slo.json @@ -0,0 +1,31 @@ +{ + "annotations": {"list": [{"builtIn": 1, "datasource": {"type": "grafana", "uid": "-- Grafana --"}, "enable": true, "hide": true, "iconColor": "#5794F2", "name": "Annotations & Alerts", "type": "dashboard"}]}, + "editable": false, + "graphTooltip": 1, + "id": null, + "links": [], + "panels": [ + {"type": "row", "title": "Service health", "gridPos": {"h": 1, "w": 24, "x": 0, "y": 0}, "panels": []}, + {"type": "stat", "title": "Healthy workloads", "description": "Ready replicas divided by desired replicas.", "gridPos": {"h": 4, "w": 6, "x": 0, "y": 1}, "datasource": {"type": "prometheus", "uid": "prometheus"}, "targets": [{"expr": "sum(kube_deployment_status_replicas_available{namespace=~\"$namespace\"}) / clamp_min(sum(kube_deployment_spec_replicas{namespace=~\"$namespace\"}), 1) * 100", "refId": "A", "legendFormat": "availability"}], "fieldConfig": {"defaults": {"unit": "percent", "decimals": 1, "thresholds": {"mode": "absolute", "steps": [{"color": "red", "value": null}, {"color": "orange", "value": 95}, {"color": "green", "value": 99}]}, "color": {"mode": "thresholds"}}, "overrides": []}, "options": {"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "orientation": "auto", "textMode": "auto", "colorMode": "background", "graphMode": "area", "justifyMode": "center"}}, + {"type": "stat", "title": "Restarts / 1h", "gridPos": {"h": 4, "w": 6, "x": 6, "y": 1}, "datasource": {"type": "prometheus", "uid": "prometheus"}, "targets": [{"expr": "sum(increase(kube_pod_container_status_restarts_total{namespace=~\"$namespace\"}[1h]))", "refId": "A"}], "fieldConfig": {"defaults": {"unit": "short", "decimals": 0, "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}, {"color": "orange", "value": 1}, {"color": "red", "value": 5}]}, "color": {"mode": "thresholds"}}, "overrides": []}, "options": {"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "colorMode": "background", "graphMode": "area", "textMode": "auto"}}, + {"type": "stat", "title": "CPU usage", "gridPos": {"h": 4, "w": 6, "x": 12, "y": 1}, "datasource": {"type": "prometheus", "uid": "prometheus"}, "targets": [{"expr": "sum(rate(container_cpu_usage_seconds_total{namespace=~\"$namespace\",container!=\"\"}[5m])) / clamp_min(sum(kube_pod_container_resource_requests{namespace=~\"$namespace\",resource=\"cpu\"}), 0.001) * 100", "refId": "A"}], "fieldConfig": {"defaults": {"unit": "percent", "decimals": 1, "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}, {"color": "orange", "value": 70}, {"color": "red", "value": 90}]}, "color": {"mode": "thresholds"}}, "overrides": []}, "options": {"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "colorMode": "background", "graphMode": "area", "textMode": "auto"}}, + {"type": "stat", "title": "Memory usage", "gridPos": {"h": 4, "w": 6, "x": 18, "y": 1}, "datasource": {"type": "prometheus", "uid": "prometheus"}, "targets": [{"expr": "sum(container_memory_working_set_bytes{namespace=~\"$namespace\",container!=\"\"}) / clamp_min(sum(kube_pod_container_resource_requests{namespace=~\"$namespace\",resource=\"memory\"}), 1) * 100", "refId": "A"}], "fieldConfig": {"defaults": {"unit": "percent", "decimals": 1, "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}, {"color": "orange", "value": 70}, {"color": "red", "value": 90}]}, "color": {"mode": "thresholds"}}, "overrides": []}, "options": {"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "colorMode": "background", "graphMode": "area", "textMode": "auto"}}, + {"type": "row", "title": "Reliability signals", "gridPos": {"h": 1, "w": 24, "x": 0, "y": 5}, "panels": []}, + {"type": "timeseries", "title": "Pod restarts by workload", "gridPos": {"h": 8, "w": 12, "x": 0, "y": 6}, "datasource": {"type": "prometheus", "uid": "prometheus"}, "targets": [{"expr": "sum by (pod) (increase(kube_pod_container_status_restarts_total{namespace=~\"$namespace\",pod=~\"$pod\"}[5m]))", "refId": "A", "legendFormat": "{{pod}}"}], "fieldConfig": {"defaults": {"unit": "short", "custom": {"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2, "fillOpacity": 12, "showPoints": "never", "spanNulls": true}}, "overrides": []}, "options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["lastNotNull", "max"]}, "tooltip": {"mode": "multi", "sort": "desc"}}}, + {"type": "timeseries", "title": "Container memory by workload", "gridPos": {"h": 8, "w": 12, "x": 12, "y": 6}, "datasource": {"type": "prometheus", "uid": "prometheus"}, "targets": [{"expr": "sum by (pod) (container_memory_working_set_bytes{namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"})", "refId": "A", "legendFormat": "{{pod}}"}], "fieldConfig": {"defaults": {"unit": "bytes", "custom": {"drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2, "fillOpacity": 12, "showPoints": "never", "spanNulls": true}}, "overrides": []}, "options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["lastNotNull", "max"]}, "tooltip": {"mode": "multi", "sort": "desc"}}} + ], + "refresh": "30s", + "schemaVersion": 39, + "style": "dark", + "tags": ["sre", "kubernetes", "production-readiness"], + "templating": {"list": [ + {"name": "namespace", "label": "Environment", "type": "query", "datasource": {"type": "prometheus", "uid": "prometheus"}, "query": "label_values(kube_namespace_labels, namespace)", "includeAll": true, "allValue": ".*", "multi": true, "refresh": 1, "sort": 1}, + {"name": "pod", "label": "Workload", "type": "query", "datasource": {"type": "prometheus", "uid": "prometheus"}, "query": "label_values(kube_pod_info{namespace=~\"$namespace\"}, pod)", "includeAll": true, "allValue": ".*", "multi": true, "refresh": 1, "sort": 1} + ]}, + "time": {"from": "now-6h", "to": "now"}, + "timepicker": {"refresh_intervals": ["30s", "1m", "5m", "15m"], "time_options": ["5m", "15m", "1h", "6h", "12h", "24h", "2d", "7d"]}, + "timezone": "browser", + "title": "Platform | Service Health", + "uid": "platform-service-health", + "version": 1 +} diff --git a/platform/monitoring/base/kube-prometheus-stack/grafana/datasources/loki.yaml b/platform/monitoring/base/kube-prometheus-stack/grafana/datasources/loki.yaml index 361d8ec..b6b316d 100644 --- a/platform/monitoring/base/kube-prometheus-stack/grafana/datasources/loki.yaml +++ b/platform/monitoring/base/kube-prometheus-stack/grafana/datasources/loki.yaml @@ -1,3 +1,4 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/yannh/kubernetes-json-schema/master/v1.32.0-standalone-strict/configmap-v1.json apiVersion: v1 kind: ConfigMap diff --git a/platform/monitoring/base/kube-prometheus-stack/grafana/datasources/prometheus.yaml b/platform/monitoring/base/kube-prometheus-stack/grafana/datasources/prometheus.yaml index f9c7bfe..081d5b2 100644 --- a/platform/monitoring/base/kube-prometheus-stack/grafana/datasources/prometheus.yaml +++ b/platform/monitoring/base/kube-prometheus-stack/grafana/datasources/prometheus.yaml @@ -1,3 +1,4 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/yannh/kubernetes-json-schema/master/v1.32.0-standalone-strict/configmap-v1.json apiVersion: v1 kind: ConfigMap