WIP: Ci integration tests

.buildkite/pipeline.yaml

@@ -25,3 +25,16 @@ steps:
       - "echo \"checking for uncommitted changes\""
       - "[[ -z $(git status -s) ]]"
     agents: { queue: standard }
+
+  - label: ":book: Verify helm-docs is up-to-date"
+    commands:
+      - "./scripts/helm-docs.sh"
+      - "echo \"checking for uncommitted changes\""
+      - "[[ -z $(git status -s) ]]"
+    agents: { queue: standard }
+
+  - label: ":jigsaw: Helm Integration"
+    commands:
+      #      - "./scripts/ci/install-helm-env.sh"
+      - "./scripts/ci/helm-integration.sh" 
+    agents: { queue: standard }

.gitignore

@@ -1 +1,5 @@
 target/*
+kind
+*.terraform/**
+*.tfstat
+*.terraform.lock.hcl

.tool-versions

@@ -0,0 +1,3 @@
+helm 3.7.2
+kubectl 1.22.5
+terraform 1.1.9

charts/sourcegraph/values.yaml

@@ -1,3 +1,4 @@
+# sample test
 # To customize these values, use an override file:
 # https://sourcegraph.com/github.com/sourcegraph/deploy-sourcegraph-helm/-/blob/charts/sourcegraph/README.md#customizations
 

scripts/ci/helm-integration.sh

@@ -0,0 +1,61 @@
+#!/bin/bash
+
+set -euf -o pipefail
+
+
+# Install asdf terraform plugin - managed by stateless agent configuration
+# asdf plugin-add terraform https://github.com/asdf-community/asdf-hashicorp.git
+#asdf plugin-add kubectl https://github.com/asdf-community/asdf-kubectl.git
+
+# Install terraform via asdf
+asdf install
+asdf reshim
+
+pushd $(pwd)
+cd scripts/ci/terraform
+
+terraform init
+terraform apply -auto-approve || true
+
+popd
+
+# checkout main branch
+git checkout main charts/sourcegraph
+
+# integration test: install chart at main branch ref
+helm upgrade \
+  --install \
+  --create-namespace -n sourcegraph \
+  --wait \
+  --set sourcegraph.localDevMode=true \
+  sourcegraph charts/sourcegraph/. || true
+
+# Set the default namespace
+kubectl config set-context --current --namespace sourcegraph
+
+# Wait for frontend pods to stabilize
+kubectl wait --for=condition=Ready --timeout=5m pod -l app=sourcegraph-frontend
+
+# checkout current branch
+git checkout HEAD charts/sourcegraph
+
+# verify git-fu 
+git status
+
+# integration test: install chart with changes in this branch
+helm upgrade \
+  --install \
+  --create-namespace -n sourcegraph \
+  --wait \
+  --set sourcegraph.localDevMode=true \
+  sourcegraph charts/sourcegraph/. || true
+
+# Wait for frontend pods to stabilize
+kubectl wait --for=condition=Ready --timeout=5m pod -l app=sourcegraph-frontend
+
+# We would want to do actual tests here ... 
+kubectl get pods -n sourcegraph
+
+# Cleanup
+cd scripts/ci/terraform
+terraform destroy -auto-approve || true

scripts/ci/terraform/main.tf

@@ -0,0 +1,74 @@
+provider "google" {
+  project = var.project
+}
+
+resource "random_id" "suffix" {
+  byte_length = 4
+}
+
+data "google_container_engine_versions" "main" {
+  location       = var.zone
+  version_prefix = "1.20."
+}
+
+data "google_service_account" "gcpapi" {
+  account_id = "${var.gcp_service_account}"
+}
+
+resource "google_container_cluster" "cluster" {
+  name               = "vault-helm-dev-${random_id.suffix.dec}"
+  project            = var.project
+  enable_legacy_abac = true
+  initial_node_count = 3
+  location           = "${var.zone}"
+  min_master_version = "${data.google_container_engine_versions.main.latest_master_version}"
+  node_version       = "${data.google_container_engine_versions.main.latest_node_version}"
+
+  node_config {
+    #service account for nodes to use
+    oauth_scopes = [
+      "https://www.googleapis.com/auth/cloud-platform",
+      "https://www.googleapis.com/auth/compute",
+      "https://www.googleapis.com/auth/devstorage.read_write",
+      "https://www.googleapis.com/auth/logging.write",
+      "https://www.googleapis.com/auth/monitoring",
+      "https://www.googleapis.com/auth/service.management.readonly",
+      "https://www.googleapis.com/auth/servicecontrol",
+      "https://www.googleapis.com/auth/trace.append",
+    ]
+
+    service_account = "${data.google_service_account.gcpapi.email}"
+  }
+}
+
+resource "null_resource" "kubectl" {
+  count = "${var.init_cli ? 1 : 0}"
+
+  triggers = {
+    cluster_id   = "${google_container_cluster.cluster.id}"
+    cluster_name = "${google_container_cluster.cluster.name}"
+    project      = var.project
+  }
+
+  # On creation, we want to setup the kubectl credentials. The easiest way
+  # to do this is to shell out to gcloud.
+  provisioner "local-exec" {
+    command = "gcloud container clusters get-credentials --zone=${var.zone} ${self.triggers.cluster_name} --project=${self.triggers.project}"
+  }
+
+  # On destroy we want to try to clean up the kubectl credentials. This
+  # might fail if the credentials are already cleaned up or something so we
+  # want this to continue on failure. Generally, this works just fine since
+  # it only operates on local data.
+  provisioner "local-exec" {
+    when       = destroy
+    on_failure = continue
+    command    = "kubectl config get-clusters | grep ${self.triggers.cluster_name} | xargs -n1 kubectl config delete-cluster"
+  }
+
+  provisioner "local-exec" {
+    when       = destroy
+    on_failure = continue
+    command    = "kubectl config get-contexts | grep ${self.triggers.cluster_name} | xargs -n1 kubectl config delete-context"
+  }
+}

scripts/ci/terraform/outputs.tf

@@ -0,0 +1,7 @@
+output "cluster_id" {
+  value = "${google_container_cluster.cluster.id}"
+}
+
+output "cluster_name" {
+  value = "${google_container_cluster.cluster.name}"
+}

scripts/ci/terraform/variables.tf

@@ -0,0 +1,28 @@
+variable "project" {
+  default = "sourcegraph-ci"
+
+  description = <<EOF
+Google Cloud Project to launch resources in. This project must have GKE
+enabled and billing activated. We can't use the GOOGLE_PROJECT environment
+variable since we need to access the project for other uses.
+EOF
+}
+
+variable "zone" {
+  default     = "us-central1-a"
+  description = "The zone to launch all the GKE nodes in."
+}
+
+variable "init_cli" {
+  default     = true
+  description = "Whether to init kubectl or not."
+}
+
+variable "gcp_service_account" {
+  default = "sourcegraph-deploy-helm-ci"
+
+  description = <<EOF
+Service account used on the nodes to manage/use the API, specifically needed
+for using auto-unseal
+EOF
+}