Add monitoring to devcontainer startup

startupscript/butane/004-git-clone-devcontainer.sh

@@ -17,19 +17,7 @@ if [[ $# -lt 1 ]]; then
     usage
 fi
 
-
-# Map the server to appropriate service path
-function get_service_url() {
-  case "$1" in
-    "dev-stable") echo "https://workbench-dev.verily.com/api/$2" ;;
-    "dev-unstable") echo "https://workbench-dev-unstable.verily.com/api/$2" ;;
-    "test") echo "https://workbench-test.verily.com/api/$2" ;;
-    "prod") echo "https://workbench.verily.com/api/$2" ;;
-    *) return 1 ;;
-  esac
-}
-readonly -f get_service_url
-
+source /home/core/service-utils.sh
 source /home/core/metadata-utils.sh
 
 # To accommodate the use of SSH URLs for public Git repositories, set the following Git configuration:
@@ -58,12 +46,8 @@ api_url="${api_url%.git}"
 private_status=$(curl --retry 5 -s "${api_url}" | jq -r ".status")
 if [[ "${PRIVATE_DEVCONTAINER_ENABLED}" == "TRUE" && "${private_status}" == 404 ]]; then
   # Get ECM service URL
-  SERVER="$(get_metadata_value "terra-cli-server" "")"
-  readonly SERVER
-  if [[ -z "${SERVER}" ]]; then
-    SERVER="prod"
-  fi
-  if ! ECM_SERVICE_URL="$(get_service_url "${SERVER}" "ecm")"; then
+  SERVER="$(get_metadata_value "terra-cli-server" "prod")"
+  if ! ECM_SERVICE_URL="$(get_service_url "ecm" "${SERVER}")"; then
     exit 1
   fi
 

startupscript/butane/monitoring-utils.sh

@@ -0,0 +1,74 @@
+#!/bin/bash
+# monitoring-utils.sh defines helper functions for notifying WSM of VM startup states.
+
+# Log an event to WSM
+function log_event() {
+    if [[ $# -lt 4 ]]; then
+        echo "usage: log_event <wsm_url> <workspace_id> <resource_id> <payload>" >&2
+        return 1
+    fi
+
+    # Input params
+    local wsm_url="$1"
+    local workspace_id="$2"
+    local resource_id="$3"
+    local payload="$4"
+    local log_url="${wsm_url}/api/workspaces/v1/${workspace_id}/resources/${resource_id}/instance-state"
+
+    # Log VM event
+    local response
+    response=$(curl -s -X POST "${log_url}" \
+        -H "Authorization: Bearer $(/home/core/wb.sh auth print-access-token)" \
+        -H "Content-Type: application/json" \
+        -d "$payload" \
+        -w "\n%{http_code}")
+
+    local http_code
+    http_code=$(echo "$response" | tail -n1)
+    local response_body
+    response_body=$(echo "$response" | head -n -1)
+
+    if [[ "$http_code" != "200" ]]; then
+        echo "Failed to record VM state. HTTP ${http_code}: ${response_body}" >&2
+        return 1
+    fi
+
+    echo "VM state recorded successfully: ${response_body}"
+}
+
+# Record devcontainer start event
+function record_devcontainer_start() {
+    if [[ $# -lt 3 ]]; then
+        echo "usage: record_devcontainer_start <wsm_url> <workspace_id> <resource_id>" >&2
+        return 1
+    fi
+
+    local payload
+    payload=$(cat <<EOF
+{
+    "event": "DEVCONTAINER_START",
+    "isSuccess": true
+}
+EOF
+)
+    log_event "$1" "$2" "$3" "$payload"
+}
+
+# Record devcontainer end event
+function record_devcontainer_end() {
+    if [[ $# -lt 4 || ("$4" != "true" && "$4" != "false") ]]; then
+        echo "usage: record_devcontainer_end <wsm_url> <workspace_id> <resource_id> <isSuccess - true/false>" >&2
+        return 1
+    fi
+
+    local success="$4"
+    local payload
+    payload=$(cat <<EOF
+{
+  "event": "DEVCONTAINER_END",
+  "isSuccess": ${success}
+}
+EOF
+)
+    log_event "$1" "$2" "$3" "$payload"
+}

startupscript/butane/pre-devcontainer.sh

@@ -1,6 +1,36 @@
 #!/bin/bash
 
-# pre-devcontainer.sh creates a file used by the devcontainer service to keep
-# track of the number of service failures.
+# pre-devcontainer.sh creates a file used by the devcontainer service for monitoring
+# and to keep track of the number of service failures.
 
-touch /tmp/devcontainer-failure-count
+set -o errexit
+set -o nounset
+set -o pipefail
+set -o xtrace
+
+# Keep track of the number of service failures
+touch /tmp/devcontainer-failure-count
+
+# Fetch and cache workspace_id
+source /home/core/metadata-utils.sh
+source /home/core/service-utils.sh
+MONITORING_UTILS_FILE="/home/core/monitoring-utils.sh"
+WORKSPACE_ID_CACHE_FILE="/tmp/workspace_id_cache"
+FIRST_BOOT_START_FILE="/home/core/first-boot-start"
+if [[ -f "${MONITORING_UTILS_FILE}" && ! -f "${FIRST_BOOT_START_FILE}" ]]; then
+    # First boot file does not exist
+    ## Cache workspace id to be used by probe-proxy-readiness.sh
+    WORKSPACE_USER_FACING_ID="$(get_metadata_value "terra-workspace-id" "")"
+    SERVER="$(get_metadata_value "terra-cli-server" "prod")"
+    WSM_SERVICE_URL="$(get_service_url "wsm" "${SERVER}")"
+    RESPONSE=$(curl -s -X GET "${WSM_SERVICE_URL}/api/workspaces/v1/workspaceByUserFacingId/${WORKSPACE_USER_FACING_ID}" \
+                -H "Authorization: Bearer $(/home/core/wb.sh auth print-access-token)")
+    WORKSPACE_ID=$(echo "${RESPONSE}" | jq -r '.id');
+    echo "${WORKSPACE_ID}" > "${WORKSPACE_ID_CACHE_FILE}"
+
+    ## Record devcontainer begin for monitoring
+    source "${MONITORING_UTILS_FILE}"
+    RESOURCE_ID="$(get_metadata_value "wb-resource-id" "")"
+    record_devcontainer_start "${WSM_SERVICE_URL}" "${WORKSPACE_ID}" "${RESOURCE_ID}"
+fi
+touch "${FIRST_BOOT_START_FILE}"

startupscript/butane/probe-proxy-readiness.sh

@@ -15,9 +15,30 @@ if docker ps -q --filter "name=proxy-agent" | grep -q . \
     && docker ps -q --filter "name=application-server" | grep -q .; then
     echo "Proxy is ready."
     status="$(get_guest_attribute "startup_script/status" "")"
+    isSuccess="false"
     if [[ "${status}" != "ERROR" ]]; then
         set_metadata "startup_script/status" "COMPLETE"
+        isSuccess="true"
     fi
+
+    FIRST_BOOT_END_FILE="/home/core/first-boot-end"
+    MONITORING_UTILS_FILE="/home/core/monitoring-utils.sh"
+    if [[ -f "${MONITORING_UTILS_FILE}" && ! -f "${FIRST_BOOT_END_FILE}" ]]; then
+        # first boot file does not exist
+        # record devcontainer end for monitoring
+        source /home/core/service-utils.sh
+        source "${MONITORING_UTILS_FILE}"
+
+        # Fetch required values
+        WORKSPACE_ID_CACHE_FILE="/tmp/workspace_id_cache"
+        WORKSPACE_ID=$(cat "${WORKSPACE_ID_CACHE_FILE}")
+        RESOURCE_ID="$(get_metadata_value "wb-resource-id" "")"
+        SERVER="$(get_metadata_value "terra-cli-server" "prod")"
+        WSM_SERVICE_URL="$(get_service_url "wsm" "${SERVER}")"
+
+        record_devcontainer_end "${WSM_SERVICE_URL}" "${WORKSPACE_ID}" "${RESOURCE_ID}" "${isSuccess}"
+    fi
+    touch "${FIRST_BOOT_END_FILE}"
 else
     echo "proxy-agent or application-server is not started"
     exit 1

startupscript/butane/service-utils.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+# service-utils defines helper functions for determining service path.
+
+# Map the server to appropriate service path
+function get_service_url() {
+  if [[ $# -lt 2 ]]; then
+    echo "usage: get_service_url <service> <server>" >&2
+    return 1
+  fi
+
+  local SERVICE="$1"
+  local SERVER="$2"
+
+  case "${SERVER}" in
+    "dev-stable") echo "https://workbench-dev.verily.com/api/${SERVICE}" ;;
+    "dev-unstable") echo "https://workbench-dev-unstable.verily.com/api/${SERVICE}" ;;
+    "test") echo "https://workbench-test.verily.com/api/${SERVICE}" ;;
+    "prod") echo "https://workbench.verily.com/api/${SERVICE}" ;;
+    *) return 1 ;;
+  esac
+}
+readonly -f get_service_url