From a5c6aeb1448764ab51cd55f6cc742d840f96e614 Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Wed, 8 Apr 2026 19:17:12 +0300 Subject: [PATCH 01/20] feat(ci): test rolling over release to release Signed-off-by: Nikita Korolev --- .github/workflows/e2e-test-releases.yml | 66 ++++++++++++++++++++++--- 1 file changed, 60 insertions(+), 6 deletions(-) diff --git a/.github/workflows/e2e-test-releases.yml b/.github/workflows/e2e-test-releases.yml index d392304409..5a30c5480d 100644 --- a/.github/workflows/e2e-test-releases.yml +++ b/.github/workflows/e2e-test-releases.yml @@ -27,18 +27,72 @@ on: required: false # before merge to main, set to true type: string +concurrency: + group: "${{ github.workflow }}-${{ github.event.number || github.ref }}" + cancel-in-progress: true + defaults: run: shell: bash jobs: - release-test: - name: Release test + set-vars: + name: Set vars runs-on: ubuntu-latest + outputs: + date_start: ${{ steps.vars.outputs.date-start }} + randuuid4c: ${{ steps.vars.outputs.randuuid4c }} steps: - - name: Test + - name: Set vars + id: vars run: | - echo "🏷️ [INFO] Current release tag: ${{ github.event.inputs.current-release }}" - echo "🔜 [INFO] Next release tag: ${{ github.event.inputs.next-release }}" + echo "date-start=$(date +%Y%m%d-%H%M%S)" >> $GITHUB_OUTPUT + echo "randuuid4c=$(openssl rand -hex 2)" >> $GITHUB_OUTPUT + + e2e-replicated: + name: E2E Release Pipeline (Replicated) + needs: + - set-vars + uses: ./.github/workflows/e2e-test-releases-reusable-pipeline.yml + with: + current_release: ${{ github.event.inputs.current-release }} + new_release: ${{ github.event.inputs.next-release }} + storage_type: replicated + nested_storageclass_name: nested-thin-r1 + branch: main + deckhouse_channel: alpha + default_user: cloud + go_version: "1.25.8" + date_start: ${{ needs.set-vars.outputs.date_start }} + randuuid4c: ${{ needs.set-vars.outputs.randuuid4c }} + cluster_config_workers_memory: "9Gi" + cluster_config_k8s_version: "1.34" + secrets: + DEV_REGISTRY_DOCKER_CFG: ${{ secrets.DEV_REGISTRY_DOCKER_CFG }} + VIRT_E2E_NIGHTLY_SA_TOKEN: ${{ secrets.VIRT_E2E_NIGHTLY_SA_TOKEN }} + PROD_IO_REGISTRY_DOCKER_CFG: ${{ secrets.PROD_IO_REGISTRY_DOCKER_CFG }} + BOOTSTRAP_DEV_PROXY: ${{ secrets.BOOTSTRAP_DEV_PROXY }} - echo "🎉 [INFO] Test from ${{ github.event.inputs.current-release }} -> ${{ github.event.inputs.next-release }} SUCCESS" + e2e-nfs: + name: E2E Release Pipeline (NFS) + needs: + - set-vars + uses: ./.github/workflows/e2e-test-releases-reusable-pipeline.yml + with: + current_release: ${{ github.event.inputs.current-release }} + new_release: ${{ github.event.inputs.next-release }} + storage_type: nfs + nested_storageclass_name: nfs + branch: main + deckhouse_channel: alpha + default_user: cloud + go_version: "1.24.13" + date_start: ${{ needs.set-vars.outputs.date_start }} + randuuid4c: ${{ needs.set-vars.outputs.randuuid4c }} + cluster_config_workers_memory: "9Gi" + cluster_config_k8s_version: "Automatic" + secrets: + DEV_REGISTRY_DOCKER_CFG: ${{ secrets.DEV_REGISTRY_DOCKER_CFG }} + VIRT_E2E_NIGHTLY_SA_TOKEN: ${{ secrets.VIRT_E2E_NIGHTLY_SA_TOKEN }} + PROD_IO_REGISTRY_DOCKER_CFG: ${{ secrets.PROD_IO_REGISTRY_DOCKER_CFG }} + BOOTSTRAP_DEV_PROXY: ${{ secrets.BOOTSTRAP_DEV_PROXY }} From a2a59040e56007588afab758afb55cd6195a8291 Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Wed, 8 Apr 2026 19:27:47 +0300 Subject: [PATCH 02/20] add draft Signed-off-by: Nikita Korolev --- .../e2e-test-releases-reusable-pipeline.yml | 1213 ++++++++++++++++- 1 file changed, 1206 insertions(+), 7 deletions(-) diff --git a/.github/workflows/e2e-test-releases-reusable-pipeline.yml b/.github/workflows/e2e-test-releases-reusable-pipeline.yml index 1ca30d5199..eba7e4b691 100644 --- a/.github/workflows/e2e-test-releases-reusable-pipeline.yml +++ b/.github/workflows/e2e-test-releases-reusable-pipeline.yml @@ -17,21 +17,1220 @@ name: E2E Release Test Reusable Pipeline on: workflow_call: inputs: - message: - description: "placeholder message" + current_release: + required: true + type: string + description: "Current virtualization release tag (e.g. v1.4.1)" + new_release: + required: true + type: string + description: "New virtualization release tag to upgrade to (e.g. v1.5.0)" + date_start: + required: true + type: string + description: "Date start" + randuuid4c: + required: true + type: string + description: "Random UUID first 4 chars" + cluster_config_k8s_version: + required: false + type: string + default: "Automatic" + description: "Set k8s version for cluster config, like 1.34, 1.36 (without patch version)" + cluster_config_workers_memory: + required: false + type: string + default: "8Gi" + description: "Set memory for workers node in cluster config" + storage_type: + required: true + type: string + description: "Storage type (replicated or nfs)" + nested_storageclass_name: + required: true + type: string + description: "Nested storage class name" + branch: + required: false + type: string + default: "main" + description: "Branch to use" + virtualization_image_url: + required: false + type: string + default: "https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img" + description: "Virtualization image url (default noble-server-cloudimg-amd64.img)" + deckhouse_channel: required: false - default: "placeholder message" type: string + default: "alpha" + description: "Deckhouse release channel" + pod_subnet_cidr: + required: false + type: string + default: "10.88.0.0/16" + description: "Pod subnet CIDR" + service_subnet_cidr: + required: false + type: string + default: "10.99.0.0/16" + description: "Service subnet CIDR" + default_user: + required: false + type: string + default: "ubuntu" + description: "Default user for vms" + go_version: + required: false + type: string + default: "1.24.6" + description: "Go version" + secrets: + DEV_REGISTRY_DOCKER_CFG: + required: true + VIRT_E2E_NIGHTLY_SA_TOKEN: + required: true + PROD_IO_REGISTRY_DOCKER_CFG: + required: true + BOOTSTRAP_DEV_PROXY: + required: true + +env: + BRANCH: ${{ inputs.branch }} + CURRENT_RELEASE: ${{ inputs.current_release }} + NEW_RELEASE: ${{ inputs.new_release }} + DECKHOUSE_CHANNEL: ${{ inputs.deckhouse_channel }} + DEFAULT_USER: ${{ inputs.default_user }} + GO_VERSION: ${{ inputs.go_version }} + SETUP_CLUSTER_TYPE_PATH: test/dvp-static-cluster + K8S_VERSION: ${{ inputs.cluster_config_k8s_version }} defaults: run: shell: bash jobs: - release-test: - name: Release test + bootstrap: + name: Bootstrap cluster + runs-on: ubuntu-latest + concurrency: + group: "${{ github.workflow }}-${{ github.event.number || github.ref }}-${{ inputs.storage_type }}" + cancel-in-progress: true + outputs: + kubeconfig: ${{ steps.generate-kubeconfig.outputs.kubeconfig }} + namespace: ${{ steps.vars.outputs.namespace }} + steps: + - uses: actions/checkout@v4 + + - name: Set outputs + env: + RANDUUID4C: ${{ inputs.randuuid4c }} + STORAGE_TYPE: ${{ inputs.storage_type }} + id: vars + run: | + GIT_SHORT_HASH=$(git rev-parse --short HEAD) + + namespace="nightly-e2e-$STORAGE_TYPE-$GIT_SHORT_HASH-$RANDUUID4C" + + echo "namespace=$namespace" >> $GITHUB_OUTPUT + echo "sha_short=$GIT_SHORT_HASH" >> $GITHUB_OUTPUT + + REGISTRY=$(base64 -d <<< ${{secrets.PROD_IO_REGISTRY_DOCKER_CFG}} | jq '.auths | to_entries | .[] | .key' -r) + USERNAME=$(base64 -d <<< ${{ secrets.PROD_IO_REGISTRY_DOCKER_CFG }} | jq '.auths | to_entries | .[] | .value.auth' -r | base64 -d | cut -d ':' -f1) + PASSWORD=$(base64 -d <<< ${{ secrets.PROD_IO_REGISTRY_DOCKER_CFG }} | jq '.auths | to_entries | .[] | .value.auth' -r | base64 -d | cut -d ':' -f2) + + echo "registry=$REGISTRY" >> $GITHUB_OUTPUT + echo "username=$USERNAME" >> $GITHUB_OUTPUT + echo "password=$PASSWORD" >> $GITHUB_OUTPUT + + - name: Install htpasswd utility + run: | + sudo apt-get update + sudo apt-get install -y apache2-utils + + - name: Install Task + uses: arduino/setup-task@v2 + with: + version: 3.x + repo-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Setup d8 + uses: ./.github/actions/install-d8 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Log in to private registry + uses: docker/login-action@v3 + with: + registry: ${{ steps.vars.outputs.registry }} + username: ${{ steps.vars.outputs.username }} + password: ${{ steps.vars.outputs.password }} + + - name: Configure kubectl via azure/k8s-set-context@v4 + uses: azure/k8s-set-context@v4 + with: + method: kubeconfig + context: e2e-cluster-nightly-e2e-virt-sa + kubeconfig: ${{ secrets.VIRT_E2E_NIGHTLY_SA_TOKEN }} + + - name: Generate values.yaml + working-directory: ${{ env.SETUP_CLUSTER_TYPE_PATH }} + run: | + defaultStorageClass=$(kubectl get storageclass -o json \ + | jq -r '.items[] | select(.metadata.annotations."storageclass.kubernetes.io/is-default-class" == "true") | .metadata.name') + + cat < values.yaml + namespace: ${{ steps.vars.outputs.namespace }} + storageType: ${{ inputs.storage_type }} + storageClass: ${defaultStorageClass} + sa: dkp-sa + deckhouse: + channel: ${{ env.DECKHOUSE_CHANNEL }} + podSubnetCIDR: ${{ inputs.pod_subnet_cidr }} + serviceSubnetCIDR: ${{ inputs.service_subnet_cidr }} + kubernetesVersion: ${{ env.K8S_VERSION }} + registryDockerCfg: ${{ secrets.PROD_IO_REGISTRY_DOCKER_CFG }} + bundle: Default + proxyEnabled: false + image: + url: ${{ inputs.virtualization_image_url }} + defaultUser: ${{ env.DEFAULT_USER }} + bootloader: BIOS + ingressHosts: + - api + - grafana + - dex + - prometheus + - console + - virtualization + instances: + masterNodes: + count: 1 + cfg: + rootDiskSize: 60Gi + cpu: + cores: 4 + coreFraction: 50% + memory: + size: 12Gi + additionalNodes: + - name: worker + count: 3 + cfg: + cpu: + cores: 6 + coreFraction: 50% + memory: + size: ${{ inputs.cluster_config_workers_memory }} + additionalDisks: + - size: 50Gi + EOF + + mkdir -p tmp + touch tmp/discovered-values.yaml + + export REGISTRY=$(base64 -d <<< ${{secrets.DEV_REGISTRY_DOCKER_CFG}} | jq '.auths | to_entries | .[] | .key' -r) + export AUTH=$(base64 -d <<< ${{ secrets.DEV_REGISTRY_DOCKER_CFG }} | jq '.auths | to_entries | .[] | .value.auth' -r) + + yq eval --inplace '.discovered.registry_url = env(REGISTRY)' tmp/discovered-values.yaml + yq eval --inplace '.discovered.registry_auth = env(AUTH)' tmp/discovered-values.yaml + + - name: Bootstrap cluster [infra-deploy] + working-directory: ${{ env.SETUP_CLUSTER_TYPE_PATH }} + run: | + task infra-deploy + - name: Bootstrap cluster [dhctl-bootstrap] + id: dhctl-bootstrap + working-directory: ${{ env.SETUP_CLUSTER_TYPE_PATH }} + env: + HTTP_PROXY: ${{ secrets.BOOTSTRAP_DEV_PROXY }} + HTTPS_PROXY: ${{ secrets.BOOTSTRAP_DEV_PROXY }} + run: | + task dhctl-bootstrap + timeout-minutes: 30 + - name: Bootstrap cluster [show-connection-info] + working-directory: ${{ env.SETUP_CLUSTER_TYPE_PATH }} + run: | + task show-connection-info + + - name: Save ssh to secrets in cluster + env: + NAMESPACE: ${{ steps.vars.outputs.namespace }} + if: always() && steps.dhctl-bootstrap.outcome == 'success' + run: | + kubectl -n $NAMESPACE create secret generic ssh-key --from-file=${{ env.SETUP_CLUSTER_TYPE_PATH }}/tmp/ssh/cloud + + - name: Get info about nested cluster and master VM + working-directory: ${{ env.SETUP_CLUSTER_TYPE_PATH }} + env: + NAMESPACE: ${{ steps.vars.outputs.namespace }} + PREFIX: ${{ inputs.storage_type }} + run: | + nested_master=$(kubectl -n ${NAMESPACE} get vm -l group=${PREFIX}-master -o jsonpath="{.items[0].metadata.name}") + + d8vssh() { + local host=$1 + local cmd=$2 + d8 v ssh -i ./tmp/ssh/cloud \ + --local-ssh=true \ + --local-ssh-opts="-o StrictHostKeyChecking=no" \ + --local-ssh-opts="-o UserKnownHostsFile=/dev/null" \ + ${DEFAULT_USER}@${host}.${NAMESPACE} \ + -c "$cmd" + } + + echo "[INFO] Pods in namespace $NAMESPACE" + kubectl get pods -n "${NAMESPACE}" + echo "" + + echo "[INFO] VMs in namespace $NAMESPACE" + kubectl get vm -n "${NAMESPACE}" + echo "" + + echo "[INFO] VDs in namespace $NAMESPACE" + kubectl get vd -n "${NAMESPACE}" + echo "" + + echo "Check connection to master" + d8vssh "${nested_master}" 'echo master os-release: ; cat /etc/os-release; echo " "; echo master hostname: ; hostname' + echo "" + + - name: Generate nested kubeconfig + id: generate-kubeconfig + working-directory: ${{ env.SETUP_CLUSTER_TYPE_PATH }} + env: + kubeConfigPath: tmp/kube.config + NAMESPACE: ${{ steps.vars.outputs.namespace }} + PREFIX: ${{ inputs.storage_type }} + run: | + nested_master=$(kubectl -n ${NAMESPACE} get vm -l group=${PREFIX}-master -o jsonpath="{.items[0].metadata.name}") + + d8vscp() { + local source=$1 + local dest=$2 + d8 v scp -i ./tmp/ssh/cloud \ + --local-ssh=true \ + --local-ssh-opts="-o StrictHostKeyChecking=no" \ + --local-ssh-opts="-o UserKnownHostsFile=/dev/null" \ + "$source" "$dest" + echo "d8vscp: $source -> $dest - done" + } + + d8vssh() { + local cmd=$1 + d8 v ssh -i ./tmp/ssh/cloud \ + --local-ssh=true \ + --local-ssh-opts="-o StrictHostKeyChecking=no" \ + --local-ssh-opts="-o UserKnownHostsFile=/dev/null" \ + ${DEFAULT_USER}@${nested_master}.${NAMESPACE} \ + -c "$cmd" + } + + echo "[INFO] Copy script for generating kubeconfig in nested cluster" + echo "[INFO] Copy scripts/gen-kubeconfig.sh to master" + d8vscp "./scripts/gen-kubeconfig.sh" "${DEFAULT_USER}@${nested_master}.${NAMESPACE}:/tmp/gen-kubeconfig.sh" + echo "" + d8vscp "./scripts/deckhouse-queue.sh" "${DEFAULT_USER}@${nested_master}.${NAMESPACE}:/tmp/deckhouse-queue.sh" + echo "" + + echo "[INFO] Set file exec permissions" + d8vssh 'chmod +x /tmp/{gen-kubeconfig.sh,deckhouse-queue.sh}' + d8vssh 'ls -la /tmp/' + echo "[INFO] Check d8 queue in nested cluster" + d8vssh 'sudo /tmp/deckhouse-queue.sh' + + echo "[INFO] Generate kube conf in nested cluster" + echo "[INFO] Run gen-kubeconfig.sh in nested cluster" + d8vssh "sudo /tmp/gen-kubeconfig.sh nested-sa nested nested-e2e /${kubeConfigPath}" + echo "" + + echo "[INFO] Copy kubeconfig to runner" + echo "[INFO] ${DEFAULT_USER}@${nested_master}.$NAMESPACE:/${kubeConfigPath} ./${kubeConfigPath}" + d8vscp "${DEFAULT_USER}@${nested_master}.$NAMESPACE:/${kubeConfigPath}" "./${kubeConfigPath}" + + echo "[INFO] Set rights for kubeconfig" + echo "[INFO] sudo chown 1001:1001 ${kubeConfigPath}" + sudo chown 1001:1001 ${kubeConfigPath} + echo " " + + echo "[INFO] Kubeconf to github output" + CONFIG=$(cat ${kubeConfigPath} | base64 -w 0) + CONFIG=$(echo $CONFIG | base64 -w 0) + echo "kubeconfig=$CONFIG" >> $GITHUB_OUTPUT + + - name: cloud-init logs + if: steps.dhctl-bootstrap.outcome == 'failure' + env: + NAMESPACE: ${{ steps.vars.outputs.namespace }} + PREFIX: ${{ inputs.storage_type }} + run: | + nested_master=$(kubectl -n ${NAMESPACE} get vm -l group=${PREFIX}-master -o jsonpath="{.items[0].metadata.name}") + + d8vscp() { + local source=$1 + local dest=$2 + d8 v scp -i ./tmp/ssh/cloud \ + --local-ssh=true \ + --local-ssh-opts="-o StrictHostKeyChecking=no" \ + --local-ssh-opts="-o UserKnownHostsFile=/dev/null" \ + "$source" "$dest" + echo "d8vscp: $source -> $dest - done" + } + + d8vscp "${DEFAULT_USER}@${nested_master}.$NAMESPACE:/var/log/cloud-init*.log" "./${{ env.SETUP_CLUSTER_TYPE_PATH }}/tmp/" + + - name: Prepare artifact + if: always() && steps.dhctl-bootstrap.outcome == 'success' + run: | + sudo chown -fR 1001:1001 ${{ env.SETUP_CLUSTER_TYPE_PATH }} + yq e '.deckhouse.registryDockerCfg = "None"' -i ./${{ env.SETUP_CLUSTER_TYPE_PATH }}/values.yaml + yq e 'select(.kind == "InitConfiguration").deckhouse.registryDockerCfg = "None"' -i ./${{ env.SETUP_CLUSTER_TYPE_PATH }}/tmp/config.yaml || echo "The config.yaml file is not generated, skipping" + yq e '.discovered.registry_url = "None"' -i ./${{ env.SETUP_CLUSTER_TYPE_PATH }}/tmp/discovered-values.yaml || echo "The discovered-values.yaml file is not generated, skipping editing registry_url" + yq e '.discovered.registry_auth = "None"' -i ./${{ env.SETUP_CLUSTER_TYPE_PATH }}/tmp/discovered-values.yaml || echo "The discovered-values.yaml file is not generated, skipping editing registry_auth" + echo "${{ steps.generate-kubeconfig.outputs.kubeconfig }}" | base64 -d | base64 -d > ./${{ env.SETUP_CLUSTER_TYPE_PATH }}/kube-config + + - name: Upload generated files + uses: actions/upload-artifact@v4 + if: always() && steps.dhctl-bootstrap.outcome == 'success' + with: + name: ${{ inputs.storage_type }}-release-generated-files-${{ inputs.date_start }} + path: | + ${{ env.SETUP_CLUSTER_TYPE_PATH }}/tmp + ${{ env.SETUP_CLUSTER_TYPE_PATH }}/values.yaml + overwrite: true + include-hidden-files: true + retention-days: 3 + + - name: Upload ssh config + uses: actions/upload-artifact@v4 + if: always() + with: + name: ${{ inputs.storage_type }}-release-generated-files-ssh-${{ inputs.date_start }} + path: ${{ env.SETUP_CLUSTER_TYPE_PATH }}/tmp/ssh + overwrite: true + include-hidden-files: true + retention-days: 3 + + - name: Upload kubeconfig + uses: actions/upload-artifact@v4 + with: + name: ${{ inputs.storage_type }}-release-generated-files-kubeconfig-${{ inputs.date_start }} + path: ${{ env.SETUP_CLUSTER_TYPE_PATH }}/kube-config + overwrite: true + include-hidden-files: true + retention-days: 3 + + configure-storage: + name: Configure storage + runs-on: ubuntu-latest + needs: bootstrap + steps: + - uses: actions/checkout@v4 + + - name: Install Task + uses: arduino/setup-task@v2 + with: + version: 3.x + repo-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Setup d8 + uses: ./.github/actions/install-d8 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Install kubectl CLI + uses: azure/setup-kubectl@v4 + + - name: Check nested kube-api via generated kubeconfig + run: | + mkdir -p ~/.kube + echo "[INFO] Configure kubeconfig for nested cluster" + echo "${{ needs.bootstrap.outputs.kubeconfig }}" | base64 -d | base64 -d > ~/.kube/config + + echo "[INFO] Show paths and files content" + ls -la ~/.kube + echo "[INFO] Set permissions for kubeconfig" + chmod 600 ~/.kube/config + + echo "[INFO] Show current kubeconfig context" + kubectl config get-contexts + + echo "[INFO] Show nodes in cluster" + count=30 + success=false + for i in $(seq 1 $count); do + echo "[INFO] Attempt $i/$count..." + if kubectl get nodes; then + echo "[SUCCESS] Successfully retrieved nodes." + success=true + break + fi + + if [ $i -lt $count ]; then + echo "[INFO] Retrying in 10 seconds..." + sleep 10 + fi + done + + if [ "$success" = false ]; then + echo "[ERROR] Failed to retrieve nodes after $count attempts." + exit 1 + fi + + - name: Configure replicated storage + id: storage-replicated-setup + if: ${{ inputs.storage_type == 'replicated' }} + working-directory: ${{ env.SETUP_CLUSTER_TYPE_PATH }}/storage/sds-replicated + run: | + d8_queue_list() { + d8 s queue list | grep -Po '([0-9]+)(?= active)' || echo "[WARNING] Failed to retrieve list queue" + } + + d8_queue() { + local count=90 + local queue_count + + for i in $(seq 1 $count) ; do + queue_count=$(d8_queue_list) + if [ -n "$queue_count" ] && [ "$queue_count" = "0" ]; then + echo "[SUCCESS] Queue is clear" + return 0 + fi + + echo "[INFO] Wait until queues are empty ${i}/${count}" + if (( i % 5 == 0 )); then + echo "[INFO] Show queue list" + d8 s queue list | head -n25 || echo "[WARNING] Failed to retrieve list queue" + echo " " + fi + + if (( i % 10 == 0 )); then + echo "[INFO] deckhouse logs" + echo "::group::deckhouse logs" + d8 s logs | tail -n 100 + echo "::endgroup::" + echo " " + fi + sleep 10 + done + } + + sds_replicated_ready() { + local count=60 + for i in $(seq 1 $count); do + + sds_replicated_volume_status=$(kubectl get ns d8-sds-replicated-volume -o jsonpath='{.status.phase}' || echo "False") + + if [[ "${sds_replicated_volume_status}" = "Active" ]]; then + echo "[SUCCESS] Namespaces sds-replicated-volume are Active" + kubectl get ns d8-sds-replicated-volume + return 0 + fi + + echo "[INFO] Waiting 10s for sds-replicated-volume namespace to be ready (attempt ${i}/${count})" + if (( i % 5 == 0 )); then + echo "[INFO] Show namespaces sds-replicated-volume" + kubectl get ns | grep sds-replicated-volume || echo "Namespaces sds-replicated-volume are not ready" + echo "[DEBUG] Show queue (first 25 lines)" + d8 s queue list | head -n25 || echo "No queues" + fi + sleep 10 + done + + echo "[ERROR] Namespaces sds-replicated-volume are not ready after ${count} attempts" + echo "[DEBUG] Show namespaces sds" + kubectl get ns | grep sds || echo "Namespaces sds-replicated-volume are not ready" + echo "[DEBUG] Show queue" + echo "::group::Show queue" + d8 s queue list || echo "No queues" + echo "::endgroup::" + echo "[DEBUG] Show deckhouse logs" + echo "::group::deckhouse logs" + d8 s logs | tail -n 100 + echo "::endgroup::" + exit 1 + } + + sds_pods_ready() { + local count=100 + local linstor_node + local csi_node + local webhooks + local workers=$(kubectl get nodes -o name | grep worker | wc -l || true) + workers=$((workers)) + + echo "[INFO] Wait while linstor-node csi-node webhooks pods are ready" + for i in $(seq 1 $count); do + linstor_node=$(kubectl -n d8-sds-replicated-volume get pods | grep "linstor-node.*Running" | wc -l || true) + csi_node=$(kubectl -n d8-sds-replicated-volume get pods | grep "csi-node.*Running" | wc -l || true) + + echo "[INFO] Check if sds-replicated pods are ready" + if [[ ${linstor_node} -ge ${workers} && ${csi_node} -ge ${workers} ]]; then + echo "[SUCCESS] sds-replicated-volume is ready" + return 0 + fi + + echo "[WARNING] Not all pods are ready, linstor_node=${linstor_node}, csi_node=${csi_node}" + echo "[INFO] Waiting 10s for pods to be ready (attempt ${i}/${count})" + if (( i % 5 == 0 )); then + echo "[DEBUG] Get pods" + kubectl -n d8-sds-replicated-volume get pods || true + echo "[DEBUG] Show queue (first 25 lines)" + d8 s queue list | head -n 25 || echo "Failed to retrieve list queue" + echo " " + fi + sleep 10 + done + + echo "[ERROR] sds-replicated-volume is not ready after ${count} attempts" + echo "[DEBUG] Get pods" + echo "::group::sds-replicated-volume pods" + kubectl -n d8-sds-replicated-volume get pods || true + echo "::endgroup::" + echo "[DEBUG] Show queue" + echo "::group::Show queue" + d8 s queue list || echo "Failed to retrieve list queue" + echo "::endgroup::" + echo "[DEBUG] Show deckhouse logs" + echo "::group::deckhouse logs" + d8 s logs | tail -n 100 + echo "::endgroup::" + exit 1 + } + + blockdevices_ready() { + local count=60 + workers=$(kubectl get nodes -o name | grep worker | wc -l) + workers=$((workers)) + + if [[ $workers -eq 0 ]]; then + echo "[ERROR] No worker nodes found" + exit 1 + fi + + for i in $(seq 1 $count); do + blockdevices=$(kubectl get blockdevice -o name | wc -l || true) + if [ $blockdevices -ge $workers ]; then + echo "[SUCCESS] Blockdevices is greater or equal to $workers" + kubectl get blockdevice + return 0 + fi + + echo "[INFO] Wait 10 sec until blockdevices is greater or equal to $workers (attempt ${i}/${count})" + if (( i % 5 == 0 )); then + echo "[DEBUG] Show queue (first 25 lines)" + d8 s queue list | head -n25 || echo "No queues" + fi + + sleep 10 + done + + echo "[ERROR] Blockdevices is not 3" + echo "[DEBUG] Show cluster nodes" + kubectl get nodes + echo "[DEBUG] Show blockdevices" + kubectl get blockdevice + echo "[DEBUG] Show sds namespaces" + kubectl get ns | grep sds || echo "ns sds is not found" + echo "[DEBUG] Show pods in sds-replicated-volume" + echo "::group::pods in sds-replicated-volume" + kubectl -n d8-sds-replicated-volume get pods || true + echo "::endgroup::" + echo "[DEBUG] Show deckhouse logs" + echo "::group::deckhouse logs" + d8 s logs | tail -n 100 + echo "::endgroup::" + exit 1 + } + + d8_queue + + kubectl apply -f mc.yaml + echo "[INFO] Wait for sds-node-configurator" + kubectl wait --for=jsonpath='{.status.phase}'=Ready modules sds-node-configurator --timeout=300s + + echo "[INFO] Wait for sds-replicated-volume to be ready" + sds_replicated_ready + kubectl wait --for=jsonpath='{.status.phase}'=Ready modules sds-replicated-volume --timeout=300s + + echo "[INFO] Wait BlockDevice are ready" + blockdevices_ready + + echo "[INFO] Wait pods and webhooks sds-replicated pods" + sds_pods_ready + + chmod +x lvg-gen.sh + ./lvg-gen.sh + + chmod +x rsc-gen.sh + ./rsc-gen.sh + + echo "[INFO] Show existing storageclasses" + if ! kubectl get storageclass | grep -q nested; then + echo "[WARNING] No nested storageclasses" + else + kubectl get storageclass | grep nested + echo "[SUCCESS] Done" + fi + + - name: Configure NFS storage + if: ${{ inputs.storage_type == 'nfs' }} + id: storage-nfs-setup + working-directory: ${{ env.SETUP_CLUSTER_TYPE_PATH }}/storage/nfs + env: + NAMESPACE: ${{ needs.bootstrap.outputs.namespace }} + run: | + nfs_ready() { + local count=90 + local controller + local csi_controller + local csi_node_desired + local csi_node_ready + + for i in $(seq 1 $count); do + echo "[INFO] Check d8-csi-nfs pods (attempt ${i}/${count})" + controller=$(kubectl -n d8-csi-nfs get deploy controller -o jsonpath='{.status.readyReplicas}' 2>/dev/null || echo "0") + csi_controller=$(kubectl -n d8-csi-nfs get deploy csi-controller -o jsonpath='{.status.readyReplicas}' 2>/dev/null || echo "0") + csi_node_desired=$(kubectl -n d8-csi-nfs get ds csi-node -o jsonpath='{.status.desiredNumberScheduled}' 2>/dev/null || echo "0") + csi_node_ready=$(kubectl -n d8-csi-nfs get ds csi-node -o jsonpath='{.status.numberReady}' 2>/dev/null || echo "0") + + if [[ "$controller" -ge 1 && "$csi_controller" -ge 1 && "$csi_node_desired" -gt 0 && "$csi_node_ready" -eq "$csi_node_desired" ]]; then + echo "[SUCCESS] NFS CSI is ready (controller=${controller}, csi-controller=${csi_controller}, csi-node=${csi_node_ready}/${csi_node_desired})" + return 0 + fi + + echo "[WARNING] NFS CSI not ready: controller=${controller}, csi-controller=${csi_controller}, csi-node=${csi_node_ready}/${csi_node_desired}" + if (( i % 5 == 0 )); then + echo "[DEBUG] Pods in d8-csi-nfs:" + kubectl -n d8-csi-nfs get pods || echo "[WARNING] Failed to retrieve pods" + echo "[DEBUG] Deployments in d8-csi-nfs:" + kubectl -n d8-csi-nfs get deploy || echo "[WARNING] Failed to retrieve deployments" + echo "[DEBUG] DaemonSets in d8-csi-nfs:" + kubectl -n d8-csi-nfs get ds || echo "[WARNING] Failed to retrieve daemonsets" + echo "[DEBUG] csi-nfs module status:" + kubectl get modules csi-nfs -o wide || echo "[WARNING] Failed to retrieve module" + fi + sleep 10 + done + + echo "[ERROR] NFS CSI did not become ready in time" + kubectl -n d8-csi-nfs get pods || true + exit 1 + } + + echo "[INFO] Apply csi-nfs ModuleConfig, ModulePullOverride, snapshot-controller" + kubectl apply -f mc.yaml + + echo "[INFO] Wait for csi-nfs module to be ready" + kubectl wait --for=jsonpath='{.status.phase}'=Ready modules csi-nfs --timeout=300s + + echo "[INFO] Wait for csi-nfs pods to be ready" + nfs_ready + + echo "[INFO] Apply NFSStorageClass" + envsubst < storageclass.yaml | kubectl apply -f - + + echo "[INFO] Configure default storage class" + ./default-sc-configure.sh + + + echo "[INFO] Show existing storageclasses" + kubectl get storageclass + + configure-virtualization: + name: Configure Virtualization (current-release) + runs-on: ubuntu-latest + needs: + - bootstrap + - configure-storage + steps: + - uses: actions/checkout@v4 + - name: Install kubectl CLI + uses: azure/setup-kubectl@v4 + - name: Setup d8 + uses: ./.github/actions/install-d8 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Check kubeconfig + run: | + echo "[INFO] Configure kube config" + mkdir -p ~/.kube + echo "${{ needs.bootstrap.outputs.kubeconfig }}" | base64 -d | base64 -d > ~/.kube/config + chmod 600 ~/.kube/config + kubectl config use-context nested-e2e-nested-sa + + - name: Configure Virtualization + run: | + REGISTRY=$(base64 -d <<< "${{secrets.DEV_REGISTRY_DOCKER_CFG}}" | jq '.auths | to_entries | .[] | .key' -r) + + echo "[INFO] Apply ModuleSource prod config" + kubectl apply -f -< ~/.kube/config + chmod 600 ~/.kube/config + + - name: "Run E2E tests on current-release (STUB)" + run: | + echo "[INFO] Current release tag: ${{ env.CURRENT_RELEASE }}" + echo "[INFO] Storage type: ${{ inputs.storage_type }}" + echo "" + echo "[INFO] Verifying virtualization module is running" + kubectl get modules virtualization || true + kubectl get mpo virtualization || true + echo "" + echo "[STUB] E2E tests on current-release ${{ env.CURRENT_RELEASE }} -- PASSED" + echo "[INFO] Resources are intentionally left in the cluster for the upgrade test" + + patch-mpo: + name: "Patch MPO to new-release: ${{ inputs.new_release }}" + runs-on: ubuntu-latest + needs: + - bootstrap + - e2e-test-current + steps: + - uses: actions/checkout@v4 + + - name: Install kubectl CLI + uses: azure/setup-kubectl@v4 + + - name: Setup d8 + uses: ./.github/actions/install-d8 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Setup kubeconfig + run: | + mkdir -p ~/.kube + echo "${{ needs.bootstrap.outputs.kubeconfig }}" | base64 -d | base64 -d > ~/.kube/config + chmod 600 ~/.kube/config + kubectl config use-context nested-e2e-nested-sa + + - name: Show current MPO state + run: | + echo "[INFO] Current ModulePullOverride before patching:" + kubectl get mpo virtualization -o yaml + + - name: "Patch ModulePullOverride to new-release: ${{ env.NEW_RELEASE }}" + run: | + echo "[INFO] Patching MPO virtualization imageTag from ${{ env.CURRENT_RELEASE }} to ${{ env.NEW_RELEASE }}" + kubectl patch mpo virtualization --type merge -p '{"spec":{"imageTag":"${{ env.NEW_RELEASE }}"}}' + + echo "[INFO] Show patched ModulePullOverride:" + kubectl get mpo virtualization -o yaml + + - name: Wait for Virtualization to be ready after upgrade + run: | + d8_queue_list() { + d8 s queue list | grep -Po '([0-9]+)(?= active)' || echo "Failed to retrieve list queue" + } + + debug_output() { + local NODES + + echo "[ERROR] Virtualization module upgrade failed" + echo "[DEBUG] Show describe virtualization module" + echo "::group::describe virtualization module" + kubectl describe modules virtualization || true + echo "::endgroup::" + echo "[DEBUG] Show namespace d8-virtualization" + kubectl get ns d8-virtualization || true + echo "[DEBUG] Show pods in namespace d8-virtualization" + kubectl -n d8-virtualization get pods || true + echo "[DEBUG] Show pvc in namespace d8-virtualization" + kubectl get pvc -n d8-virtualization || true + echo "[DEBUG] Show cluster StorageClasses" + kubectl get storageclasses || true + echo "[DEBUG] Show cluster nodes" + kubectl get node + echo "[DEBUG] Show queue (first 25 lines)" + d8 s queue list | head -n 25 || echo "[WARNING] Failed to retrieve list queue" + echo "[DEBUG] Show deckhouse logs" + echo "::group::deckhouse logs" + d8 s logs | tail -n 100 + echo "::endgroup::" + } + + d8_queue() { + local count=90 + local queue_count + + for i in $(seq 1 $count) ; do + queue_count=$(d8_queue_list) + if [ -n "$queue_count" ] && [ "$queue_count" = "0" ]; then + echo "[SUCCESS] Queue is clear" + return 0 + fi + + echo "[INFO] Wait until queues are empty ${i}/${count}" + if (( i % 5 == 0 )); then + echo "[INFO] Show queue list" + d8 s queue list | head -n25 || echo "[WARNING] Failed to retrieve list queue" + echo " " + fi + + if (( i % 10 == 0 )); then + echo "[INFO] deckhouse logs" + echo "::group::deckhouse logs" + d8 s logs | tail -n 100 + echo "::endgroup::" + echo " " + fi + sleep 10 + done + } + + virtualization_ready() { + local count=90 + local virtualization_status + + for i in $(seq 1 $count) ; do + virtualization_status=$(kubectl get modules virtualization -o jsonpath='{.status.phase}') + if [ "$virtualization_status" == "Ready" ]; then + echo "[SUCCESS] Virtualization module is ready after upgrade to ${{ env.NEW_RELEASE }}" + kubectl get modules virtualization + kubectl -n d8-virtualization get pods + kubectl get vmclass || echo "[WARNING] no vmclasses found" + return 0 + fi + + echo "[INFO] Waiting 10s for Virtualization module to be ready (attempt $i/$count)" + + if (( i % 5 == 0 )); then + echo " " + echo "[DEBUG] Show additional info" + kubectl get ns d8-virtualization || echo "[WARNING] Namespace virtualization is not ready" + echo " " + kubectl -n d8-virtualization get pods || echo "[WARNING] Pods in namespace virtualization is not ready" + kubectl get pvc -n d8-virtualization || echo "[WARNING] PVC in namespace virtualization is not ready" + echo " " + fi + sleep 10 + done + + debug_output + exit 1 + } + + virt_handler_ready() { + local count=180 + local virt_handler_ready + local workers + local time_wait=10 + + workers=$(kubectl get nodes -o name | grep worker | wc -l || true) + workers=$((workers)) + + for i in $(seq 1 $count); do + virt_handler_ready=$(kubectl -n d8-virtualization get pods | grep "virt-handler.*Running" | wc -l || true) + + if [[ $virt_handler_ready -ge $workers ]]; then + echo "[SUCCESS] virt-handlers pods are ready" + return 0 + fi + + echo "[INFO] virt-handler pods $virt_handler_ready/$workers " + echo "[INFO] Wait ${time_wait}s virt-handler pods are ready (attempt $i/$count)" + if (( i % 5 == 0 )); then + echo "[DEBUG] Show pods in namespace d8-virtualization" + echo "::group::virtualization pods" + kubectl -n d8-virtualization get pods || echo "No pods in virtualization namespace found" + echo "::endgroup::" + echo "[DEBUG] Show cluster nodes" + echo "::group::cluster nodes" + kubectl get node + echo "::endgroup::" + fi + sleep ${time_wait} + done + + debug_output + exit 1 + } + + echo " " + echo "[INFO] Waiting for Virtualization module to be ready after upgrade to ${{ env.NEW_RELEASE }}" + d8_queue + + virtualization_ready + + echo "[INFO] Checking Virtualization module deployments" + kubectl -n d8-virtualization wait --for=condition=Available deploy --all --timeout 900s + echo "[INFO] Checking virt-handler pods" + virt_handler_ready + + - name: Show MPO state after upgrade + run: | + echo "[INFO] ModulePullOverride after upgrade:" + kubectl get mpo virtualization -o yaml + echo "[INFO] Virtualization module status:" + kubectl get modules virtualization + + e2e-test-new: + name: "E2E test (new-release: ${{ inputs.new_release }})" runs-on: ubuntu-latest + needs: + - bootstrap + - patch-mpo steps: - - name: Print message + - uses: actions/checkout@v4 + + - name: Install kubectl CLI + uses: azure/setup-kubectl@v4 + + - name: Setup kubeconfig + run: | + mkdir -p ~/.kube + echo "${{ needs.bootstrap.outputs.kubeconfig }}" | base64 -d | base64 -d > ~/.kube/config + chmod 600 ~/.kube/config + + - name: "Run E2E tests on new-release (STUB)" run: | - echo "${{ inputs.message }}" + echo "[INFO] New release tag: ${{ env.NEW_RELEASE }}" + echo "[INFO] Storage type: ${{ inputs.storage_type }}" + echo "" + echo "[INFO] Verifying virtualization module is running with new release" + kubectl get modules virtualization || true + kubectl get mpo virtualization || true + echo "" + echo "[STUB] E2E tests on new-release ${{ env.NEW_RELEASE }} -- PASSED" + echo "[INFO] Cluster is intentionally left running (no cleanup)" From 01b17165d24113ebec6181e566d4a9e4a50fad23 Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Thu, 9 Apr 2026 15:11:26 +0300 Subject: [PATCH 03/20] add sds local Signed-off-by: Nikita Korolev --- .../e2e-test-releases-reusable-pipeline.yml | 129 +++++++++++++++++- .github/workflows/e2e-test-releases.yml | 29 +--- .../charts/infra/templates/nfs/svc.yaml | 2 +- .../charts/infra/templates/nfs/vm.yaml | 2 +- .../storage/sds-local-volume/lsc-gen.sh | 109 +++++++++++++++ .../storage/sds-local-volume/mc.yaml | 8 ++ 6 files changed, 244 insertions(+), 35 deletions(-) create mode 100644 test/dvp-static-cluster/storage/sds-local-volume/lsc-gen.sh create mode 100644 test/dvp-static-cluster/storage/sds-local-volume/mc.yaml diff --git a/.github/workflows/e2e-test-releases-reusable-pipeline.yml b/.github/workflows/e2e-test-releases-reusable-pipeline.yml index eba7e4b691..1424388080 100644 --- a/.github/workflows/e2e-test-releases-reusable-pipeline.yml +++ b/.github/workflows/e2e-test-releases-reusable-pipeline.yml @@ -224,7 +224,7 @@ jobs: memory: size: ${{ inputs.cluster_config_workers_memory }} additionalDisks: - - size: 50Gi + - size: 100Gi EOF mkdir -p tmp @@ -480,7 +480,7 @@ jobs: - name: Configure replicated storage id: storage-replicated-setup - if: ${{ inputs.storage_type == 'replicated' }} + if: ${{ inputs.storage_type == 'replicated' || inputs.storage_type == 'mixed'}} working-directory: ${{ env.SETUP_CLUSTER_TYPE_PATH }}/storage/sds-replicated run: | d8_queue_list() { @@ -644,6 +644,18 @@ jobs: exit 1 } + lvg_ready() { + echo "[INFO] Wait for generated LVMVolumeGroup resources to be Ready" + if ! kubectl wait -f sds-lvg.yaml --for=jsonpath='{.status.phase}'=Ready --timeout=300s; then + echo "[ERROR] Generated LVMVolumeGroup resources did not become Ready" + kubectl get lvmvolumegroup -o wide || true + exit 1 + fi + + echo "[SUCCESS] Generated LVMVolumeGroup resources are Ready" + kubectl get lvmvolumegroup -o wide + } + d8_queue kubectl apply -f mc.yaml @@ -662,6 +674,7 @@ jobs: chmod +x lvg-gen.sh ./lvg-gen.sh + lvg_ready chmod +x rsc-gen.sh ./rsc-gen.sh @@ -674,12 +687,113 @@ jobs: echo "[SUCCESS] Done" fi + - name: Configure sds-local-volume + if: ${{ inputs.storage_type == 'local' || inputs.storage_type == 'mixed' }} + working-directory: ${{ env.SETUP_CLUSTER_TYPE_PATH }}/storage/sds-local-volume + run: | + d8_queue_list() { + d8 s queue list | grep -Po '([0-9]+)(?= active)' || echo "[WARNING] Failed to retrieve list queue" + } + + d8_queue() { + local count=90 + local queue_count + + for i in $(seq 1 $count) ; do + queue_count=$(d8_queue_list) + if [ -n "$queue_count" ] && [ "$queue_count" = "0" ]; then + echo "[SUCCESS] Queue is clear" + return 0 + fi + + echo "[INFO] Wait until queues are empty ${i}/${count}" + if (( i % 5 == 0 )); then + echo "[INFO] Show queue list" + d8 s queue list | head -n25 || echo "[WARNING] Failed to retrieve list queue" + echo " " + fi + + if (( i % 10 == 0 )); then + echo "[INFO] deckhouse logs" + echo "::group::deckhouse logs" + d8 s logs | tail -n 100 + echo "::endgroup::" + echo " " + fi + sleep 10 + done + } + + sds_local_volume_ready() { + local count=90 + local local_volume_status + local csi_node_desired + local csi_node_ready + local deploy_count + local controller_ready + + for i in $(seq 1 $count); do + local_volume_status=$(kubectl get modules sds-local-volume -o jsonpath='{.status.phase}' 2>/dev/null || echo "False") + csi_node_desired=$(kubectl -n d8-sds-local-volume get ds csi-node -o jsonpath='{.status.desiredNumberScheduled}' 2>/dev/null || echo "0") + csi_node_ready=$(kubectl -n d8-sds-local-volume get ds csi-node -o jsonpath='{.status.numberReady}' 2>/dev/null || echo "0") + deploy_count=$(kubectl -n d8-sds-local-volume get deploy -o name 2>/dev/null | wc -l | tr -d ' ') + controller_ready=false + + if [[ "${deploy_count}" -gt 0 ]] && kubectl -n d8-sds-local-volume wait --for=condition=Available deploy --all --timeout=10s >/dev/null 2>&1; then + controller_ready=true + fi + + if [[ "${local_volume_status}" == "Ready" && "${csi_node_desired}" -gt 0 && "${csi_node_ready}" -eq "${csi_node_desired}" && "${controller_ready}" == "true" ]]; then + echo "[SUCCESS] sds-local-volume is ready (module=${local_volume_status}, csi-node=${csi_node_ready}/${csi_node_desired}, deployments=${deploy_count})" + kubectl get modules sds-local-volume + kubectl -n d8-sds-local-volume get pods + return 0 + fi + + echo "[INFO] Waiting for sds-local-volume to be ready (attempt ${i}/${count})" + echo "[WARNING] Current state: module=${local_volume_status}, csi-node=${csi_node_ready}/${csi_node_desired}, deployments=${deploy_count}, controller_ready=${controller_ready}" + if (( i % 5 == 0 )); then + kubectl get ns d8-sds-local-volume || true + kubectl get modules sds-local-volume -o wide || true + kubectl -n d8-sds-local-volume get pods || true + kubectl -n d8-sds-local-volume get ds || true + kubectl -n d8-sds-local-volume get deploy || true + d8 s queue list | head -n 25 || true + fi + sleep 10 + done + + echo "[ERROR] sds-local-volume did not become ready in time" + kubectl get modules sds-local-volume -o wide || true + kubectl -n d8-sds-local-volume get pods || true + d8 s queue list || true + echo "::group::deckhouse logs" + d8 s logs | tail -n 100 + echo "::endgroup::" + exit 1 + } + + echo "[INFO] Apply sds-local-volume ModuleConfig" + kubectl apply -f mc.yaml + + echo "[INFO] Wait for sds-local-volume module queue" + d8_queue + kubectl wait --for=jsonpath='{.status.phase}'=Ready modules sds-local-volume --timeout=300s + sds_local_volume_ready + + chmod +x ./lsc-gen.sh + ./lsc-gen.sh + + echo "[INFO] Show resulting local storage classes" + kubectl get localstorageclass || true + - name: Configure NFS storage - if: ${{ inputs.storage_type == 'nfs' }} + if: ${{ inputs.storage_type == 'nfs' || inputs.storage_type == 'mixed' }} id: storage-nfs-setup working-directory: ${{ env.SETUP_CLUSTER_TYPE_PATH }}/storage/nfs env: NAMESPACE: ${{ needs.bootstrap.outputs.namespace }} + STORAGE_TYPE: ${{ inputs.storage_type }} run: | nfs_ready() { local count=90 @@ -730,10 +844,11 @@ jobs: echo "[INFO] Apply NFSStorageClass" envsubst < storageclass.yaml | kubectl apply -f - - - echo "[INFO] Configure default storage class" - ./default-sc-configure.sh - + + if [[ "${STORAGE_TYPE}" == "mixed" ]]; then + echo "[INFO] Configure default storage class as ${STORAGE_TYPE}" + ./default-sc-configure.sh + fi echo "[INFO] Show existing storageclasses" kubectl get storageclass diff --git a/.github/workflows/e2e-test-releases.yml b/.github/workflows/e2e-test-releases.yml index 5a30c5480d..3c7c9b16c3 100644 --- a/.github/workflows/e2e-test-releases.yml +++ b/.github/workflows/e2e-test-releases.yml @@ -20,7 +20,7 @@ on: current-release: description: "Current release tag, like v1.4.1" required: false # before merge to main, set to true - default: "v1.4.1" + default: "v1.4.1" # before merge to main, remove type: string next-release: description: "Next release like v1.5.0, should be greater than current-release" @@ -50,14 +50,14 @@ jobs: echo "randuuid4c=$(openssl rand -hex 2)" >> $GITHUB_OUTPUT e2e-replicated: - name: E2E Release Pipeline (Replicated) + name: E2E Release Pipeline (Replicated + NFS) needs: - set-vars uses: ./.github/workflows/e2e-test-releases-reusable-pipeline.yml with: current_release: ${{ github.event.inputs.current-release }} new_release: ${{ github.event.inputs.next-release }} - storage_type: replicated + storage_type: mixed nested_storageclass_name: nested-thin-r1 branch: main deckhouse_channel: alpha @@ -73,26 +73,3 @@ jobs: PROD_IO_REGISTRY_DOCKER_CFG: ${{ secrets.PROD_IO_REGISTRY_DOCKER_CFG }} BOOTSTRAP_DEV_PROXY: ${{ secrets.BOOTSTRAP_DEV_PROXY }} - e2e-nfs: - name: E2E Release Pipeline (NFS) - needs: - - set-vars - uses: ./.github/workflows/e2e-test-releases-reusable-pipeline.yml - with: - current_release: ${{ github.event.inputs.current-release }} - new_release: ${{ github.event.inputs.next-release }} - storage_type: nfs - nested_storageclass_name: nfs - branch: main - deckhouse_channel: alpha - default_user: cloud - go_version: "1.24.13" - date_start: ${{ needs.set-vars.outputs.date_start }} - randuuid4c: ${{ needs.set-vars.outputs.randuuid4c }} - cluster_config_workers_memory: "9Gi" - cluster_config_k8s_version: "Automatic" - secrets: - DEV_REGISTRY_DOCKER_CFG: ${{ secrets.DEV_REGISTRY_DOCKER_CFG }} - VIRT_E2E_NIGHTLY_SA_TOKEN: ${{ secrets.VIRT_E2E_NIGHTLY_SA_TOKEN }} - PROD_IO_REGISTRY_DOCKER_CFG: ${{ secrets.PROD_IO_REGISTRY_DOCKER_CFG }} - BOOTSTRAP_DEV_PROXY: ${{ secrets.BOOTSTRAP_DEV_PROXY }} diff --git a/test/dvp-static-cluster/charts/infra/templates/nfs/svc.yaml b/test/dvp-static-cluster/charts/infra/templates/nfs/svc.yaml index 3e71f64488..975e9f6ad9 100644 --- a/test/dvp-static-cluster/charts/infra/templates/nfs/svc.yaml +++ b/test/dvp-static-cluster/charts/infra/templates/nfs/svc.yaml @@ -1,4 +1,4 @@ -{{ if eq .Values.storageType "nfs" }} +{{ if or (eq .Values.storageType "nfs") (eq .Values.storageType "mixed") }} --- apiVersion: v1 kind: Service diff --git a/test/dvp-static-cluster/charts/infra/templates/nfs/vm.yaml b/test/dvp-static-cluster/charts/infra/templates/nfs/vm.yaml index b7d77a81a7..718390449e 100644 --- a/test/dvp-static-cluster/charts/infra/templates/nfs/vm.yaml +++ b/test/dvp-static-cluster/charts/infra/templates/nfs/vm.yaml @@ -1,4 +1,4 @@ -{{ if eq .Values.storageType "nfs" }} +{{ if or (eq .Values.storageType "nfs") (eq .Values.storageType "mixed") }} --- apiVersion: virtualization.deckhouse.io/v1alpha2 kind: VirtualDisk diff --git a/test/dvp-static-cluster/storage/sds-local-volume/lsc-gen.sh b/test/dvp-static-cluster/storage/sds-local-volume/lsc-gen.sh new file mode 100644 index 0000000000..a818b26f77 --- /dev/null +++ b/test/dvp-static-cluster/storage/sds-local-volume/lsc-gen.sh @@ -0,0 +1,109 @@ +#!/usr/bin/env bash + +# Copyright 2026 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +script_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) +lvg_generator_script="${script_dir}/../sds-replicated/lvg-gen.sh" +manifest=sds-local-lsc.yaml +localStorageClassName=nested-local-thin +targetThinPoolName=thin-data + +discover_lvgs() { + kubectl get lvmvolumegroup -o json | jq -rc \ + --arg targetThinPoolName "${targetThinPoolName}" ' + .items[] + | select((.spec.thinPools // []) | any(.name == $targetThinPoolName)) + | {name: .metadata.name} + ' +} + +lvgs=$(discover_lvgs) + +if [[ -z "${lvgs}" ]]; then + echo "[WARNING] No LVMVolumeGroup resources with thin pool ${targetThinPoolName} found" + echo "[INFO] Trying to create missing LVMVolumeGroup resources via ${lvg_generator_script}" + kubectl get lvmvolumegroup -o wide || true + + if [[ ! -x "${lvg_generator_script}" ]]; then + chmod +x "${lvg_generator_script}" + fi + + "${lvg_generator_script}" + + if [[ -f "sds-lvg.yaml" ]]; then + echo "[INFO] Wait for generated LVMVolumeGroup resources to become Ready" + kubectl wait -f "sds-lvg.yaml" --for=jsonpath='{.status.phase}'=Ready --timeout=300s + fi + + lvgs=$(discover_lvgs) +fi + +if [[ -z "${lvgs}" ]]; then + echo "[ERROR] No LVMVolumeGroup resources with thin pool ${targetThinPoolName} found after creation attempt" + kubectl get lvmvolumegroup -o wide || true + exit 1 +fi + +cat << EOF > "${manifest}" +--- +apiVersion: storage.deckhouse.io/v1alpha1 +kind: LocalStorageClass +metadata: + name: ${localStorageClassName} +spec: + lvm: + type: Thin + lvmVolumeGroups: +EOF + +for lvg in ${lvgs}; do + lvg_name=$(echo "${lvg}" | jq -r '.name') + echo "[INFO] Add LVMVolumeGroup ${lvg_name} to LocalStorageClass" +cat << EOF >> "${manifest}" + - name: ${lvg_name} + thin: + poolName: ${targetThinPoolName} +EOF +done + +cat << EOF >> "${manifest}" + reclaimPolicy: Delete + volumeBindingMode: WaitForFirstConsumer +EOF + +kubectl apply -f "${manifest}" + +for i in $(seq 1 60); do + lsc_phase=$(kubectl get localstorageclass "${localStorageClassName}" -o jsonpath='{.status.phase}' 2>/dev/null || true) + if [[ "${lsc_phase}" == "Created" ]]; then + echo "[SUCCESS] LocalStorageClass ${localStorageClassName} is Created" + kubectl get localstorageclass "${localStorageClassName}" -o yaml + kubectl get storageclass "${localStorageClassName}" + exit 0 + fi + + echo "[INFO] Waiting for LocalStorageClass ${localStorageClassName} to become Created (attempt ${i}/60)" + if (( i % 5 == 0 )); then + kubectl get localstorageclass "${localStorageClassName}" -o yaml || true + fi + sleep 10 +done + +echo "[ERROR] LocalStorageClass ${localStorageClassName} was not created in time" +kubectl get localstorageclass "${localStorageClassName}" -o yaml || true +kubectl get storageclass || true +exit 1 diff --git a/test/dvp-static-cluster/storage/sds-local-volume/mc.yaml b/test/dvp-static-cluster/storage/sds-local-volume/mc.yaml new file mode 100644 index 0000000000..ffe4020fea --- /dev/null +++ b/test/dvp-static-cluster/storage/sds-local-volume/mc.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: deckhouse.io/v1alpha1 +kind: ModuleConfig +metadata: + name: sds-local-volume +spec: + version: 2 + enabled: true From 3e1220cfad123042a4bd60e21eb2c9d5823b542f Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Thu, 9 Apr 2026 15:28:01 +0300 Subject: [PATCH 04/20] separate sds-configurator deploy scripts Signed-off-by: Nikita Korolev --- .github/workflows/e2e-reusable-pipeline.yml | 5 +++-- .../e2e-test-releases-reusable-pipeline.yml | 18 +++--------------- .../storage/sds-local-volume/lsc-gen.sh | 7 +------ .../lvg-gen.sh | 13 +++++++++++-- .../storage/sds-node-configurator/mc.yaml | 8 ++++++++ .../storage/sds-replicated/mc.yaml | 9 --------- 6 files changed, 26 insertions(+), 34 deletions(-) rename test/dvp-static-cluster/storage/{sds-replicated => sds-node-configurator}/lvg-gen.sh (79%) mode change 100755 => 100644 create mode 100644 test/dvp-static-cluster/storage/sds-node-configurator/mc.yaml diff --git a/.github/workflows/e2e-reusable-pipeline.yml b/.github/workflows/e2e-reusable-pipeline.yml index 17cb9b31b5..0d28246853 100644 --- a/.github/workflows/e2e-reusable-pipeline.yml +++ b/.github/workflows/e2e-reusable-pipeline.yml @@ -658,6 +658,7 @@ jobs: d8_queue + kubectl apply -f ../sds-node-configurator/mc.yaml kubectl apply -f mc.yaml echo "[INFO] Wait for sds-node-configurator" kubectl wait --for=jsonpath='{.status.phase}'=Ready modules sds-node-configurator --timeout=300s @@ -672,8 +673,8 @@ jobs: echo "[INFO] Wait pods and webhooks sds-replicated pods" sds_pods_ready - chmod +x lvg-gen.sh - ./lvg-gen.sh + chmod +x ../sds-node-configurator/lvg-gen.sh + ../sds-node-configurator/lvg-gen.sh chmod +x rsc-gen.sh ./rsc-gen.sh diff --git a/.github/workflows/e2e-test-releases-reusable-pipeline.yml b/.github/workflows/e2e-test-releases-reusable-pipeline.yml index 1424388080..cba39f836b 100644 --- a/.github/workflows/e2e-test-releases-reusable-pipeline.yml +++ b/.github/workflows/e2e-test-releases-reusable-pipeline.yml @@ -644,20 +644,9 @@ jobs: exit 1 } - lvg_ready() { - echo "[INFO] Wait for generated LVMVolumeGroup resources to be Ready" - if ! kubectl wait -f sds-lvg.yaml --for=jsonpath='{.status.phase}'=Ready --timeout=300s; then - echo "[ERROR] Generated LVMVolumeGroup resources did not become Ready" - kubectl get lvmvolumegroup -o wide || true - exit 1 - fi - - echo "[SUCCESS] Generated LVMVolumeGroup resources are Ready" - kubectl get lvmvolumegroup -o wide - } - d8_queue + kubectl apply -f ../sds-node-configurator/mc.yaml kubectl apply -f mc.yaml echo "[INFO] Wait for sds-node-configurator" kubectl wait --for=jsonpath='{.status.phase}'=Ready modules sds-node-configurator --timeout=300s @@ -672,9 +661,8 @@ jobs: echo "[INFO] Wait pods and webhooks sds-replicated pods" sds_pods_ready - chmod +x lvg-gen.sh - ./lvg-gen.sh - lvg_ready + chmod +x ../sds-node-configurator/lvg-gen.sh + ../sds-node-configurator/lvg-gen.sh chmod +x rsc-gen.sh ./rsc-gen.sh diff --git a/test/dvp-static-cluster/storage/sds-local-volume/lsc-gen.sh b/test/dvp-static-cluster/storage/sds-local-volume/lsc-gen.sh index a818b26f77..f533dabbe9 100644 --- a/test/dvp-static-cluster/storage/sds-local-volume/lsc-gen.sh +++ b/test/dvp-static-cluster/storage/sds-local-volume/lsc-gen.sh @@ -17,7 +17,7 @@ set -euo pipefail script_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) -lvg_generator_script="${script_dir}/../sds-replicated/lvg-gen.sh" +lvg_generator_script="${script_dir}/../sds-node-configurator/lvg-gen.sh" manifest=sds-local-lsc.yaml localStorageClassName=nested-local-thin targetThinPoolName=thin-data @@ -44,11 +44,6 @@ if [[ -z "${lvgs}" ]]; then "${lvg_generator_script}" - if [[ -f "sds-lvg.yaml" ]]; then - echo "[INFO] Wait for generated LVMVolumeGroup resources to become Ready" - kubectl wait -f "sds-lvg.yaml" --for=jsonpath='{.status.phase}'=Ready --timeout=300s - fi - lvgs=$(discover_lvgs) fi diff --git a/test/dvp-static-cluster/storage/sds-replicated/lvg-gen.sh b/test/dvp-static-cluster/storage/sds-node-configurator/lvg-gen.sh old mode 100755 new mode 100644 similarity index 79% rename from test/dvp-static-cluster/storage/sds-replicated/lvg-gen.sh rename to test/dvp-static-cluster/storage/sds-node-configurator/lvg-gen.sh index 6e92143f03..b26de284b8 --- a/test/dvp-static-cluster/storage/sds-replicated/lvg-gen.sh +++ b/test/dvp-static-cluster/storage/sds-node-configurator/lvg-gen.sh @@ -14,12 +14,15 @@ # See the License for the specific language governing permissions and # limitations under the License. -manifest=sds-lvg.yaml +set -euo pipefail + +script_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) +manifest="${script_dir}/sds-lvg.yaml" LVMVG_SIZE=45Gi devs=$(kubectl get blockdevices.storage.deckhouse.io -o json | jq '.items[] | {name: .metadata.name, node: .status.nodeName, dev_path: .status.path}' -rc) -rm -rf "${manifest}" +rm -f "${manifest}" echo detected block devices: "$devs" @@ -55,3 +58,9 @@ EOF done kubectl apply -f "${manifest}" + +echo "[INFO] Wait for generated LVMVolumeGroup resources to become Ready" +kubectl wait -f "${manifest}" --for=jsonpath='{.status.phase}'=Ready --timeout=300s + +echo "[SUCCESS] Generated LVMVolumeGroup resources are Ready" +kubectl get lvmvolumegroup -o wide diff --git a/test/dvp-static-cluster/storage/sds-node-configurator/mc.yaml b/test/dvp-static-cluster/storage/sds-node-configurator/mc.yaml new file mode 100644 index 0000000000..d448935b67 --- /dev/null +++ b/test/dvp-static-cluster/storage/sds-node-configurator/mc.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: deckhouse.io/v1alpha1 +kind: ModuleConfig +metadata: + name: sds-node-configurator +spec: + version: 1 + enabled: true diff --git a/test/dvp-static-cluster/storage/sds-replicated/mc.yaml b/test/dvp-static-cluster/storage/sds-replicated/mc.yaml index c220c01f7f..8e5440ef9a 100644 --- a/test/dvp-static-cluster/storage/sds-replicated/mc.yaml +++ b/test/dvp-static-cluster/storage/sds-replicated/mc.yaml @@ -1,12 +1,3 @@ ---- -apiVersion: deckhouse.io/v1alpha1 -kind: ModuleConfig -metadata: - name: sds-node-configurator -spec: - version: 1 - enabled: true ---- apiVersion: deckhouse.io/v1alpha1 kind: ModuleConfig metadata: From bd7ed7881c98e46dbd26f305fafe2b0fb03c3204 Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Thu, 9 Apr 2026 16:33:13 +0300 Subject: [PATCH 05/20] add build for releases Signed-off-by: Nikita Korolev --- .github/workflows/e2e-test-releases.yml | 80 ++++++++++++++++++++++++- 1 file changed, 79 insertions(+), 1 deletion(-) diff --git a/.github/workflows/e2e-test-releases.yml b/.github/workflows/e2e-test-releases.yml index 3c7c9b16c3..fba5d56294 100644 --- a/.github/workflows/e2e-test-releases.yml +++ b/.github/workflows/e2e-test-releases.yml @@ -26,9 +26,14 @@ on: description: "Next release like v1.5.0, should be greater than current-release" required: false # before merge to main, set to true type: string + enableBuild: + description: "Build release images before E2E tests" + required: false + default: false + type: boolean concurrency: - group: "${{ github.workflow }}-${{ github.event.number || github.ref }}" + group: "${{ github.workflow }}-${{ github.event.inputs.current-release }}-${{ github.event.inputs.next-release || 'no-next' }}" cancel-in-progress: true defaults: @@ -49,10 +54,83 @@ jobs: echo "date-start=$(date +%Y%m%d-%H%M%S)" >> $GITHUB_OUTPUT echo "randuuid4c=$(openssl rand -hex 2)" >> $GITHUB_OUTPUT + prepare-release-matrix: + name: Prepare release matrix + if: ${{ inputs.enableBuild }} + runs-on: ubuntu-latest + outputs: + list: ${{ steps.releases.outputs.list }} + steps: + - name: Prepare release refs + id: releases + env: + CURRENT_RELEASE: ${{ github.event.inputs.current-release }} + NEXT_RELEASE: ${{ github.event.inputs.next-release }} + run: | + if [[ -z "${CURRENT_RELEASE}" ]]; then + echo "::error title=Current release is required::Set workflow input 'current-release' to a git ref or tag that can be checked out." + exit 1 + fi + + releases=$(jq -cn \ + --arg current "${CURRENT_RELEASE}" \ + --arg next "${NEXT_RELEASE}" \ + '[$current, $next] | map(select(. != "")) | unique') + + echo "list=${releases}" >> "$GITHUB_OUTPUT" + + build-release-images: + name: Build release image (${{ matrix.release_tag }}) + if: ${{ inputs.enableBuild }} + runs-on: [self-hosted, large] + needs: + - prepare-release-matrix + strategy: + matrix: + release_tag: ${{ fromJSON(needs.prepare-release-matrix.outputs.list) }} + steps: + - name: Setup Docker config + run: | + echo "DOCKER_CONFIG=$(mktemp -d)" >> $GITHUB_ENV + + - name: Print vars + run: | + echo MODULES_REGISTRY=${{ vars.DEV_REGISTRY }} + echo MODULES_MODULE_SOURCE=${{ vars.DEV_MODULE_SOURCE }} + echo MODULES_MODULE_NAME=${{ vars.MODULE_NAME }} + echo MODULES_MODULE_TAG=${{ matrix.release_tag }} + echo DOCKER_CONFIG=$DOCKER_CONFIG + + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ matrix.release_tag }} + + - uses: deckhouse/modules-actions/setup@v2 + with: + registry: ${{ vars.DEV_REGISTRY }} + registry_login: ${{ vars.DEV_MODULES_REGISTRY_LOGIN }} + registry_password: ${{ secrets.DEV_MODULES_REGISTRY_PASSWORD }} + + - uses: deckhouse/modules-actions/build@v4 + with: + module_source: ${{ vars.DEV_MODULE_SOURCE }} + module_name: ${{ vars.MODULE_NAME }} + module_tag: ${{ matrix.release_tag }} + source_repo: ${{ secrets.SOURCE_REPO_GIT }} + source_repo_ssh_key: ${{ secrets.SOURCE_REPO_SSH_KEY }} + + - name: Cleanup Docker config + if: ${{ always() }} + run: | + rm -rf "$DOCKER_CONFIG" + e2e-replicated: name: E2E Release Pipeline (Replicated + NFS) + if: ${{ always() && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') }} needs: - set-vars + - build-release-images uses: ./.github/workflows/e2e-test-releases-reusable-pipeline.yml with: current_release: ${{ github.event.inputs.current-release }} From edd8b0cb27f01c9c46b019348822e352acb42416 Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Thu, 9 Apr 2026 16:55:15 +0300 Subject: [PATCH 06/20] refactor Signed-off-by: Nikita Korolev --- .../e2e-test-releases-reusable-pipeline.yml | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/.github/workflows/e2e-test-releases-reusable-pipeline.yml b/.github/workflows/e2e-test-releases-reusable-pipeline.yml index cba39f836b..43a43b09a4 100644 --- a/.github/workflows/e2e-test-releases-reusable-pipeline.yml +++ b/.github/workflows/e2e-test-releases-reusable-pipeline.yml @@ -664,6 +664,7 @@ jobs: chmod +x ../sds-node-configurator/lvg-gen.sh ../sds-node-configurator/lvg-gen.sh + echo "[INFO] Configur ReplicatedStorageClass and set default nested-thin-r1" chmod +x rsc-gen.sh ./rsc-gen.sh @@ -897,7 +898,7 @@ jobs: storage: persistentVolumeClaim: size: 10Gi - storageClassName: ${{ inputs.nested_storageclass_name }} + storageClassName: nested-thin-r1 type: PersistentVolumeClaim virtualMachineCIDRs: - 192.168.10.0/24 @@ -1094,7 +1095,7 @@ jobs: echo "[INFO] Checking virt-handler pods " virt_handler_ready - e2e-test-current: + test-current-release: name: "E2E test (current-release: ${{ inputs.current_release }})" runs-on: ubuntu-latest needs: @@ -1105,6 +1106,11 @@ jobs: - name: Install kubectl CLI uses: azure/setup-kubectl@v4 + + - name: Setup d8 + uses: ./.github/actions/install-d8 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Setup kubeconfig run: | @@ -1129,7 +1135,7 @@ jobs: runs-on: ubuntu-latest needs: - bootstrap - - e2e-test-current + - test-current-release steps: - uses: actions/checkout@v4 @@ -1308,7 +1314,7 @@ jobs: echo "[INFO] Virtualization module status:" kubectl get modules virtualization - e2e-test-new: + test-new-release: name: "E2E test (new-release: ${{ inputs.new_release }})" runs-on: ubuntu-latest needs: From 9420f5cc81f983455c3db80b17d43590bca11fb3 Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Thu, 9 Apr 2026 18:28:07 +0300 Subject: [PATCH 07/20] add release tests Signed-off-by: Nikita Korolev --- .../e2e-test-releases-reusable-pipeline.yml | 73 ++++++- test/e2e/release/current_release_smoke.go | 200 ++++++++++++++++++ test/e2e/release/release_suite_test.go | 67 ++++++ 3 files changed, 335 insertions(+), 5 deletions(-) create mode 100644 test/e2e/release/current_release_smoke.go create mode 100644 test/e2e/release/release_suite_test.go diff --git a/.github/workflows/e2e-test-releases-reusable-pipeline.yml b/.github/workflows/e2e-test-releases-reusable-pipeline.yml index 43a43b09a4..19c1229ce6 100644 --- a/.github/workflows/e2e-test-releases-reusable-pipeline.yml +++ b/.github/workflows/e2e-test-releases-reusable-pipeline.yml @@ -1104,6 +1104,11 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Set up Go ${{ env.GO_VERSION }} + uses: actions/setup-go@v5 + with: + go-version: "${{ env.GO_VERSION }}" + - name: Install kubectl CLI uses: azure/setup-kubectl@v4 @@ -1117,18 +1122,76 @@ jobs: mkdir -p ~/.kube echo "${{ needs.bootstrap.outputs.kubeconfig }}" | base64 -d | base64 -d > ~/.kube/config chmod 600 ~/.kube/config + kubectl config use-context nested-e2e-nested-sa + + - name: Install ginkgo + working-directory: ./test/e2e/ + run: | + echo "Install ginkgo" + go install tool + + - name: Download dependencies + working-directory: ./test/e2e/ + run: | + echo "Download dependencies" + go mod download + + - name: Create vmclass for release e2e tests + run: | + if ! (kubectl get vmclass generic-for-e2e 2>/dev/null); then + kubectl get vmclass/generic -o json | jq 'del(.status) | del(.metadata) | .metadata = {"name":"generic-for-e2e","annotations":{"virtualmachineclass.virtualization.deckhouse.io/is-default-class":"true"}} ' | kubectl create -f - + fi + + echo "[INFO] Showing existing vmclasses" + kubectl get vmclass - - name: "Run E2E tests on current-release (STUB)" + - name: "Run E2E tests on current-release" + id: release-e2e + env: + POST_CLEANUP: no + PRECREATED_CVI_CLEANUP: no + CSI: ${{ inputs.storage_type }} + STORAGE_CLASS_NAME: ${{ inputs.nested_storageclass_name }} run: | echo "[INFO] Current release tag: ${{ env.CURRENT_RELEASE }}" echo "[INFO] Storage type: ${{ inputs.storage_type }}" echo "" echo "[INFO] Verifying virtualization module is running" - kubectl get modules virtualization || true - kubectl get mpo virtualization || true + kubectl get modules virtualization + kubectl get mpo virtualization echo "" - echo "[STUB] E2E tests on current-release ${{ env.CURRENT_RELEASE }} -- PASSED" - echo "[INFO] Resources are intentionally left in the cluster for the upgrade test" + echo "[INFO] Running dedicated release suite" + echo "[INFO] Resources will be intentionally left in the cluster for the upgrade test" + cd ./test/e2e/ + GINKGO_RESULT=$(mktemp -p "$RUNNER_TEMP") + junit_report="release_current_suite.xml" + set +e + go tool ginkgo \ + -v --race --timeout=45m \ + --junit-report="$junit_report" \ + ./release | tee "$GINKGO_RESULT" + GINKGO_EXIT_CODE=$? + set -e + echo "[INFO] Exit code: $GINKGO_EXIT_CODE" + exit $GINKGO_EXIT_CODE + + - name: Upload current-release test results + uses: actions/upload-artifact@v4 + if: always() && steps.release-e2e.outcome != 'skipped' + with: + name: current-release-e2e-results-${{ github.run_id }} + path: test/e2e/release_current_suite.xml + if-no-files-found: ignore + retention-days: 3 + + - name: Upload resources from failed current-release tests + uses: actions/upload-artifact@v4 + if: always() && steps.release-e2e.outcome != 'skipped' + with: + name: current-release-resources-from-failed-tests-${{ github.run_id }} + path: ${{ runner.temp }}/e2e_failed__* + if-no-files-found: ignore + retention-days: 3 patch-mpo: name: "Patch MPO to new-release: ${{ inputs.new_release }}" diff --git a/test/e2e/release/current_release_smoke.go b/test/e2e/release/current_release_smoke.go new file mode 100644 index 0000000000..5674446124 --- /dev/null +++ b/test/e2e/release/current_release_smoke.go @@ -0,0 +1,200 @@ +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package release + +import ( + "context" + "fmt" + "strings" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/utils/ptr" + crclient "sigs.k8s.io/controller-runtime/pkg/client" + + vdbuilder "github.com/deckhouse/virtualization-controller/pkg/builder/vd" + vmbuilder "github.com/deckhouse/virtualization-controller/pkg/builder/vm" + "github.com/deckhouse/virtualization/api/core/v1alpha2" + "github.com/deckhouse/virtualization/test/e2e/internal/framework" + "github.com/deckhouse/virtualization/test/e2e/internal/object" + "github.com/deckhouse/virtualization/test/e2e/internal/util" +) + +const ( + replicatedStorageClass = "nested-thin-r1" + localThinStorageClass = "nested-local-thin" + diskCountCommand = "lsblk -dn -o TYPE | grep -c '^disk$'" +) + +var _ = Describe("CurrentReleaseSmoke", func() { + It("should validate alpine virtual machines on current release", func() { + f := framework.NewFramework("release-current") + DeferCleanup(f.After) + f.Before() + + test := newCurrentReleaseSmokeTest(f) + + By("Creating root and hotplug virtual disks") + Expect(f.CreateWithDeferredDeletion(context.Background(), test.diskObjects()...)).To(Succeed()) + util.UntilObjectPhase(string(v1alpha2.DiskReady), framework.LongTimeout, test.diskObjects()...) + + By("Creating virtual machines") + Expect(f.CreateWithDeferredDeletion(context.Background(), test.vmObjects()...)).To(Succeed()) + util.UntilObjectPhase(string(v1alpha2.MachineRunning), framework.LongTimeout, test.vmOneHotplug, test.vmTwoHotplug) + util.UntilObjectPhase(string(v1alpha2.MachineStopped), framework.MiddleTimeout, test.vmAlwaysOff) + + By("Starting the manual-policy virtual machine") + util.StartVirtualMachine(f, test.vmAlwaysOff) + util.UntilObjectPhase(string(v1alpha2.MachineRunning), framework.LongTimeout, test.vmAlwaysOff) + + By("Attaching hotplug disks") + Expect(f.CreateWithDeferredDeletion(context.Background(), test.vmbdaOneHotplug, test.vmbdaReplicated, test.vmbdaLocalThin)).To(Succeed()) + util.UntilObjectPhase( + string(v1alpha2.BlockDeviceAttachmentPhaseAttached), + framework.LongTimeout, + test.vmbdaOneHotplug, + test.vmbdaReplicated, + test.vmbdaLocalThin, + ) + + By("Waiting for guest agent and SSH access") + test.expectGuestReady(test.vmAlwaysOff) + test.expectGuestReady(test.vmOneHotplug) + test.expectGuestReady(test.vmTwoHotplug) + + By("Checking attached disks inside guests") + test.expectDiskCount(test.vmAlwaysOff, 1) + test.expectDiskCount(test.vmOneHotplug, 2) + test.expectDiskCount(test.vmTwoHotplug, 3) + }) +}) + +type currentReleaseSmokeTest struct { + framework *framework.Framework + + vmAlwaysOff *v1alpha2.VirtualMachine + vmOneHotplug *v1alpha2.VirtualMachine + vmTwoHotplug *v1alpha2.VirtualMachine + + rootAlwaysOff *v1alpha2.VirtualDisk + rootOneHotplug *v1alpha2.VirtualDisk + rootTwoHotplug *v1alpha2.VirtualDisk + + hotplugOne *v1alpha2.VirtualDisk + hotplugReplicated *v1alpha2.VirtualDisk + hotplugLocalThin *v1alpha2.VirtualDisk + + vmbdaOneHotplug *v1alpha2.VirtualMachineBlockDeviceAttachment + vmbdaReplicated *v1alpha2.VirtualMachineBlockDeviceAttachment + vmbdaLocalThin *v1alpha2.VirtualMachineBlockDeviceAttachment +} + +func newCurrentReleaseSmokeTest(f *framework.Framework) *currentReleaseSmokeTest { + test := ¤tReleaseSmokeTest{framework: f} + namespace := f.Namespace().Name + + test.rootAlwaysOff = newRootDisk("vd-root-always-off", namespace) + test.rootOneHotplug = newRootDisk("vd-root-one-hotplug", namespace) + test.rootTwoHotplug = newRootDisk("vd-root-two-hotplug", namespace) + + test.hotplugOne = newHotplugDisk("vd-hotplug", namespace, replicatedStorageClass) + test.hotplugReplicated = newHotplugDisk("vd-hotplug-replicated", namespace, replicatedStorageClass) + test.hotplugLocalThin = newHotplugDisk("vd-hotplug-local-thin", namespace, localThinStorageClass) + + test.vmAlwaysOff = newVM("vm-always-off", namespace, v1alpha2.ManualPolicy, test.rootAlwaysOff.Name) + test.vmOneHotplug = newVM("vm-one-hotplug", namespace, v1alpha2.AlwaysOnUnlessStoppedManually, test.rootOneHotplug.Name) + test.vmTwoHotplug = newVM("vm-two-hotplug", namespace, v1alpha2.AlwaysOnPolicy, test.rootTwoHotplug.Name) + + test.vmbdaOneHotplug = object.NewVMBDAFromDisk("vmbda", test.vmOneHotplug.Name, test.hotplugOne) + test.vmbdaReplicated = object.NewVMBDAFromDisk("vmbda1", test.vmTwoHotplug.Name, test.hotplugReplicated) + test.vmbdaLocalThin = object.NewVMBDAFromDisk("vmbda2", test.vmTwoHotplug.Name, test.hotplugLocalThin) + + return test +} + +func newRootDisk(name, namespace string) *v1alpha2.VirtualDisk { + return object.NewVDFromCVI( + name, + namespace, + object.PrecreatedCVIAlpineBIOS, + vdbuilder.WithStorageClass(ptr.To(replicatedStorageClass)), + vdbuilder.WithSize(ptr.To(resource.MustParse("350Mi"))), + ) +} + +func newHotplugDisk(name, namespace, storageClass string) *v1alpha2.VirtualDisk { + return object.NewBlankVD( + name, + namespace, + ptr.To(storageClass), + ptr.To(resource.MustParse("100Mi")), + ) +} + +func newVM(name, namespace string, runPolicy v1alpha2.RunPolicy, rootDiskName string) *v1alpha2.VirtualMachine { + return vmbuilder.New( + vmbuilder.WithName(name), + vmbuilder.WithNamespace(namespace), + vmbuilder.WithCPU(1, ptr.To("20%")), + vmbuilder.WithMemory(resource.MustParse("512Mi")), + vmbuilder.WithLiveMigrationPolicy(v1alpha2.AlwaysSafeMigrationPolicy), + vmbuilder.WithVirtualMachineClass(object.DefaultVMClass), + vmbuilder.WithProvisioningUserData(object.AlpineCloudInit), + vmbuilder.WithRunPolicy(runPolicy), + vmbuilder.WithBlockDeviceRefs(v1alpha2.BlockDeviceSpecRef{ + Kind: v1alpha2.DiskDevice, + Name: rootDiskName, + }), + ) +} + +func (t *currentReleaseSmokeTest) diskObjects() []crclient.Object { + return []crclient.Object{ + t.rootAlwaysOff, + t.rootOneHotplug, + t.rootTwoHotplug, + t.hotplugOne, + t.hotplugReplicated, + t.hotplugLocalThin, + } +} + +func (t *currentReleaseSmokeTest) vmObjects() []crclient.Object { + return []crclient.Object{ + t.vmAlwaysOff, + t.vmOneHotplug, + t.vmTwoHotplug, + } +} + +func (t *currentReleaseSmokeTest) expectGuestReady(vm *v1alpha2.VirtualMachine) { + By(fmt.Sprintf("Waiting for guest agent on %s", vm.Name)) + util.UntilVMAgentReady(crclient.ObjectKeyFromObject(vm), framework.LongTimeout) + + By(fmt.Sprintf("Waiting for SSH access on %s", vm.Name)) + util.UntilSSHReady(t.framework, vm, framework.LongTimeout) +} + +func (t *currentReleaseSmokeTest) expectDiskCount(vm *v1alpha2.VirtualMachine, expectedCount int) { + Eventually(func(g Gomega) { + output, err := t.framework.SSHCommand(vm.Name, vm.Namespace, diskCountCommand, framework.WithSSHTimeout(10*time.Second)) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(strings.TrimSpace(output)).To(Equal(fmt.Sprintf("%d", expectedCount))) + }).WithTimeout(framework.LongTimeout).WithPolling(time.Second).Should(Succeed()) +} diff --git a/test/e2e/release/release_suite_test.go b/test/e2e/release/release_suite_test.go new file mode 100644 index 0000000000..13335f8984 --- /dev/null +++ b/test/e2e/release/release_suite_test.go @@ -0,0 +1,67 @@ +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package release + +import ( + "context" + "fmt" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + + "github.com/deckhouse/virtualization/api/core/v1alpha2" + "github.com/deckhouse/virtualization/test/e2e/controller" + "github.com/deckhouse/virtualization/test/e2e/internal/config" + "github.com/deckhouse/virtualization/test/e2e/internal/framework" + "github.com/deckhouse/virtualization/test/e2e/internal/precreatedcvi" + "github.com/deckhouse/virtualization/test/e2e/internal/util" +) + +var cviManager = precreatedcvi.NewManager() + +func TestRelease(t *testing.T) { + log.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) + RegisterFailHandler(Fail) + RunSpecs(t, "Release Tests") +} + +var _ = SynchronizedBeforeSuite(func() { + controller.NewBeforeProcess1Body() + + By("Creating or reusing precreated CVIs") + err := cviManager.Bootstrap(context.Background()) + Expect(err).NotTo(HaveOccurred()) + + cvis := cviManager.CVIsAsObjects() + By(fmt.Sprintf("Waiting for all %d precreated CVIs to be ready", len(cvis))) + util.UntilObjectPhase(string(v1alpha2.ImageReady), framework.LongTimeout, cvis...) +}, func() {}) + +var _ = SynchronizedAfterSuite(func() { + if !config.IsCleanUpNeeded() || !config.IsPrecreatedCVICleanupNeeded() { + return + } + + By("Cleaning up precreated CVIs") + err := cviManager.Cleanup(context.Background()) + Expect(err).NotTo(HaveOccurred(), "Failed to delete precreated CVIs") +}, func() { + controller.NewAfterAllProcessBody() +}) From ad9845ff820b995802e1fd8a121d1b14c6fbad87 Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Fri, 10 Apr 2026 10:18:57 +0300 Subject: [PATCH 08/20] add labels test=release Signed-off-by: Nikita Korolev --- .github/workflows/e2e-test-releases-reusable-pipeline.yml | 6 +++--- .github/workflows/e2e-test-releases.yml | 1 - .../charts/infra/templates/_helpers.tpl | 8 ++++++++ test/dvp-static-cluster/charts/infra/templates/ns.yaml | 2 +- test/dvp-static-cluster/charts/infra/templates/vmc.yaml | 2 +- 5 files changed, 13 insertions(+), 6 deletions(-) diff --git a/.github/workflows/e2e-test-releases-reusable-pipeline.yml b/.github/workflows/e2e-test-releases-reusable-pipeline.yml index 19c1229ce6..8996432779 100644 --- a/.github/workflows/e2e-test-releases-reusable-pipeline.yml +++ b/.github/workflows/e2e-test-releases-reusable-pipeline.yml @@ -131,7 +131,7 @@ jobs: run: | GIT_SHORT_HASH=$(git rev-parse --short HEAD) - namespace="nightly-e2e-$STORAGE_TYPE-$GIT_SHORT_HASH-$RANDUUID4C" + namespace="release-test-$STORAGE_TYPE-$GIT_SHORT_HASH-$RANDUUID4C" echo "namespace=$namespace" >> $GITHUB_OUTPUT echo "sha_short=$GIT_SHORT_HASH" >> $GITHUB_OUTPUT @@ -833,7 +833,7 @@ jobs: echo "[INFO] Apply NFSStorageClass" envsubst < storageclass.yaml | kubectl apply -f - - + if [[ "${STORAGE_TYPE}" == "mixed" ]]; then echo "[INFO] Configure default storage class as ${STORAGE_TYPE}" ./default-sc-configure.sh @@ -1111,7 +1111,7 @@ jobs: - name: Install kubectl CLI uses: azure/setup-kubectl@v4 - + - name: Setup d8 uses: ./.github/actions/install-d8 env: diff --git a/.github/workflows/e2e-test-releases.yml b/.github/workflows/e2e-test-releases.yml index fba5d56294..3bea15f484 100644 --- a/.github/workflows/e2e-test-releases.yml +++ b/.github/workflows/e2e-test-releases.yml @@ -150,4 +150,3 @@ jobs: VIRT_E2E_NIGHTLY_SA_TOKEN: ${{ secrets.VIRT_E2E_NIGHTLY_SA_TOKEN }} PROD_IO_REGISTRY_DOCKER_CFG: ${{ secrets.PROD_IO_REGISTRY_DOCKER_CFG }} BOOTSTRAP_DEV_PROXY: ${{ secrets.BOOTSTRAP_DEV_PROXY }} - diff --git a/test/dvp-static-cluster/charts/infra/templates/_helpers.tpl b/test/dvp-static-cluster/charts/infra/templates/_helpers.tpl index 4a5e99da43..fb991483fa 100644 --- a/test/dvp-static-cluster/charts/infra/templates/_helpers.tpl +++ b/test/dvp-static-cluster/charts/infra/templates/_helpers.tpl @@ -1,3 +1,11 @@ +{{- define "infra.test-label" -}} +{{- if contains "release" .Values.namespace -}} +test: release +{{- else -}} +test: nightly-e2e +{{- end -}} +{{- end }} + {{- define "infra.vm-labels" -}} {{- $prefix := regexReplaceAll "-\\d+$" . "" -}} vm: {{ . }} diff --git a/test/dvp-static-cluster/charts/infra/templates/ns.yaml b/test/dvp-static-cluster/charts/infra/templates/ns.yaml index 259ba5c4aa..c7437157c6 100644 --- a/test/dvp-static-cluster/charts/infra/templates/ns.yaml +++ b/test/dvp-static-cluster/charts/infra/templates/ns.yaml @@ -3,4 +3,4 @@ kind: Namespace metadata: name: {{ .Values.namespace }} labels: - test: nightly-e2e + {{- include "infra.test-label" . | nindent 4 }} diff --git a/test/dvp-static-cluster/charts/infra/templates/vmc.yaml b/test/dvp-static-cluster/charts/infra/templates/vmc.yaml index 796d266391..19dd8b8e90 100644 --- a/test/dvp-static-cluster/charts/infra/templates/vmc.yaml +++ b/test/dvp-static-cluster/charts/infra/templates/vmc.yaml @@ -3,7 +3,7 @@ kind: VirtualMachineClass metadata: name: {{ include "infra.vmclass-name" . }} labels: - test: nightly-e2e + {{- include "infra.test-label" . | nindent 4 }} spec: cpu: type: Discovery From 5acb34e5ee378d70acf0b759ba8a64b8e2e793ea Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Fri, 10 Apr 2026 10:27:35 +0300 Subject: [PATCH 09/20] add default vers release Signed-off-by: Nikita Korolev --- .github/workflows/e2e-test-releases.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/e2e-test-releases.yml b/.github/workflows/e2e-test-releases.yml index 3bea15f484..9b1fc3a347 100644 --- a/.github/workflows/e2e-test-releases.yml +++ b/.github/workflows/e2e-test-releases.yml @@ -20,11 +20,12 @@ on: current-release: description: "Current release tag, like v1.4.1" required: false # before merge to main, set to true - default: "v1.4.1" # before merge to main, remove + default: "v1.6.3-rc.3" # before merge to main, remove type: string next-release: description: "Next release like v1.5.0, should be greater than current-release" required: false # before merge to main, set to true + default: "v1.7.0" # before merge to main, remove type: string enableBuild: description: "Build release images before E2E tests" From dac74640a6662a79aba6b3d9a5a6b1aa31faa3d8 Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Fri, 10 Apr 2026 11:43:09 +0300 Subject: [PATCH 10/20] rm annotation for caps master-0 Signed-off-by: Nikita Korolev --- .../charts/cluster-config/templates/master-nodes.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/dvp-static-cluster/charts/cluster-config/templates/master-nodes.yaml b/test/dvp-static-cluster/charts/cluster-config/templates/master-nodes.yaml index 191afcd96e..ab59713c38 100644 --- a/test/dvp-static-cluster/charts/cluster-config/templates/master-nodes.yaml +++ b/test/dvp-static-cluster/charts/cluster-config/templates/master-nodes.yaml @@ -4,6 +4,9 @@ {{- $totalNodes = add $totalNodes .count -}} {{- end -}} +{{- $masterCount := $.Values.instances.masterNodes.count | int -}} +{{- if gt $masterCount 1 -}} + {{- $staticCount := sub $masterCount 1 -}} --- apiVersion: deckhouse.io/v1 kind: NodeGroup @@ -29,16 +32,12 @@ spec: matchLabels: role: master -{{- range $_, $i := untilStep 0 (.Values.instances.masterNodes.count | int) 1}} +{{- range $_, $i := untilStep 1 (.Values.instances.masterNodes.count | int) 1}} {{- $vmName := printf "%s-master-%d" $.Values.storageType $i }} --- apiVersion: deckhouse.io/v1alpha1 kind: StaticInstance metadata: - {{- if eq $i 0 }} - annotations: - static.node.deckhouse.io/skip-bootstrap-phase: "" - {{- end }} name: {{ $vmName }} labels: role: master @@ -48,3 +47,4 @@ spec: kind: SSHCredentials name: mvp-static {{- end }} +{{- end }} From 1a13002f5b4f8dfc6371fec29acba972da2b4c06 Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Fri, 10 Apr 2026 13:22:45 +0300 Subject: [PATCH 11/20] fix step 'Run E2E tests on current-release' Signed-off-by: Nikita Korolev --- .../e2e-test-releases-reusable-pipeline.yml | 47 ++++++++++- .github/workflows/e2e-test-releases.yml | 2 +- test/e2e/release/current_release_smoke.go | 82 ++++++++++++++++--- 3 files changed, 116 insertions(+), 15 deletions(-) diff --git a/.github/workflows/e2e-test-releases-reusable-pipeline.yml b/.github/workflows/e2e-test-releases-reusable-pipeline.yml index 8996432779..f699e49244 100644 --- a/.github/workflows/e2e-test-releases-reusable-pipeline.yml +++ b/.github/workflows/e2e-test-releases-reusable-pipeline.yml @@ -834,7 +834,7 @@ jobs: echo "[INFO] Apply NFSStorageClass" envsubst < storageclass.yaml | kubectl apply -f - - if [[ "${STORAGE_TYPE}" == "mixed" ]]; then + if [[ "${STORAGE_TYPE}" != "mixed" ]]; then echo "[INFO] Configure default storage class as ${STORAGE_TYPE}" ./default-sc-configure.sh fi @@ -1152,6 +1152,7 @@ jobs: PRECREATED_CVI_CLEANUP: no CSI: ${{ inputs.storage_type }} STORAGE_CLASS_NAME: ${{ inputs.nested_storageclass_name }} + E2E_CONFIG: ${{ github.workspace }}/test/e2e/default_config.yaml run: | echo "[INFO] Current release tag: ${{ env.CURRENT_RELEASE }}" echo "[INFO] Storage type: ${{ inputs.storage_type }}" @@ -1164,9 +1165,9 @@ jobs: echo "[INFO] Resources will be intentionally left in the cluster for the upgrade test" cd ./test/e2e/ GINKGO_RESULT=$(mktemp -p "$RUNNER_TEMP") - junit_report="release_current_suite.xml" + junit_report="$GITHUB_WORKSPACE/test/e2e/release_current_suite.xml" set +e - go tool ginkgo \ + POST_CLEANUP=no go tool ginkgo \ -v --race --timeout=45m \ --junit-report="$junit_report" \ ./release | tee "$GINKGO_RESULT" @@ -1215,7 +1216,6 @@ jobs: mkdir -p ~/.kube echo "${{ needs.bootstrap.outputs.kubeconfig }}" | base64 -d | base64 -d > ~/.kube/config chmod 600 ~/.kube/config - kubectl config use-context nested-e2e-nested-sa - name: Show current MPO state run: | @@ -1406,3 +1406,42 @@ jobs: echo "" echo "[STUB] E2E tests on new-release ${{ env.NEW_RELEASE }} -- PASSED" echo "[INFO] Cluster is intentionally left running (no cleanup)" + + undeploy-cluster: + name: Undeploy cluster + runs-on: ubuntu-latest + needs: + - bootstrap + - test-new-release + if: always() + steps: + - uses: actions/checkout@v4 + + - name: Setup d8 + uses: ./.github/actions/install-d8 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Install Task + uses: arduino/setup-task@v2 + with: + version: 3.x + repo-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Download artifacts + uses: actions/download-artifact@v5 + with: + name: ${{ inputs.storage_type }}-release-generated-files-${{ inputs.date_start }} + path: ${{ env.SETUP_CLUSTER_TYPE_PATH }}/ + + - name: Configure kubectl via azure/k8s-set-context@v4 + uses: azure/k8s-set-context@v4 + with: + method: kubeconfig + context: e2e-cluster-nightly-e2e-virt-sa + kubeconfig: ${{ secrets.VIRT_E2E_NIGHTLY_SA_TOKEN }} + + - name: infra-undeploy + working-directory: ${{ env.SETUP_CLUSTER_TYPE_PATH }} + run: | + task infra-undeploy diff --git a/.github/workflows/e2e-test-releases.yml b/.github/workflows/e2e-test-releases.yml index 9b1fc3a347..da0a210804 100644 --- a/.github/workflows/e2e-test-releases.yml +++ b/.github/workflows/e2e-test-releases.yml @@ -127,7 +127,7 @@ jobs: rm -rf "$DOCKER_CONFIG" e2e-replicated: - name: E2E Release Pipeline (Replicated + NFS) + name: E2E Release Pipeline (Replicated + Local + NFS) if: ${{ always() && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') }} needs: - set-vars diff --git a/test/e2e/release/current_release_smoke.go b/test/e2e/release/current_release_smoke.go index 5674446124..0fd3601a7d 100644 --- a/test/e2e/release/current_release_smoke.go +++ b/test/e2e/release/current_release_smoke.go @@ -18,8 +18,8 @@ package release import ( "context" + "encoding/json" "fmt" - "strings" "time" . "github.com/onsi/ginkgo/v2" @@ -31,6 +31,7 @@ import ( vdbuilder "github.com/deckhouse/virtualization-controller/pkg/builder/vd" vmbuilder "github.com/deckhouse/virtualization-controller/pkg/builder/vm" "github.com/deckhouse/virtualization/api/core/v1alpha2" + "github.com/deckhouse/virtualization/test/e2e/internal/config" "github.com/deckhouse/virtualization/test/e2e/internal/framework" "github.com/deckhouse/virtualization/test/e2e/internal/object" "github.com/deckhouse/virtualization/test/e2e/internal/util" @@ -39,20 +40,30 @@ import ( const ( replicatedStorageClass = "nested-thin-r1" localThinStorageClass = "nested-local-thin" - diskCountCommand = "lsblk -dn -o TYPE | grep -c '^disk$'" + lsblkJSONCommand = "lsblk --bytes --json --nodeps --output NAME,SIZE,TYPE,MOUNTPOINTS" + minDataDiskSizeBytes = int64(50 * 1024 * 1024) ) var _ = Describe("CurrentReleaseSmoke", func() { It("should validate alpine virtual machines on current release", func() { f := framework.NewFramework("release-current") - DeferCleanup(f.After) + if config.IsCleanUpNeeded() { + DeferCleanup(f.After) + } else { + // Keep created resources after a successful run when POST_CLEANUP=no, + // but still preserve failure dumps if the spec breaks. + DeferCleanup(func() { + if CurrentSpecReport().Failed() { + f.After() + } + }) + } f.Before() test := newCurrentReleaseSmokeTest(f) By("Creating root and hotplug virtual disks") Expect(f.CreateWithDeferredDeletion(context.Background(), test.diskObjects()...)).To(Succeed()) - util.UntilObjectPhase(string(v1alpha2.DiskReady), framework.LongTimeout, test.diskObjects()...) By("Creating virtual machines") Expect(f.CreateWithDeferredDeletion(context.Background(), test.vmObjects()...)).To(Succeed()) @@ -73,15 +84,18 @@ var _ = Describe("CurrentReleaseSmoke", func() { test.vmbdaLocalThin, ) + By("Waiting for all disks to become ready after consumers appear") + util.UntilObjectPhase(string(v1alpha2.DiskReady), framework.LongTimeout, test.diskObjects()...) + By("Waiting for guest agent and SSH access") test.expectGuestReady(test.vmAlwaysOff) test.expectGuestReady(test.vmOneHotplug) test.expectGuestReady(test.vmTwoHotplug) By("Checking attached disks inside guests") - test.expectDiskCount(test.vmAlwaysOff, 1) - test.expectDiskCount(test.vmOneHotplug, 2) - test.expectDiskCount(test.vmTwoHotplug, 3) + test.expectAdditionalDiskCount(test.vmAlwaysOff, 0) + test.expectAdditionalDiskCount(test.vmOneHotplug, 1) + test.expectAdditionalDiskCount(test.vmTwoHotplug, 2) }) }) @@ -105,6 +119,17 @@ type currentReleaseSmokeTest struct { vmbdaLocalThin *v1alpha2.VirtualMachineBlockDeviceAttachment } +type lsblkOutput struct { + BlockDevices []lsblkDevice `json:"blockdevices"` +} + +type lsblkDevice struct { + Name string `json:"name"` + Size int64 `json:"size"` + Type string `json:"type"` + Mountpoints []string `json:"mountpoints"` +} + func newCurrentReleaseSmokeTest(f *framework.Framework) *currentReleaseSmokeTest { test := ¤tReleaseSmokeTest{framework: f} namespace := f.Namespace().Name @@ -191,10 +216,47 @@ func (t *currentReleaseSmokeTest) expectGuestReady(vm *v1alpha2.VirtualMachine) util.UntilSSHReady(t.framework, vm, framework.LongTimeout) } -func (t *currentReleaseSmokeTest) expectDiskCount(vm *v1alpha2.VirtualMachine, expectedCount int) { +func (t *currentReleaseSmokeTest) expectAdditionalDiskCount(vm *v1alpha2.VirtualMachine, expectedCount int) { Eventually(func(g Gomega) { - output, err := t.framework.SSHCommand(vm.Name, vm.Namespace, diskCountCommand, framework.WithSSHTimeout(10*time.Second)) + output, err := t.framework.SSHCommand(vm.Name, vm.Namespace, lsblkJSONCommand, framework.WithSSHTimeout(10*time.Second)) + g.Expect(err).NotTo(HaveOccurred()) + + disks, err := parseLSBLKOutput(output) g.Expect(err).NotTo(HaveOccurred()) - g.Expect(strings.TrimSpace(output)).To(Equal(fmt.Sprintf("%d", expectedCount))) + + count := 0 + for _, disk := range disks { + if disk.Type != "disk" { + continue + } + if disk.Size <= minDataDiskSizeBytes { + continue + } + if hasMountpoint(disk.Mountpoints, "/") { + continue + } + count++ + } + + g.Expect(count).To(Equal(expectedCount)) }).WithTimeout(framework.LongTimeout).WithPolling(time.Second).Should(Succeed()) } + +func parseLSBLKOutput(raw string) ([]lsblkDevice, error) { + var output lsblkOutput + if err := json.Unmarshal([]byte(raw), &output); err != nil { + return nil, fmt.Errorf("parse lsblk json: %w", err) + } + + return output.BlockDevices, nil +} + +func hasMountpoint(mountpoints []string, expected string) bool { + for _, mountpoint := range mountpoints { + if mountpoint == expected { + return true + } + } + + return false +} From ca68a91a74963e73cae44dbd20ca0f6fea6fc2c2 Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Mon, 13 Apr 2026 11:25:05 +0300 Subject: [PATCH 12/20] remove undeploy vms Signed-off-by: Nikita Korolev --- test/e2e/release/current_release_smoke.go | 411 +++++++++++++++++----- 1 file changed, 330 insertions(+), 81 deletions(-) diff --git a/test/e2e/release/current_release_smoke.go b/test/e2e/release/current_release_smoke.go index 0fd3601a7d..bd28bf71f5 100644 --- a/test/e2e/release/current_release_smoke.go +++ b/test/e2e/release/current_release_smoke.go @@ -20,18 +20,19 @@ import ( "context" "encoding/json" "fmt" + "strings" "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/utils/ptr" crclient "sigs.k8s.io/controller-runtime/pkg/client" vdbuilder "github.com/deckhouse/virtualization-controller/pkg/builder/vd" vmbuilder "github.com/deckhouse/virtualization-controller/pkg/builder/vm" "github.com/deckhouse/virtualization/api/core/v1alpha2" - "github.com/deckhouse/virtualization/test/e2e/internal/config" "github.com/deckhouse/virtualization/test/e2e/internal/framework" "github.com/deckhouse/virtualization/test/e2e/internal/object" "github.com/deckhouse/virtualization/test/e2e/internal/util" @@ -42,22 +43,14 @@ const ( localThinStorageClass = "nested-local-thin" lsblkJSONCommand = "lsblk --bytes --json --nodeps --output NAME,SIZE,TYPE,MOUNTPOINTS" minDataDiskSizeBytes = int64(50 * 1024 * 1024) + defaultRootDiskSize = "350Mi" + defaultDataDiskSize = "100Mi" + iperfDurationSeconds = 5 ) var _ = Describe("CurrentReleaseSmoke", func() { - It("should validate alpine virtual machines on current release", func() { + It("should validate current release virtual machines", func() { f := framework.NewFramework("release-current") - if config.IsCleanUpNeeded() { - DeferCleanup(f.After) - } else { - // Keep created resources after a successful run when POST_CLEANUP=no, - // but still preserve failure dumps if the spec breaks. - DeferCleanup(func() { - if CurrentSpecReport().Failed() { - f.After() - } - }) - } f.Before() test := newCurrentReleaseSmokeTest(f) @@ -67,56 +60,81 @@ var _ = Describe("CurrentReleaseSmoke", func() { By("Creating virtual machines") Expect(f.CreateWithDeferredDeletion(context.Background(), test.vmObjects()...)).To(Succeed()) - util.UntilObjectPhase(string(v1alpha2.MachineRunning), framework.LongTimeout, test.vmOneHotplug, test.vmTwoHotplug) - util.UntilObjectPhase(string(v1alpha2.MachineStopped), framework.MiddleTimeout, test.vmAlwaysOff) + if runningVMs := test.initialRunningVMObjects(); len(runningVMs) > 0 { + util.UntilObjectPhase(string(v1alpha2.MachineRunning), framework.LongTimeout, runningVMs...) + } + if stoppedVMs := test.initialStoppedVMObjects(); len(stoppedVMs) > 0 { + util.UntilObjectPhase(string(v1alpha2.MachineStopped), framework.MiddleTimeout, stoppedVMs...) + } - By("Starting the manual-policy virtual machine") - util.StartVirtualMachine(f, test.vmAlwaysOff) - util.UntilObjectPhase(string(v1alpha2.MachineRunning), framework.LongTimeout, test.vmAlwaysOff) + By("Starting manual-policy virtual machines") + for _, vmScenario := range test.manualStartVMs() { + util.StartVirtualMachine(f, vmScenario.vm) + } + if startedVMs := test.manualStartVMObjects(); len(startedVMs) > 0 { + util.UntilObjectPhase(string(v1alpha2.MachineRunning), framework.LongTimeout, startedVMs...) + } By("Attaching hotplug disks") - Expect(f.CreateWithDeferredDeletion(context.Background(), test.vmbdaOneHotplug, test.vmbdaReplicated, test.vmbdaLocalThin)).To(Succeed()) - util.UntilObjectPhase( - string(v1alpha2.BlockDeviceAttachmentPhaseAttached), - framework.LongTimeout, - test.vmbdaOneHotplug, - test.vmbdaReplicated, - test.vmbdaLocalThin, - ) + Expect(f.CreateWithDeferredDeletion(context.Background(), test.attachmentObjects()...)).To(Succeed()) + util.UntilObjectPhase(string(v1alpha2.BlockDeviceAttachmentPhaseAttached), framework.LongTimeout, test.attachmentObjects()...) By("Waiting for all disks to become ready after consumers appear") util.UntilObjectPhase(string(v1alpha2.DiskReady), framework.LongTimeout, test.diskObjects()...) By("Waiting for guest agent and SSH access") - test.expectGuestReady(test.vmAlwaysOff) - test.expectGuestReady(test.vmOneHotplug) - test.expectGuestReady(test.vmTwoHotplug) + for _, vmScenario := range test.vms { + test.expectGuestReady(vmScenario.vm) + } By("Checking attached disks inside guests") - test.expectAdditionalDiskCount(test.vmAlwaysOff, 0) - test.expectAdditionalDiskCount(test.vmOneHotplug, 1) - test.expectAdditionalDiskCount(test.vmTwoHotplug, 2) + for _, vmScenario := range test.vms { + test.expectAdditionalDiskCount(vmScenario.vm, vmScenario.expectedAdditionalDisks) + } + + By("Running iperf smoke check between alpine guests") + test.expectIPerfConnectivity() }) }) type currentReleaseSmokeTest struct { - framework *framework.Framework + framework *framework.Framework + vms []*vmScenario + dataDisks []*dataDiskScenario + attachments []*attachmentScenario + vmByName map[string]*vmScenario + dataDiskByName map[string]*dataDiskScenario + iperfClient *vmScenario + iperfServer *vmScenario +} - vmAlwaysOff *v1alpha2.VirtualMachine - vmOneHotplug *v1alpha2.VirtualMachine - vmTwoHotplug *v1alpha2.VirtualMachine +type vmScenario struct { + name string + rootDiskName string + cviName string + cloudInit string + runPolicy v1alpha2.RunPolicy + rootDiskSize string + expectedAdditionalDisks int + + rootDisk *v1alpha2.VirtualDisk + vm *v1alpha2.VirtualMachine +} - rootAlwaysOff *v1alpha2.VirtualDisk - rootOneHotplug *v1alpha2.VirtualDisk - rootTwoHotplug *v1alpha2.VirtualDisk +type dataDiskScenario struct { + name string + storageClass string + size string - hotplugOne *v1alpha2.VirtualDisk - hotplugReplicated *v1alpha2.VirtualDisk - hotplugLocalThin *v1alpha2.VirtualDisk + disk *v1alpha2.VirtualDisk +} + +type attachmentScenario struct { + name string + vmName string + diskName string - vmbdaOneHotplug *v1alpha2.VirtualMachineBlockDeviceAttachment - vmbdaReplicated *v1alpha2.VirtualMachineBlockDeviceAttachment - vmbdaLocalThin *v1alpha2.VirtualMachineBlockDeviceAttachment + attachment *v1alpha2.VirtualMachineBlockDeviceAttachment } type lsblkOutput struct { @@ -130,49 +148,180 @@ type lsblkDevice struct { Mountpoints []string `json:"mountpoints"` } +type iperfReport struct { + End iperfReportEnd `json:"end"` +} + +type iperfReportEnd struct { + SumSent iperfReportSummary `json:"sum_sent"` + SumReceived iperfReportSummary `json:"sum_received"` +} + +type iperfReportSummary struct { + Bytes int64 `json:"bytes"` + BitsPerSecond float64 `json:"bits_per_second"` +} + func newCurrentReleaseSmokeTest(f *framework.Framework) *currentReleaseSmokeTest { - test := ¤tReleaseSmokeTest{framework: f} + test := ¤tReleaseSmokeTest{ + framework: f, + vmByName: make(map[string]*vmScenario), + dataDiskByName: make(map[string]*dataDiskScenario), + } namespace := f.Namespace().Name - test.rootAlwaysOff = newRootDisk("vd-root-always-off", namespace) - test.rootOneHotplug = newRootDisk("vd-root-one-hotplug", namespace) - test.rootTwoHotplug = newRootDisk("vd-root-two-hotplug", namespace) + test.vms = []*vmScenario{ + { + name: "vm-alpine-manual", + rootDiskName: "vd-root-alpine-manual", + cviName: object.PrecreatedCVIAlpineBIOS, + cloudInit: object.AlpineCloudInit, + runPolicy: v1alpha2.ManualPolicy, + rootDiskSize: defaultRootDiskSize, + expectedAdditionalDisks: 0, + }, + { + name: "vm-alpine-single-hotplug", + rootDiskName: "vd-root-alpine-single-hotplug", + cviName: object.PrecreatedCVIAlpineBIOS, + cloudInit: object.AlpineCloudInit, + runPolicy: v1alpha2.AlwaysOnUnlessStoppedManually, + rootDiskSize: defaultRootDiskSize, + expectedAdditionalDisks: 1, + }, + { + name: "vm-alpine-double-hotplug", + rootDiskName: "vd-root-alpine-double-hotplug", + cviName: object.PrecreatedCVIAlpineBIOS, + cloudInit: object.AlpineCloudInit, + runPolicy: v1alpha2.AlwaysOnPolicy, + rootDiskSize: defaultRootDiskSize, + expectedAdditionalDisks: 2, + }, + { + name: "vm-ubuntu-replicated-five", + rootDiskName: "vd-root-ubuntu-replicated-five", + cviName: object.PrecreatedCVIUbuntu, + cloudInit: object.UbuntuCloudInit, + runPolicy: v1alpha2.AlwaysOnUnlessStoppedManually, + expectedAdditionalDisks: 5, + }, + { + name: "vm-ubuntu-mixed-five", + rootDiskName: "vd-root-ubuntu-mixed-five", + cviName: object.PrecreatedCVIUbuntu, + cloudInit: object.UbuntuCloudInit, + runPolicy: v1alpha2.AlwaysOnUnlessStoppedManually, + expectedAdditionalDisks: 5, + }, + { + name: "vm-alpine-iperf-client", + rootDiskName: "vd-root-alpine-iperf-client", + cviName: object.PrecreatedCVIAlpineBIOS, + cloudInit: object.AlpineCloudInit, + runPolicy: v1alpha2.AlwaysOnUnlessStoppedManually, + rootDiskSize: defaultRootDiskSize, + expectedAdditionalDisks: 2, + }, + { + name: "vm-alpine-iperf-server", + rootDiskName: "vd-root-alpine-iperf-server", + cviName: object.PrecreatedCVIAlpineBIOS, + cloudInit: object.PerfCloudInit, + runPolicy: v1alpha2.AlwaysOnUnlessStoppedManually, + rootDiskSize: defaultRootDiskSize, + expectedAdditionalDisks: 0, + }, + } + + test.dataDisks = []*dataDiskScenario{ + {name: "vd-data-alpine-single-hotplug-01-repl", storageClass: replicatedStorageClass, size: defaultDataDiskSize}, + {name: "vd-data-alpine-double-hotplug-01-repl", storageClass: replicatedStorageClass, size: defaultDataDiskSize}, + {name: "vd-data-alpine-double-hotplug-02-local", storageClass: localThinStorageClass, size: defaultDataDiskSize}, + {name: "vd-data-ubuntu-replicated-five-01-repl", storageClass: replicatedStorageClass, size: defaultDataDiskSize}, + {name: "vd-data-ubuntu-replicated-five-02-repl", storageClass: replicatedStorageClass, size: defaultDataDiskSize}, + {name: "vd-data-ubuntu-replicated-five-03-repl", storageClass: replicatedStorageClass, size: defaultDataDiskSize}, + {name: "vd-data-ubuntu-replicated-five-04-repl", storageClass: replicatedStorageClass, size: defaultDataDiskSize}, + {name: "vd-data-ubuntu-replicated-five-05-repl", storageClass: replicatedStorageClass, size: defaultDataDiskSize}, + {name: "vd-data-ubuntu-mixed-five-01-repl", storageClass: replicatedStorageClass, size: defaultDataDiskSize}, + {name: "vd-data-ubuntu-mixed-five-02-repl", storageClass: replicatedStorageClass, size: defaultDataDiskSize}, + {name: "vd-data-ubuntu-mixed-five-03-local", storageClass: localThinStorageClass, size: defaultDataDiskSize}, + {name: "vd-data-ubuntu-mixed-five-04-local", storageClass: localThinStorageClass, size: defaultDataDiskSize}, + {name: "vd-data-ubuntu-mixed-five-05-local", storageClass: localThinStorageClass, size: defaultDataDiskSize}, + {name: "vd-data-alpine-iperf-client-01-repl", storageClass: replicatedStorageClass, size: defaultDataDiskSize}, + {name: "vd-data-alpine-iperf-client-02-repl", storageClass: replicatedStorageClass, size: defaultDataDiskSize}, + } + + test.attachments = []*attachmentScenario{ + {name: "vmbda-alpine-single-hotplug-01", vmName: "vm-alpine-single-hotplug", diskName: "vd-data-alpine-single-hotplug-01-repl"}, + {name: "vmbda-alpine-double-hotplug-01", vmName: "vm-alpine-double-hotplug", diskName: "vd-data-alpine-double-hotplug-01-repl"}, + {name: "vmbda-alpine-double-hotplug-02", vmName: "vm-alpine-double-hotplug", diskName: "vd-data-alpine-double-hotplug-02-local"}, + {name: "vmbda-ubuntu-replicated-five-01", vmName: "vm-ubuntu-replicated-five", diskName: "vd-data-ubuntu-replicated-five-01-repl"}, + {name: "vmbda-ubuntu-replicated-five-02", vmName: "vm-ubuntu-replicated-five", diskName: "vd-data-ubuntu-replicated-five-02-repl"}, + {name: "vmbda-ubuntu-replicated-five-03", vmName: "vm-ubuntu-replicated-five", diskName: "vd-data-ubuntu-replicated-five-03-repl"}, + {name: "vmbda-ubuntu-replicated-five-04", vmName: "vm-ubuntu-replicated-five", diskName: "vd-data-ubuntu-replicated-five-04-repl"}, + {name: "vmbda-ubuntu-replicated-five-05", vmName: "vm-ubuntu-replicated-five", diskName: "vd-data-ubuntu-replicated-five-05-repl"}, + {name: "vmbda-ubuntu-mixed-five-01", vmName: "vm-ubuntu-mixed-five", diskName: "vd-data-ubuntu-mixed-five-01-repl"}, + {name: "vmbda-ubuntu-mixed-five-02", vmName: "vm-ubuntu-mixed-five", diskName: "vd-data-ubuntu-mixed-five-02-repl"}, + {name: "vmbda-ubuntu-mixed-five-03", vmName: "vm-ubuntu-mixed-five", diskName: "vd-data-ubuntu-mixed-five-03-local"}, + {name: "vmbda-ubuntu-mixed-five-04", vmName: "vm-ubuntu-mixed-five", diskName: "vd-data-ubuntu-mixed-five-04-local"}, + {name: "vmbda-ubuntu-mixed-five-05", vmName: "vm-ubuntu-mixed-five", diskName: "vd-data-ubuntu-mixed-five-05-local"}, + {name: "vmbda-alpine-iperf-client-01", vmName: "vm-alpine-iperf-client", diskName: "vd-data-alpine-iperf-client-01-repl"}, + {name: "vmbda-alpine-iperf-client-02", vmName: "vm-alpine-iperf-client", diskName: "vd-data-alpine-iperf-client-02-repl"}, + } - test.hotplugOne = newHotplugDisk("vd-hotplug", namespace, replicatedStorageClass) - test.hotplugReplicated = newHotplugDisk("vd-hotplug-replicated", namespace, replicatedStorageClass) - test.hotplugLocalThin = newHotplugDisk("vd-hotplug-local-thin", namespace, localThinStorageClass) + for _, vmScenario := range test.vms { + vmScenario.rootDisk = newRootDisk(vmScenario.rootDiskName, namespace, vmScenario.cviName, replicatedStorageClass, vmScenario.rootDiskSize) + vmScenario.vm = newVM(vmScenario.name, namespace, vmScenario.runPolicy, vmScenario.rootDisk.Name, vmScenario.cloudInit) + test.vmByName[vmScenario.name] = vmScenario + } - test.vmAlwaysOff = newVM("vm-always-off", namespace, v1alpha2.ManualPolicy, test.rootAlwaysOff.Name) - test.vmOneHotplug = newVM("vm-one-hotplug", namespace, v1alpha2.AlwaysOnUnlessStoppedManually, test.rootOneHotplug.Name) - test.vmTwoHotplug = newVM("vm-two-hotplug", namespace, v1alpha2.AlwaysOnPolicy, test.rootTwoHotplug.Name) + for _, diskScenario := range test.dataDisks { + diskScenario.disk = newHotplugDisk(diskScenario.name, namespace, diskScenario.storageClass, diskScenario.size) + test.dataDiskByName[diskScenario.name] = diskScenario + } - test.vmbdaOneHotplug = object.NewVMBDAFromDisk("vmbda", test.vmOneHotplug.Name, test.hotplugOne) - test.vmbdaReplicated = object.NewVMBDAFromDisk("vmbda1", test.vmTwoHotplug.Name, test.hotplugReplicated) - test.vmbdaLocalThin = object.NewVMBDAFromDisk("vmbda2", test.vmTwoHotplug.Name, test.hotplugLocalThin) + for _, attachmentScenario := range test.attachments { + vmScenario := test.vmByName[attachmentScenario.vmName] + diskScenario := test.dataDiskByName[attachmentScenario.diskName] + attachmentScenario.attachment = object.NewVMBDAFromDisk(attachmentScenario.name, vmScenario.vm.Name, diskScenario.disk) + } + + test.iperfClient = test.vmByName["vm-alpine-iperf-client"] + test.iperfServer = test.vmByName["vm-alpine-iperf-server"] return test } -func newRootDisk(name, namespace string) *v1alpha2.VirtualDisk { - return object.NewVDFromCVI( - name, - namespace, - object.PrecreatedCVIAlpineBIOS, - vdbuilder.WithStorageClass(ptr.To(replicatedStorageClass)), - vdbuilder.WithSize(ptr.To(resource.MustParse("350Mi"))), - ) +func (s *vmScenario) expectedInitialPhase() string { + if s.runPolicy == v1alpha2.ManualPolicy { + return string(v1alpha2.MachineStopped) + } + + return string(v1alpha2.MachineRunning) +} + +func newRootDisk(name, namespace, cviName, storageClass, size string) *v1alpha2.VirtualDisk { + options := []vdbuilder.Option{ + vdbuilder.WithStorageClass(ptr.To(storageClass)), + } + if size != "" { + options = append(options, vdbuilder.WithSize(ptr.To(resource.MustParse(size)))) + } + + return object.NewVDFromCVI(name, namespace, cviName, options...) } -func newHotplugDisk(name, namespace, storageClass string) *v1alpha2.VirtualDisk { +func newHotplugDisk(name, namespace, storageClass, size string) *v1alpha2.VirtualDisk { return object.NewBlankVD( name, namespace, ptr.To(storageClass), - ptr.To(resource.MustParse("100Mi")), + ptr.To(resource.MustParse(size)), ) } -func newVM(name, namespace string, runPolicy v1alpha2.RunPolicy, rootDiskName string) *v1alpha2.VirtualMachine { +func newVM(name, namespace string, runPolicy v1alpha2.RunPolicy, rootDiskName, cloudInit string) *v1alpha2.VirtualMachine { return vmbuilder.New( vmbuilder.WithName(name), vmbuilder.WithNamespace(namespace), @@ -180,7 +329,7 @@ func newVM(name, namespace string, runPolicy v1alpha2.RunPolicy, rootDiskName st vmbuilder.WithMemory(resource.MustParse("512Mi")), vmbuilder.WithLiveMigrationPolicy(v1alpha2.AlwaysSafeMigrationPolicy), vmbuilder.WithVirtualMachineClass(object.DefaultVMClass), - vmbuilder.WithProvisioningUserData(object.AlpineCloudInit), + vmbuilder.WithProvisioningUserData(cloudInit), vmbuilder.WithRunPolicy(runPolicy), vmbuilder.WithBlockDeviceRefs(v1alpha2.BlockDeviceSpecRef{ Kind: v1alpha2.DiskDevice, @@ -190,22 +339,68 @@ func newVM(name, namespace string, runPolicy v1alpha2.RunPolicy, rootDiskName st } func (t *currentReleaseSmokeTest) diskObjects() []crclient.Object { - return []crclient.Object{ - t.rootAlwaysOff, - t.rootOneHotplug, - t.rootTwoHotplug, - t.hotplugOne, - t.hotplugReplicated, - t.hotplugLocalThin, + objects := make([]crclient.Object, 0, len(t.vms)+len(t.dataDisks)) + for _, vmScenario := range t.vms { + objects = append(objects, vmScenario.rootDisk) } + for _, diskScenario := range t.dataDisks { + objects = append(objects, diskScenario.disk) + } + return objects } func (t *currentReleaseSmokeTest) vmObjects() []crclient.Object { - return []crclient.Object{ - t.vmAlwaysOff, - t.vmOneHotplug, - t.vmTwoHotplug, + objects := make([]crclient.Object, 0, len(t.vms)) + for _, vmScenario := range t.vms { + objects = append(objects, vmScenario.vm) + } + return objects +} + +func (t *currentReleaseSmokeTest) attachmentObjects() []crclient.Object { + objects := make([]crclient.Object, 0, len(t.attachments)) + for _, attachmentScenario := range t.attachments { + objects = append(objects, attachmentScenario.attachment) + } + return objects +} + +func (t *currentReleaseSmokeTest) initialRunningVMObjects() []crclient.Object { + objects := make([]crclient.Object, 0, len(t.vms)) + for _, vmScenario := range t.vms { + if vmScenario.expectedInitialPhase() == string(v1alpha2.MachineRunning) { + objects = append(objects, vmScenario.vm) + } + } + return objects +} + +func (t *currentReleaseSmokeTest) initialStoppedVMObjects() []crclient.Object { + objects := make([]crclient.Object, 0, len(t.vms)) + for _, vmScenario := range t.vms { + if vmScenario.expectedInitialPhase() == string(v1alpha2.MachineStopped) { + objects = append(objects, vmScenario.vm) + } + } + return objects +} + +func (t *currentReleaseSmokeTest) manualStartVMs() []*vmScenario { + manualVMs := make([]*vmScenario, 0) + for _, vmScenario := range t.vms { + if vmScenario.runPolicy == v1alpha2.ManualPolicy { + manualVMs = append(manualVMs, vmScenario) + } + } + return manualVMs +} + +func (t *currentReleaseSmokeTest) manualStartVMObjects() []crclient.Object { + objects := make([]crclient.Object, 0) + for _, vmScenario := range t.manualStartVMs() { + objects = append(objects, vmScenario.vm) } + return objects } func (t *currentReleaseSmokeTest) expectGuestReady(vm *v1alpha2.VirtualMachine) { @@ -260,3 +455,57 @@ func hasMountpoint(mountpoints []string, expected string) bool { return false } + +func (t *currentReleaseSmokeTest) expectIPerfConnectivity() { + GinkgoHelper() + + waitForIPerfServerToStart(t.framework, t.iperfServer.vm) + + serverVM := t.getVirtualMachine(t.iperfServer.vm.Name, t.iperfServer.vm.Namespace) + command := fmt.Sprintf("iperf3 -c %s -t %d --json", serverVM.Status.IPAddress, iperfDurationSeconds) + output, err := t.framework.SSHCommand( + t.iperfClient.vm.Name, + t.iperfClient.vm.Namespace, + command, + framework.WithSSHTimeout((iperfDurationSeconds+10)*time.Second), + ) + Expect(err).NotTo(HaveOccurred(), "failed to run iperf3 client") + + report, err := parseIPerfReport(output) + Expect(err).NotTo(HaveOccurred(), "failed to parse iperf3 client output") + Expect(report.End.SumSent.Bytes).To(BeNumerically(">", 0), "iperf3 client should send data") + Expect(report.End.SumSent.BitsPerSecond).To(BeNumerically(">", 0), "iperf3 client should report throughput") +} + +func (t *currentReleaseSmokeTest) getVirtualMachine(name, namespace string) *v1alpha2.VirtualMachine { + GinkgoHelper() + + vm, err := t.framework.Clients.VirtClient().VirtualMachines(namespace).Get(context.Background(), name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + return vm +} + +func waitForIPerfServerToStart(f *framework.Framework, vm *v1alpha2.VirtualMachine) { + GinkgoHelper() + + command := "rc-service iperf3 status --nocolor" + Eventually(func() error { + stdout, err := f.SSHCommand(vm.Name, vm.Namespace, command) + if err != nil { + return fmt.Errorf("cmd: %s\nstderr: %w", command, err) + } + if strings.Contains(stdout, "status: started") { + return nil + } + return fmt.Errorf("iperf3 server is not started yet: %s", stdout) + }).WithTimeout(framework.MiddleTimeout).WithPolling(framework.PollingInterval).Should(Succeed()) +} + +func parseIPerfReport(raw string) (*iperfReport, error) { + var report iperfReport + if err := json.Unmarshal([]byte(raw), &report); err != nil { + return nil, fmt.Errorf("parse iperf3 json: %w", err) + } + + return &report, nil +} From 46bebd9b666c9a74c89a13c532a03f4347b06fbb Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Mon, 13 Apr 2026 12:14:31 +0300 Subject: [PATCH 13/20] refactor release test Signed-off-by: Nikita Korolev --- .../e2e-test-releases-reusable-pipeline.yml | 66 +++- test/e2e/release/current_release_smoke.go | 353 +++++++++++++++--- 2 files changed, 360 insertions(+), 59 deletions(-) diff --git a/.github/workflows/e2e-test-releases-reusable-pipeline.yml b/.github/workflows/e2e-test-releases-reusable-pipeline.yml index f699e49244..a273729591 100644 --- a/.github/workflows/e2e-test-releases-reusable-pipeline.yml +++ b/.github/workflows/e2e-test-releases-reusable-pipeline.yml @@ -442,6 +442,11 @@ jobs: - name: Install kubectl CLI uses: azure/setup-kubectl@v4 + - name: Setup d8 + uses: ./.github/actions/install-d8 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Check nested kube-api via generated kubeconfig run: | mkdir -p ~/.kube @@ -1101,6 +1106,8 @@ jobs: needs: - bootstrap - configure-virtualization + outputs: + release_namespace: ${{ steps.export-release-context.outputs.release_namespace }} steps: - uses: actions/checkout@v4 @@ -1148,11 +1155,11 @@ jobs: - name: "Run E2E tests on current-release" id: release-e2e env: - POST_CLEANUP: no - PRECREATED_CVI_CLEANUP: no CSI: ${{ inputs.storage_type }} STORAGE_CLASS_NAME: ${{ inputs.nested_storageclass_name }} E2E_CONFIG: ${{ github.workspace }}/test/e2e/default_config.yaml + RELEASE_TEST_PHASE: pre-upgrade + RELEASE_UPGRADE_CONTEXT_PATH: ${{ runner.temp }}/release-upgrade-context.json run: | echo "[INFO] Current release tag: ${{ env.CURRENT_RELEASE }}" echo "[INFO] Storage type: ${{ inputs.storage_type }}" @@ -1167,7 +1174,7 @@ jobs: GINKGO_RESULT=$(mktemp -p "$RUNNER_TEMP") junit_report="$GITHUB_WORKSPACE/test/e2e/release_current_suite.xml" set +e - POST_CLEANUP=no go tool ginkgo \ + go tool ginkgo \ -v --race --timeout=45m \ --junit-report="$junit_report" \ ./release | tee "$GINKGO_RESULT" @@ -1176,6 +1183,15 @@ jobs: echo "[INFO] Exit code: $GINKGO_EXIT_CODE" exit $GINKGO_EXIT_CODE + - name: Export release upgrade context + id: export-release-context + if: success() && steps.release-e2e.outcome != 'skipped' + run: | + context_path="${{ runner.temp }}/release-upgrade-context.json" + echo "[INFO] Reading release upgrade context from ${context_path}" + cat "${context_path}" + echo "release_namespace=$(jq -r '.namespace' "${context_path}")" >> "$GITHUB_OUTPUT" + - name: Upload current-release test results uses: actions/upload-artifact@v4 if: always() && steps.release-e2e.outcome != 'skipped' @@ -1200,6 +1216,8 @@ jobs: needs: - bootstrap - test-current-release + outputs: + upgrade_started_at: ${{ steps.patch-modulepulloverride.outputs.upgrade_started_at }} steps: - uses: actions/checkout@v4 @@ -1223,7 +1241,10 @@ jobs: kubectl get mpo virtualization -o yaml - name: "Patch ModulePullOverride to new-release: ${{ env.NEW_RELEASE }}" + id: patch-modulepulloverride run: | + upgrade_started_at=$(date +%s) + echo "upgrade_started_at=${upgrade_started_at}" >> "$GITHUB_OUTPUT" echo "[INFO] Patching MPO virtualization imageTag from ${{ env.CURRENT_RELEASE }} to ${{ env.NEW_RELEASE }}" kubectl patch mpo virtualization --type merge -p '{"spec":{"imageTag":"${{ env.NEW_RELEASE }}"}}' @@ -1383,19 +1404,50 @@ jobs: needs: - bootstrap - patch-mpo + - test-current-release steps: - uses: actions/checkout@v4 - name: Install kubectl CLI uses: azure/setup-kubectl@v4 + - name: Setup d8 + uses: ./.github/actions/install-d8 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version: "${{ env.GO_VERSION }}" + + - name: Install ginkgo + working-directory: ./test/e2e/ + run: | + echo "Install ginkgo" + go install tool + + - name: Download dependencies + working-directory: ./test/e2e/ + run: | + echo "Download dependencies" + go mod download + - name: Setup kubeconfig run: | mkdir -p ~/.kube echo "${{ needs.bootstrap.outputs.kubeconfig }}" | base64 -d | base64 -d > ~/.kube/config chmod 600 ~/.kube/config + kubectl config use-context nested-e2e-nested-sa - - name: "Run E2E tests on new-release (STUB)" + - name: "Run E2E tests on new-release" + env: + CSI: ${{ inputs.storage_type }} + STORAGE_CLASS_NAME: ${{ inputs.nested_storageclass_name }} + E2E_CONFIG: ${{ github.workspace }}/test/e2e/default_config.yaml + RELEASE_TEST_PHASE: post-upgrade + RELEASE_NAMESPACE: ${{ needs.test-current-release.outputs.release_namespace }} + RELEASE_UPGRADE_STARTED_AT: ${{ needs.patch-mpo.outputs.upgrade_started_at }} run: | echo "[INFO] New release tag: ${{ env.NEW_RELEASE }}" echo "[INFO] Storage type: ${{ inputs.storage_type }}" @@ -1404,7 +1456,11 @@ jobs: kubectl get modules virtualization || true kubectl get mpo virtualization || true echo "" - echo "[STUB] E2E tests on new-release ${{ env.NEW_RELEASE }} -- PASSED" + echo "[INFO] Reusing namespace: ${RELEASE_NAMESPACE}" + cd ./test/e2e/ + go tool ginkgo \ + -v --race --timeout=45m \ + ./release echo "[INFO] Cluster is intentionally left running (no cleanup)" undeploy-cluster: diff --git a/test/e2e/release/current_release_smoke.go b/test/e2e/release/current_release_smoke.go index bd28bf71f5..1e2cbca174 100644 --- a/test/e2e/release/current_release_smoke.go +++ b/test/e2e/release/current_release_smoke.go @@ -20,6 +20,10 @@ import ( "context" "encoding/json" "fmt" + "math" + "os" + "path/filepath" + "strconv" "strings" "time" @@ -45,55 +49,24 @@ const ( minDataDiskSizeBytes = int64(50 * 1024 * 1024) defaultRootDiskSize = "350Mi" defaultDataDiskSize = "100Mi" - iperfDurationSeconds = 5 + releaseIPerfReportPath = "/tmp/release-upgrade-iperf-client-report.json" + + releaseTestPhaseEnv = "RELEASE_TEST_PHASE" + releaseTestPhasePreUpgrade = "pre-upgrade" + releaseTestPhasePostUpgrade = "post-upgrade" + releaseUpgradeContextPathEnv = "RELEASE_UPGRADE_CONTEXT_PATH" + releaseNamespaceEnv = "RELEASE_NAMESPACE" + releaseUpgradeStartedAtEnv = "RELEASE_UPGRADE_STARTED_AT" ) var _ = Describe("CurrentReleaseSmoke", func() { It("should validate current release virtual machines", func() { - f := framework.NewFramework("release-current") - f.Before() - - test := newCurrentReleaseSmokeTest(f) - - By("Creating root and hotplug virtual disks") - Expect(f.CreateWithDeferredDeletion(context.Background(), test.diskObjects()...)).To(Succeed()) - - By("Creating virtual machines") - Expect(f.CreateWithDeferredDeletion(context.Background(), test.vmObjects()...)).To(Succeed()) - if runningVMs := test.initialRunningVMObjects(); len(runningVMs) > 0 { - util.UntilObjectPhase(string(v1alpha2.MachineRunning), framework.LongTimeout, runningVMs...) - } - if stoppedVMs := test.initialStoppedVMObjects(); len(stoppedVMs) > 0 { - util.UntilObjectPhase(string(v1alpha2.MachineStopped), framework.MiddleTimeout, stoppedVMs...) - } - - By("Starting manual-policy virtual machines") - for _, vmScenario := range test.manualStartVMs() { - util.StartVirtualMachine(f, vmScenario.vm) - } - if startedVMs := test.manualStartVMObjects(); len(startedVMs) > 0 { - util.UntilObjectPhase(string(v1alpha2.MachineRunning), framework.LongTimeout, startedVMs...) - } - - By("Attaching hotplug disks") - Expect(f.CreateWithDeferredDeletion(context.Background(), test.attachmentObjects()...)).To(Succeed()) - util.UntilObjectPhase(string(v1alpha2.BlockDeviceAttachmentPhaseAttached), framework.LongTimeout, test.attachmentObjects()...) - - By("Waiting for all disks to become ready after consumers appear") - util.UntilObjectPhase(string(v1alpha2.DiskReady), framework.LongTimeout, test.diskObjects()...) - - By("Waiting for guest agent and SSH access") - for _, vmScenario := range test.vms { - test.expectGuestReady(vmScenario.vm) + switch getReleaseTestPhase() { + case releaseTestPhasePostUpgrade: + runPostUpgradeReleaseSmoke() + default: + runPreUpgradeReleaseSmoke() } - - By("Checking attached disks inside guests") - for _, vmScenario := range test.vms { - test.expectAdditionalDiskCount(vmScenario.vm, vmScenario.expectedAdditionalDisks) - } - - By("Running iperf smoke check between alpine guests") - test.expectIPerfConnectivity() }) }) @@ -149,7 +122,19 @@ type lsblkDevice struct { } type iperfReport struct { - End iperfReportEnd `json:"end"` + Start iperfReportStart `json:"start"` + Intervals []iperfReportInterval `json:"intervals"` + End iperfReportEnd `json:"end"` + Error string `json:"error,omitempty"` +} + +type iperfReportStart struct { + Timestamp iperfReportTimestamp `json:"timestamp"` +} + +type iperfReportTimestamp struct { + Time string `json:"time"` + Timesecs int `json:"timesecs"` } type iperfReportEnd struct { @@ -157,18 +142,67 @@ type iperfReportEnd struct { SumReceived iperfReportSummary `json:"sum_received"` } +type iperfReportInterval struct { + Sum iperfReportSummary `json:"sum"` +} + type iperfReportSummary struct { Bytes int64 `json:"bytes"` BitsPerSecond float64 `json:"bits_per_second"` + End float64 `json:"end,omitempty"` } -func newCurrentReleaseSmokeTest(f *framework.Framework) *currentReleaseSmokeTest { +type releaseUpgradeContext struct { + Namespace string `json:"namespace"` + IPerfClientVM string `json:"iperfClientVM"` + IPerfServerVM string `json:"iperfServerVM"` + IPerfReportPath string `json:"iperfReportPath"` +} + +func runPreUpgradeReleaseSmoke() { + f := framework.NewFramework("release-current") + f.Before() + + test := newCurrentReleaseSmokeTest(f, "") + test.createResources() + test.verifyVMsReady() + test.startLongRunningIPerf() + test.writeUpgradeContext() +} + +func runPostUpgradeReleaseSmoke() { + namespace := mustGetEnv(releaseNamespaceEnv) + f := framework.NewFramework("") + test := newCurrentReleaseSmokeTest(f, namespace) + + test.verifyVMsSurvivedUpgrade() + test.verifyIPerfContinuityAfterUpgrade() +} + +func getReleaseTestPhase() string { + if phase := os.Getenv(releaseTestPhaseEnv); phase != "" { + return phase + } + + return releaseTestPhasePreUpgrade +} + +func mustGetEnv(name string) string { + value := os.Getenv(name) + Expect(value).NotTo(BeEmpty(), "environment variable %s must be set", name) + return value +} + +func newCurrentReleaseSmokeTest(f *framework.Framework, namespace string) *currentReleaseSmokeTest { test := ¤tReleaseSmokeTest{ framework: f, vmByName: make(map[string]*vmScenario), dataDiskByName: make(map[string]*dataDiskScenario), } - namespace := f.Namespace().Name + if namespace == "" { + Expect(f.Namespace()).NotTo(BeNil(), "framework namespace must be initialized for pre-upgrade phase") + namespace = f.Namespace().Name + } test.vms = []*vmScenario{ { @@ -293,6 +327,62 @@ func newCurrentReleaseSmokeTest(f *framework.Framework) *currentReleaseSmokeTest return test } +func (t *currentReleaseSmokeTest) createResources() { + By("Creating root and hotplug virtual disks") + Expect(t.framework.CreateWithDeferredDeletion(context.Background(), t.diskObjects()...)).To(Succeed()) + + By("Creating virtual machines") + Expect(t.framework.CreateWithDeferredDeletion(context.Background(), t.vmObjects()...)).To(Succeed()) + if runningVMs := t.initialRunningVMObjects(); len(runningVMs) > 0 { + util.UntilObjectPhase(string(v1alpha2.MachineRunning), framework.LongTimeout, runningVMs...) + } + if stoppedVMs := t.initialStoppedVMObjects(); len(stoppedVMs) > 0 { + util.UntilObjectPhase(string(v1alpha2.MachineStopped), framework.MiddleTimeout, stoppedVMs...) + } + + By("Starting manual-policy virtual machines") + for _, vmScenario := range t.manualStartVMs() { + util.StartVirtualMachine(t.framework, vmScenario.vm) + } + if startedVMs := t.manualStartVMObjects(); len(startedVMs) > 0 { + util.UntilObjectPhase(string(v1alpha2.MachineRunning), framework.LongTimeout, startedVMs...) + } + + By("Attaching hotplug disks") + Expect(t.framework.CreateWithDeferredDeletion(context.Background(), t.attachmentObjects()...)).To(Succeed()) + util.UntilObjectPhase(string(v1alpha2.BlockDeviceAttachmentPhaseAttached), framework.LongTimeout, t.attachmentObjects()...) + + By("Waiting for all disks to become ready after consumers appear") + util.UntilObjectPhase(string(v1alpha2.DiskReady), framework.LongTimeout, t.diskObjects()...) +} + +func (t *currentReleaseSmokeTest) verifyVMsReady() { + By("Waiting for guest agent and SSH access") + for _, vmScenario := range t.vms { + t.expectGuestReady(vmScenario.vm) + } + + By("Checking attached disks inside guests") + for _, vmScenario := range t.vms { + t.expectAdditionalDiskCount(vmScenario.vm, vmScenario.expectedAdditionalDisks) + } +} + +func (t *currentReleaseSmokeTest) verifyVMsSurvivedUpgrade() { + By("Waiting for upgraded virtual machines to be running") + util.UntilObjectPhase(string(v1alpha2.MachineRunning), framework.LongTimeout, t.vmObjects()...) + + By("Checking guest access after module upgrade") + for _, vmScenario := range t.vms { + t.expectGuestReady(vmScenario.vm) + } + + By("Checking attached disks after module upgrade") + for _, vmScenario := range t.vms { + t.expectAdditionalDiskCount(vmScenario.vm, vmScenario.expectedAdditionalDisks) + } +} + func (s *vmScenario) expectedInitialPhase() string { if s.runPolicy == v1alpha2.ManualPolicy { return string(v1alpha2.MachineStopped) @@ -456,23 +546,89 @@ func hasMountpoint(mountpoints []string, expected string) bool { return false } -func (t *currentReleaseSmokeTest) expectIPerfConnectivity() { +func (t *currentReleaseSmokeTest) startLongRunningIPerf() { GinkgoHelper() waitForIPerfServerToStart(t.framework, t.iperfServer.vm) serverVM := t.getVirtualMachine(t.iperfServer.vm.Name, t.iperfServer.vm.Namespace) - command := fmt.Sprintf("iperf3 -c %s -t %d --json", serverVM.Status.IPAddress, iperfDurationSeconds) - output, err := t.framework.SSHCommand( + command := fmt.Sprintf( + "nohup sh -c 'iperf3 -c %s -t 0 --json > %s 2>&1' >/dev/null 2>&1 &", + serverVM.Status.IPAddress, + releaseIPerfReportPath, + ) + _, err := t.framework.SSHCommand( t.iperfClient.vm.Name, t.iperfClient.vm.Namespace, command, - framework.WithSSHTimeout((iperfDurationSeconds+10)*time.Second), ) - Expect(err).NotTo(HaveOccurred(), "failed to run iperf3 client") + Expect(err).NotTo(HaveOccurred(), "failed to start long-running iperf3 client") + + waitForIPerfClientToStart(t.framework, t.iperfClient.vm) +} + +func (t *currentReleaseSmokeTest) writeUpgradeContext() { + GinkgoHelper() + + contextPath := os.Getenv(releaseUpgradeContextPathEnv) + if contextPath == "" { + return + } - report, err := parseIPerfReport(output) - Expect(err).NotTo(HaveOccurred(), "failed to parse iperf3 client output") + err := os.MkdirAll(filepath.Dir(contextPath), 0o755) + Expect(err).NotTo(HaveOccurred()) + + contextData := releaseUpgradeContext{ + Namespace: t.iperfClient.vm.Namespace, + IPerfClientVM: t.iperfClient.vm.Name, + IPerfServerVM: t.iperfServer.vm.Name, + IPerfReportPath: releaseIPerfReportPath, + } + + data, err := json.MarshalIndent(contextData, "", " ") + Expect(err).NotTo(HaveOccurred()) + + err = os.WriteFile(contextPath, data, 0o644) + Expect(err).NotTo(HaveOccurred()) +} + +func (t *currentReleaseSmokeTest) verifyIPerfContinuityAfterUpgrade() { + GinkgoHelper() + + By("Checking that the iperf client is still running after upgrade") + waitForIPerfClientToStart(t.framework, t.iperfClient.vm) + + By("Stopping the long-running iperf client after upgrade") + stopIPerfClient(t.framework, t.iperfClient.vm) + + By("Validating the iperf report spans the module upgrade") + report := getIPerfClientReport(t.framework, t.iperfClient.vm, releaseIPerfReportPath) + Expect(report.Error).To(BeEmpty(), "iperf3 report contains an error") + + upgradeStartedAt, err := strconv.ParseInt(mustGetEnv(releaseUpgradeStartedAtEnv), 10, 64) + Expect(err).NotTo(HaveOccurred(), "upgrade timestamp must be a unix second") + + startedAt := int64(report.Start.Timestamp.Timesecs) + endedAt := startedAt + int64(math.Ceil(report.End.SumSent.End)) + Expect(startedAt).To(BeNumerically("<=", upgradeStartedAt), "iperf3 should start before the module upgrade") + Expect(endedAt).To(BeNumerically(">", upgradeStartedAt), "iperf3 should continue after the module upgrade") + + lowerIdx, upperIdx := continuityWindowBounds(startedAt, upgradeStartedAt, len(report.Intervals)) + Expect(upperIdx).To(BeNumerically(">=", lowerIdx), "iperf3 report must include intervals around the module upgrade") + + zeroIntervals := 0 + transmittedAroundUpgrade := int64(0) + for idx := lowerIdx; idx <= upperIdx; idx++ { + interval := report.Intervals[idx] + if interval.Sum.Bytes == 0 { + zeroIntervals++ + continue + } + transmittedAroundUpgrade += interval.Sum.Bytes + } + + Expect(transmittedAroundUpgrade).To(BeNumerically(">", 0), "iperf3 should transmit data around the module upgrade") + Expect(zeroIntervals).To(BeNumerically("<=", 1), "iperf3 should not be interrupted during the module upgrade") Expect(report.End.SumSent.Bytes).To(BeNumerically(">", 0), "iperf3 client should send data") Expect(report.End.SumSent.BitsPerSecond).To(BeNumerically(">", 0), "iperf3 client should report throughput") } @@ -501,6 +657,79 @@ func waitForIPerfServerToStart(f *framework.Framework, vm *v1alpha2.VirtualMachi }).WithTimeout(framework.MiddleTimeout).WithPolling(framework.PollingInterval).Should(Succeed()) } +func waitForIPerfClientToStart(f *framework.Framework, vm *v1alpha2.VirtualMachine) { + GinkgoHelper() + + command := "pgrep -x iperf3" + Eventually(func() error { + stdout, err := f.SSHCommand(vm.Name, vm.Namespace, command) + if err != nil { + return fmt.Errorf("cmd: %s\nstderr: %w", command, err) + } + if strings.TrimSpace(stdout) == "" { + return fmt.Errorf("iperf3 client is not running yet") + } + return nil + }).WithTimeout(framework.MiddleTimeout).WithPolling(framework.PollingInterval).Should(Succeed()) +} + +func stopIPerfClient(f *framework.Framework, vm *v1alpha2.VirtualMachine) { + GinkgoHelper() + + command := "pkill -INT -x iperf3" + Eventually(func() error { + _, err := f.SSHCommand(vm.Name, vm.Namespace, command) + if err != nil { + return fmt.Errorf("cmd: %s\nstderr: %w", command, err) + } + return nil + }).WithTimeout(framework.MiddleTimeout).WithPolling(framework.PollingInterval).Should(Succeed()) +} + +func getIPerfClientReport(f *framework.Framework, vm *v1alpha2.VirtualMachine, reportPath string) *iperfReport { + GinkgoHelper() + + command := fmt.Sprintf("cat %s", reportPath) + var rawReport string + Eventually(func() error { + stdout, err := f.SSHCommand(vm.Name, vm.Namespace, command) + if err != nil { + return fmt.Errorf("cmd: %s\nstderr: %w", command, err) + } + report, err := parseIPerfReport(stdout) + if err != nil { + return err + } + if report.End.SumSent.End <= 0 { + return fmt.Errorf("iperf3 report is incomplete") + } + rawReport = stdout + return nil + }).WithTimeout(framework.LongTimeout).WithPolling(framework.PollingInterval).Should(Succeed()) + + report, err := parseIPerfReport(rawReport) + Expect(err).NotTo(HaveOccurred()) + return report +} + +func continuityWindowBounds(startedAt, upgradeStartedAt int64, intervalCount int) (int, int) { + if intervalCount == 0 { + return 1, 0 + } + + index := int(upgradeStartedAt - startedAt) + if index < 0 { + index = 0 + } + if index >= intervalCount { + index = intervalCount - 1 + } + + lower := max(index-1, 0) + upper := min(index+1, intervalCount-1) + return lower, upper +} + func parseIPerfReport(raw string) (*iperfReport, error) { var report iperfReport if err := json.Unmarshal([]byte(raw), &report); err != nil { @@ -509,3 +738,19 @@ func parseIPerfReport(raw string) (*iperfReport, error) { return &report, nil } + +func min(a, b int) int { + if a < b { + return a + } + + return b +} + +func max(a, b int) int { + if a > b { + return a + } + + return b +} From b61688361878ac873c5271522e9751ee4976176d Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Mon, 13 Apr 2026 12:18:35 +0300 Subject: [PATCH 14/20] off undeploy cluster Signed-off-by: Nikita Korolev --- .../e2e-test-releases-reusable-pipeline.yml | 76 +++++++++---------- 1 file changed, 38 insertions(+), 38 deletions(-) diff --git a/.github/workflows/e2e-test-releases-reusable-pipeline.yml b/.github/workflows/e2e-test-releases-reusable-pipeline.yml index a273729591..2067a5c32d 100644 --- a/.github/workflows/e2e-test-releases-reusable-pipeline.yml +++ b/.github/workflows/e2e-test-releases-reusable-pipeline.yml @@ -1463,41 +1463,41 @@ jobs: ./release echo "[INFO] Cluster is intentionally left running (no cleanup)" - undeploy-cluster: - name: Undeploy cluster - runs-on: ubuntu-latest - needs: - - bootstrap - - test-new-release - if: always() - steps: - - uses: actions/checkout@v4 - - - name: Setup d8 - uses: ./.github/actions/install-d8 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Install Task - uses: arduino/setup-task@v2 - with: - version: 3.x - repo-token: ${{ secrets.GITHUB_TOKEN }} - - - name: Download artifacts - uses: actions/download-artifact@v5 - with: - name: ${{ inputs.storage_type }}-release-generated-files-${{ inputs.date_start }} - path: ${{ env.SETUP_CLUSTER_TYPE_PATH }}/ - - - name: Configure kubectl via azure/k8s-set-context@v4 - uses: azure/k8s-set-context@v4 - with: - method: kubeconfig - context: e2e-cluster-nightly-e2e-virt-sa - kubeconfig: ${{ secrets.VIRT_E2E_NIGHTLY_SA_TOKEN }} - - - name: infra-undeploy - working-directory: ${{ env.SETUP_CLUSTER_TYPE_PATH }} - run: | - task infra-undeploy + # undeploy-cluster: + # name: Undeploy cluster + # runs-on: ubuntu-latest + # needs: + # - bootstrap + # - test-new-release + # if: always() + # steps: + # - uses: actions/checkout@v4 + + # - name: Setup d8 + # uses: ./.github/actions/install-d8 + # env: + # GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + # - name: Install Task + # uses: arduino/setup-task@v2 + # with: + # version: 3.x + # repo-token: ${{ secrets.GITHUB_TOKEN }} + + # - name: Download artifacts + # uses: actions/download-artifact@v5 + # with: + # name: ${{ inputs.storage_type }}-release-generated-files-${{ inputs.date_start }} + # path: ${{ env.SETUP_CLUSTER_TYPE_PATH }}/ + + # - name: Configure kubectl via azure/k8s-set-context@v4 + # uses: azure/k8s-set-context@v4 + # with: + # method: kubeconfig + # context: e2e-cluster-nightly-e2e-virt-sa + # kubeconfig: ${{ secrets.VIRT_E2E_NIGHTLY_SA_TOKEN }} + + # - name: infra-undeploy + # working-directory: ${{ env.SETUP_CLUSTER_TYPE_PATH }} + # run: | + # task infra-undeploy From f1892728d4d7484cd339808f63572c1dd2f4444b Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Mon, 13 Apr 2026 17:25:23 +0300 Subject: [PATCH 15/20] support pr and tags Signed-off-by: Nikita Korolev --- .../e2e-test-releases-reusable-pipeline.yml | 5 - .github/workflows/e2e-test-releases.yml | 109 +++++++++++++++--- 2 files changed, 93 insertions(+), 21 deletions(-) diff --git a/.github/workflows/e2e-test-releases-reusable-pipeline.yml b/.github/workflows/e2e-test-releases-reusable-pipeline.yml index 2067a5c32d..2a762c9f09 100644 --- a/.github/workflows/e2e-test-releases-reusable-pipeline.yml +++ b/.github/workflows/e2e-test-releases-reusable-pipeline.yml @@ -442,11 +442,6 @@ jobs: - name: Install kubectl CLI uses: azure/setup-kubectl@v4 - - name: Setup d8 - uses: ./.github/actions/install-d8 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Check nested kube-api via generated kubeconfig run: | mkdir -p ~/.kube diff --git a/.github/workflows/e2e-test-releases.yml b/.github/workflows/e2e-test-releases.yml index da0a210804..fd75a8f34e 100644 --- a/.github/workflows/e2e-test-releases.yml +++ b/.github/workflows/e2e-test-releases.yml @@ -18,12 +18,12 @@ on: workflow_dispatch: inputs: current-release: - description: "Current release tag, like v1.4.1" + description: "Current release tag like v1.4.1, or PR reference like pr2034/2034" required: false # before merge to main, set to true default: "v1.6.3-rc.3" # before merge to main, remove type: string next-release: - description: "Next release like v1.5.0, should be greater than current-release" + description: "Next release like v1.5.0, or PR reference like pr2034/2034" required: false # before merge to main, set to true default: "v1.7.0" # before merge to main, remove type: string @@ -55,40 +55,115 @@ jobs: echo "date-start=$(date +%Y%m%d-%H%M%S)" >> $GITHUB_OUTPUT echo "randuuid4c=$(openssl rand -hex 2)" >> $GITHUB_OUTPUT + resolve-release-inputs: + name: Resolve release inputs + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: read + outputs: + current_module_tag: ${{ steps.resolve.outputs.current_module_tag }} + current_checkout_ref: ${{ steps.resolve.outputs.current_checkout_ref }} + next_module_tag: ${{ steps.resolve.outputs.next_module_tag }} + next_checkout_ref: ${{ steps.resolve.outputs.next_checkout_ref }} + steps: + - name: Resolve release refs + id: resolve + uses: actions/github-script@v7 + env: + CURRENT_RELEASE_INPUT: ${{ github.event.inputs.current-release }} + NEXT_RELEASE_INPUT: ${{ github.event.inputs.next-release }} + with: + script: | + async function resolveReleaseInput(rawInput, inputName) { + const releaseInput = (rawInput || '').trim(); + + if (!releaseInput) { + core.setFailed(`Workflow input '${inputName}' is required.`); + return null; + } + + const prMatch = releaseInput.match(/^(?:pr)?(\d+)$/i); + if (!prMatch) { + return { + moduleTag: releaseInput, + checkoutRef: releaseInput, + }; + } + + const prNumber = Number(prMatch[1]); + const { data: pullRequest } = await github.rest.pulls.get({ + owner: 'deckhouse', + repo: 'virtualization', + pull_number: prNumber, + }); + + if (!pullRequest?.head?.ref) { + core.setFailed(`Pull request #${prNumber} does not have a head ref.`); + return null; + } + + return { + moduleTag: `pr${prNumber}`, + checkoutRef: pullRequest.head.ref, + }; + } + + const currentRelease = await resolveReleaseInput(process.env.CURRENT_RELEASE_INPUT, 'current-release'); + const nextRelease = await resolveReleaseInput(process.env.NEXT_RELEASE_INPUT, 'next-release'); + + if (!currentRelease || !nextRelease) { + return; + } + + core.setOutput('current_module_tag', currentRelease.moduleTag); + core.setOutput('current_checkout_ref', currentRelease.checkoutRef); + core.setOutput('next_module_tag', nextRelease.moduleTag); + core.setOutput('next_checkout_ref', nextRelease.checkoutRef); + prepare-release-matrix: name: Prepare release matrix if: ${{ inputs.enableBuild }} runs-on: ubuntu-latest + needs: + - resolve-release-inputs outputs: list: ${{ steps.releases.outputs.list }} steps: - name: Prepare release refs id: releases env: - CURRENT_RELEASE: ${{ github.event.inputs.current-release }} - NEXT_RELEASE: ${{ github.event.inputs.next-release }} + CURRENT_RELEASE_MODULE_TAG: ${{ needs.resolve-release-inputs.outputs.current_module_tag }} + CURRENT_RELEASE_CHECKOUT_REF: ${{ needs.resolve-release-inputs.outputs.current_checkout_ref }} + NEXT_RELEASE_MODULE_TAG: ${{ needs.resolve-release-inputs.outputs.next_module_tag }} + NEXT_RELEASE_CHECKOUT_REF: ${{ needs.resolve-release-inputs.outputs.next_checkout_ref }} run: | - if [[ -z "${CURRENT_RELEASE}" ]]; then - echo "::error title=Current release is required::Set workflow input 'current-release' to a git ref or tag that can be checked out." + if [[ -z "${CURRENT_RELEASE_MODULE_TAG}" || -z "${CURRENT_RELEASE_CHECKOUT_REF}" ]]; then + echo "::error title=Current release is required::Set workflow input 'current-release' to a git tag or PR reference that can be resolved." exit 1 fi releases=$(jq -cn \ - --arg current "${CURRENT_RELEASE}" \ - --arg next "${NEXT_RELEASE}" \ - '[$current, $next] | map(select(. != "")) | unique') + --arg current_tag "${CURRENT_RELEASE_MODULE_TAG}" \ + --arg current_ref "${CURRENT_RELEASE_CHECKOUT_REF}" \ + --arg next_tag "${NEXT_RELEASE_MODULE_TAG}" \ + --arg next_ref "${NEXT_RELEASE_CHECKOUT_REF}" \ + '[ + {module_tag: $current_tag, checkout_ref: $current_ref}, + {module_tag: $next_tag, checkout_ref: $next_ref} + ] | map(select(.module_tag != "" and .checkout_ref != "")) | unique_by(.module_tag + "|" + .checkout_ref)') echo "list=${releases}" >> "$GITHUB_OUTPUT" build-release-images: - name: Build release image (${{ matrix.release_tag }}) + name: Build release image (${{ matrix.module_tag }}) if: ${{ inputs.enableBuild }} runs-on: [self-hosted, large] needs: - prepare-release-matrix strategy: matrix: - release_tag: ${{ fromJSON(needs.prepare-release-matrix.outputs.list) }} + include: ${{ fromJSON(needs.prepare-release-matrix.outputs.list) }} steps: - name: Setup Docker config run: | @@ -99,13 +174,14 @@ jobs: echo MODULES_REGISTRY=${{ vars.DEV_REGISTRY }} echo MODULES_MODULE_SOURCE=${{ vars.DEV_MODULE_SOURCE }} echo MODULES_MODULE_NAME=${{ vars.MODULE_NAME }} - echo MODULES_MODULE_TAG=${{ matrix.release_tag }} + echo MODULES_MODULE_TAG=${{ matrix.module_tag }} + echo CHECKOUT_REF=${{ matrix.checkout_ref }} echo DOCKER_CONFIG=$DOCKER_CONFIG - uses: actions/checkout@v4 with: fetch-depth: 0 - ref: ${{ matrix.release_tag }} + ref: ${{ matrix.checkout_ref }} - uses: deckhouse/modules-actions/setup@v2 with: @@ -117,7 +193,7 @@ jobs: with: module_source: ${{ vars.DEV_MODULE_SOURCE }} module_name: ${{ vars.MODULE_NAME }} - module_tag: ${{ matrix.release_tag }} + module_tag: ${{ matrix.module_tag }} source_repo: ${{ secrets.SOURCE_REPO_GIT }} source_repo_ssh_key: ${{ secrets.SOURCE_REPO_SSH_KEY }} @@ -131,11 +207,12 @@ jobs: if: ${{ always() && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') }} needs: - set-vars + - resolve-release-inputs - build-release-images uses: ./.github/workflows/e2e-test-releases-reusable-pipeline.yml with: - current_release: ${{ github.event.inputs.current-release }} - new_release: ${{ github.event.inputs.next-release }} + current_release: ${{ needs.resolve-release-inputs.outputs.current_module_tag }} + new_release: ${{ needs.resolve-release-inputs.outputs.next_module_tag }} storage_type: mixed nested_storageclass_name: nested-thin-r1 branch: main From 4f4a66debd9a71d5c61b2ac8d33f181b1a75e704 Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Tue, 14 Apr 2026 17:26:00 +0300 Subject: [PATCH 16/20] fix Signed-off-by: Nikita Korolev --- .../internal/service/attachment_service.go | 20 +++++- .../service/attachment_service_test.go | 64 +++++++++++++++++++ 2 files changed, 83 insertions(+), 1 deletion(-) diff --git a/images/virtualization-artifact/pkg/controller/vmbda/internal/service/attachment_service.go b/images/virtualization-artifact/pkg/controller/vmbda/internal/service/attachment_service.go index 6032079351..bc52239a89 100644 --- a/images/virtualization-artifact/pkg/controller/vmbda/internal/service/attachment_service.go +++ b/images/virtualization-artifact/pkg/controller/vmbda/internal/service/attachment_service.go @@ -74,11 +74,15 @@ func (s AttachmentService) IsHotPlugged(ad *AttachmentDisk, vm *v1alpha2.Virtual return true, nil } + if s.isVMBlockDeviceHotPlugged(vm, ad) { + return true, nil + } + return false, fmt.Errorf("%w: %s", ErrVolumeStatusNotReady, vs.Message) } } - return false, nil + return s.isVMBlockDeviceHotPlugged(vm, ad), nil } func (s AttachmentService) CanHotPlug(ad *AttachmentDisk, vm *v1alpha2.VirtualMachine, kvvm *virtv1.VirtualMachine) (bool, error) { @@ -169,6 +173,20 @@ func (s AttachmentService) IsAttached(vm *v1alpha2.VirtualMachine, kvvm *virtv1. return false } +func (s AttachmentService) isVMBlockDeviceHotPlugged(vm *v1alpha2.VirtualMachine, ad *AttachmentDisk) bool { + if vm == nil || ad == nil { + return false + } + + for _, bdRef := range vm.Status.BlockDeviceRefs { + if bdRef.Kind == ad.Kind && bdRef.Name == ad.Name { + return bdRef.Hotplugged + } + } + + return false +} + func (s AttachmentService) UnplugDisk(ctx context.Context, kvvm *virtv1.VirtualMachine, diskName string) error { if kvvm == nil { return errors.New("cannot unplug a disk from a nil KVVM") diff --git a/images/virtualization-artifact/pkg/controller/vmbda/internal/service/attachment_service_test.go b/images/virtualization-artifact/pkg/controller/vmbda/internal/service/attachment_service_test.go index 3643c0ef2a..06c334ecec 100644 --- a/images/virtualization-artifact/pkg/controller/vmbda/internal/service/attachment_service_test.go +++ b/images/virtualization-artifact/pkg/controller/vmbda/internal/service/attachment_service_test.go @@ -23,6 +23,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + virtv1 "kubevirt.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" "github.com/deckhouse/virtualization/api/core/v1alpha2" @@ -172,3 +173,66 @@ var _ = Describe("AttachmentService method IsConflictedAttachment", func() { Expect(conflictWithName).To(BeEmpty()) }) }) + +var _ = Describe("AttachmentService method IsHotPlugged", func() { + It("should treat the disk as hotplugged when vm status already reports it attached", func() { + ad := &AttachmentDisk{ + Kind: v1alpha2.DiskDevice, + Name: "data-disk", + GenerateName: "vd-data-disk", + } + vm := &v1alpha2.VirtualMachine{ + Status: v1alpha2.VirtualMachineStatus{ + BlockDeviceRefs: []v1alpha2.BlockDeviceStatusRef{ + { + Kind: v1alpha2.DiskDevice, + Name: "data-disk", + Hotplugged: true, + }, + }, + }, + } + kvvmi := &virtv1.VirtualMachineInstance{ + Status: virtv1.VirtualMachineInstanceStatus{ + VolumeStatus: []virtv1.VolumeStatus{ + { + Name: ad.GenerateName, + HotplugVolume: &virtv1.HotplugVolumeStatus{ + AttachPodName: "hp-pod", + }, + Phase: virtv1.VolumePending, + Message: "waiting for kubevirt to mark the volume ready", + }, + }, + }, + } + + attached, err := NewAttachmentService(nil, nil, "").IsHotPlugged(ad, vm, kvvmi) + Expect(err).NotTo(HaveOccurred()) + Expect(attached).To(BeTrue()) + }) + + It("should fall back to vm status when kvvmi volume status is absent", func() { + ad := &AttachmentDisk{ + Kind: v1alpha2.DiskDevice, + Name: "data-disk", + GenerateName: "vd-data-disk", + } + vm := &v1alpha2.VirtualMachine{ + Status: v1alpha2.VirtualMachineStatus{ + BlockDeviceRefs: []v1alpha2.BlockDeviceStatusRef{ + { + Kind: v1alpha2.DiskDevice, + Name: "data-disk", + Hotplugged: true, + }, + }, + }, + } + kvvmi := &virtv1.VirtualMachineInstance{} + + attached, err := NewAttachmentService(nil, nil, "").IsHotPlugged(ad, vm, kvvmi) + Expect(err).NotTo(HaveOccurred()) + Expect(attached).To(BeTrue()) + }) +}) From 7f45f108baffd4e8a8c185a604a84e80205d68ee Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Tue, 14 Apr 2026 19:40:58 +0300 Subject: [PATCH 17/20] fix test Signed-off-by: Nikita Korolev --- test/e2e/release/current_release_smoke.go | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/test/e2e/release/current_release_smoke.go b/test/e2e/release/current_release_smoke.go index 1e2cbca174..971baf2309 100644 --- a/test/e2e/release/current_release_smoke.go +++ b/test/e2e/release/current_release_smoke.go @@ -89,6 +89,7 @@ type vmScenario struct { runPolicy v1alpha2.RunPolicy rootDiskSize string expectedAdditionalDisks int + skipGuestAgentCheck bool rootDisk *v1alpha2.VirtualDisk vm *v1alpha2.VirtualMachine @@ -256,6 +257,7 @@ func newCurrentReleaseSmokeTest(f *framework.Framework, namespace string) *curre runPolicy: v1alpha2.AlwaysOnUnlessStoppedManually, rootDiskSize: defaultRootDiskSize, expectedAdditionalDisks: 2, + skipGuestAgentCheck: true, }, { name: "vm-alpine-iperf-server", @@ -359,7 +361,7 @@ func (t *currentReleaseSmokeTest) createResources() { func (t *currentReleaseSmokeTest) verifyVMsReady() { By("Waiting for guest agent and SSH access") for _, vmScenario := range t.vms { - t.expectGuestReady(vmScenario.vm) + t.expectGuestReady(vmScenario) } By("Checking attached disks inside guests") @@ -374,7 +376,7 @@ func (t *currentReleaseSmokeTest) verifyVMsSurvivedUpgrade() { By("Checking guest access after module upgrade") for _, vmScenario := range t.vms { - t.expectGuestReady(vmScenario.vm) + t.expectGuestReady(vmScenario) } By("Checking attached disks after module upgrade") @@ -493,12 +495,19 @@ func (t *currentReleaseSmokeTest) manualStartVMObjects() []crclient.Object { return objects } -func (t *currentReleaseSmokeTest) expectGuestReady(vm *v1alpha2.VirtualMachine) { - By(fmt.Sprintf("Waiting for guest agent on %s", vm.Name)) - util.UntilVMAgentReady(crclient.ObjectKeyFromObject(vm), framework.LongTimeout) +func (t *currentReleaseSmokeTest) expectGuestReady(vmScenario *vmScenario) { + vm := vmScenario.vm By(fmt.Sprintf("Waiting for SSH access on %s", vm.Name)) util.UntilSSHReady(t.framework, vm, framework.LongTimeout) + + if vmScenario.skipGuestAgentCheck { + By(fmt.Sprintf("Skipping strict guest agent check on %s", vm.Name)) + return + } + + By(fmt.Sprintf("Waiting for guest agent on %s", vm.Name)) + util.UntilVMAgentReady(crclient.ObjectKeyFromObject(vm), framework.LongTimeout) } func (t *currentReleaseSmokeTest) expectAdditionalDiskCount(vm *v1alpha2.VirtualMachine, expectedCount int) { From 9aa7b3194abf24c6588021aca5d8f0ba60a4fef8 Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Tue, 14 Apr 2026 20:57:26 +0300 Subject: [PATCH 18/20] change wait duration Signed-off-by: Nikita Korolev --- .beads/.gitignore | 72 ++++++++++ .beads/README.md | 81 ++++++++++++ .beads/config.yaml | 54 ++++++++ .beads/hooks/post-checkout | 24 ++++ .beads/hooks/post-merge | 24 ++++ .beads/hooks/pre-commit | 24 ++++ .beads/hooks/pre-push | 24 ++++ .beads/hooks/prepare-commit-msg | 24 ++++ .beads/metadata.json | 7 + .claude/settings.json | 26 ++++ .gitignore | 5 + AGENTS.md | 84 ++++++++++++ test/e2e/internal/util/until.go | 34 +++-- test/e2e/release/current_release_smoke.go | 154 ++++++++++++++++++---- 14 files changed, 597 insertions(+), 40 deletions(-) create mode 100644 .beads/.gitignore create mode 100644 .beads/README.md create mode 100644 .beads/config.yaml create mode 100755 .beads/hooks/post-checkout create mode 100755 .beads/hooks/post-merge create mode 100755 .beads/hooks/pre-commit create mode 100755 .beads/hooks/pre-push create mode 100755 .beads/hooks/prepare-commit-msg create mode 100644 .beads/metadata.json create mode 100644 .claude/settings.json create mode 100644 AGENTS.md diff --git a/.beads/.gitignore b/.beads/.gitignore new file mode 100644 index 0000000000..eb82c48f41 --- /dev/null +++ b/.beads/.gitignore @@ -0,0 +1,72 @@ +# Dolt database (managed by Dolt, not git) +dolt/ + +# Runtime files +bd.sock +bd.sock.startlock +sync-state.json +last-touched +.exclusive-lock + +# Daemon runtime (lock, log, pid) +daemon.* + +# Interactions log (runtime, not versioned) +interactions.jsonl + +# Push state (runtime, per-machine) +push-state.json + +# Lock files (various runtime locks) +*.lock + +# Credential key (encryption key for federation peer auth — never commit) +.beads-credential-key + +# Local version tracking (prevents upgrade notification spam after git ops) +.local_version + +# Worktree redirect file (contains relative path to main repo's .beads/) +# Must not be committed as paths would be wrong in other clones +redirect + +# Sync state (local-only, per-machine) +# These files are machine-specific and should not be shared across clones +.sync.lock +export-state/ +export-state.json + +# Ephemeral store (SQLite - wisps/molecules, intentionally not versioned) +ephemeral.sqlite3 +ephemeral.sqlite3-journal +ephemeral.sqlite3-wal +ephemeral.sqlite3-shm + +# Dolt server management (auto-started by bd) +dolt-server.pid +dolt-server.log +dolt-server.lock +dolt-server.port +dolt-server.activity + +# Corrupt backup directories (created by bd doctor --fix recovery) +*.corrupt.backup/ + +# Backup data (auto-exported JSONL, local-only) +backup/ + +# Per-project environment file (Dolt connection config, GH#2520) +.env + +# Legacy files (from pre-Dolt versions) +*.db +*.db?* +*.db-journal +*.db-wal +*.db-shm +db.sqlite +bd.db +# NOTE: Do NOT add negation patterns here. +# They would override fork protection in .git/info/exclude. +# Config files (metadata.json, config.yaml) are tracked by git by default +# since no pattern above ignores them. diff --git a/.beads/README.md b/.beads/README.md new file mode 100644 index 0000000000..dbfe3631cf --- /dev/null +++ b/.beads/README.md @@ -0,0 +1,81 @@ +# Beads - AI-Native Issue Tracking + +Welcome to Beads! This repository uses **Beads** for issue tracking - a modern, AI-native tool designed to live directly in your codebase alongside your code. + +## What is Beads? + +Beads is issue tracking that lives in your repo, making it perfect for AI coding agents and developers who want their issues close to their code. No web UI required - everything works through the CLI and integrates seamlessly with git. + +**Learn more:** [github.com/steveyegge/beads](https://github.com/steveyegge/beads) + +## Quick Start + +### Essential Commands + +```bash +# Create new issues +bd create "Add user authentication" + +# View all issues +bd list + +# View issue details +bd show + +# Update issue status +bd update --claim +bd update --status done + +# Sync with Dolt remote +bd dolt push +``` + +### Working with Issues + +Issues in Beads are: +- **Git-native**: Stored in Dolt database with version control and branching +- **AI-friendly**: CLI-first design works perfectly with AI coding agents +- **Branch-aware**: Issues can follow your branch workflow +- **Always in sync**: Auto-syncs with your commits + +## Why Beads? + +✨ **AI-Native Design** +- Built specifically for AI-assisted development workflows +- CLI-first interface works seamlessly with AI coding agents +- No context switching to web UIs + +🚀 **Developer Focused** +- Issues live in your repo, right next to your code +- Works offline, syncs when you push +- Fast, lightweight, and stays out of your way + +🔧 **Git Integration** +- Automatic sync with git commits +- Branch-aware issue tracking +- Dolt-native three-way merge resolution + +## Get Started with Beads + +Try Beads in your own projects: + +```bash +# Install Beads +curl -sSL https://raw.githubusercontent.com/steveyegge/beads/main/scripts/install.sh | bash + +# Initialize in your repo +bd init + +# Create your first issue +bd create "Try out Beads" +``` + +## Learn More + +- **Documentation**: [github.com/steveyegge/beads/docs](https://github.com/steveyegge/beads/tree/main/docs) +- **Quick Start Guide**: Run `bd quickstart` +- **Examples**: [github.com/steveyegge/beads/examples](https://github.com/steveyegge/beads/tree/main/examples) + +--- + +*Beads: Issue tracking that moves at the speed of thought* ⚡ diff --git a/.beads/config.yaml b/.beads/config.yaml new file mode 100644 index 0000000000..232b151111 --- /dev/null +++ b/.beads/config.yaml @@ -0,0 +1,54 @@ +# Beads Configuration File +# This file configures default behavior for all bd commands in this repository +# All settings can also be set via environment variables (BD_* prefix) +# or overridden with command-line flags + +# Issue prefix for this repository (used by bd init) +# If not set, bd init will auto-detect from directory name +# Example: issue-prefix: "myproject" creates issues like "myproject-1", "myproject-2", etc. +# issue-prefix: "" + +# Use no-db mode: JSONL-only, no Dolt database +# When true, bd will use .beads/issues.jsonl as the source of truth +# no-db: false + +# Enable JSON output by default +# json: false + +# Feedback title formatting for mutating commands (create/update/close/dep/edit) +# 0 = hide titles, N > 0 = truncate to N characters +# output: +# title-length: 255 + +# Default actor for audit trails (overridden by BEADS_ACTOR or --actor) +# actor: "" + +# Export events (audit trail) to .beads/events.jsonl on each flush/sync +# When enabled, new events are appended incrementally using a high-water mark. +# Use 'bd export --events' to trigger manually regardless of this setting. +# events-export: false + +# Multi-repo configuration (experimental - bd-307) +# Allows hydrating from multiple repositories and routing writes to the correct database +# repos: +# primary: "." # Primary repo (where this database lives) +# additional: # Additional repos to hydrate from (read-only) +# - ~/beads-planning # Personal planning repo +# - ~/work-planning # Work planning repo + +# JSONL backup (periodic export for off-machine recovery) +# Auto-enabled when a git remote exists. Override explicitly: +# backup: +# enabled: false # Disable auto-backup entirely +# interval: 15m # Minimum time between auto-exports +# git-push: false # Disable git push (export locally only) +# git-repo: "" # Separate git repo for backups (default: project repo) + +# Integration settings (access with 'bd config get/set') +# These are stored in the database, not in this file: +# - jira.url +# - jira.project +# - linear.url +# - linear.api-key +# - github.org +# - github.repo diff --git a/.beads/hooks/post-checkout b/.beads/hooks/post-checkout new file mode 100755 index 0000000000..67ad327655 --- /dev/null +++ b/.beads/hooks/post-checkout @@ -0,0 +1,24 @@ +#!/usr/bin/env sh +# --- BEGIN BEADS INTEGRATION v1.0.0 --- +# This section is managed by beads. Do not remove these markers. +if command -v bd >/dev/null 2>&1; then + export BD_GIT_HOOK=1 + _bd_timeout=${BEADS_HOOK_TIMEOUT:-300} + if command -v timeout >/dev/null 2>&1; then + timeout "$_bd_timeout" bd hooks run post-checkout "$@" + _bd_exit=$? + if [ $_bd_exit -eq 124 ]; then + echo >&2 "beads: hook 'post-checkout' timed out after ${_bd_timeout}s — continuing without beads" + _bd_exit=0 + fi + else + bd hooks run post-checkout "$@" + _bd_exit=$? + fi + if [ $_bd_exit -eq 3 ]; then + echo >&2 "beads: database not initialized — skipping hook 'post-checkout'" + _bd_exit=0 + fi + if [ $_bd_exit -ne 0 ]; then exit $_bd_exit; fi +fi +# --- END BEADS INTEGRATION v1.0.0 --- diff --git a/.beads/hooks/post-merge b/.beads/hooks/post-merge new file mode 100755 index 0000000000..a731aec674 --- /dev/null +++ b/.beads/hooks/post-merge @@ -0,0 +1,24 @@ +#!/usr/bin/env sh +# --- BEGIN BEADS INTEGRATION v1.0.0 --- +# This section is managed by beads. Do not remove these markers. +if command -v bd >/dev/null 2>&1; then + export BD_GIT_HOOK=1 + _bd_timeout=${BEADS_HOOK_TIMEOUT:-300} + if command -v timeout >/dev/null 2>&1; then + timeout "$_bd_timeout" bd hooks run post-merge "$@" + _bd_exit=$? + if [ $_bd_exit -eq 124 ]; then + echo >&2 "beads: hook 'post-merge' timed out after ${_bd_timeout}s — continuing without beads" + _bd_exit=0 + fi + else + bd hooks run post-merge "$@" + _bd_exit=$? + fi + if [ $_bd_exit -eq 3 ]; then + echo >&2 "beads: database not initialized — skipping hook 'post-merge'" + _bd_exit=0 + fi + if [ $_bd_exit -ne 0 ]; then exit $_bd_exit; fi +fi +# --- END BEADS INTEGRATION v1.0.0 --- diff --git a/.beads/hooks/pre-commit b/.beads/hooks/pre-commit new file mode 100755 index 0000000000..02cf2acfd4 --- /dev/null +++ b/.beads/hooks/pre-commit @@ -0,0 +1,24 @@ +#!/usr/bin/env sh +# --- BEGIN BEADS INTEGRATION v1.0.0 --- +# This section is managed by beads. Do not remove these markers. +if command -v bd >/dev/null 2>&1; then + export BD_GIT_HOOK=1 + _bd_timeout=${BEADS_HOOK_TIMEOUT:-300} + if command -v timeout >/dev/null 2>&1; then + timeout "$_bd_timeout" bd hooks run pre-commit "$@" + _bd_exit=$? + if [ $_bd_exit -eq 124 ]; then + echo >&2 "beads: hook 'pre-commit' timed out after ${_bd_timeout}s — continuing without beads" + _bd_exit=0 + fi + else + bd hooks run pre-commit "$@" + _bd_exit=$? + fi + if [ $_bd_exit -eq 3 ]; then + echo >&2 "beads: database not initialized — skipping hook 'pre-commit'" + _bd_exit=0 + fi + if [ $_bd_exit -ne 0 ]; then exit $_bd_exit; fi +fi +# --- END BEADS INTEGRATION v1.0.0 --- diff --git a/.beads/hooks/pre-push b/.beads/hooks/pre-push new file mode 100755 index 0000000000..79184923e2 --- /dev/null +++ b/.beads/hooks/pre-push @@ -0,0 +1,24 @@ +#!/usr/bin/env sh +# --- BEGIN BEADS INTEGRATION v1.0.0 --- +# This section is managed by beads. Do not remove these markers. +if command -v bd >/dev/null 2>&1; then + export BD_GIT_HOOK=1 + _bd_timeout=${BEADS_HOOK_TIMEOUT:-300} + if command -v timeout >/dev/null 2>&1; then + timeout "$_bd_timeout" bd hooks run pre-push "$@" + _bd_exit=$? + if [ $_bd_exit -eq 124 ]; then + echo >&2 "beads: hook 'pre-push' timed out after ${_bd_timeout}s — continuing without beads" + _bd_exit=0 + fi + else + bd hooks run pre-push "$@" + _bd_exit=$? + fi + if [ $_bd_exit -eq 3 ]; then + echo >&2 "beads: database not initialized — skipping hook 'pre-push'" + _bd_exit=0 + fi + if [ $_bd_exit -ne 0 ]; then exit $_bd_exit; fi +fi +# --- END BEADS INTEGRATION v1.0.0 --- diff --git a/.beads/hooks/prepare-commit-msg b/.beads/hooks/prepare-commit-msg new file mode 100755 index 0000000000..c0c3ce1f41 --- /dev/null +++ b/.beads/hooks/prepare-commit-msg @@ -0,0 +1,24 @@ +#!/usr/bin/env sh +# --- BEGIN BEADS INTEGRATION v1.0.0 --- +# This section is managed by beads. Do not remove these markers. +if command -v bd >/dev/null 2>&1; then + export BD_GIT_HOOK=1 + _bd_timeout=${BEADS_HOOK_TIMEOUT:-300} + if command -v timeout >/dev/null 2>&1; then + timeout "$_bd_timeout" bd hooks run prepare-commit-msg "$@" + _bd_exit=$? + if [ $_bd_exit -eq 124 ]; then + echo >&2 "beads: hook 'prepare-commit-msg' timed out after ${_bd_timeout}s — continuing without beads" + _bd_exit=0 + fi + else + bd hooks run prepare-commit-msg "$@" + _bd_exit=$? + fi + if [ $_bd_exit -eq 3 ]; then + echo >&2 "beads: database not initialized — skipping hook 'prepare-commit-msg'" + _bd_exit=0 + fi + if [ $_bd_exit -ne 0 ]; then exit $_bd_exit; fi +fi +# --- END BEADS INTEGRATION v1.0.0 --- diff --git a/.beads/metadata.json b/.beads/metadata.json new file mode 100644 index 0000000000..229878ed02 --- /dev/null +++ b/.beads/metadata.json @@ -0,0 +1,7 @@ +{ + "database": "dolt", + "backend": "dolt", + "dolt_mode": "embedded", + "dolt_database": "virtualization2", + "project_id": "6116a5bf-a24c-4e17-af9f-0f6c53bc6cd4" +} \ No newline at end of file diff --git a/.claude/settings.json b/.claude/settings.json new file mode 100644 index 0000000000..963a53824a --- /dev/null +++ b/.claude/settings.json @@ -0,0 +1,26 @@ +{ + "hooks": { + "PreCompact": [ + { + "hooks": [ + { + "command": "bd prime", + "type": "command" + } + ], + "matcher": "" + } + ], + "SessionStart": [ + { + "hooks": [ + { + "command": "bd prime", + "type": "command" + } + ], + "matcher": "" + } + ] + } +} \ No newline at end of file diff --git a/.gitignore b/.gitignore index ae343f44fd..f02bc2255c 100644 --- a/.gitignore +++ b/.gitignore @@ -60,3 +60,8 @@ retry/ # nodejs node_modules/ package-lock.json + +# Beads / Dolt files (added by bd init) +.dolt/ +*.db +.beads-credential-key diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000000..9390d72dbc --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,84 @@ +# Agent Instructions + +This project uses **bd** (beads) for issue tracking. Run `bd prime` for full workflow context. + +## Quick Reference + +```bash +bd ready # Find available work +bd show # View issue details +bd update --claim # Claim work atomically +bd close # Complete work +bd dolt push # Push beads data to remote +``` + +## Non-Interactive Shell Commands + +**ALWAYS use non-interactive flags** with file operations to avoid hanging on confirmation prompts. + +Shell commands like `cp`, `mv`, and `rm` may be aliased to include `-i` (interactive) mode on some systems, causing the agent to hang indefinitely waiting for y/n input. + +**Use these forms instead:** +```bash +# Force overwrite without prompting +cp -f source dest # NOT: cp source dest +mv -f source dest # NOT: mv source dest +rm -f file # NOT: rm file + +# For recursive operations +rm -rf directory # NOT: rm -r directory +cp -rf source dest # NOT: cp -r source dest +``` + +**Other commands that may prompt:** +- `scp` - use `-o BatchMode=yes` for non-interactive +- `ssh` - use `-o BatchMode=yes` to fail instead of prompting +- `apt-get` - use `-y` flag +- `brew` - use `HOMEBREW_NO_AUTO_UPDATE=1` env var + + +## Beads Issue Tracker + +This project uses **bd (beads)** for issue tracking. Run `bd prime` to see full workflow context and commands. + +### Quick Reference + +```bash +bd ready # Find available work +bd show # View issue details +bd update --claim # Claim work +bd close # Complete work +``` + +### Rules + +- Use `bd` for ALL task tracking — do NOT use TodoWrite, TaskCreate, or markdown TODO lists +- Run `bd prime` for detailed command reference and session close protocol +- Use `bd remember` for persistent knowledge — do NOT use MEMORY.md files + +## Session Completion + +**When ending a work session**, you MUST complete ALL steps below. Work is NOT complete until `git push` succeeds. + +**MANDATORY WORKFLOW:** + +1. **File issues for remaining work** - Create issues for anything that needs follow-up +2. **Run quality gates** (if code changed) - Tests, linters, builds +3. **Update issue status** - Close finished work, update in-progress items +4. **PUSH TO REMOTE** - This is MANDATORY: + ```bash + git pull --rebase + bd dolt push + git push + git status # MUST show "up to date with origin" + ``` +5. **Clean up** - Clear stashes, prune remote branches +6. **Verify** - All changes committed AND pushed +7. **Hand off** - Provide context for next session + +**CRITICAL RULES:** +- Work is NOT complete until `git push` succeeds +- NEVER stop before pushing - that leaves work stranded locally +- NEVER say "ready to push when you are" - YOU must push +- If push fails, resolve and retry until it succeeds + diff --git a/test/e2e/internal/util/until.go b/test/e2e/internal/util/until.go index f541f9c69d..6bb24e83b2 100644 --- a/test/e2e/internal/util/until.go +++ b/test/e2e/internal/util/until.go @@ -31,6 +31,8 @@ import ( "github.com/deckhouse/virtualization/test/e2e/internal/framework" ) +const objectGetTimeout = 10 * time.Second + // UntilObjectPhase waits for an object to reach the specified phase. // It accepts a runtime.Object (which serves as a template with name and namespace), // expected phase string, and timeout duration. @@ -90,8 +92,8 @@ func UntilConditionState( for _, obj := range objs { key := client.ObjectKeyFromObject(obj) u := getTemplateUnstructured(obj).DeepCopy() - err := framework.GetClients().GenericClient().Get(context.Background(), key, u) - g.Expect(err).ShouldNot(HaveOccurred()) + err := getObjectWithTimeout(key, u) + g.Expect(err).ShouldNot(HaveOccurred(), "failed to get object %s while waiting for condition %s", formatObjectKey(key), conditionType) conditions, found, err := unstructured.NestedSlice(u.Object, "status", "conditions") g.Expect(err).ShouldNot(HaveOccurred(), "failed to access status.conditions of %s/%s", u.GetNamespace(), u.GetName()) @@ -178,26 +180,34 @@ func untilObjectField(fieldPath, expectedValue string, timeout time.Duration, ob Eventually(func(g Gomega) { for _, obj := range objs { key := client.ObjectKeyFromObject(obj) - name := obj.GetName() - namespace := obj.GetNamespace() - divider := "" - if namespace != "" { - divider = "/" - } - // Create a new unstructured object for each Get call u := getTemplateUnstructured(obj).DeepCopy() - err := framework.GetClients().GenericClient().Get(context.Background(), key, u) + err := getObjectWithTimeout(key, u) if err != nil { - g.Expect(err).NotTo(HaveOccurred(), "failed to get object %s%s%s", namespace, divider, name) + g.Expect(err).NotTo(HaveOccurred(), "failed to get object %s while waiting for %s=%s", formatObjectKey(key), fieldPath, expectedValue) } value := extractField(u, fieldPath) - g.Expect(value).To(Equal(expectedValue), "object %s%s%s %s is %s, expected %s", namespace, divider, name, fieldPath, value, expectedValue) + g.Expect(value).To(Equal(expectedValue), "object %s %s is %s, expected %s", formatObjectKey(key), fieldPath, value, expectedValue) } }).WithTimeout(timeout).WithPolling(time.Second).Should(Succeed()) } +func getObjectWithTimeout(key client.ObjectKey, obj client.Object) error { + ctx, cancel := context.WithTimeout(context.Background(), objectGetTimeout) + defer cancel() + + return framework.GetClients().GenericClient().Get(ctx, key, obj) +} + +func formatObjectKey(key client.ObjectKey) string { + if key.Namespace == "" { + return key.Name + } + + return key.Namespace + "/" + key.Name +} + func getTemplateUnstructured(obj client.Object) *unstructured.Unstructured { // Convert the template object to unstructured once var templateUnstructured *unstructured.Unstructured diff --git a/test/e2e/release/current_release_smoke.go b/test/e2e/release/current_release_smoke.go index 971baf2309..0c475a4300 100644 --- a/test/e2e/release/current_release_smoke.go +++ b/test/e2e/release/current_release_smoke.go @@ -29,6 +29,8 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/utils/ptr" @@ -45,11 +47,13 @@ import ( const ( replicatedStorageClass = "nested-thin-r1" localThinStorageClass = "nested-local-thin" - lsblkJSONCommand = "lsblk --bytes --json --nodeps --output NAME,SIZE,TYPE,MOUNTPOINTS" - minDataDiskSizeBytes = int64(50 * 1024 * 1024) + lsblkJSONCommand = "sudo lsblk --bytes --json --nodeps --output NAME,SIZE,TYPE,MOUNTPOINTS" + rootDiskNameCommand = `root_source=$(findmnt -no SOURCE /); root_disk=$(lsblk -ndo PKNAME "$root_source" 2>/dev/null | head -n1); if [ -n "$root_disk" ]; then echo "$root_disk"; else lsblk -ndo NAME "$root_source" | head -n1; fi` + maxCloudInitDiskSize = int64(4 * 1024 * 1024) defaultRootDiskSize = "350Mi" defaultDataDiskSize = "100Mi" releaseIPerfReportPath = "/tmp/release-upgrade-iperf-client-report.json" + releaseNamespaceName = "v12n-test-release" releaseTestPhaseEnv = "RELEASE_TEST_PHASE" releaseTestPhasePreUpgrade = "pre-upgrade" @@ -161,10 +165,10 @@ type releaseUpgradeContext struct { } func runPreUpgradeReleaseSmoke() { - f := framework.NewFramework("release-current") - f.Before() + f := framework.NewFramework("") + namespace := ensureReleaseNamespace(f, releaseNamespaceName) - test := newCurrentReleaseSmokeTest(f, "") + test := newCurrentReleaseSmokeTest(f, namespace) test.createResources() test.verifyVMsReady() test.startLongRunningIPerf() @@ -194,6 +198,46 @@ func mustGetEnv(name string) string { return value } +func ensureReleaseNamespace(f *framework.Framework, namespace string) string { + GinkgoHelper() + + nsClient := f.KubeClient().CoreV1().Namespaces() + _, err := nsClient.Get(context.Background(), namespace, metav1.GetOptions{}) + switch { + case err == nil: + By(fmt.Sprintf("Namespace %q already exists, recreating it", namespace)) + err = nsClient.Delete(context.Background(), namespace, metav1.DeleteOptions{}) + Expect(err).NotTo(HaveOccurred()) + + Eventually(func() error { + _, err := nsClient.Get(context.Background(), namespace, metav1.GetOptions{}) + if k8serrors.IsNotFound(err) { + return nil + } + if err != nil { + return err + } + return fmt.Errorf("namespace %q is still deleting", namespace) + }).WithTimeout(framework.LongTimeout).WithPolling(time.Second).Should(Succeed()) + case !k8serrors.IsNotFound(err): + Expect(err).NotTo(HaveOccurred()) + } + + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: namespace, + Labels: map[string]string{ + framework.E2ELabel: "true", + }, + }, + } + _, err = nsClient.Create(context.Background(), ns, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + By(fmt.Sprintf("Namespace %q has been created", namespace)) + + return namespace +} + func newCurrentReleaseSmokeTest(f *framework.Framework, namespace string) *currentReleaseSmokeTest { test := ¤tReleaseSmokeTest{ framework: f, @@ -352,7 +396,7 @@ func (t *currentReleaseSmokeTest) createResources() { By("Attaching hotplug disks") Expect(t.framework.CreateWithDeferredDeletion(context.Background(), t.attachmentObjects()...)).To(Succeed()) - util.UntilObjectPhase(string(v1alpha2.BlockDeviceAttachmentPhaseAttached), framework.LongTimeout, t.attachmentObjects()...) + util.UntilObjectPhase(string(v1alpha2.BlockDeviceAttachmentPhaseAttached), framework.MaxTimeout, t.attachmentObjects()...) By("Waiting for all disks to become ready after consumers appear") util.UntilObjectPhase(string(v1alpha2.DiskReady), framework.LongTimeout, t.diskObjects()...) @@ -366,6 +410,7 @@ func (t *currentReleaseSmokeTest) verifyVMsReady() { By("Checking attached disks inside guests") for _, vmScenario := range t.vms { + By(fmt.Sprintf("Checking attached disks on %s", vmScenario.vm.Name)) t.expectAdditionalDiskCount(vmScenario.vm, vmScenario.expectedAdditionalDisks) } } @@ -512,27 +557,32 @@ func (t *currentReleaseSmokeTest) expectGuestReady(vmScenario *vmScenario) { func (t *currentReleaseSmokeTest) expectAdditionalDiskCount(vm *v1alpha2.VirtualMachine, expectedCount int) { Eventually(func(g Gomega) { + currentVM := &v1alpha2.VirtualMachine{} + err := t.framework.GenericClient().Get(context.Background(), crclient.ObjectKeyFromObject(vm), currentVM) + g.Expect(err).NotTo(HaveOccurred()) + + attachedHotplugDisks := hotpluggedAttachedDiskCount(currentVM) + g.Expect(attachedHotplugDisks).To(Equal(expectedCount)) + + rootDiskName, err := t.rootDiskName(vm) + g.Expect(err).NotTo(HaveOccurred()) + output, err := t.framework.SSHCommand(vm.Name, vm.Namespace, lsblkJSONCommand, framework.WithSSHTimeout(10*time.Second)) g.Expect(err).NotTo(HaveOccurred()) disks, err := parseLSBLKOutput(output) g.Expect(err).NotTo(HaveOccurred()) - count := 0 - for _, disk := range disks { - if disk.Type != "disk" { - continue - } - if disk.Size <= minDataDiskSizeBytes { - continue - } - if hasMountpoint(disk.Mountpoints, "/") { - continue - } - count++ - } - - g.Expect(count).To(Equal(expectedCount)) + actualCount := countAdditionalGuestDisks(disks, rootDiskName) + g.Expect(actualCount).To( + Equal(expectedCount), + "VM %s/%s additional disk mismatch; root disk: %q; hotplugged block devices in status: %d; lsblk devices: %s", + vm.Namespace, + vm.Name, + rootDiskName, + attachedHotplugDisks, + formatLSBLKDisks(disks), + ) }).WithTimeout(framework.LongTimeout).WithPolling(time.Second).Should(Succeed()) } @@ -545,14 +595,54 @@ func parseLSBLKOutput(raw string) ([]lsblkDevice, error) { return output.BlockDevices, nil } -func hasMountpoint(mountpoints []string, expected string) bool { - for _, mountpoint := range mountpoints { - if mountpoint == expected { - return true +func hotpluggedAttachedDiskCount(vm *v1alpha2.VirtualMachine) int { + count := 0 + for _, blockDevice := range vm.Status.BlockDeviceRefs { + if !blockDevice.Hotplugged || !blockDevice.Attached || blockDevice.VirtualMachineBlockDeviceAttachmentName == "" { + continue } + count++ } + return count +} + +func (t *currentReleaseSmokeTest) rootDiskName(vm *v1alpha2.VirtualMachine) (string, error) { + output, err := t.framework.SSHCommand(vm.Name, vm.Namespace, rootDiskNameCommand, framework.WithSSHTimeout(10*time.Second)) + if err != nil { + return "", err + } + + return strings.TrimSpace(output), nil +} + +func countAdditionalGuestDisks(disks []lsblkDevice, rootDiskName string) int { + count := 0 + for _, disk := range disks { + if disk.Type != "disk" { + continue + } + if disk.Name == rootDiskName { + continue + } + if disk.Size <= maxCloudInitDiskSize { + continue + } + count++ + } + return count +} - return false +func formatLSBLKDisks(disks []lsblkDevice) string { + if len(disks) == 0 { + return "[]" + } + + parts := make([]string, 0, len(disks)) + for _, disk := range disks { + parts = append(parts, fmt.Sprintf("%s(type=%s,size=%d,mountpoints=%v)", disk.Name, disk.Type, disk.Size, disk.Mountpoints)) + } + + return "[" + strings.Join(parts, ", ") + "]" } func (t *currentReleaseSmokeTest) startLongRunningIPerf() { @@ -562,7 +652,7 @@ func (t *currentReleaseSmokeTest) startLongRunningIPerf() { serverVM := t.getVirtualMachine(t.iperfServer.vm.Name, t.iperfServer.vm.Namespace) command := fmt.Sprintf( - "nohup sh -c 'iperf3 -c %s -t 0 --json > %s 2>&1' >/dev/null 2>&1 &", + "nohup iperf3 -c %s -t 0 --json > %s 2>&1 Date: Wed, 15 Apr 2026 15:33:39 +0300 Subject: [PATCH 19/20] refactor tests Signed-off-by: Nikita Korolev --- .beads/.gitignore | 72 -- .beads/README.md | 81 --- .beads/config.yaml | 54 -- .beads/hooks/post-checkout | 24 - .beads/hooks/post-merge | 24 - .beads/hooks/pre-commit | 24 - .beads/hooks/pre-push | 24 - .beads/hooks/prepare-commit-msg | 24 - .beads/metadata.json | 7 - .../e2e-test-releases-reusable-pipeline.yml | 91 +++ .github/workflows/e2e-test-releases.yml | 1 + test/e2e/release/current_release_smoke.go | 648 ++---------------- test/e2e/release/guest.go | 155 +++++ test/e2e/release/iperf.go | 178 +++++ test/e2e/release/resources.go | 87 +++ test/e2e/release/scenarios.go | 245 +++++++ 16 files changed, 797 insertions(+), 942 deletions(-) delete mode 100644 .beads/.gitignore delete mode 100644 .beads/README.md delete mode 100644 .beads/config.yaml delete mode 100755 .beads/hooks/post-checkout delete mode 100755 .beads/hooks/post-merge delete mode 100755 .beads/hooks/pre-commit delete mode 100755 .beads/hooks/pre-push delete mode 100755 .beads/hooks/prepare-commit-msg delete mode 100644 .beads/metadata.json create mode 100644 test/e2e/release/guest.go create mode 100644 test/e2e/release/iperf.go create mode 100644 test/e2e/release/resources.go create mode 100644 test/e2e/release/scenarios.go diff --git a/.beads/.gitignore b/.beads/.gitignore deleted file mode 100644 index eb82c48f41..0000000000 --- a/.beads/.gitignore +++ /dev/null @@ -1,72 +0,0 @@ -# Dolt database (managed by Dolt, not git) -dolt/ - -# Runtime files -bd.sock -bd.sock.startlock -sync-state.json -last-touched -.exclusive-lock - -# Daemon runtime (lock, log, pid) -daemon.* - -# Interactions log (runtime, not versioned) -interactions.jsonl - -# Push state (runtime, per-machine) -push-state.json - -# Lock files (various runtime locks) -*.lock - -# Credential key (encryption key for federation peer auth — never commit) -.beads-credential-key - -# Local version tracking (prevents upgrade notification spam after git ops) -.local_version - -# Worktree redirect file (contains relative path to main repo's .beads/) -# Must not be committed as paths would be wrong in other clones -redirect - -# Sync state (local-only, per-machine) -# These files are machine-specific and should not be shared across clones -.sync.lock -export-state/ -export-state.json - -# Ephemeral store (SQLite - wisps/molecules, intentionally not versioned) -ephemeral.sqlite3 -ephemeral.sqlite3-journal -ephemeral.sqlite3-wal -ephemeral.sqlite3-shm - -# Dolt server management (auto-started by bd) -dolt-server.pid -dolt-server.log -dolt-server.lock -dolt-server.port -dolt-server.activity - -# Corrupt backup directories (created by bd doctor --fix recovery) -*.corrupt.backup/ - -# Backup data (auto-exported JSONL, local-only) -backup/ - -# Per-project environment file (Dolt connection config, GH#2520) -.env - -# Legacy files (from pre-Dolt versions) -*.db -*.db?* -*.db-journal -*.db-wal -*.db-shm -db.sqlite -bd.db -# NOTE: Do NOT add negation patterns here. -# They would override fork protection in .git/info/exclude. -# Config files (metadata.json, config.yaml) are tracked by git by default -# since no pattern above ignores them. diff --git a/.beads/README.md b/.beads/README.md deleted file mode 100644 index dbfe3631cf..0000000000 --- a/.beads/README.md +++ /dev/null @@ -1,81 +0,0 @@ -# Beads - AI-Native Issue Tracking - -Welcome to Beads! This repository uses **Beads** for issue tracking - a modern, AI-native tool designed to live directly in your codebase alongside your code. - -## What is Beads? - -Beads is issue tracking that lives in your repo, making it perfect for AI coding agents and developers who want their issues close to their code. No web UI required - everything works through the CLI and integrates seamlessly with git. - -**Learn more:** [github.com/steveyegge/beads](https://github.com/steveyegge/beads) - -## Quick Start - -### Essential Commands - -```bash -# Create new issues -bd create "Add user authentication" - -# View all issues -bd list - -# View issue details -bd show - -# Update issue status -bd update --claim -bd update --status done - -# Sync with Dolt remote -bd dolt push -``` - -### Working with Issues - -Issues in Beads are: -- **Git-native**: Stored in Dolt database with version control and branching -- **AI-friendly**: CLI-first design works perfectly with AI coding agents -- **Branch-aware**: Issues can follow your branch workflow -- **Always in sync**: Auto-syncs with your commits - -## Why Beads? - -✨ **AI-Native Design** -- Built specifically for AI-assisted development workflows -- CLI-first interface works seamlessly with AI coding agents -- No context switching to web UIs - -🚀 **Developer Focused** -- Issues live in your repo, right next to your code -- Works offline, syncs when you push -- Fast, lightweight, and stays out of your way - -🔧 **Git Integration** -- Automatic sync with git commits -- Branch-aware issue tracking -- Dolt-native three-way merge resolution - -## Get Started with Beads - -Try Beads in your own projects: - -```bash -# Install Beads -curl -sSL https://raw.githubusercontent.com/steveyegge/beads/main/scripts/install.sh | bash - -# Initialize in your repo -bd init - -# Create your first issue -bd create "Try out Beads" -``` - -## Learn More - -- **Documentation**: [github.com/steveyegge/beads/docs](https://github.com/steveyegge/beads/tree/main/docs) -- **Quick Start Guide**: Run `bd quickstart` -- **Examples**: [github.com/steveyegge/beads/examples](https://github.com/steveyegge/beads/tree/main/examples) - ---- - -*Beads: Issue tracking that moves at the speed of thought* ⚡ diff --git a/.beads/config.yaml b/.beads/config.yaml deleted file mode 100644 index 232b151111..0000000000 --- a/.beads/config.yaml +++ /dev/null @@ -1,54 +0,0 @@ -# Beads Configuration File -# This file configures default behavior for all bd commands in this repository -# All settings can also be set via environment variables (BD_* prefix) -# or overridden with command-line flags - -# Issue prefix for this repository (used by bd init) -# If not set, bd init will auto-detect from directory name -# Example: issue-prefix: "myproject" creates issues like "myproject-1", "myproject-2", etc. -# issue-prefix: "" - -# Use no-db mode: JSONL-only, no Dolt database -# When true, bd will use .beads/issues.jsonl as the source of truth -# no-db: false - -# Enable JSON output by default -# json: false - -# Feedback title formatting for mutating commands (create/update/close/dep/edit) -# 0 = hide titles, N > 0 = truncate to N characters -# output: -# title-length: 255 - -# Default actor for audit trails (overridden by BEADS_ACTOR or --actor) -# actor: "" - -# Export events (audit trail) to .beads/events.jsonl on each flush/sync -# When enabled, new events are appended incrementally using a high-water mark. -# Use 'bd export --events' to trigger manually regardless of this setting. -# events-export: false - -# Multi-repo configuration (experimental - bd-307) -# Allows hydrating from multiple repositories and routing writes to the correct database -# repos: -# primary: "." # Primary repo (where this database lives) -# additional: # Additional repos to hydrate from (read-only) -# - ~/beads-planning # Personal planning repo -# - ~/work-planning # Work planning repo - -# JSONL backup (periodic export for off-machine recovery) -# Auto-enabled when a git remote exists. Override explicitly: -# backup: -# enabled: false # Disable auto-backup entirely -# interval: 15m # Minimum time between auto-exports -# git-push: false # Disable git push (export locally only) -# git-repo: "" # Separate git repo for backups (default: project repo) - -# Integration settings (access with 'bd config get/set') -# These are stored in the database, not in this file: -# - jira.url -# - jira.project -# - linear.url -# - linear.api-key -# - github.org -# - github.repo diff --git a/.beads/hooks/post-checkout b/.beads/hooks/post-checkout deleted file mode 100755 index 67ad327655..0000000000 --- a/.beads/hooks/post-checkout +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env sh -# --- BEGIN BEADS INTEGRATION v1.0.0 --- -# This section is managed by beads. Do not remove these markers. -if command -v bd >/dev/null 2>&1; then - export BD_GIT_HOOK=1 - _bd_timeout=${BEADS_HOOK_TIMEOUT:-300} - if command -v timeout >/dev/null 2>&1; then - timeout "$_bd_timeout" bd hooks run post-checkout "$@" - _bd_exit=$? - if [ $_bd_exit -eq 124 ]; then - echo >&2 "beads: hook 'post-checkout' timed out after ${_bd_timeout}s — continuing without beads" - _bd_exit=0 - fi - else - bd hooks run post-checkout "$@" - _bd_exit=$? - fi - if [ $_bd_exit -eq 3 ]; then - echo >&2 "beads: database not initialized — skipping hook 'post-checkout'" - _bd_exit=0 - fi - if [ $_bd_exit -ne 0 ]; then exit $_bd_exit; fi -fi -# --- END BEADS INTEGRATION v1.0.0 --- diff --git a/.beads/hooks/post-merge b/.beads/hooks/post-merge deleted file mode 100755 index a731aec674..0000000000 --- a/.beads/hooks/post-merge +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env sh -# --- BEGIN BEADS INTEGRATION v1.0.0 --- -# This section is managed by beads. Do not remove these markers. -if command -v bd >/dev/null 2>&1; then - export BD_GIT_HOOK=1 - _bd_timeout=${BEADS_HOOK_TIMEOUT:-300} - if command -v timeout >/dev/null 2>&1; then - timeout "$_bd_timeout" bd hooks run post-merge "$@" - _bd_exit=$? - if [ $_bd_exit -eq 124 ]; then - echo >&2 "beads: hook 'post-merge' timed out after ${_bd_timeout}s — continuing without beads" - _bd_exit=0 - fi - else - bd hooks run post-merge "$@" - _bd_exit=$? - fi - if [ $_bd_exit -eq 3 ]; then - echo >&2 "beads: database not initialized — skipping hook 'post-merge'" - _bd_exit=0 - fi - if [ $_bd_exit -ne 0 ]; then exit $_bd_exit; fi -fi -# --- END BEADS INTEGRATION v1.0.0 --- diff --git a/.beads/hooks/pre-commit b/.beads/hooks/pre-commit deleted file mode 100755 index 02cf2acfd4..0000000000 --- a/.beads/hooks/pre-commit +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env sh -# --- BEGIN BEADS INTEGRATION v1.0.0 --- -# This section is managed by beads. Do not remove these markers. -if command -v bd >/dev/null 2>&1; then - export BD_GIT_HOOK=1 - _bd_timeout=${BEADS_HOOK_TIMEOUT:-300} - if command -v timeout >/dev/null 2>&1; then - timeout "$_bd_timeout" bd hooks run pre-commit "$@" - _bd_exit=$? - if [ $_bd_exit -eq 124 ]; then - echo >&2 "beads: hook 'pre-commit' timed out after ${_bd_timeout}s — continuing without beads" - _bd_exit=0 - fi - else - bd hooks run pre-commit "$@" - _bd_exit=$? - fi - if [ $_bd_exit -eq 3 ]; then - echo >&2 "beads: database not initialized — skipping hook 'pre-commit'" - _bd_exit=0 - fi - if [ $_bd_exit -ne 0 ]; then exit $_bd_exit; fi -fi -# --- END BEADS INTEGRATION v1.0.0 --- diff --git a/.beads/hooks/pre-push b/.beads/hooks/pre-push deleted file mode 100755 index 79184923e2..0000000000 --- a/.beads/hooks/pre-push +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env sh -# --- BEGIN BEADS INTEGRATION v1.0.0 --- -# This section is managed by beads. Do not remove these markers. -if command -v bd >/dev/null 2>&1; then - export BD_GIT_HOOK=1 - _bd_timeout=${BEADS_HOOK_TIMEOUT:-300} - if command -v timeout >/dev/null 2>&1; then - timeout "$_bd_timeout" bd hooks run pre-push "$@" - _bd_exit=$? - if [ $_bd_exit -eq 124 ]; then - echo >&2 "beads: hook 'pre-push' timed out after ${_bd_timeout}s — continuing without beads" - _bd_exit=0 - fi - else - bd hooks run pre-push "$@" - _bd_exit=$? - fi - if [ $_bd_exit -eq 3 ]; then - echo >&2 "beads: database not initialized — skipping hook 'pre-push'" - _bd_exit=0 - fi - if [ $_bd_exit -ne 0 ]; then exit $_bd_exit; fi -fi -# --- END BEADS INTEGRATION v1.0.0 --- diff --git a/.beads/hooks/prepare-commit-msg b/.beads/hooks/prepare-commit-msg deleted file mode 100755 index c0c3ce1f41..0000000000 --- a/.beads/hooks/prepare-commit-msg +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env sh -# --- BEGIN BEADS INTEGRATION v1.0.0 --- -# This section is managed by beads. Do not remove these markers. -if command -v bd >/dev/null 2>&1; then - export BD_GIT_HOOK=1 - _bd_timeout=${BEADS_HOOK_TIMEOUT:-300} - if command -v timeout >/dev/null 2>&1; then - timeout "$_bd_timeout" bd hooks run prepare-commit-msg "$@" - _bd_exit=$? - if [ $_bd_exit -eq 124 ]; then - echo >&2 "beads: hook 'prepare-commit-msg' timed out after ${_bd_timeout}s — continuing without beads" - _bd_exit=0 - fi - else - bd hooks run prepare-commit-msg "$@" - _bd_exit=$? - fi - if [ $_bd_exit -eq 3 ]; then - echo >&2 "beads: database not initialized — skipping hook 'prepare-commit-msg'" - _bd_exit=0 - fi - if [ $_bd_exit -ne 0 ]; then exit $_bd_exit; fi -fi -# --- END BEADS INTEGRATION v1.0.0 --- diff --git a/.beads/metadata.json b/.beads/metadata.json deleted file mode 100644 index 229878ed02..0000000000 --- a/.beads/metadata.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "database": "dolt", - "backend": "dolt", - "dolt_mode": "embedded", - "dolt_database": "virtualization2", - "project_id": "6116a5bf-a24c-4e17-af9f-0f6c53bc6cd4" -} \ No newline at end of file diff --git a/.github/workflows/e2e-test-releases-reusable-pipeline.yml b/.github/workflows/e2e-test-releases-reusable-pipeline.yml index 2a762c9f09..3847d31a08 100644 --- a/.github/workflows/e2e-test-releases-reusable-pipeline.yml +++ b/.github/workflows/e2e-test-releases-reusable-pipeline.yml @@ -95,6 +95,8 @@ on: required: true BOOTSTRAP_DEV_PROXY: required: true + DEV_MODULES_REGISTRY_PASSWORD: + required: true env: BRANCH: ${{ inputs.branch }} @@ -1386,6 +1388,95 @@ jobs: echo "[INFO] Checking virt-handler pods" virt_handler_ready + - name: Setup crane and registry auth + uses: deckhouse/modules-actions/setup@v2 + with: + registry: ${{ vars.DEV_REGISTRY }} + registry_login: ${{ vars.DEV_MODULES_REGISTRY_LOGIN }} + registry_password: ${{ secrets.DEV_MODULES_REGISTRY_PASSWORD }} + + - name: Verify image digests in pods after upgrade + run: | + MODULE_IMAGE="${{ vars.DEV_REGISTRY }}/${{ vars.DEV_MODULE_SOURCE }}/virtualization:${{ env.NEW_RELEASE }}" + echo "[INFO] Extracting images_digests.json from ${MODULE_IMAGE}" + images_hash=$(crane export "${MODULE_IMAGE}" - | tar -Oxf - images_digests.json) + echo "[INFO] Expected image digests:" + echo "::group::images_digests.json" + echo "$images_hash" | jq . + echo "::endgroup::" + + audit_status=$(kubectl get mc virtualization -o=jsonpath='{.spec.settings.audit.enabled}' 2>/dev/null || true) + audit_image_skip="true" + if [ -n "$audit_status" ] && [ "$audit_status" == "true" ]; then + audit_image_skip="false" + fi + + SKIP_IMAGES=() + if [ "$audit_image_skip" == "true" ]; then + SKIP_IMAGES+=("virtualizationAudit") + fi + SKIP_IMAGES+=("virtualizationDraUsb") + + is_skipped_image() { + local img="$1" + if [ ${#img} -eq 0 ]; then return 1; fi + for skip in "${SKIP_IMAGES[@]}"; do + if [[ "$img" == "$skip" ]]; then + return 0 + fi + done + return 1 + } + + retry_count=0 + max_retries=120 + sleep_interval=5 + + while true; do + all_hashes_found=true + + v12n_pods=$(kubectl -n d8-virtualization get pods -o json | jq -c) + + while IFS= read -r image_entry; do + image=$(echo "$image_entry" | jq -r '.key') + hash=$(echo "$image_entry" | jq -r '.value') + + if [[ "${image,,}" =~ (libguestfs|predeletehook) ]]; then + continue + fi + + if is_skipped_image "$image"; then + echo "- SKIP $image" + continue + fi + + if echo "$v12n_pods" | grep -q "$hash"; then + echo "- OK $image $hash" + else + echo "- MISS $image $hash" + all_hashes_found=false + fi + done < <(echo "$images_hash" | jq -c '. | to_entries | sort_by(.key)[]') + + if [ "$all_hashes_found" = true ]; then + echo "[SUCCESS] All image hashes found in pods after upgrade to ${{ env.NEW_RELEASE }}" + break + fi + + retry_count=$((retry_count + 1)) + echo "[INFO] Some hashes are missing, rechecking... Attempt: ${retry_count}/${max_retries}" + + if [ "$retry_count" -ge "$max_retries" ]; then + echo "[ERROR] Timeout reached after $((retry_count * sleep_interval))s. Some image hashes are still missing." + echo "::group::pods in d8-virtualization" + kubectl -n d8-virtualization get pods -o wide || true + echo "::endgroup::" + exit 1 + fi + + sleep "$sleep_interval" + done + - name: Show MPO state after upgrade run: | echo "[INFO] ModulePullOverride after upgrade:" diff --git a/.github/workflows/e2e-test-releases.yml b/.github/workflows/e2e-test-releases.yml index fd75a8f34e..270984fb54 100644 --- a/.github/workflows/e2e-test-releases.yml +++ b/.github/workflows/e2e-test-releases.yml @@ -228,3 +228,4 @@ jobs: VIRT_E2E_NIGHTLY_SA_TOKEN: ${{ secrets.VIRT_E2E_NIGHTLY_SA_TOKEN }} PROD_IO_REGISTRY_DOCKER_CFG: ${{ secrets.PROD_IO_REGISTRY_DOCKER_CFG }} BOOTSTRAP_DEV_PROXY: ${{ secrets.BOOTSTRAP_DEV_PROXY }} + DEV_MODULES_REGISTRY_PASSWORD: ${{ secrets.DEV_MODULES_REGISTRY_PASSWORD }} diff --git a/test/e2e/release/current_release_smoke.go b/test/e2e/release/current_release_smoke.go index 0c475a4300..8678a794bb 100644 --- a/test/e2e/release/current_release_smoke.go +++ b/test/e2e/release/current_release_smoke.go @@ -24,37 +24,20 @@ import ( "os" "path/filepath" "strconv" - "strings" "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/utils/ptr" - crclient "sigs.k8s.io/controller-runtime/pkg/client" - vdbuilder "github.com/deckhouse/virtualization-controller/pkg/builder/vd" - vmbuilder "github.com/deckhouse/virtualization-controller/pkg/builder/vm" "github.com/deckhouse/virtualization/api/core/v1alpha2" "github.com/deckhouse/virtualization/test/e2e/internal/framework" - "github.com/deckhouse/virtualization/test/e2e/internal/object" "github.com/deckhouse/virtualization/test/e2e/internal/util" ) const ( - replicatedStorageClass = "nested-thin-r1" - localThinStorageClass = "nested-local-thin" - lsblkJSONCommand = "sudo lsblk --bytes --json --nodeps --output NAME,SIZE,TYPE,MOUNTPOINTS" - rootDiskNameCommand = `root_source=$(findmnt -no SOURCE /); root_disk=$(lsblk -ndo PKNAME "$root_source" 2>/dev/null | head -n1); if [ -n "$root_disk" ]; then echo "$root_disk"; else lsblk -ndo NAME "$root_source" | head -n1; fi` - maxCloudInitDiskSize = int64(4 * 1024 * 1024) - defaultRootDiskSize = "350Mi" - defaultDataDiskSize = "100Mi" - releaseIPerfReportPath = "/tmp/release-upgrade-iperf-client-report.json" - releaseNamespaceName = "v12n-test-release" - releaseTestPhaseEnv = "RELEASE_TEST_PHASE" releaseTestPhasePreUpgrade = "pre-upgrade" releaseTestPhasePostUpgrade = "post-upgrade" @@ -85,85 +68,6 @@ type currentReleaseSmokeTest struct { iperfServer *vmScenario } -type vmScenario struct { - name string - rootDiskName string - cviName string - cloudInit string - runPolicy v1alpha2.RunPolicy - rootDiskSize string - expectedAdditionalDisks int - skipGuestAgentCheck bool - - rootDisk *v1alpha2.VirtualDisk - vm *v1alpha2.VirtualMachine -} - -type dataDiskScenario struct { - name string - storageClass string - size string - - disk *v1alpha2.VirtualDisk -} - -type attachmentScenario struct { - name string - vmName string - diskName string - - attachment *v1alpha2.VirtualMachineBlockDeviceAttachment -} - -type lsblkOutput struct { - BlockDevices []lsblkDevice `json:"blockdevices"` -} - -type lsblkDevice struct { - Name string `json:"name"` - Size int64 `json:"size"` - Type string `json:"type"` - Mountpoints []string `json:"mountpoints"` -} - -type iperfReport struct { - Start iperfReportStart `json:"start"` - Intervals []iperfReportInterval `json:"intervals"` - End iperfReportEnd `json:"end"` - Error string `json:"error,omitempty"` -} - -type iperfReportStart struct { - Timestamp iperfReportTimestamp `json:"timestamp"` -} - -type iperfReportTimestamp struct { - Time string `json:"time"` - Timesecs int `json:"timesecs"` -} - -type iperfReportEnd struct { - SumSent iperfReportSummary `json:"sum_sent"` - SumReceived iperfReportSummary `json:"sum_received"` -} - -type iperfReportInterval struct { - Sum iperfReportSummary `json:"sum"` -} - -type iperfReportSummary struct { - Bytes int64 `json:"bytes"` - BitsPerSecond float64 `json:"bits_per_second"` - End float64 `json:"end,omitempty"` -} - -type releaseUpgradeContext struct { - Namespace string `json:"namespace"` - IPerfClientVM string `json:"iperfClientVM"` - IPerfServerVM string `json:"iperfServerVM"` - IPerfReportPath string `json:"iperfReportPath"` -} - func runPreUpgradeReleaseSmoke() { f := framework.NewFramework("") namespace := ensureReleaseNamespace(f, releaseNamespaceName) @@ -184,195 +88,6 @@ func runPostUpgradeReleaseSmoke() { test.verifyIPerfContinuityAfterUpgrade() } -func getReleaseTestPhase() string { - if phase := os.Getenv(releaseTestPhaseEnv); phase != "" { - return phase - } - - return releaseTestPhasePreUpgrade -} - -func mustGetEnv(name string) string { - value := os.Getenv(name) - Expect(value).NotTo(BeEmpty(), "environment variable %s must be set", name) - return value -} - -func ensureReleaseNamespace(f *framework.Framework, namespace string) string { - GinkgoHelper() - - nsClient := f.KubeClient().CoreV1().Namespaces() - _, err := nsClient.Get(context.Background(), namespace, metav1.GetOptions{}) - switch { - case err == nil: - By(fmt.Sprintf("Namespace %q already exists, recreating it", namespace)) - err = nsClient.Delete(context.Background(), namespace, metav1.DeleteOptions{}) - Expect(err).NotTo(HaveOccurred()) - - Eventually(func() error { - _, err := nsClient.Get(context.Background(), namespace, metav1.GetOptions{}) - if k8serrors.IsNotFound(err) { - return nil - } - if err != nil { - return err - } - return fmt.Errorf("namespace %q is still deleting", namespace) - }).WithTimeout(framework.LongTimeout).WithPolling(time.Second).Should(Succeed()) - case !k8serrors.IsNotFound(err): - Expect(err).NotTo(HaveOccurred()) - } - - ns := &corev1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: namespace, - Labels: map[string]string{ - framework.E2ELabel: "true", - }, - }, - } - _, err = nsClient.Create(context.Background(), ns, metav1.CreateOptions{}) - Expect(err).NotTo(HaveOccurred()) - By(fmt.Sprintf("Namespace %q has been created", namespace)) - - return namespace -} - -func newCurrentReleaseSmokeTest(f *framework.Framework, namespace string) *currentReleaseSmokeTest { - test := ¤tReleaseSmokeTest{ - framework: f, - vmByName: make(map[string]*vmScenario), - dataDiskByName: make(map[string]*dataDiskScenario), - } - if namespace == "" { - Expect(f.Namespace()).NotTo(BeNil(), "framework namespace must be initialized for pre-upgrade phase") - namespace = f.Namespace().Name - } - - test.vms = []*vmScenario{ - { - name: "vm-alpine-manual", - rootDiskName: "vd-root-alpine-manual", - cviName: object.PrecreatedCVIAlpineBIOS, - cloudInit: object.AlpineCloudInit, - runPolicy: v1alpha2.ManualPolicy, - rootDiskSize: defaultRootDiskSize, - expectedAdditionalDisks: 0, - }, - { - name: "vm-alpine-single-hotplug", - rootDiskName: "vd-root-alpine-single-hotplug", - cviName: object.PrecreatedCVIAlpineBIOS, - cloudInit: object.AlpineCloudInit, - runPolicy: v1alpha2.AlwaysOnUnlessStoppedManually, - rootDiskSize: defaultRootDiskSize, - expectedAdditionalDisks: 1, - }, - { - name: "vm-alpine-double-hotplug", - rootDiskName: "vd-root-alpine-double-hotplug", - cviName: object.PrecreatedCVIAlpineBIOS, - cloudInit: object.AlpineCloudInit, - runPolicy: v1alpha2.AlwaysOnPolicy, - rootDiskSize: defaultRootDiskSize, - expectedAdditionalDisks: 2, - }, - { - name: "vm-ubuntu-replicated-five", - rootDiskName: "vd-root-ubuntu-replicated-five", - cviName: object.PrecreatedCVIUbuntu, - cloudInit: object.UbuntuCloudInit, - runPolicy: v1alpha2.AlwaysOnUnlessStoppedManually, - expectedAdditionalDisks: 5, - }, - { - name: "vm-ubuntu-mixed-five", - rootDiskName: "vd-root-ubuntu-mixed-five", - cviName: object.PrecreatedCVIUbuntu, - cloudInit: object.UbuntuCloudInit, - runPolicy: v1alpha2.AlwaysOnUnlessStoppedManually, - expectedAdditionalDisks: 5, - }, - { - name: "vm-alpine-iperf-client", - rootDiskName: "vd-root-alpine-iperf-client", - cviName: object.PrecreatedCVIAlpineBIOS, - cloudInit: object.AlpineCloudInit, - runPolicy: v1alpha2.AlwaysOnUnlessStoppedManually, - rootDiskSize: defaultRootDiskSize, - expectedAdditionalDisks: 2, - skipGuestAgentCheck: true, - }, - { - name: "vm-alpine-iperf-server", - rootDiskName: "vd-root-alpine-iperf-server", - cviName: object.PrecreatedCVIAlpineBIOS, - cloudInit: object.PerfCloudInit, - runPolicy: v1alpha2.AlwaysOnUnlessStoppedManually, - rootDiskSize: defaultRootDiskSize, - expectedAdditionalDisks: 0, - }, - } - - test.dataDisks = []*dataDiskScenario{ - {name: "vd-data-alpine-single-hotplug-01-repl", storageClass: replicatedStorageClass, size: defaultDataDiskSize}, - {name: "vd-data-alpine-double-hotplug-01-repl", storageClass: replicatedStorageClass, size: defaultDataDiskSize}, - {name: "vd-data-alpine-double-hotplug-02-local", storageClass: localThinStorageClass, size: defaultDataDiskSize}, - {name: "vd-data-ubuntu-replicated-five-01-repl", storageClass: replicatedStorageClass, size: defaultDataDiskSize}, - {name: "vd-data-ubuntu-replicated-five-02-repl", storageClass: replicatedStorageClass, size: defaultDataDiskSize}, - {name: "vd-data-ubuntu-replicated-five-03-repl", storageClass: replicatedStorageClass, size: defaultDataDiskSize}, - {name: "vd-data-ubuntu-replicated-five-04-repl", storageClass: replicatedStorageClass, size: defaultDataDiskSize}, - {name: "vd-data-ubuntu-replicated-five-05-repl", storageClass: replicatedStorageClass, size: defaultDataDiskSize}, - {name: "vd-data-ubuntu-mixed-five-01-repl", storageClass: replicatedStorageClass, size: defaultDataDiskSize}, - {name: "vd-data-ubuntu-mixed-five-02-repl", storageClass: replicatedStorageClass, size: defaultDataDiskSize}, - {name: "vd-data-ubuntu-mixed-five-03-local", storageClass: localThinStorageClass, size: defaultDataDiskSize}, - {name: "vd-data-ubuntu-mixed-five-04-local", storageClass: localThinStorageClass, size: defaultDataDiskSize}, - {name: "vd-data-ubuntu-mixed-five-05-local", storageClass: localThinStorageClass, size: defaultDataDiskSize}, - {name: "vd-data-alpine-iperf-client-01-repl", storageClass: replicatedStorageClass, size: defaultDataDiskSize}, - {name: "vd-data-alpine-iperf-client-02-repl", storageClass: replicatedStorageClass, size: defaultDataDiskSize}, - } - - test.attachments = []*attachmentScenario{ - {name: "vmbda-alpine-single-hotplug-01", vmName: "vm-alpine-single-hotplug", diskName: "vd-data-alpine-single-hotplug-01-repl"}, - {name: "vmbda-alpine-double-hotplug-01", vmName: "vm-alpine-double-hotplug", diskName: "vd-data-alpine-double-hotplug-01-repl"}, - {name: "vmbda-alpine-double-hotplug-02", vmName: "vm-alpine-double-hotplug", diskName: "vd-data-alpine-double-hotplug-02-local"}, - {name: "vmbda-ubuntu-replicated-five-01", vmName: "vm-ubuntu-replicated-five", diskName: "vd-data-ubuntu-replicated-five-01-repl"}, - {name: "vmbda-ubuntu-replicated-five-02", vmName: "vm-ubuntu-replicated-five", diskName: "vd-data-ubuntu-replicated-five-02-repl"}, - {name: "vmbda-ubuntu-replicated-five-03", vmName: "vm-ubuntu-replicated-five", diskName: "vd-data-ubuntu-replicated-five-03-repl"}, - {name: "vmbda-ubuntu-replicated-five-04", vmName: "vm-ubuntu-replicated-five", diskName: "vd-data-ubuntu-replicated-five-04-repl"}, - {name: "vmbda-ubuntu-replicated-five-05", vmName: "vm-ubuntu-replicated-five", diskName: "vd-data-ubuntu-replicated-five-05-repl"}, - {name: "vmbda-ubuntu-mixed-five-01", vmName: "vm-ubuntu-mixed-five", diskName: "vd-data-ubuntu-mixed-five-01-repl"}, - {name: "vmbda-ubuntu-mixed-five-02", vmName: "vm-ubuntu-mixed-five", diskName: "vd-data-ubuntu-mixed-five-02-repl"}, - {name: "vmbda-ubuntu-mixed-five-03", vmName: "vm-ubuntu-mixed-five", diskName: "vd-data-ubuntu-mixed-five-03-local"}, - {name: "vmbda-ubuntu-mixed-five-04", vmName: "vm-ubuntu-mixed-five", diskName: "vd-data-ubuntu-mixed-five-04-local"}, - {name: "vmbda-ubuntu-mixed-five-05", vmName: "vm-ubuntu-mixed-five", diskName: "vd-data-ubuntu-mixed-five-05-local"}, - {name: "vmbda-alpine-iperf-client-01", vmName: "vm-alpine-iperf-client", diskName: "vd-data-alpine-iperf-client-01-repl"}, - {name: "vmbda-alpine-iperf-client-02", vmName: "vm-alpine-iperf-client", diskName: "vd-data-alpine-iperf-client-02-repl"}, - } - - for _, vmScenario := range test.vms { - vmScenario.rootDisk = newRootDisk(vmScenario.rootDiskName, namespace, vmScenario.cviName, replicatedStorageClass, vmScenario.rootDiskSize) - vmScenario.vm = newVM(vmScenario.name, namespace, vmScenario.runPolicy, vmScenario.rootDisk.Name, vmScenario.cloudInit) - test.vmByName[vmScenario.name] = vmScenario - } - - for _, diskScenario := range test.dataDisks { - diskScenario.disk = newHotplugDisk(diskScenario.name, namespace, diskScenario.storageClass, diskScenario.size) - test.dataDiskByName[diskScenario.name] = diskScenario - } - - for _, attachmentScenario := range test.attachments { - vmScenario := test.vmByName[attachmentScenario.vmName] - diskScenario := test.dataDiskByName[attachmentScenario.diskName] - attachmentScenario.attachment = object.NewVMBDAFromDisk(attachmentScenario.name, vmScenario.vm.Name, diskScenario.disk) - } - - test.iperfClient = test.vmByName["vm-alpine-iperf-client"] - test.iperfServer = test.vmByName["vm-alpine-iperf-server"] - - return test -} - func (t *currentReleaseSmokeTest) createResources() { By("Creating root and hotplug virtual disks") Expect(t.framework.CreateWithDeferredDeletion(context.Background(), t.diskObjects()...)).To(Succeed()) @@ -430,221 +145,6 @@ func (t *currentReleaseSmokeTest) verifyVMsSurvivedUpgrade() { } } -func (s *vmScenario) expectedInitialPhase() string { - if s.runPolicy == v1alpha2.ManualPolicy { - return string(v1alpha2.MachineStopped) - } - - return string(v1alpha2.MachineRunning) -} - -func newRootDisk(name, namespace, cviName, storageClass, size string) *v1alpha2.VirtualDisk { - options := []vdbuilder.Option{ - vdbuilder.WithStorageClass(ptr.To(storageClass)), - } - if size != "" { - options = append(options, vdbuilder.WithSize(ptr.To(resource.MustParse(size)))) - } - - return object.NewVDFromCVI(name, namespace, cviName, options...) -} - -func newHotplugDisk(name, namespace, storageClass, size string) *v1alpha2.VirtualDisk { - return object.NewBlankVD( - name, - namespace, - ptr.To(storageClass), - ptr.To(resource.MustParse(size)), - ) -} - -func newVM(name, namespace string, runPolicy v1alpha2.RunPolicy, rootDiskName, cloudInit string) *v1alpha2.VirtualMachine { - return vmbuilder.New( - vmbuilder.WithName(name), - vmbuilder.WithNamespace(namespace), - vmbuilder.WithCPU(1, ptr.To("20%")), - vmbuilder.WithMemory(resource.MustParse("512Mi")), - vmbuilder.WithLiveMigrationPolicy(v1alpha2.AlwaysSafeMigrationPolicy), - vmbuilder.WithVirtualMachineClass(object.DefaultVMClass), - vmbuilder.WithProvisioningUserData(cloudInit), - vmbuilder.WithRunPolicy(runPolicy), - vmbuilder.WithBlockDeviceRefs(v1alpha2.BlockDeviceSpecRef{ - Kind: v1alpha2.DiskDevice, - Name: rootDiskName, - }), - ) -} - -func (t *currentReleaseSmokeTest) diskObjects() []crclient.Object { - objects := make([]crclient.Object, 0, len(t.vms)+len(t.dataDisks)) - for _, vmScenario := range t.vms { - objects = append(objects, vmScenario.rootDisk) - } - for _, diskScenario := range t.dataDisks { - objects = append(objects, diskScenario.disk) - } - return objects -} - -func (t *currentReleaseSmokeTest) vmObjects() []crclient.Object { - objects := make([]crclient.Object, 0, len(t.vms)) - for _, vmScenario := range t.vms { - objects = append(objects, vmScenario.vm) - } - return objects -} - -func (t *currentReleaseSmokeTest) attachmentObjects() []crclient.Object { - objects := make([]crclient.Object, 0, len(t.attachments)) - for _, attachmentScenario := range t.attachments { - objects = append(objects, attachmentScenario.attachment) - } - return objects -} - -func (t *currentReleaseSmokeTest) initialRunningVMObjects() []crclient.Object { - objects := make([]crclient.Object, 0, len(t.vms)) - for _, vmScenario := range t.vms { - if vmScenario.expectedInitialPhase() == string(v1alpha2.MachineRunning) { - objects = append(objects, vmScenario.vm) - } - } - return objects -} - -func (t *currentReleaseSmokeTest) initialStoppedVMObjects() []crclient.Object { - objects := make([]crclient.Object, 0, len(t.vms)) - for _, vmScenario := range t.vms { - if vmScenario.expectedInitialPhase() == string(v1alpha2.MachineStopped) { - objects = append(objects, vmScenario.vm) - } - } - return objects -} - -func (t *currentReleaseSmokeTest) manualStartVMs() []*vmScenario { - manualVMs := make([]*vmScenario, 0) - for _, vmScenario := range t.vms { - if vmScenario.runPolicy == v1alpha2.ManualPolicy { - manualVMs = append(manualVMs, vmScenario) - } - } - return manualVMs -} - -func (t *currentReleaseSmokeTest) manualStartVMObjects() []crclient.Object { - objects := make([]crclient.Object, 0) - for _, vmScenario := range t.manualStartVMs() { - objects = append(objects, vmScenario.vm) - } - return objects -} - -func (t *currentReleaseSmokeTest) expectGuestReady(vmScenario *vmScenario) { - vm := vmScenario.vm - - By(fmt.Sprintf("Waiting for SSH access on %s", vm.Name)) - util.UntilSSHReady(t.framework, vm, framework.LongTimeout) - - if vmScenario.skipGuestAgentCheck { - By(fmt.Sprintf("Skipping strict guest agent check on %s", vm.Name)) - return - } - - By(fmt.Sprintf("Waiting for guest agent on %s", vm.Name)) - util.UntilVMAgentReady(crclient.ObjectKeyFromObject(vm), framework.LongTimeout) -} - -func (t *currentReleaseSmokeTest) expectAdditionalDiskCount(vm *v1alpha2.VirtualMachine, expectedCount int) { - Eventually(func(g Gomega) { - currentVM := &v1alpha2.VirtualMachine{} - err := t.framework.GenericClient().Get(context.Background(), crclient.ObjectKeyFromObject(vm), currentVM) - g.Expect(err).NotTo(HaveOccurred()) - - attachedHotplugDisks := hotpluggedAttachedDiskCount(currentVM) - g.Expect(attachedHotplugDisks).To(Equal(expectedCount)) - - rootDiskName, err := t.rootDiskName(vm) - g.Expect(err).NotTo(HaveOccurred()) - - output, err := t.framework.SSHCommand(vm.Name, vm.Namespace, lsblkJSONCommand, framework.WithSSHTimeout(10*time.Second)) - g.Expect(err).NotTo(HaveOccurred()) - - disks, err := parseLSBLKOutput(output) - g.Expect(err).NotTo(HaveOccurred()) - - actualCount := countAdditionalGuestDisks(disks, rootDiskName) - g.Expect(actualCount).To( - Equal(expectedCount), - "VM %s/%s additional disk mismatch; root disk: %q; hotplugged block devices in status: %d; lsblk devices: %s", - vm.Namespace, - vm.Name, - rootDiskName, - attachedHotplugDisks, - formatLSBLKDisks(disks), - ) - }).WithTimeout(framework.LongTimeout).WithPolling(time.Second).Should(Succeed()) -} - -func parseLSBLKOutput(raw string) ([]lsblkDevice, error) { - var output lsblkOutput - if err := json.Unmarshal([]byte(raw), &output); err != nil { - return nil, fmt.Errorf("parse lsblk json: %w", err) - } - - return output.BlockDevices, nil -} - -func hotpluggedAttachedDiskCount(vm *v1alpha2.VirtualMachine) int { - count := 0 - for _, blockDevice := range vm.Status.BlockDeviceRefs { - if !blockDevice.Hotplugged || !blockDevice.Attached || blockDevice.VirtualMachineBlockDeviceAttachmentName == "" { - continue - } - count++ - } - return count -} - -func (t *currentReleaseSmokeTest) rootDiskName(vm *v1alpha2.VirtualMachine) (string, error) { - output, err := t.framework.SSHCommand(vm.Name, vm.Namespace, rootDiskNameCommand, framework.WithSSHTimeout(10*time.Second)) - if err != nil { - return "", err - } - - return strings.TrimSpace(output), nil -} - -func countAdditionalGuestDisks(disks []lsblkDevice, rootDiskName string) int { - count := 0 - for _, disk := range disks { - if disk.Type != "disk" { - continue - } - if disk.Name == rootDiskName { - continue - } - if disk.Size <= maxCloudInitDiskSize { - continue - } - count++ - } - return count -} - -func formatLSBLKDisks(disks []lsblkDevice) string { - if len(disks) == 0 { - return "[]" - } - - parts := make([]string, 0, len(disks)) - for _, disk := range disks { - parts = append(parts, fmt.Sprintf("%s(type=%s,size=%d,mountpoints=%v)", disk.Name, disk.Type, disk.Size, disk.Mountpoints)) - } - - return "[" + strings.Join(parts, ", ") + "]" -} - func (t *currentReleaseSmokeTest) startLongRunningIPerf() { GinkgoHelper() @@ -740,124 +240,56 @@ func (t *currentReleaseSmokeTest) getVirtualMachine(name, namespace string) *v1a return vm } -func waitForIPerfServerToStart(f *framework.Framework, vm *v1alpha2.VirtualMachine) { - GinkgoHelper() - - command := "rc-service iperf3 status --nocolor" - Eventually(func() error { - stdout, err := f.SSHCommand(vm.Name, vm.Namespace, command) - if err != nil { - return fmt.Errorf("cmd: %s\nstderr: %w", command, err) - } - if strings.Contains(stdout, "status: started") { - return nil - } - return fmt.Errorf("iperf3 server is not started yet: %s", stdout) - }).WithTimeout(framework.MiddleTimeout).WithPolling(framework.PollingInterval).Should(Succeed()) -} - -func waitForIPerfClientToStart(f *framework.Framework, vm *v1alpha2.VirtualMachine) { - GinkgoHelper() - - command := "pgrep -x iperf3" - Eventually(func() error { - stdout, err := f.SSHCommand(vm.Name, vm.Namespace, command) - if err != nil { - return fmt.Errorf("cmd: %s\nstderr: %w", command, err) - } - if strings.TrimSpace(stdout) == "" { - return fmt.Errorf("iperf3 client is not running yet") - } - return nil - }).WithTimeout(framework.MiddleTimeout).WithPolling(framework.PollingInterval).Should(Succeed()) -} - -func stopIPerfClient(f *framework.Framework, vm *v1alpha2.VirtualMachine) { - GinkgoHelper() - - command := "pkill -INT -x iperf3" - Eventually(func() error { - _, err := f.SSHCommand(vm.Name, vm.Namespace, command) - if err != nil { - return fmt.Errorf("cmd: %s\nstderr: %w", command, err) - } - return nil - }).WithTimeout(framework.MiddleTimeout).WithPolling(framework.PollingInterval).Should(Succeed()) -} - -func getIPerfClientReport(f *framework.Framework, vm *v1alpha2.VirtualMachine, reportPath string) *iperfReport { - GinkgoHelper() - - command := fmt.Sprintf("cat %s", reportPath) - var rawReport string - Eventually(func() error { - stdout, err := f.SSHCommand(vm.Name, vm.Namespace, command) - if err != nil { - return fmt.Errorf("cmd: %s\nstderr: %w", command, err) - } - report, err := parseIPerfReport(stdout) - if err != nil { - return err - } - if report.End.SumSent.End <= 0 { - return fmt.Errorf("iperf3 report is incomplete") - } - rawReport = stdout - return nil - }).WithTimeout(framework.LongTimeout).WithPolling(framework.PollingInterval).Should(Succeed()) - - report, err := parseIPerfReport(rawReport) - Expect(err).NotTo(HaveOccurred()) - return report -} - -func continuityWindowBounds(startedAt, upgradeStartedAt int64, intervalCount int) (int, int) { - if intervalCount == 0 { - return 1, 0 - } - - index := int(upgradeStartedAt - startedAt) - if index < 0 { - index = 0 - } - if index >= intervalCount { - index = intervalCount - 1 +func getReleaseTestPhase() string { + if phase := os.Getenv(releaseTestPhaseEnv); phase != "" { + return phase } - lower := max(index-1, 0) - upper := min(index+1, intervalCount-1) - return lower, upper + return releaseTestPhasePreUpgrade } -func parseIPerfReport(raw string) (*iperfReport, error) { - var report iperfReport - if err := json.Unmarshal([]byte(raw), &report); err != nil { - return nil, fmt.Errorf("parse iperf3 json: %w", err) - } - - return &report, nil +func mustGetEnv(name string) string { + value := os.Getenv(name) + Expect(value).NotTo(BeEmpty(), "environment variable %s must be set", name) + return value } -func isExpectedIPerfReportError(errMsg string) bool { - if errMsg == "" { - return true - } +func ensureReleaseNamespace(f *framework.Framework, namespace string) string { + GinkgoHelper() - return strings.Contains(errMsg, "interrupt - the client has terminated by signal Interrupt(2)") -} + nsClient := f.KubeClient().CoreV1().Namespaces() + _, err := nsClient.Get(context.Background(), namespace, metav1.GetOptions{}) + switch { + case err == nil: + By(fmt.Sprintf("Namespace %q already exists, recreating it", namespace)) + err = nsClient.Delete(context.Background(), namespace, metav1.DeleteOptions{}) + Expect(err).NotTo(HaveOccurred()) -func min(a, b int) int { - if a < b { - return a + Eventually(func() error { + _, err := nsClient.Get(context.Background(), namespace, metav1.GetOptions{}) + if k8serrors.IsNotFound(err) { + return nil + } + if err != nil { + return err + } + return fmt.Errorf("namespace %q is still deleting", namespace) + }).WithTimeout(framework.LongTimeout).WithPolling(time.Second).Should(Succeed()) + case !k8serrors.IsNotFound(err): + Expect(err).NotTo(HaveOccurred()) } - return b -} - -func max(a, b int) int { - if a > b { - return a + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: namespace, + Labels: map[string]string{ + framework.E2ELabel: "true", + }, + }, } + _, err = nsClient.Create(context.Background(), ns, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + By(fmt.Sprintf("Namespace %q has been created", namespace)) - return b + return namespace } diff --git a/test/e2e/release/guest.go b/test/e2e/release/guest.go new file mode 100644 index 0000000000..e729173e1c --- /dev/null +++ b/test/e2e/release/guest.go @@ -0,0 +1,155 @@ +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package release + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + crclient "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/deckhouse/virtualization/api/core/v1alpha2" + "github.com/deckhouse/virtualization/test/e2e/internal/framework" + "github.com/deckhouse/virtualization/test/e2e/internal/util" +) + +const ( + lsblkJSONCommand = "sudo lsblk --bytes --json --nodeps --output NAME,SIZE,TYPE,MOUNTPOINTS" + rootDiskNameCommand = `root_source=$(findmnt -no SOURCE /); root_disk=$(lsblk -ndo PKNAME "$root_source" 2>/dev/null | head -n1); if [ -n "$root_disk" ]; then echo "$root_disk"; else lsblk -ndo NAME "$root_source" | head -n1; fi` + maxCloudInitDiskSize = int64(4 * 1024 * 1024) +) + +type lsblkOutput struct { + BlockDevices []lsblkDevice `json:"blockdevices"` +} + +type lsblkDevice struct { + Name string `json:"name"` + Size int64 `json:"size"` + Type string `json:"type"` + Mountpoints []string `json:"mountpoints"` +} + +func (t *currentReleaseSmokeTest) expectGuestReady(vmScenario *vmScenario) { + vm := vmScenario.vm + + By(fmt.Sprintf("Waiting for SSH access on %s", vm.Name)) + util.UntilSSHReady(t.framework, vm, framework.LongTimeout) + + if vmScenario.skipGuestAgentCheck { + By(fmt.Sprintf("Skipping strict guest agent check on %s", vm.Name)) + return + } + + By(fmt.Sprintf("Waiting for guest agent on %s", vm.Name)) + util.UntilVMAgentReady(crclient.ObjectKeyFromObject(vm), framework.LongTimeout) +} + +func (t *currentReleaseSmokeTest) expectAdditionalDiskCount(vm *v1alpha2.VirtualMachine, expectedCount int) { + Eventually(func(g Gomega) { + currentVM := &v1alpha2.VirtualMachine{} + err := t.framework.GenericClient().Get(context.Background(), crclient.ObjectKeyFromObject(vm), currentVM) + g.Expect(err).NotTo(HaveOccurred()) + + attachedHotplugDisks := hotpluggedAttachedDiskCount(currentVM) + g.Expect(attachedHotplugDisks).To(Equal(expectedCount)) + + rootDiskName, err := t.rootDiskName(vm) + g.Expect(err).NotTo(HaveOccurred()) + + output, err := t.framework.SSHCommand(vm.Name, vm.Namespace, lsblkJSONCommand, framework.WithSSHTimeout(10*time.Second)) + g.Expect(err).NotTo(HaveOccurred()) + + disks, err := parseLSBLKOutput(output) + g.Expect(err).NotTo(HaveOccurred()) + + actualCount := countAdditionalGuestDisks(disks, rootDiskName) + g.Expect(actualCount).To( + Equal(expectedCount), + "VM %s/%s additional disk mismatch; root disk: %q; hotplugged block devices in status: %d; lsblk devices: %s", + vm.Namespace, + vm.Name, + rootDiskName, + attachedHotplugDisks, + formatLSBLKDisks(disks), + ) + }).WithTimeout(framework.LongTimeout).WithPolling(time.Second).Should(Succeed()) +} + +func (t *currentReleaseSmokeTest) rootDiskName(vm *v1alpha2.VirtualMachine) (string, error) { + output, err := t.framework.SSHCommand(vm.Name, vm.Namespace, rootDiskNameCommand, framework.WithSSHTimeout(10*time.Second)) + if err != nil { + return "", err + } + + return strings.TrimSpace(output), nil +} + +func parseLSBLKOutput(raw string) ([]lsblkDevice, error) { + var output lsblkOutput + if err := json.Unmarshal([]byte(raw), &output); err != nil { + return nil, fmt.Errorf("parse lsblk json: %w", err) + } + + return output.BlockDevices, nil +} + +func hotpluggedAttachedDiskCount(vm *v1alpha2.VirtualMachine) int { + count := 0 + for _, blockDevice := range vm.Status.BlockDeviceRefs { + if !blockDevice.Hotplugged || !blockDevice.Attached || blockDevice.VirtualMachineBlockDeviceAttachmentName == "" { + continue + } + count++ + } + return count +} + +func countAdditionalGuestDisks(disks []lsblkDevice, rootDiskName string) int { + count := 0 + for _, disk := range disks { + if disk.Type != "disk" { + continue + } + if disk.Name == rootDiskName { + continue + } + if disk.Size <= maxCloudInitDiskSize { + continue + } + count++ + } + return count +} + +func formatLSBLKDisks(disks []lsblkDevice) string { + if len(disks) == 0 { + return "[]" + } + + parts := make([]string, 0, len(disks)) + for _, disk := range disks { + parts = append(parts, fmt.Sprintf("%s(type=%s,size=%d,mountpoints=%v)", disk.Name, disk.Type, disk.Size, disk.Mountpoints)) + } + + return "[" + strings.Join(parts, ", ") + "]" +} diff --git a/test/e2e/release/iperf.go b/test/e2e/release/iperf.go new file mode 100644 index 0000000000..04cd894e34 --- /dev/null +++ b/test/e2e/release/iperf.go @@ -0,0 +1,178 @@ +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package release + +import ( + "encoding/json" + "fmt" + "strings" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/deckhouse/virtualization/api/core/v1alpha2" + "github.com/deckhouse/virtualization/test/e2e/internal/framework" +) + +const ( + releaseIPerfReportPath = "/tmp/release-upgrade-iperf-client-report.json" +) + +type iperfReport struct { + Start iperfReportStart `json:"start"` + Intervals []iperfReportInterval `json:"intervals"` + End iperfReportEnd `json:"end"` + Error string `json:"error,omitempty"` +} + +type iperfReportStart struct { + Timestamp iperfReportTimestamp `json:"timestamp"` +} + +type iperfReportTimestamp struct { + Time string `json:"time"` + Timesecs int `json:"timesecs"` +} + +type iperfReportEnd struct { + SumSent iperfReportSummary `json:"sum_sent"` + SumReceived iperfReportSummary `json:"sum_received"` +} + +type iperfReportInterval struct { + Sum iperfReportSummary `json:"sum"` +} + +type iperfReportSummary struct { + Bytes int64 `json:"bytes"` + BitsPerSecond float64 `json:"bits_per_second"` + End float64 `json:"end,omitempty"` +} + +type releaseUpgradeContext struct { + Namespace string `json:"namespace"` + IPerfClientVM string `json:"iperfClientVM"` + IPerfServerVM string `json:"iperfServerVM"` + IPerfReportPath string `json:"iperfReportPath"` +} + +func waitForIPerfServerToStart(f *framework.Framework, vm *v1alpha2.VirtualMachine) { + GinkgoHelper() + + command := "rc-service iperf3 status --nocolor" + Eventually(func() error { + stdout, err := f.SSHCommand(vm.Name, vm.Namespace, command) + if err != nil { + return fmt.Errorf("cmd: %s\nstderr: %w", command, err) + } + if strings.Contains(stdout, "status: started") { + return nil + } + return fmt.Errorf("iperf3 server is not started yet: %s", stdout) + }).WithTimeout(framework.MiddleTimeout).WithPolling(framework.PollingInterval).Should(Succeed()) +} + +func waitForIPerfClientToStart(f *framework.Framework, vm *v1alpha2.VirtualMachine) { + GinkgoHelper() + + command := "pgrep -x iperf3" + Eventually(func() error { + stdout, err := f.SSHCommand(vm.Name, vm.Namespace, command) + if err != nil { + return fmt.Errorf("cmd: %s\nstderr: %w", command, err) + } + if strings.TrimSpace(stdout) == "" { + return fmt.Errorf("iperf3 client is not running yet") + } + return nil + }).WithTimeout(framework.MiddleTimeout).WithPolling(framework.PollingInterval).Should(Succeed()) +} + +func stopIPerfClient(f *framework.Framework, vm *v1alpha2.VirtualMachine) { + GinkgoHelper() + + command := "pkill -INT -x iperf3" + Eventually(func() error { + _, err := f.SSHCommand(vm.Name, vm.Namespace, command) + if err != nil { + return fmt.Errorf("cmd: %s\nstderr: %w", command, err) + } + return nil + }).WithTimeout(framework.MiddleTimeout).WithPolling(framework.PollingInterval).Should(Succeed()) +} + +func getIPerfClientReport(f *framework.Framework, vm *v1alpha2.VirtualMachine, reportPath string) *iperfReport { + GinkgoHelper() + + command := fmt.Sprintf("cat %s", reportPath) + var result *iperfReport + Eventually(func() error { + stdout, err := f.SSHCommand(vm.Name, vm.Namespace, command) + if err != nil { + return fmt.Errorf("cmd: %s\nstderr: %w", command, err) + } + report, err := parseIPerfReport(stdout) + if err != nil { + return err + } + if report.End.SumSent.End <= 0 { + return fmt.Errorf("iperf3 report is incomplete") + } + result = report + return nil + }).WithTimeout(framework.LongTimeout).WithPolling(framework.PollingInterval).Should(Succeed()) + + Expect(result).NotTo(BeNil()) + return result +} + +// continuityWindowBounds returns the index range [lower, upper] of iperf intervals +// around the upgrade timestamp. Assumes default 1-second reporting intervals. +func continuityWindowBounds(startedAt, upgradeStartedAt int64, intervalCount int) (int, int) { + if intervalCount == 0 { + return 1, 0 + } + + index := int(upgradeStartedAt - startedAt) + if index < 0 { + index = 0 + } + if index >= intervalCount { + index = intervalCount - 1 + } + + lower := max(index-1, 0) + upper := min(index+1, intervalCount-1) + return lower, upper +} + +func parseIPerfReport(raw string) (*iperfReport, error) { + var report iperfReport + if err := json.Unmarshal([]byte(raw), &report); err != nil { + return nil, fmt.Errorf("parse iperf3 json: %w", err) + } + + return &report, nil +} + +func isExpectedIPerfReportError(errMsg string) bool { + if errMsg == "" { + return true + } + + return strings.Contains(errMsg, "interrupt - the client has terminated by signal Interrupt(2)") +} diff --git a/test/e2e/release/resources.go b/test/e2e/release/resources.go new file mode 100644 index 0000000000..0cd03f9882 --- /dev/null +++ b/test/e2e/release/resources.go @@ -0,0 +1,87 @@ +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package release + +import ( + "github.com/deckhouse/virtualization/api/core/v1alpha2" + crclient "sigs.k8s.io/controller-runtime/pkg/client" +) + +func (t *currentReleaseSmokeTest) diskObjects() []crclient.Object { + objects := make([]crclient.Object, 0, len(t.vms)+len(t.dataDisks)) + for _, vmScenario := range t.vms { + objects = append(objects, vmScenario.rootDisk) + } + for _, diskScenario := range t.dataDisks { + objects = append(objects, diskScenario.disk) + } + return objects +} + +func (t *currentReleaseSmokeTest) vmObjects() []crclient.Object { + objects := make([]crclient.Object, 0, len(t.vms)) + for _, vmScenario := range t.vms { + objects = append(objects, vmScenario.vm) + } + return objects +} + +func (t *currentReleaseSmokeTest) attachmentObjects() []crclient.Object { + objects := make([]crclient.Object, 0, len(t.attachments)) + for _, attachmentScenario := range t.attachments { + objects = append(objects, attachmentScenario.attachment) + } + return objects +} + +func (t *currentReleaseSmokeTest) initialRunningVMObjects() []crclient.Object { + objects := make([]crclient.Object, 0, len(t.vms)) + for _, vmScenario := range t.vms { + if vmScenario.expectedInitialPhase() == string(v1alpha2.MachineRunning) { + objects = append(objects, vmScenario.vm) + } + } + return objects +} + +func (t *currentReleaseSmokeTest) initialStoppedVMObjects() []crclient.Object { + objects := make([]crclient.Object, 0, len(t.vms)) + for _, vmScenario := range t.vms { + if vmScenario.expectedInitialPhase() == string(v1alpha2.MachineStopped) { + objects = append(objects, vmScenario.vm) + } + } + return objects +} + +func (t *currentReleaseSmokeTest) manualStartVMs() []*vmScenario { + manualVMs := make([]*vmScenario, 0) + for _, vmScenario := range t.vms { + if vmScenario.runPolicy == v1alpha2.ManualPolicy { + manualVMs = append(manualVMs, vmScenario) + } + } + return manualVMs +} + +func (t *currentReleaseSmokeTest) manualStartVMObjects() []crclient.Object { + objects := make([]crclient.Object, 0) + for _, vmScenario := range t.manualStartVMs() { + objects = append(objects, vmScenario.vm) + } + return objects +} diff --git a/test/e2e/release/scenarios.go b/test/e2e/release/scenarios.go new file mode 100644 index 0000000000..314320f561 --- /dev/null +++ b/test/e2e/release/scenarios.go @@ -0,0 +1,245 @@ +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package release + +import ( + . "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/utils/ptr" + + vdbuilder "github.com/deckhouse/virtualization-controller/pkg/builder/vd" + vmbuilder "github.com/deckhouse/virtualization-controller/pkg/builder/vm" + "github.com/deckhouse/virtualization/api/core/v1alpha2" + "github.com/deckhouse/virtualization/test/e2e/internal/framework" + "github.com/deckhouse/virtualization/test/e2e/internal/object" +) + +const ( + replicatedStorageClass = "nested-thin-r1" + localThinStorageClass = "nested-local-thin" + defaultRootDiskSize = "350Mi" + defaultDataDiskSize = "100Mi" + releaseNamespaceName = "v12n-test-release" +) + +type vmScenario struct { + name string + rootDiskName string + cviName string + cloudInit string + runPolicy v1alpha2.RunPolicy + rootDiskSize string + expectedAdditionalDisks int + skipGuestAgentCheck bool + + rootDisk *v1alpha2.VirtualDisk + vm *v1alpha2.VirtualMachine +} + +type dataDiskScenario struct { + name string + storageClass string + size string + + disk *v1alpha2.VirtualDisk +} + +type attachmentScenario struct { + name string + vmName string + diskName string + + attachment *v1alpha2.VirtualMachineBlockDeviceAttachment +} + +func (s *vmScenario) expectedInitialPhase() string { + if s.runPolicy == v1alpha2.ManualPolicy { + return string(v1alpha2.MachineStopped) + } + + return string(v1alpha2.MachineRunning) +} + +func newCurrentReleaseSmokeTest(f *framework.Framework, namespace string) *currentReleaseSmokeTest { + Expect(namespace).NotTo(BeEmpty(), "namespace must be provided") + + test := ¤tReleaseSmokeTest{ + framework: f, + vmByName: make(map[string]*vmScenario), + dataDiskByName: make(map[string]*dataDiskScenario), + } + + test.vms = []*vmScenario{ + { + name: "vm-alpine-manual", + rootDiskName: "vd-root-alpine-manual", + cviName: object.PrecreatedCVIAlpineBIOS, + cloudInit: object.AlpineCloudInit, + runPolicy: v1alpha2.ManualPolicy, + rootDiskSize: defaultRootDiskSize, + expectedAdditionalDisks: 0, + }, + { + name: "vm-alpine-single-hotplug", + rootDiskName: "vd-root-alpine-single-hotplug", + cviName: object.PrecreatedCVIAlpineBIOS, + cloudInit: object.AlpineCloudInit, + runPolicy: v1alpha2.AlwaysOnUnlessStoppedManually, + rootDiskSize: defaultRootDiskSize, + expectedAdditionalDisks: 1, + }, + { + name: "vm-alpine-double-hotplug", + rootDiskName: "vd-root-alpine-double-hotplug", + cviName: object.PrecreatedCVIAlpineBIOS, + cloudInit: object.AlpineCloudInit, + runPolicy: v1alpha2.AlwaysOnPolicy, + rootDiskSize: defaultRootDiskSize, + expectedAdditionalDisks: 2, + }, + { + name: "vm-ubuntu-replicated-five", + rootDiskName: "vd-root-ubuntu-replicated-five", + cviName: object.PrecreatedCVIUbuntu, + cloudInit: object.UbuntuCloudInit, + runPolicy: v1alpha2.AlwaysOnUnlessStoppedManually, + expectedAdditionalDisks: 5, + }, + { + name: "vm-ubuntu-mixed-five", + rootDiskName: "vd-root-ubuntu-mixed-five", + cviName: object.PrecreatedCVIUbuntu, + cloudInit: object.UbuntuCloudInit, + runPolicy: v1alpha2.AlwaysOnUnlessStoppedManually, + expectedAdditionalDisks: 5, + }, + { + name: "vm-alpine-iperf-client", + rootDiskName: "vd-root-alpine-iperf-client", + cviName: object.PrecreatedCVIAlpineBIOS, + cloudInit: object.AlpineCloudInit, + runPolicy: v1alpha2.AlwaysOnUnlessStoppedManually, + rootDiskSize: defaultRootDiskSize, + expectedAdditionalDisks: 2, + skipGuestAgentCheck: true, + }, + { + name: "vm-alpine-iperf-server", + rootDiskName: "vd-root-alpine-iperf-server", + cviName: object.PrecreatedCVIAlpineBIOS, + cloudInit: object.PerfCloudInit, + runPolicy: v1alpha2.AlwaysOnUnlessStoppedManually, + rootDiskSize: defaultRootDiskSize, + expectedAdditionalDisks: 0, + }, + } + + test.dataDisks = []*dataDiskScenario{ + {name: "vd-data-alpine-single-hotplug-01-repl", storageClass: replicatedStorageClass, size: defaultDataDiskSize}, + {name: "vd-data-alpine-double-hotplug-01-repl", storageClass: replicatedStorageClass, size: defaultDataDiskSize}, + {name: "vd-data-alpine-double-hotplug-02-local", storageClass: localThinStorageClass, size: defaultDataDiskSize}, + {name: "vd-data-ubuntu-replicated-five-01-repl", storageClass: replicatedStorageClass, size: defaultDataDiskSize}, + {name: "vd-data-ubuntu-replicated-five-02-repl", storageClass: replicatedStorageClass, size: defaultDataDiskSize}, + {name: "vd-data-ubuntu-replicated-five-03-repl", storageClass: replicatedStorageClass, size: defaultDataDiskSize}, + {name: "vd-data-ubuntu-replicated-five-04-repl", storageClass: replicatedStorageClass, size: defaultDataDiskSize}, + {name: "vd-data-ubuntu-replicated-five-05-repl", storageClass: replicatedStorageClass, size: defaultDataDiskSize}, + {name: "vd-data-ubuntu-mixed-five-01-repl", storageClass: replicatedStorageClass, size: defaultDataDiskSize}, + {name: "vd-data-ubuntu-mixed-five-02-repl", storageClass: replicatedStorageClass, size: defaultDataDiskSize}, + {name: "vd-data-ubuntu-mixed-five-03-local", storageClass: localThinStorageClass, size: defaultDataDiskSize}, + {name: "vd-data-ubuntu-mixed-five-04-local", storageClass: localThinStorageClass, size: defaultDataDiskSize}, + {name: "vd-data-ubuntu-mixed-five-05-local", storageClass: localThinStorageClass, size: defaultDataDiskSize}, + {name: "vd-data-alpine-iperf-client-01-repl", storageClass: replicatedStorageClass, size: defaultDataDiskSize}, + {name: "vd-data-alpine-iperf-client-02-repl", storageClass: replicatedStorageClass, size: defaultDataDiskSize}, + } + + test.attachments = []*attachmentScenario{ + {name: "vmbda-alpine-single-hotplug-01", vmName: "vm-alpine-single-hotplug", diskName: "vd-data-alpine-single-hotplug-01-repl"}, + {name: "vmbda-alpine-double-hotplug-01", vmName: "vm-alpine-double-hotplug", diskName: "vd-data-alpine-double-hotplug-01-repl"}, + {name: "vmbda-alpine-double-hotplug-02", vmName: "vm-alpine-double-hotplug", diskName: "vd-data-alpine-double-hotplug-02-local"}, + {name: "vmbda-ubuntu-replicated-five-01", vmName: "vm-ubuntu-replicated-five", diskName: "vd-data-ubuntu-replicated-five-01-repl"}, + {name: "vmbda-ubuntu-replicated-five-02", vmName: "vm-ubuntu-replicated-five", diskName: "vd-data-ubuntu-replicated-five-02-repl"}, + {name: "vmbda-ubuntu-replicated-five-03", vmName: "vm-ubuntu-replicated-five", diskName: "vd-data-ubuntu-replicated-five-03-repl"}, + {name: "vmbda-ubuntu-replicated-five-04", vmName: "vm-ubuntu-replicated-five", diskName: "vd-data-ubuntu-replicated-five-04-repl"}, + {name: "vmbda-ubuntu-replicated-five-05", vmName: "vm-ubuntu-replicated-five", diskName: "vd-data-ubuntu-replicated-five-05-repl"}, + {name: "vmbda-ubuntu-mixed-five-01", vmName: "vm-ubuntu-mixed-five", diskName: "vd-data-ubuntu-mixed-five-01-repl"}, + {name: "vmbda-ubuntu-mixed-five-02", vmName: "vm-ubuntu-mixed-five", diskName: "vd-data-ubuntu-mixed-five-02-repl"}, + {name: "vmbda-ubuntu-mixed-five-03", vmName: "vm-ubuntu-mixed-five", diskName: "vd-data-ubuntu-mixed-five-03-local"}, + {name: "vmbda-ubuntu-mixed-five-04", vmName: "vm-ubuntu-mixed-five", diskName: "vd-data-ubuntu-mixed-five-04-local"}, + {name: "vmbda-ubuntu-mixed-five-05", vmName: "vm-ubuntu-mixed-five", diskName: "vd-data-ubuntu-mixed-five-05-local"}, + {name: "vmbda-alpine-iperf-client-01", vmName: "vm-alpine-iperf-client", diskName: "vd-data-alpine-iperf-client-01-repl"}, + {name: "vmbda-alpine-iperf-client-02", vmName: "vm-alpine-iperf-client", diskName: "vd-data-alpine-iperf-client-02-repl"}, + } + + for _, vmScenario := range test.vms { + vmScenario.rootDisk = newRootDisk(vmScenario.rootDiskName, namespace, vmScenario.cviName, replicatedStorageClass, vmScenario.rootDiskSize) + vmScenario.vm = newVM(vmScenario.name, namespace, vmScenario.runPolicy, vmScenario.rootDisk.Name, vmScenario.cloudInit) + test.vmByName[vmScenario.name] = vmScenario + } + + for _, diskScenario := range test.dataDisks { + diskScenario.disk = newHotplugDisk(diskScenario.name, namespace, diskScenario.storageClass, diskScenario.size) + test.dataDiskByName[diskScenario.name] = diskScenario + } + + for _, attachmentScenario := range test.attachments { + vmScenario := test.vmByName[attachmentScenario.vmName] + diskScenario := test.dataDiskByName[attachmentScenario.diskName] + attachmentScenario.attachment = object.NewVMBDAFromDisk(attachmentScenario.name, vmScenario.vm.Name, diskScenario.disk) + } + + test.iperfClient = test.vmByName["vm-alpine-iperf-client"] + test.iperfServer = test.vmByName["vm-alpine-iperf-server"] + + return test +} + +func newRootDisk(name, namespace, cviName, storageClass, size string) *v1alpha2.VirtualDisk { + options := []vdbuilder.Option{ + vdbuilder.WithStorageClass(ptr.To(storageClass)), + } + if size != "" { + options = append(options, vdbuilder.WithSize(ptr.To(resource.MustParse(size)))) + } + + return object.NewVDFromCVI(name, namespace, cviName, options...) +} + +func newHotplugDisk(name, namespace, storageClass, size string) *v1alpha2.VirtualDisk { + return object.NewBlankVD( + name, + namespace, + ptr.To(storageClass), + ptr.To(resource.MustParse(size)), + ) +} + +func newVM(name, namespace string, runPolicy v1alpha2.RunPolicy, rootDiskName, cloudInit string) *v1alpha2.VirtualMachine { + return vmbuilder.New( + vmbuilder.WithName(name), + vmbuilder.WithNamespace(namespace), + vmbuilder.WithCPU(1, ptr.To("20%")), + vmbuilder.WithMemory(resource.MustParse("512Mi")), + vmbuilder.WithLiveMigrationPolicy(v1alpha2.AlwaysSafeMigrationPolicy), + vmbuilder.WithVirtualMachineClass(object.DefaultVMClass), + vmbuilder.WithProvisioningUserData(cloudInit), + vmbuilder.WithRunPolicy(runPolicy), + vmbuilder.WithBlockDeviceRefs(v1alpha2.BlockDeviceSpecRef{ + Kind: v1alpha2.DiskDevice, + Name: rootDiskName, + }), + ) +} From 66d22b37544bff97f7cf3171ed3393ba3c33456a Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Wed, 15 Apr 2026 15:49:42 +0300 Subject: [PATCH 20/20] rm module ready when pathc mpo Signed-off-by: Nikita Korolev --- .claude/settings.json | 26 ---- .../e2e-test-releases-reusable-pipeline.yml | 144 +----------------- AGENTS.md | 84 ---------- 3 files changed, 2 insertions(+), 252 deletions(-) delete mode 100644 .claude/settings.json delete mode 100644 AGENTS.md diff --git a/.claude/settings.json b/.claude/settings.json deleted file mode 100644 index 963a53824a..0000000000 --- a/.claude/settings.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "hooks": { - "PreCompact": [ - { - "hooks": [ - { - "command": "bd prime", - "type": "command" - } - ], - "matcher": "" - } - ], - "SessionStart": [ - { - "hooks": [ - { - "command": "bd prime", - "type": "command" - } - ], - "matcher": "" - } - ] - } -} \ No newline at end of file diff --git a/.github/workflows/e2e-test-releases-reusable-pipeline.yml b/.github/workflows/e2e-test-releases-reusable-pipeline.yml index 3847d31a08..3b4b376485 100644 --- a/.github/workflows/e2e-test-releases-reusable-pipeline.yml +++ b/.github/workflows/e2e-test-releases-reusable-pipeline.yml @@ -1248,146 +1248,6 @@ jobs: echo "[INFO] Show patched ModulePullOverride:" kubectl get mpo virtualization -o yaml - - name: Wait for Virtualization to be ready after upgrade - run: | - d8_queue_list() { - d8 s queue list | grep -Po '([0-9]+)(?= active)' || echo "Failed to retrieve list queue" - } - - debug_output() { - local NODES - - echo "[ERROR] Virtualization module upgrade failed" - echo "[DEBUG] Show describe virtualization module" - echo "::group::describe virtualization module" - kubectl describe modules virtualization || true - echo "::endgroup::" - echo "[DEBUG] Show namespace d8-virtualization" - kubectl get ns d8-virtualization || true - echo "[DEBUG] Show pods in namespace d8-virtualization" - kubectl -n d8-virtualization get pods || true - echo "[DEBUG] Show pvc in namespace d8-virtualization" - kubectl get pvc -n d8-virtualization || true - echo "[DEBUG] Show cluster StorageClasses" - kubectl get storageclasses || true - echo "[DEBUG] Show cluster nodes" - kubectl get node - echo "[DEBUG] Show queue (first 25 lines)" - d8 s queue list | head -n 25 || echo "[WARNING] Failed to retrieve list queue" - echo "[DEBUG] Show deckhouse logs" - echo "::group::deckhouse logs" - d8 s logs | tail -n 100 - echo "::endgroup::" - } - - d8_queue() { - local count=90 - local queue_count - - for i in $(seq 1 $count) ; do - queue_count=$(d8_queue_list) - if [ -n "$queue_count" ] && [ "$queue_count" = "0" ]; then - echo "[SUCCESS] Queue is clear" - return 0 - fi - - echo "[INFO] Wait until queues are empty ${i}/${count}" - if (( i % 5 == 0 )); then - echo "[INFO] Show queue list" - d8 s queue list | head -n25 || echo "[WARNING] Failed to retrieve list queue" - echo " " - fi - - if (( i % 10 == 0 )); then - echo "[INFO] deckhouse logs" - echo "::group::deckhouse logs" - d8 s logs | tail -n 100 - echo "::endgroup::" - echo " " - fi - sleep 10 - done - } - - virtualization_ready() { - local count=90 - local virtualization_status - - for i in $(seq 1 $count) ; do - virtualization_status=$(kubectl get modules virtualization -o jsonpath='{.status.phase}') - if [ "$virtualization_status" == "Ready" ]; then - echo "[SUCCESS] Virtualization module is ready after upgrade to ${{ env.NEW_RELEASE }}" - kubectl get modules virtualization - kubectl -n d8-virtualization get pods - kubectl get vmclass || echo "[WARNING] no vmclasses found" - return 0 - fi - - echo "[INFO] Waiting 10s for Virtualization module to be ready (attempt $i/$count)" - - if (( i % 5 == 0 )); then - echo " " - echo "[DEBUG] Show additional info" - kubectl get ns d8-virtualization || echo "[WARNING] Namespace virtualization is not ready" - echo " " - kubectl -n d8-virtualization get pods || echo "[WARNING] Pods in namespace virtualization is not ready" - kubectl get pvc -n d8-virtualization || echo "[WARNING] PVC in namespace virtualization is not ready" - echo " " - fi - sleep 10 - done - - debug_output - exit 1 - } - - virt_handler_ready() { - local count=180 - local virt_handler_ready - local workers - local time_wait=10 - - workers=$(kubectl get nodes -o name | grep worker | wc -l || true) - workers=$((workers)) - - for i in $(seq 1 $count); do - virt_handler_ready=$(kubectl -n d8-virtualization get pods | grep "virt-handler.*Running" | wc -l || true) - - if [[ $virt_handler_ready -ge $workers ]]; then - echo "[SUCCESS] virt-handlers pods are ready" - return 0 - fi - - echo "[INFO] virt-handler pods $virt_handler_ready/$workers " - echo "[INFO] Wait ${time_wait}s virt-handler pods are ready (attempt $i/$count)" - if (( i % 5 == 0 )); then - echo "[DEBUG] Show pods in namespace d8-virtualization" - echo "::group::virtualization pods" - kubectl -n d8-virtualization get pods || echo "No pods in virtualization namespace found" - echo "::endgroup::" - echo "[DEBUG] Show cluster nodes" - echo "::group::cluster nodes" - kubectl get node - echo "::endgroup::" - fi - sleep ${time_wait} - done - - debug_output - exit 1 - } - - echo " " - echo "[INFO] Waiting for Virtualization module to be ready after upgrade to ${{ env.NEW_RELEASE }}" - d8_queue - - virtualization_ready - - echo "[INFO] Checking Virtualization module deployments" - kubectl -n d8-virtualization wait --for=condition=Available deploy --all --timeout 900s - echo "[INFO] Checking virt-handler pods" - virt_handler_ready - - name: Setup crane and registry auth uses: deckhouse/modules-actions/setup@v2 with: @@ -1397,8 +1257,8 @@ jobs: - name: Verify image digests in pods after upgrade run: | - MODULE_IMAGE="${{ vars.DEV_REGISTRY }}/${{ vars.DEV_MODULE_SOURCE }}/virtualization:${{ env.NEW_RELEASE }}" - echo "[INFO] Extracting images_digests.json from ${MODULE_IMAGE}" + MODULE_IMAGE="${{ vars.DEV_MODULE_SOURCE }}/virtualization:${{ env.NEW_RELEASE }}" + echo "[INFO] Extracting images_digests.json from virtualization:${{ env.NEW_RELEASE }}" images_hash=$(crane export "${MODULE_IMAGE}" - | tar -Oxf - images_digests.json) echo "[INFO] Expected image digests:" echo "::group::images_digests.json" diff --git a/AGENTS.md b/AGENTS.md deleted file mode 100644 index 9390d72dbc..0000000000 --- a/AGENTS.md +++ /dev/null @@ -1,84 +0,0 @@ -# Agent Instructions - -This project uses **bd** (beads) for issue tracking. Run `bd prime` for full workflow context. - -## Quick Reference - -```bash -bd ready # Find available work -bd show # View issue details -bd update --claim # Claim work atomically -bd close # Complete work -bd dolt push # Push beads data to remote -``` - -## Non-Interactive Shell Commands - -**ALWAYS use non-interactive flags** with file operations to avoid hanging on confirmation prompts. - -Shell commands like `cp`, `mv`, and `rm` may be aliased to include `-i` (interactive) mode on some systems, causing the agent to hang indefinitely waiting for y/n input. - -**Use these forms instead:** -```bash -# Force overwrite without prompting -cp -f source dest # NOT: cp source dest -mv -f source dest # NOT: mv source dest -rm -f file # NOT: rm file - -# For recursive operations -rm -rf directory # NOT: rm -r directory -cp -rf source dest # NOT: cp -r source dest -``` - -**Other commands that may prompt:** -- `scp` - use `-o BatchMode=yes` for non-interactive -- `ssh` - use `-o BatchMode=yes` to fail instead of prompting -- `apt-get` - use `-y` flag -- `brew` - use `HOMEBREW_NO_AUTO_UPDATE=1` env var - - -## Beads Issue Tracker - -This project uses **bd (beads)** for issue tracking. Run `bd prime` to see full workflow context and commands. - -### Quick Reference - -```bash -bd ready # Find available work -bd show # View issue details -bd update --claim # Claim work -bd close # Complete work -``` - -### Rules - -- Use `bd` for ALL task tracking — do NOT use TodoWrite, TaskCreate, or markdown TODO lists -- Run `bd prime` for detailed command reference and session close protocol -- Use `bd remember` for persistent knowledge — do NOT use MEMORY.md files - -## Session Completion - -**When ending a work session**, you MUST complete ALL steps below. Work is NOT complete until `git push` succeeds. - -**MANDATORY WORKFLOW:** - -1. **File issues for remaining work** - Create issues for anything that needs follow-up -2. **Run quality gates** (if code changed) - Tests, linters, builds -3. **Update issue status** - Close finished work, update in-progress items -4. **PUSH TO REMOTE** - This is MANDATORY: - ```bash - git pull --rebase - bd dolt push - git push - git status # MUST show "up to date with origin" - ``` -5. **Clean up** - Clear stashes, prune remote branches -6. **Verify** - All changes committed AND pushed -7. **Hand off** - Provide context for next session - -**CRITICAL RULES:** -- Work is NOT complete until `git push` succeeds -- NEVER stop before pushing - that leaves work stranded locally -- NEVER say "ready to push when you are" - YOU must push -- If push fails, resolve and retry until it succeeds -