Harden deploy migration flow during local and Azure deploys

cmd/deploy_azure.go

@@ -3,7 +3,6 @@ package cmd
 import (
 	"encoding/json"
 	"fmt"
-	"net/http"
 	"os"
 	"path/filepath"
 	"strings"
@@ -304,14 +303,10 @@ func runDeployAzure(cmd *cobra.Command, args []string) error {
 
 	if backendReady {
 		fmt.Println("   ✅ Backend is responding!")
-		fmt.Println("\n🔄 Triggering database migration...")
-		httpClient := &http.Client{Timeout: 5 * time.Second}
-		resp, err := httpClient.Get(deployment.BackendEndpoint + "/proceed-db-migration")
-		if err == nil {
-			resp.Body.Close()
-			fmt.Println("   ✅ Migration triggered")
-		} else {
-			fmt.Printf("   ⚠️  Migration may need manual trigger: %v\n", err)
+		if err := triggerAndWaitForMigration(deployment.BackendEndpoint); err != nil {
+			fmt.Printf("   ⚠️  %v\n", err)
+			fmt.Printf("   Trigger migration manually if needed: GET %s/proceed-db-migration\n", deployment.BackendEndpoint)
+			fmt.Println("   Migration may still be running — proceeding anyway")
 		}
 	} else {
 		fmt.Println("   Backend not ready after 30 attempts.")

cmd/deploy_local.go

@@ -8,7 +8,6 @@ import (
 	"strings"
 	"time"
 
-	"github.com/DevExpGBB/gh-devlake/internal/devlake"
 	dockerpkg "github.com/DevExpGBB/gh-devlake/internal/docker"
 	"github.com/DevExpGBB/gh-devlake/internal/download"
 	"github.com/DevExpGBB/gh-devlake/internal/gitclone"
@@ -196,17 +195,10 @@ func runDeployLocal(cmd *cobra.Command, args []string) error {
 		}
 		cfgURL = backendURL
 
-		fmt.Println("\n🔄 Triggering database migration...")
-		migClient := devlake.NewClient(backendURL)
-		if err := migClient.TriggerMigration(); err != nil {
-			fmt.Printf("   ⚠️  Migration may need manual trigger: %v\n", err)
-		} else {
-			fmt.Println("   ✅ Migration triggered")
-			fmt.Println("\n⏳ Waiting for migration to complete...")
-			if err := waitForMigration(backendURL, 60, 5*time.Second); err != nil {
-				fmt.Printf("   ⚠️  %v\n", err)
-				fmt.Println("   Migration may still be running — proceeding anyway")
-			}
+		if err := triggerAndWaitForMigration(backendURL); err != nil {
+			fmt.Printf("   ⚠️  %v\n", err)
+			fmt.Printf("   Trigger migration manually if needed: GET %s/proceed-db-migration\n", backendURL)
+			fmt.Println("   Migration may still be running — proceeding anyway")
 		}
 
 		if !deployLocalQuiet {

cmd/helpers.go

@@ -2,7 +2,9 @@ package cmd
 
 import (
 	"encoding/json"
+	"errors"
 	"fmt"
+	"io"
 	"net/http"
 	"os"
 	"strings"
@@ -221,19 +223,73 @@ func waitForReadyAny(baseURLs []string, maxAttempts int, interval time.Duration)
 // During migration the API returns 428 (Precondition Required).
 func waitForMigration(baseURL string, maxAttempts int, interval time.Duration) error {
 	httpClient := &http.Client{Timeout: 5 * time.Second}
+	lastStatus := 0
 	for attempt := 1; attempt <= maxAttempts; attempt++ {
 		resp, err := httpClient.Get(baseURL + "/ping")
 		if err == nil {
+			lastStatus = resp.StatusCode
+			_, _ = io.Copy(io.Discard, resp.Body)
 			resp.Body.Close()
 			if resp.StatusCode == http.StatusOK {
 				fmt.Println("   ✅ Migration complete!")
 				return nil
 			}
+		} else if resp != nil {
+			lastStatus = resp.StatusCode
+			_, _ = io.Copy(io.Discard, resp.Body)
+			resp.Body.Close()
+		}
+		statusSuffix := ""
+		if lastStatus != 0 {
+			statusSuffix = fmt.Sprintf(", status=%d", lastStatus)
 		}
-		fmt.Printf("   Migrating... (%d/%d)\n", attempt, maxAttempts)
+		fmt.Printf("   Migrating... (%d/%d%s)\n", attempt, maxAttempts, statusSuffix)
 		time.Sleep(interval)
 	}
-	return fmt.Errorf("migration did not complete after %d attempts", maxAttempts)
+	statusSuffix := ""
+	if lastStatus != 0 {
+		statusSuffix = fmt.Sprintf(" (last status %d)", lastStatus)
+	}
+	return fmt.Errorf("migration did not complete after %d attempts%s", maxAttempts, statusSuffix)
+}
+
+func triggerAndWaitForMigration(baseURL string) error {
+	return triggerAndWaitForMigrationWithClient(devlake.NewClientWithTimeout(baseURL, 5*time.Second), 3, 10*time.Second, 60, 5*time.Second)
+}
+
+func triggerAndWaitForMigrationWithClient(devlakeClient *devlake.Client, triggerAttempts int, triggerInterval time.Duration, waitAttempts int, waitInterval time.Duration) error {
+	fmt.Println("\n🔄 Triggering database migration...")
+
+	var lastErr error
+	for attempt := 1; attempt <= triggerAttempts; attempt++ {
+		err := devlakeClient.TriggerMigration()
+		if err == nil {
+			lastErr = nil
+			fmt.Println("   ✅ Migration triggered")
+			break
+		}
+		lastErr = err
+		fmt.Printf("   ⚠️  Trigger attempt %d/%d failed: %v\n", attempt, triggerAttempts, err)
+		if attempt < triggerAttempts {
+			fmt.Println("   DevLake may still be starting or migration may already be running — retrying...")
+			time.Sleep(triggerInterval)
+		}
+	}
+
+	fmt.Println("\n⏳ Waiting for migration to complete...")
+	if lastErr != nil {
+		fmt.Println("   Continuing to monitor migration status anyway...")
+	}
+	if err := waitForMigration(devlakeClient.BaseURL, waitAttempts, waitInterval); err != nil {
+		if lastErr != nil {
+			return errors.Join(
+				fmt.Errorf("migration trigger failed earlier: %w", lastErr),
+				fmt.Errorf("waiting for migration completion: %w", err),
+			)
+		}
+		return err
+	}
+	return nil
 }
 
 // ── Scope orchestration ─────────────────────────────────────────

cmd/helpers_migration_test.go

@@ -0,0 +1,189 @@
+package cmd
+
+import (
+	"errors"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/DevExpGBB/gh-devlake/internal/devlake"
+)
+
+func TestTriggerAndWaitForMigrationWithClient_CompletesAfterTriggerTimeout(t *testing.T) {
+	triggerCalls := 0
+	pingCalls := 0
+	var mu sync.Mutex
+
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch r.URL.Path {
+		case "/proceed-db-migration":
+			mu.Lock()
+			triggerCalls++
+			mu.Unlock()
+			time.Sleep(25 * time.Millisecond)
+			w.WriteHeader(http.StatusOK)
+		case "/ping":
+			mu.Lock()
+			pingCalls++
+			currentPingCalls := pingCalls
+			mu.Unlock()
+			if currentPingCalls == 1 {
+				w.WriteHeader(http.StatusPreconditionRequired)
+				return
+			}
+			w.WriteHeader(http.StatusOK)
+		default:
+			http.NotFound(w, r)
+		}
+	}))
+	defer srv.Close()
+
+	client := &devlake.Client{
+		BaseURL: srv.URL,
+		HTTPClient: &http.Client{
+			Timeout: 5 * time.Millisecond,
+		},
+	}
+
+	err := triggerAndWaitForMigrationWithClient(client, 1, time.Millisecond, 3, time.Millisecond)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	mu.Lock()
+	gotTriggerCalls := triggerCalls
+	gotPingCalls := pingCalls
+	mu.Unlock()
+	if gotTriggerCalls != 1 {
+		t.Fatalf("trigger calls = %d, want 1", gotTriggerCalls)
+	}
+	if gotPingCalls != 2 {
+		t.Fatalf("ping calls = %d, want 2", gotPingCalls)
+	}
+}
+
+func TestTriggerAndWaitForMigrationWithClient_RetriesBeforeWaiting(t *testing.T) {
+	triggerCalls := 0
+	pingCalls := 0
+
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch r.URL.Path {
+		case "/proceed-db-migration":
+			triggerCalls++
+			if triggerCalls == 1 {
+				w.WriteHeader(http.StatusServiceUnavailable)
+				return
+			}
+			w.WriteHeader(http.StatusOK)
+		case "/ping":
+			pingCalls++
+			w.WriteHeader(http.StatusOK)
+		default:
+			http.NotFound(w, r)
+		}
+	}))
+	defer srv.Close()
+
+	client := devlake.NewClient(srv.URL)
+
+	err := triggerAndWaitForMigrationWithClient(client, 2, time.Millisecond, 2, time.Millisecond)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if triggerCalls != 2 {
+		t.Fatalf("trigger calls = %d, want 2", triggerCalls)
+	}
+	if pingCalls != 1 {
+		t.Fatalf("ping calls = %d, want 1", pingCalls)
+	}
+}
+
+func TestTriggerAndWaitForMigrationWithClient_TriggerEventuallySucceedsBeforeWaitFails(t *testing.T) {
+	triggerCalls := 0
+	pingCalls := 0
+
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch r.URL.Path {
+		case "/proceed-db-migration":
+			triggerCalls++
+			if triggerCalls == 1 {
+				w.WriteHeader(http.StatusServiceUnavailable)
+				return
+			}
+			w.WriteHeader(http.StatusOK)
+		case "/ping":
+			pingCalls++
+			w.WriteHeader(http.StatusPreconditionRequired)
+		default:
+			http.NotFound(w, r)
+		}
+	}))
+	defer srv.Close()
+
+	client := devlake.NewClient(srv.URL)
+
+	err := triggerAndWaitForMigrationWithClient(client, 2, 5*time.Millisecond, 2, 5*time.Millisecond)
+	if err == nil {
+		t.Fatal("expected error, got nil")
+	}
+	if strings.Contains(err.Error(), "migration trigger failed earlier") {
+		t.Fatalf("unexpected trigger failure in error: %v", err)
+	}
+	if !strings.Contains(err.Error(), "migration did not complete after 2 attempts") {
+		t.Fatalf("expected wait failure in error, got: %v", err)
+	}
+	if triggerCalls != 2 {
+		t.Fatalf("trigger calls = %d, want 2", triggerCalls)
+	}
+	if pingCalls != 2 {
+		t.Fatalf("ping calls = %d, want 2", pingCalls)
+	}
+}
+
+func TestTriggerAndWaitForMigrationWithClient_JoinsTriggerAndWaitFailures(t *testing.T) {
+	triggerCalls := 0
+	pingCalls := 0
+
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch r.URL.Path {
+		case "/proceed-db-migration":
+			triggerCalls++
+			w.WriteHeader(http.StatusServiceUnavailable)
+		case "/ping":
+			pingCalls++
+			w.WriteHeader(http.StatusPreconditionRequired)
+		default:
+			http.NotFound(w, r)
+		}
+	}))
+	defer srv.Close()
+
+	client := devlake.NewClient(srv.URL)
+
+	err := triggerAndWaitForMigrationWithClient(client, 1, 5*time.Millisecond, 1, 5*time.Millisecond)
+	if err == nil {
+		t.Fatal("expected error, got nil")
+	}
+	if !strings.Contains(err.Error(), "migration trigger failed earlier: GET /proceed-db-migration") {
+		t.Fatalf("expected joined trigger failure in error, got: %v", err)
+	}
+	if !strings.Contains(err.Error(), "waiting for migration completion: migration did not complete after 1 attempts") {
+		t.Fatalf("expected joined wait failure in error, got: %v", err)
+	}
+
+	var joined interface{ Unwrap() []error }
+	if !errors.As(err, &joined) {
+		t.Fatalf("expected joined error, got: %T", err)
+	}
+	if len(joined.Unwrap()) != 2 {
+		t.Fatalf("joined error count = %d, want 2", len(joined.Unwrap()))
+	}
+	if triggerCalls != 1 {
+		t.Fatalf("trigger calls = %d, want 1", triggerCalls)
+	}
+	if pingCalls != 1 {
+		t.Fatalf("ping calls = %d, want 1", pingCalls)
+	}
+}