diff --git a/README.md b/README.md index 1304623..1dcc4c1 100644 --- a/README.md +++ b/README.md @@ -56,13 +56,13 @@ gh devlake deploy local --dir ./devlake gh devlake configure full ``` -After setup, open Grafana at **http://localhost:3002** (admin / admin). DORA and Copilot dashboards will populate after the first sync completes. +After setup, open the URL bundle printed by `gh devlake deploy local`. Local deploys normally use `8080/3002/4000`, and automatically fall back once to `8085/3004/4004` when the default ports are already in use. DORA and Copilot dashboards will populate after the first sync completes. | Service | URL | |---------|-----| -| Grafana | http://localhost:3002 (admin/admin) | -| Config UI | http://localhost:4000 | -| Backend API | http://localhost:8080 | +| Grafana | http://localhost:3002 or http://localhost:3004 (admin/admin) | +| Config UI | http://localhost:4000 or http://localhost:4004 | +| Backend API | http://localhost:8080 or http://localhost:8085 | --- diff --git a/cmd/deploy_azure.go b/cmd/deploy_azure.go index 14e9a12..36983b8 100644 --- a/cmd/deploy_azure.go +++ b/cmd/deploy_azure.go @@ -144,7 +144,9 @@ func runDeployAzure(cmd *cobra.Command, args []string) error { fmt.Println("\nšŸ”‘ Checking Azure CLI login...") acct, err := azure.CheckLogin() if err != nil { - fmt.Println(" Not logged in. Running az login...") + // Bounded recovery: Auto-login (single attempt) + fmt.Println(" āŒ Not logged in") + fmt.Println("\nšŸ”§ Recovery: Running az login...") if loginErr := azure.Login(); loginErr != nil { return fmt.Errorf("az login failed: %w", loginErr) } @@ -152,6 +154,7 @@ func runDeployAzure(cmd *cobra.Command, args []string) error { if err != nil { return fmt.Errorf("still not logged in after az login: %w", err) } + fmt.Println(" āœ… Recovery successful") } fmt.Printf(" Logged in as: %s\n", acct.User.Name) @@ -240,9 +243,12 @@ func runDeployAzure(cmd *cobra.Command, args []string) error { fmt.Println("\nšŸ—„ļø Checking MySQL state...") state, err := azure.MySQLState(mysqlName, azureRG) if err == nil && state == "Stopped" { - fmt.Println(" MySQL is stopped. Starting...") + // Bounded recovery: Start stopped MySQL (single attempt) + fmt.Println(" āŒ MySQL is stopped") + fmt.Println("\nšŸ”§ Recovery: Starting MySQL...") if err := azure.MySQLStart(mysqlName, azureRG); err != nil { fmt.Printf(" āš ļø Could not start MySQL: %v\n", err) + fmt.Println(" Continuing deployment — MySQL may start later") } else { fmt.Println(" Waiting 30s for MySQL...") time.Sleep(30 * time.Second) @@ -258,11 +264,14 @@ func runDeployAzure(cmd *cobra.Command, args []string) error { kvName := fmt.Sprintf("%skv%s", azureBaseName, suffix) found, _ := azure.CheckSoftDeletedKeyVault(kvName) if found { - fmt.Printf("\nšŸ”‘ Key Vault %q found in soft-deleted state, purging...\n", kvName) + // Bounded recovery: Purge soft-deleted Key Vault (single attempt) + fmt.Printf("\nšŸ”‘ Key Vault conflict detected\n") + fmt.Printf(" Key Vault %q is in soft-deleted state\n", kvName) + fmt.Println("\nšŸ”§ Recovery: Purging soft-deleted Key Vault...") if err := azure.PurgeKeyVault(kvName, azureLocation); err != nil { return fmt.Errorf("failed to purge soft-deleted Key Vault %q: %w\nManual fix: az keyvault purge --name %s --location %s", kvName, err, kvName, azureLocation) } - fmt.Println(" āœ… Key Vault purged") + fmt.Println(" āœ… Key Vault purged — deployment can proceed") } // ── Deploy infrastructure ── diff --git a/cmd/deploy_errors.go b/cmd/deploy_errors.go new file mode 100644 index 0000000..9cc042d --- /dev/null +++ b/cmd/deploy_errors.go @@ -0,0 +1,304 @@ +package cmd + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "regexp" + "strings" +) + +// DeployErrorClass represents a known failure class during deployment. +type DeployErrorClass string + +const ( + ErrorClassDockerPortConflict DeployErrorClass = "docker_port_conflict" + ErrorClassDockerBindFailed DeployErrorClass = "docker_bind_failed" + ErrorClassAzureAuth DeployErrorClass = "azure_auth" + ErrorClassAzureMySQLStopped DeployErrorClass = "azure_mysql_stopped" + ErrorClassAzureKeyVault DeployErrorClass = "azure_keyvault_softdelete" + ErrorClassUnknown DeployErrorClass = "unknown" +) + +// DeployError represents a classified deployment error with recovery context. +type DeployError struct { + Class DeployErrorClass + OriginalErr error + Port string // For port conflict errors + Container string // For port conflict errors + ComposeFile string // For port conflict errors + Message string // Human-readable classification +} + +// classifyDockerComposeError inspects a docker compose error and returns +// a classified error with recovery context. This covers: +// - "port is already allocated" +// - "Bind for 0.0.0.0:PORT" +// - "ports are not available" / "Ports are not available" +// - "address already in use" +// - "failed programming external connectivity" +func classifyDockerComposeError(err error) *DeployError { + if err == nil { + return nil + } + + errStr := err.Error() + errStrLower := strings.ToLower(errStr) + + // Port conflict patterns (case-insensitive) + portConflictPatterns := []string{ + "port is already allocated", + "bind for", + "ports are not available", + "address already in use", + "failed programming external connectivity", + } + + isPortConflict := false + for _, pattern := range portConflictPatterns { + if strings.Contains(errStrLower, pattern) { + isPortConflict = true + break + } + } + + if !isPortConflict { + return &DeployError{ + Class: ErrorClassUnknown, + OriginalErr: err, + Message: "Docker Compose failed", + } + } + + // Extract port number from various error formats + port := extractPortFromError(errStr) + + result := &DeployError{ + Class: ErrorClassDockerPortConflict, + OriginalErr: err, + Port: port, + Message: "Docker port conflict detected", + } + + // Try to identify owning container + if port != "" { + container, composeFile := findPortOwner(port) + result.Container = container + result.ComposeFile = composeFile + } + + return result +} + +// extractPortFromError extracts the port number from various Docker error formats: +// - "Bind for 0.0.0.0:8080: failed: port is already allocated" +// - "Error response from daemon: Ports are not available: exposing port TCP 0.0.0.0:8080" +// - "bind: address already in use (listening on [::]:8080)" +// - "failed programming external connectivity on endpoint devlake (8080/tcp)" +func extractPortFromError(errStr string) string { + // Pattern 1: "Bind for 0.0.0.0:PORT" (case-insensitive using regexp) + re := regexp.MustCompile(`(?i)bind for 0\.0\.0\.0:(\d+)`) + if matches := re.FindStringSubmatch(errStr); len(matches) > 1 { + port := matches[1] + if isValidPort(port) { + return port + } + } + + // Pattern 2: "exposing port TCP 0.0.0.0:PORT" + if idx := strings.Index(errStr, "0.0.0.0:"); idx != -1 { + rest := errStr[idx+len("0.0.0.0:"):] + if end := strings.IndexAny(rest, " ->\n"); end > 0 { + port := rest[:end] + if isValidPort(port) { + return port + } + } + } + + // Pattern 3: "listening on [::]:PORT" or "[::]PORT" + if idx := strings.Index(errStr, "[::]"); idx != -1 { + rest := errStr[idx+len("[::]"):] + // Skip potential colon separator + if strings.HasPrefix(rest, ":") { + rest = rest[1:] + } + if end := strings.IndexAny(rest, " )\n"); end > 0 { + port := rest[:end] + if isValidPort(port) { + return port + } + } + // If no delimiter found, but there are digits, use them + if len(rest) > 0 { + for i, ch := range rest { + if ch < '0' || ch > '9' { + if i > 0 { + port := rest[:i] + if isValidPort(port) { + return port + } + } + break + } + } + } + } + + // Pattern 4: "(PORT/tcp)" or "(PORT/udp)" in endpoint errors + if idx := strings.Index(errStr, "("); idx != -1 { + rest := errStr[idx+1:] + if end := strings.Index(rest, "/tcp)"); end > 0 { + port := strings.TrimSpace(rest[:end]) + if isValidPort(port) { + return port + } + } + if end := strings.Index(rest, "/udp)"); end > 0 { + port := strings.TrimSpace(rest[:end]) + if isValidPort(port) { + return port + } + } + } + + // Pattern 5: Generic port number extraction (last resort) + // Look for sequences like ":8080" or " 8080 " in the error + for _, candidate := range strings.Fields(errStr) { + // Try splitting by colons + if strings.Contains(candidate, ":") { + parts := strings.Split(candidate, ":") + for _, part := range parts { + part = strings.Trim(part, "(),[]") + if isValidPort(part) { + return part + } + } + } + // Try the field itself (for cases like "[::] 3002") + cleaned := strings.Trim(candidate, "(),[]") + if isValidPort(cleaned) { + return cleaned + } + } + + return "" +} + +// isValidPort checks if a string looks like a valid port number (all digits, 1-65535). +func isValidPort(s string) bool { + if len(s) < 1 || len(s) > 5 { + return false + } + for _, ch := range s { + if ch < '0' || ch > '9' { + return false + } + } + // Parse to int and validate range 1-65535 + port := 0 + for _, ch := range s { + port = port*10 + int(ch-'0') + } + return port >= 1 && port <= 65535 +} + +// findPortOwner queries Docker to find which container is using the specified port. +// Returns (containerName, composeFilePath). +func findPortOwner(port string) (string, string) { + out, err := exec.Command( + "docker", + "ps", + "--filter", "publish="+port, + "--format", "{{.Names}}\t{{.Label \"com.docker.compose.project.config_files\"}}\t{{.Label \"com.docker.compose.project.working_dir\"}}", + ).Output() + + if err != nil || len(strings.TrimSpace(string(out))) == 0 { + return "", "" + } + + lines := strings.Split(strings.TrimSpace(string(out)), "\n") + parts := strings.SplitN(lines[0], "\t", 3) + + containerName := parts[0] + configFiles := "" + workDir := "" + + if len(parts) >= 2 { + configFiles = strings.TrimSpace(parts[1]) + } + if len(parts) == 3 { + workDir = strings.TrimSpace(parts[2]) + } + + // Prefer the exact compose file path Docker recorded + if configFiles != "" { + configFile := strings.Split(configFiles, ";")[0] + configFile = strings.TrimSpace(configFile) + if configFile != "" { + if _, statErr := os.Stat(configFile); statErr == nil { + return containerName, configFile + } + } + } + + // Fallback: assume docker-compose.yml under working_dir + if workDir != "" { + composePath := filepath.Join(workDir, "docker-compose.yml") + if _, statErr := os.Stat(composePath); statErr == nil { + return containerName, composePath + } + } + + return containerName, "" +} + +// printDockerPortConflictError prints a user-friendly error message for port conflicts +// with actionable remediation steps. +// If customHeader is provided, it replaces the default "Port conflict detected" header. +// If nextSteps is provided, it replaces the default "Then re-run: gh devlake deploy local" text. +func printDockerPortConflictError(de *DeployError, customHeader string, nextSteps string) { + // Print header + if customHeader != "" { + // Normalize header to ensure consistent spacing (blank line before emoji-prefixed steps) + normalizedHeader := customHeader + if !strings.HasPrefix(normalizedHeader, "\n") { + normalizedHeader = "\n" + normalizedHeader + } + fmt.Println(normalizedHeader) + } else { + if de.Port != "" { + fmt.Printf("\nāŒ Port conflict detected: port %s is already in use.\n", de.Port) + } else { + fmt.Println("\nāŒ Port conflict detected: a required port is already in use.") + } + } + + // Print container info and stop commands + if de.Container != "" { + fmt.Printf(" Container holding the port: %s\n", de.Container) + + if de.ComposeFile != "" { + fmt.Println(" Stop it with:") + fmt.Printf(" docker compose -f \"%s\" down\n", de.ComposeFile) + } else { + fmt.Println(" Stop it with:") + fmt.Printf(" docker stop %s\n", de.Container) + } + } else if de.Port != "" { + fmt.Println(" Find what's using it:") + fmt.Println(" docker ps --format \"table {{.Names}}\\t{{.Ports}}\"") + } + + // Print next steps + if nextSteps != "" { + fmt.Println(nextSteps) + } else { + fmt.Println(" Then re-run:") + fmt.Println(" gh devlake deploy local") + } + + fmt.Println("\nšŸ’” To clean up partial artifacts:") + fmt.Println(" gh devlake cleanup --local --force") +} diff --git a/cmd/deploy_errors_test.go b/cmd/deploy_errors_test.go new file mode 100644 index 0000000..e746142 --- /dev/null +++ b/cmd/deploy_errors_test.go @@ -0,0 +1,200 @@ +package cmd + +import ( + "errors" + "testing" +) + +func TestExtractPortFromError(t *testing.T) { + tests := []struct { + name string + errStr string + wantPort string + }{ + { + name: "bind for pattern", + errStr: "Error response from daemon: driver failed: Bind for 0.0.0.0:8080 failed: port is already allocated", + wantPort: "8080", + }, + { + name: "exposing port pattern", + errStr: "Error response from daemon: Ports are not available: exposing port TCP 0.0.0.0:8080", + wantPort: "8080", + }, + { + name: "ipv6 listening pattern with colon", + errStr: "bind: address already in use (listening on [::]:8080)", + wantPort: "8080", + }, + { + name: "ipv6 listening pattern without colon", + errStr: "bind: address already in use (listening on [::] 3002)", + wantPort: "3002", + }, + { + name: "port in error context", + errStr: "failed programming external connectivity on endpoint devlake (8080/tcp)", + wantPort: "8080", + }, + { + name: "alternate port 8085", + errStr: "Error response from daemon: driver failed: Bind for 0.0.0.0:8085 failed: port is already allocated", + wantPort: "8085", + }, + { + name: "grafana port 3002", + errStr: "Bind for 0.0.0.0:3002: failed: port is already allocated", + wantPort: "3002", + }, + { + name: "config-ui port 4000", + errStr: "Ports are not available: exposing port TCP 0.0.0.0:4000", + wantPort: "4000", + }, + { + name: "no port in error", + errStr: "Error response from daemon: some other error", + wantPort: "", + }, + { + name: "empty error", + errStr: "", + wantPort: "", + }, + { + name: "port 0 should be rejected", + errStr: "Error response from daemon: Bind for 0.0.0.0:0 failed", + wantPort: "", + }, + { + name: "port 00000 should be rejected", + errStr: "Error response from daemon: Bind for 0.0.0.0:00000 failed", + wantPort: "", + }, + { + name: "port 65536 should be rejected", + errStr: "Error response from daemon: Bind for 0.0.0.0:65536 failed", + wantPort: "", + }, + { + name: "case insensitive bind for with uppercase", + errStr: "Error response from daemon: BIND FOR 0.0.0.0:8080 failed", + wantPort: "8080", + }, + { + name: "non-ASCII characters before match", + errStr: "ć‚Øćƒ©ćƒ¼: Bind for 0.0.0.0:8080 failed: port is already allocated", + wantPort: "8080", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := extractPortFromError(tt.errStr) + if got != tt.wantPort { + t.Errorf("extractPortFromError() = %q, want %q", got, tt.wantPort) + } + }) + } +} + +func TestClassifyDockerComposeError(t *testing.T) { + tests := []struct { + name string + err error + wantClass DeployErrorClass + wantPort string + wantNilErr bool + }{ + { + name: "port already allocated", + err: errors.New("Error response from daemon: driver failed: Bind for 0.0.0.0:8080 failed: port is already allocated"), + wantClass: ErrorClassDockerPortConflict, + wantPort: "8080", + }, + { + name: "ports are not available", + err: errors.New("Error response from daemon: Ports are not available: exposing port TCP 0.0.0.0:8080"), + wantClass: ErrorClassDockerPortConflict, + wantPort: "8080", + }, + { + name: "address already in use", + err: errors.New("bind: address already in use (listening on [::]:8080)"), + wantClass: ErrorClassDockerPortConflict, + wantPort: "8080", + }, + { + name: "failed programming external connectivity", + err: errors.New("failed programming external connectivity on endpoint devlake"), + wantClass: ErrorClassDockerPortConflict, + wantPort: "", + }, + { + name: "unknown docker error", + err: errors.New("Error response from daemon: some other error"), + wantClass: ErrorClassUnknown, + wantPort: "", + }, + { + name: "nil error", + err: nil, + wantNilErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := classifyDockerComposeError(tt.err) + + if tt.wantNilErr { + if got != nil { + t.Errorf("classifyDockerComposeError() = %v, want nil", got) + } + return + } + + if got == nil { + t.Fatal("classifyDockerComposeError() returned nil, want non-nil") + } + + if got.Class != tt.wantClass { + t.Errorf("classifyDockerComposeError().Class = %v, want %v", got.Class, tt.wantClass) + } + + if got.Port != tt.wantPort { + t.Errorf("classifyDockerComposeError().Port = %q, want %q", got.Port, tt.wantPort) + } + + if got.OriginalErr != tt.err { + t.Errorf("classifyDockerComposeError().OriginalErr = %v, want %v", got.OriginalErr, tt.err) + } + }) + } +} + +func TestClassifyDockerComposeError_AllPatterns(t *testing.T) { + // Test that all documented port conflict patterns are recognized + patterns := []string{ + "port is already allocated", + "Bind for 0.0.0.0:8080", + "ports are not available", + "address already in use", + "failed programming external connectivity", + } + + for _, pattern := range patterns { + t.Run(pattern, func(t *testing.T) { + err := errors.New("Error: " + pattern) + got := classifyDockerComposeError(err) + + if got == nil { + t.Fatal("classifyDockerComposeError() returned nil, want non-nil") + } + + if got.Class != ErrorClassDockerPortConflict { + t.Errorf("Pattern %q not classified as port conflict, got %v", pattern, got.Class) + } + }) + } +} diff --git a/cmd/deploy_local.go b/cmd/deploy_local.go index 917282c..f0b66ab 100644 --- a/cmd/deploy_local.go +++ b/cmd/deploy_local.go @@ -1,20 +1,24 @@ package cmd import ( + "encoding/json" "fmt" "os" "os/exec" "path/filepath" + "regexp" + "strconv" "strings" "time" + "github.com/spf13/cobra" + "github.com/DevExpGBB/gh-devlake/internal/devlake" dockerpkg "github.com/DevExpGBB/gh-devlake/internal/docker" "github.com/DevExpGBB/gh-devlake/internal/download" "github.com/DevExpGBB/gh-devlake/internal/gitclone" "github.com/DevExpGBB/gh-devlake/internal/prompt" "github.com/DevExpGBB/gh-devlake/internal/secrets" - "github.com/spf13/cobra" ) const ( @@ -45,6 +49,11 @@ Image source (interactive prompt or flags): fork Clone a DevLake repo and build images from source custom Use your own docker-compose.yml already in the target directory +For official and fork deployments, if the default local port bundle +(8080/3002/4000) is already in use, the CLI automatically retries once with +alternate ports (8085/3004/4004). Custom deployments require manual port +conflict resolution. + Example: gh devlake deploy local gh devlake deploy local --version v1.0.2 --dir ./devlake @@ -190,7 +199,11 @@ func runDeployLocal(cmd *cobra.Command, args []string) error { if deployLocalSource == "fork" { services = []string{"mysql", "devlake", "grafana", "config-ui"} } - backendURL, err := startLocalContainers(absDir, buildImages, services...) + + // Allow alternate port bundle for official/fork (not custom) + allowPortFallback := deployLocalSource != "custom" + + backendURL, err := startLocalContainers(absDir, buildImages, allowPortFallback, services...) if err != nil { return err } @@ -212,8 +225,11 @@ func runDeployLocal(cmd *cobra.Command, args []string) error { if !deployLocalQuiet { printBanner("āœ… DevLake is running!") fmt.Printf("\n Backend API: %s\n", backendURL) - fmt.Println(" Config UI: http://localhost:4000") - fmt.Println(" Grafana: http://localhost:3002 (admin/admin)") + // Infer companion URLs based on compose file ports + composePath := findComposeFile(absDir) + grafanaURL, configUIURL := inferCompanionURLs(backendURL, composePath) + fmt.Printf(" Config UI: %s\n", configUIURL) + fmt.Printf(" Grafana: %s (admin/admin)\n", grafanaURL) fmt.Println("\nTo stop/remove DevLake:") fmt.Printf(" cd \"%s\" && gh devlake cleanup\n", absDir) } @@ -424,10 +440,12 @@ func copyDir(src, dst string) error { // startLocalContainers runs docker compose up -d and polls until DevLake is healthy. // If build is true, images are rebuilt from local Dockerfiles (fork mode). +// If allowPortFallback is true, the function will retry once with alternate ports (8085/3004/4004) +// when a port conflict is detected on the default bundle (8080/3002/4000). // If services are specified, only those services are started (used by fork mode // to avoid starting unnecessary services like postgres/authproxy). // Returns the backend URL on success. -func startLocalContainers(dir string, build bool, services ...string) (string, error) { +func startLocalContainers(dir string, build, allowPortFallback bool, services ...string) (string, error) { absDir, _ := filepath.Abs(dir) if build { fmt.Printf("\n🐳 Building and starting containers in %s...\n", absDir) @@ -435,101 +453,168 @@ func startLocalContainers(dir string, build bool, services ...string) (string, e } else { fmt.Printf("\n🐳 Starting containers in %s...\n", absDir) } - if err := dockerpkg.ComposeUp(absDir, build, services...); err != nil { - // Give a friendlier error for port conflicts - errStr := err.Error() - if strings.Contains(errStr, "port is already allocated") || strings.Contains(errStr, "Bind for") { - // Extract the port number from the error - port := "" - if idx := strings.Index(errStr, "Bind for 0.0.0.0:"); idx != -1 { - rest := errStr[idx+len("Bind for 0.0.0.0:"):] - if end := strings.IndexAny(rest, " \n"); end > 0 { - port = rest[:end] - } - } + // Attempt 1: Default ports + err := dockerpkg.ComposeUp(absDir, build, services...) + if err == nil { + fmt.Println(" āœ… Containers starting") + return waitAndDetectBackendURL(absDir) + } + + // Classify the error + deployErr := classifyDockerComposeError(err) + if deployErr == nil || deployErr.Class != ErrorClassDockerPortConflict { + // Not a port conflict or unknown error - print general cleanup and fail + fmt.Println("\nšŸ’” To clean up partial artifacts:") + fmt.Println(" gh devlake cleanup --local --force") + return "", fmt.Errorf("starting containers: %w", err) + } + + // Port conflict detected + if !allowPortFallback { + // Custom deployments don't get auto-fallback - print friendly error + printDockerPortConflictError(deployErr, "", "") + return "", fmt.Errorf("port conflict — stop the conflicting container and retry: %w", err) + } + + // Bounded recovery: Try alternate port bundle once + // Find compose file + composePath := findComposeFile(absDir) + if composePath == "" { + // No compose file found - can't perform automatic recovery + printDockerPortConflictError(deployErr, "", "") + return "", fmt.Errorf("port conflict, and no compose file found for automatic recovery: %w", err) + } + + // Detect which port bundle the compose file is using + bundle := detectPortBundle(composePath) + + switch bundle { + case portBundleAlternate: + // Compose file is already on alternate ports - can't fallback further + // Build custom header with port/container info + header := "\nāŒ Port conflict detected on alternate ports (8085/3004/4004)" + if deployErr.Port != "" { + header += fmt.Sprintf("\n Port %s is in use", deployErr.Port) + if deployErr.Container != "" { + header += fmt.Sprintf(" by container: %s", deployErr.Container) + } + } + nextSteps := " The alternate port bundle is already in use.\n Free ports 8085/3004/4004, then retry deployment." + printDockerPortConflictError(deployErr, header, nextSteps) + return "", fmt.Errorf("port conflict on alternate ports: %w", err) + + case portBundleCustom: + // Custom ports - don't attempt automatic rewrite + // Build custom header with port/container info + header := "\nāŒ Port conflict detected on custom ports" + if deployErr.Port != "" { + header += fmt.Sprintf("\n Port %s is in use", deployErr.Port) + if deployErr.Container != "" { + header += fmt.Sprintf(" by container: %s", deployErr.Container) + } + } + nextSteps := " Edit your compose file to use different host ports, or stop the conflicting container." + printDockerPortConflictError(deployErr, header, nextSteps) + return "", fmt.Errorf("port conflict on custom ports: %w", err) + + case portBundleDefault: + // Compose file has default ports - try rewriting to alternate bundle + fmt.Println("\nšŸ”§ Port conflict detected on default ports (8080/3002/4000)") + if deployErr.Port != "" { + fmt.Printf(" Port %s is in use", deployErr.Port) + if deployErr.Container != "" { + fmt.Printf(" by container: %s", deployErr.Container) + } fmt.Println() - if port != "" { - fmt.Printf("āŒ Port conflict: %s is already in use.\n", port) + } + fmt.Println("\nšŸ”„ Retrying with alternate ports (8085/3004/4004)...") + + if err := rewriteComposePorts(composePath); err != nil { + fmt.Printf(" āš ļø Could not rewrite ports: %v\n", err) + printDockerPortConflictError(deployErr, "", "") + return "", fmt.Errorf("port conflict and failed to apply alternate ports: %w", err) + } + + fmt.Println(" āœ… Ports updated in compose file") + + // Attempt 2: Retry with alternate ports + fmt.Println(" Starting containers with alternate ports...") + err = dockerpkg.ComposeUp(absDir, build, services...) + if err != nil { + // Second attempt failed - classify again + retryErr := classifyDockerComposeError(err) + if retryErr != nil && retryErr.Class == ErrorClassDockerPortConflict { + // Build header that indicates both bundles failed + header := "\nāŒ Alternate ports are also in use." + nextSteps := " Both default (8080/3002/4000) and alternate (8085/3004/4004) port bundles are occupied.\n Free at least one bundle, then retry deployment." + printDockerPortConflictError(retryErr, header, nextSteps) } else { - fmt.Println("āŒ Port conflict: a required port is already in use.") + fmt.Println("\nšŸ’” To clean up partial artifacts:") + fmt.Println(" gh devlake cleanup --local --force") } + return "", fmt.Errorf("deployment failed after port fallback: %w", err) + } - // Ask Docker which container owns the port - conflictCmd := "" - if port != "" { - out, dockerErr := exec.Command( - "docker", - "ps", - "--filter", - "publish="+port, - "--format", - "{{.Names}}\t{{.Label \"com.docker.compose.project.config_files\"}}\t{{.Label \"com.docker.compose.project.working_dir\"}}", - ).Output() - if dockerErr == nil && len(strings.TrimSpace(string(out))) > 0 { - lines := strings.Split(strings.TrimSpace(string(out)), "\n") - // Use the first match - parts := strings.SplitN(lines[0], "\t", 3) - containerName := parts[0] - configFiles := "" - workDir := "" - if len(parts) >= 2 { - configFiles = strings.TrimSpace(parts[1]) - } - if len(parts) == 3 { - workDir = strings.TrimSpace(parts[2]) - } - fmt.Printf(" Container holding the port: %s\n", containerName) - // Prefer the exact compose file path Docker recorded (most reliable). - if configFiles != "" { - configFile := strings.Split(configFiles, ";")[0] - configFile = strings.TrimSpace(configFile) - if configFile != "" { - if _, statErr := os.Stat(configFile); statErr == nil { - fmt.Println("\n Stop it with:") - fmt.Printf(" docker compose -f \"%s\" down\n", configFile) - conflictCmd = fmt.Sprintf("docker compose -f \"%s\" down", configFile) - } else { - fmt.Println("\n Stop it with:") - fmt.Printf(" docker stop %s\n", containerName) - fmt.Printf("\n āš ļø Compose file not found at: %s\n", configFile) - fmt.Println(" (It may have been moved/deleted since the container was created.)") - conflictCmd = "docker stop " + containerName - } - } - } else if workDir != "" { - // Fallback for older Docker versions: assume docker-compose.yml under working_dir. - composePath := filepath.Join(workDir, "docker-compose.yml") - if _, statErr := os.Stat(composePath); statErr == nil { - fmt.Println("\n Stop it with:") - fmt.Printf(" docker compose -f \"%s\" down\n", composePath) - conflictCmd = fmt.Sprintf("docker compose -f \"%s\" down", composePath) - } - } - if conflictCmd == "" { - fmt.Println("\n Stop it with:") - fmt.Printf(" docker stop %s\n", containerName) - conflictCmd = "docker stop " + containerName - } - } - } - if conflictCmd == "" { - fmt.Println("\n Find what's using it:") - fmt.Println(" docker ps --format \"table {{.Names}}\\t{{.Ports}}\"") - } - fmt.Println("\n Then re-run:") - fmt.Println(" gh devlake init") - fmt.Println("\nšŸ’” To clean up partial artifacts:") - fmt.Println(" gh devlake cleanup --local --force") - return "", fmt.Errorf("port conflict — stop the conflicting container and retry") + fmt.Println(" āœ… Containers starting on alternate ports") + return waitAndDetectBackendURL(absDir) + } + + return "", fmt.Errorf("unexpected port bundle detection result") +} + +// findComposeFile returns the path to a compose file in the given directory for introspection. +// Checks for compose files in Docker Compose's default auto-detection order: +// compose.yaml → compose.yml → docker-compose.yaml → docker-compose.yml. +// Also checks docker-compose-dev.yml as a CLI-only fallback for port extraction/UX +// (not part of Docker's auto-detection set, but useful for fork/dev deployments). +// Note: docker.ComposeUp runs "docker compose up" without -f, so it uses Docker's default +// auto-detection order. This function is for port extraction, port rewriting, and UX output. +func findComposeFile(dir string) string { + // Check in the same order Docker Compose auto-detects compose files + candidates := []string{ + "compose.yaml", + "compose.yml", + "docker-compose.yaml", + "docker-compose.yml", + "docker-compose-dev.yml", // CLI-only fallback for introspection (not in Docker's set) + } + + for _, filename := range candidates { + composePath := filepath.Join(dir, filename) + if _, err := os.Stat(composePath); err == nil { + return composePath } - fmt.Println("\nšŸ’” To clean up partial artifacts:") - fmt.Println(" gh devlake cleanup --local --force") - return "", err } - fmt.Println(" āœ… Containers starting") - backendURLCandidates := []string{"http://localhost:8080", "http://localhost:8085"} + // Return empty string if no compose file found + return "" +} + +// waitAndDetectBackendURL polls the backend URL extracted from the compose file. +// Falls back to probing both 8080 and 8085 if extraction fails or no compose file is found. +func waitAndDetectBackendURL(dir string) (string, error) { + composePath := findComposeFile(dir) + + var ports map[string]int + if composePath != "" { + // Try to extract the actual backend port from the compose file (when present) + ports = extractServicePorts(composePath, "devlake") + } else { + // No compose file detected that Docker would use; rely on default port probing + ports = map[string]int{} + } + + var backendURLCandidates []string + + if backendPort, ok := ports["devlake"]; ok { + // Use the extracted port + backendURLCandidates = []string{fmt.Sprintf("http://localhost:%d", backendPort)} + } else { + // Fall back to probing both default and alternate ports + backendURLCandidates = []string{"http://localhost:8080", "http://localhost:8085"} + } + fmt.Println("\nā³ Waiting for DevLake to be ready...") fmt.Println(" Giving MySQL time to initialize (this takes ~30s on first run)...") time.Sleep(30 * time.Second) @@ -540,3 +625,227 @@ func startLocalContainers(dir string, build bool, services ...string) (string, e } return backendURL, nil } + +// portBundle represents the detected port configuration in a compose file +type portBundle int + +const ( + portBundleDefault portBundle = iota // 8080/3002/4000 + portBundleAlternate // 8085/3004/4004 + portBundleCustom // Other custom ports +) + +// detectPortBundle analyzes a compose file to determine which port bundle it uses. +// Returns: +// - portBundleDefault if compose file contains any of 8080:8080, 3002:3002, or 4000:4000 +// - portBundleAlternate if compose file contains any of 8085:8080, 3004:3002, or 4004:4000 +// - portBundleCustom if compose file has other custom host ports +func detectPortBundle(composePath string) portBundle { + data, err := os.ReadFile(composePath) + if err != nil { + return portBundleDefault // Assume default if we can't read - let rewriteComposePorts surface the I/O error + } + + content := string(data) + + // Use regex to match port mappings as list items (avoiding substring matches like 18080:8080) + // Pattern: start of line, optional whitespace, dash, optional whitespace, optional quotes, port mapping + defaultPatterns := []*regexp.Regexp{ + regexp.MustCompile(`(?m)^\s*-\s*["']?8080:8080["']?`), + regexp.MustCompile(`(?m)^\s*-\s*["']?3002:3002["']?`), + regexp.MustCompile(`(?m)^\s*-\s*["']?4000:4000["']?`), + } + for _, re := range defaultPatterns { + if re.MatchString(content) { + return portBundleDefault + } + } + + // Check for alternate port bundle + alternatePatterns := []*regexp.Regexp{ + regexp.MustCompile(`(?m)^\s*-\s*["']?8085:8080["']?`), + regexp.MustCompile(`(?m)^\s*-\s*["']?3004:3002["']?`), + regexp.MustCompile(`(?m)^\s*-\s*["']?4004:4000["']?`), + } + for _, re := range alternatePatterns { + if re.MatchString(content) { + return portBundleAlternate + } + } + + // If neither default nor alternate, it's custom + return portBundleCustom +} + +// rewriteComposePorts rewrites the port mappings in a docker-compose.yml file +// from the default bundle (8080/3002/4000) to the alternate bundle (8085/3004/4004). +// Uses regex with proper boundaries to avoid rewriting custom ports like 18080:8080. +func rewriteComposePorts(composePath string) error { + data, err := os.ReadFile(composePath) + if err != nil { + return fmt.Errorf("reading compose file: %w", err) + } + + content := string(data) + modified := content + + // Port mapping patterns with regex boundaries + // Match: "- 8080:8080" or "- "8080:8080" or "- '8080:8080'" at start of list item + // Avoid: "- 18080:8080" (custom host port that contains 8080) + portReplacements := []struct { + pattern string + replacement string + }{ + // Backend: 8080:8080 -> 8085:8080 + {`(?m)(^\s*-\s*)["']?8080:8080["']?`, `${1}8085:8080`}, + // Grafana: 3002:3002 -> 3004:3002 + {`(?m)(^\s*-\s*)["']?3002:3002["']?`, `${1}3004:3002`}, + // Config UI: 4000:4000 -> 4004:4000 + {`(?m)(^\s*-\s*)["']?4000:4000["']?`, `${1}4004:4000`}, + } + + for _, repl := range portReplacements { + re := regexp.MustCompile(repl.pattern) + modified = re.ReplaceAllString(modified, repl.replacement) + } + + if modified == content { + return fmt.Errorf("no port mappings found to rewrite (expected 8080/3002/4000)") + } + + // Preserve original file permissions for atomic write + var originalMode os.FileMode = 0644 + if info, err := os.Stat(composePath); err == nil { + originalMode = info.Mode() + } + + // Create a backup of the original compose file so users can revert if needed + backupPath := composePath + ".bak" + if _, statErr := os.Stat(backupPath); os.IsNotExist(statErr) { + if err := os.WriteFile(backupPath, data, originalMode); err != nil { + return fmt.Errorf("creating compose backup %s: %w", backupPath, err) + } + } + + // Write updated content atomically via temp file + rename to avoid partial writes + dir := filepath.Dir(composePath) + tmpFile, err := os.CreateTemp(dir, ".compose-*.tmp") + if err != nil { + return fmt.Errorf("creating temp compose file: %w", err) + } + tmpPath := tmpFile.Name() + // Ensure temp file is removed on failure paths + defer os.Remove(tmpPath) + + if _, err := tmpFile.Write([]byte(modified)); err != nil { + tmpFile.Close() + return fmt.Errorf("writing temp compose file: %w", err) + } + + if err := tmpFile.Close(); err != nil { + return fmt.Errorf("closing temp compose file: %w", err) + } + + // Set the original file permissions on the temp file before replacing + if err := os.Chmod(tmpPath, originalMode); err != nil { + return fmt.Errorf("setting compose file permissions: %w", err) + } + + // On Windows, os.Rename fails if destination exists, so remove it first + if err := os.Remove(composePath); err != nil { + return fmt.Errorf("removing original compose file: %w", err) + } + + if err := os.Rename(tmpPath, composePath); err != nil { + return fmt.Errorf("replacing compose file atomically: %w", err) + } + + return nil +} + +// extractServicePorts parses the docker-compose.yml file and extracts host ports for the specified services. +// Returns a map of service name to host port (e.g., "devlake" -> 8080). +// Returns empty map if parsing fails or service not found. +func extractServicePorts(composePath string, serviceNames ...string) map[string]int { + result := make(map[string]int) + + // Use docker compose config to parse the compose file reliably + cmd := exec.Command("docker", "compose", "-f", composePath, "config", "--format", "json") + output, err := cmd.Output() + if err != nil { + // Silently return empty map - caller will fall back to default behavior + return result + } + + var config struct { + Services map[string]struct { + Ports []struct { + Published string `json:"published,omitempty"` + Target int `json:"target,omitempty"` + } `json:"ports,omitempty"` + } `json:"services"` + } + + if err := json.Unmarshal(output, &config); err != nil { + return result + } + + // Extract host ports for requested services + for _, serviceName := range serviceNames { + service, exists := config.Services[serviceName] + if !exists { + continue + } + + // Find the first published port mapping + for _, port := range service.Ports { + if port.Published != "" { + // Published can be a port number as string + if hostPort, err := strconv.Atoi(port.Published); err == nil { + result[serviceName] = hostPort + break + } + } + } + } + + return result +} + +// inferCompanionURLs extracts and returns the actual Grafana and Config UI URLs from the compose file. +// Falls back to inferring from backend URL if extraction fails. +func inferCompanionURLs(backendURL string, composePath string) (grafanaURL, configUIURL string) { + // Try to extract actual ports from compose file + ports := extractServicePorts(composePath, "grafana", "config-ui") + + if grafanaPort, ok := ports["grafana"]; ok { + grafanaURL = fmt.Sprintf("http://localhost:%d", grafanaPort) + } + if configUIPort, ok := ports["config-ui"]; ok { + configUIURL = fmt.Sprintf("http://localhost:%d", configUIPort) + } + + // If extraction succeeded for both, return + if grafanaURL != "" && configUIURL != "" { + return grafanaURL, configUIURL + } + + // Fall back to inference from backend URL + if strings.Contains(backendURL, ":8085") { + if grafanaURL == "" { + grafanaURL = "http://localhost:3004" + } + if configUIURL == "" { + configUIURL = "http://localhost:4004" + } + } else { + if grafanaURL == "" { + grafanaURL = "http://localhost:3002" + } + if configUIURL == "" { + configUIURL = "http://localhost:4000" + } + } + + return grafanaURL, configUIURL +} diff --git a/cmd/deploy_local_test.go b/cmd/deploy_local_test.go index b575b10..6048e0c 100644 --- a/cmd/deploy_local_test.go +++ b/cmd/deploy_local_test.go @@ -1,6 +1,11 @@ package cmd -import "testing" +import ( + "os" + "path/filepath" + "strings" + "testing" +) func TestRewritePoetryInstallLine_RewritesInstallerLine(t *testing.T) { input := "FROM python:3.9-slim-bookworm\nRUN curl -sSL https://install.python-poetry.org | python3 -\n" @@ -38,3 +43,424 @@ func TestRewritePoetryInstallLine_NoChangeWhenLineMissing(t *testing.T) { t.Fatalf("content changed unexpectedly") } } + +func TestRewriteComposePorts(t *testing.T) { + tests := []struct { + name string + input string + wantContain []string + wantErr bool + }{ + { + name: "standard docker-compose format", + input: `version: '3' +services: + devlake: + ports: + - 8080:8080 + grafana: + ports: + - 3002:3002 + config-ui: + ports: + - 4000:4000 +`, + wantContain: []string{"8085:8080", "3004:3002", "4004:4000"}, + wantErr: false, + }, + { + name: "quoted port mappings", + input: `version: '3' +services: + devlake: + ports: + - "8080:8080" + grafana: + ports: + - "3002:3002" + config-ui: + ports: + - "4000:4000" +`, + wantContain: []string{"8085:8080", "3004:3002", "4004:4000"}, + wantErr: false, + }, + { + name: "single-quoted port mappings", + input: `version: '3' +services: + devlake: + ports: + - '8080:8080' + grafana: + ports: + - '3002:3002' + config-ui: + ports: + - '4000:4000' +`, + wantContain: []string{"8085:8080", "3004:3002", "4004:4000"}, + wantErr: false, + }, + { + name: "mixed port formats", + input: `version: '3' +services: + devlake: + ports: + - 8080:8080 + grafana: + ports: + - "3002:3002" + config-ui: + ports: + - '4000:4000' +`, + wantContain: []string{"8085:8080", "3004:3002", "4004:4000"}, + wantErr: false, + }, + { + name: "no matching ports", + input: `version: '3' +services: + mysql: + ports: + - 3306:3306 +`, + wantErr: true, + }, + { + name: "already rewritten ports (alternate bundle)", + input: `version: '3' +services: + devlake: + ports: + - 8085:8080 + grafana: + ports: + - 3004:3002 + config-ui: + ports: + - 4004:4000 +`, + wantErr: true, // No changes made + }, + { + name: "custom host port should not be rewritten", + input: `version: '3' +services: + devlake: + ports: + - 18080:8080 + grafana: + ports: + - 13002:3002 + config-ui: + ports: + - 14000:4000 +`, + wantErr: true, // No matching ports to rewrite + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Create temp file + tmpDir := t.TempDir() + composePath := filepath.Join(tmpDir, "docker-compose.yml") + if err := os.WriteFile(composePath, []byte(tt.input), 0644); err != nil { + t.Fatalf("Failed to write test compose file: %v", err) + } + + // Run rewrite + err := rewriteComposePorts(composePath) + + if tt.wantErr { + if err == nil { + t.Errorf("rewriteComposePorts() expected error, got nil") + } + return + } + + if err != nil { + t.Errorf("rewriteComposePorts() unexpected error: %v", err) + return + } + + // Read result + result, err := os.ReadFile(composePath) + if err != nil { + t.Fatalf("Failed to read result: %v", err) + } + resultStr := string(result) + + // Check expected content + for _, want := range tt.wantContain { + if !strings.Contains(resultStr, want) { + t.Errorf("rewriteComposePorts() result missing %q\nResult:\n%s", want, resultStr) + } + } + + // Ensure old ports are gone + oldPorts := []string{"8080:8080", "3002:3002", "4000:4000"} + for _, old := range oldPorts { + if strings.Contains(resultStr, old) { + t.Errorf("rewriteComposePorts() result still contains old port %q", old) + } + } + }) + } +} + +func TestRewriteComposePorts_FileNotFound(t *testing.T) { + err := rewriteComposePorts("/nonexistent/path/docker-compose.yml") + if err == nil { + t.Error("rewriteComposePorts() expected error for nonexistent file, got nil") + } + if !strings.Contains(err.Error(), "reading compose file") { + t.Errorf("rewriteComposePorts() error = %v, want error about reading compose file", err) + } +} + +func TestDetectPortBundle(t *testing.T) { + tests := []struct { + name string + input string + want portBundle + }{ + { + name: "default port bundle", + input: `version: '3' +services: + devlake: + ports: + - 8080:8080 + grafana: + ports: + - 3002:3002 + config-ui: + ports: + - 4000:4000 +`, + want: portBundleDefault, + }, + { + name: "alternate port bundle", + input: `version: '3' +services: + devlake: + ports: + - 8085:8080 + grafana: + ports: + - 3004:3002 + config-ui: + ports: + - 4004:4000 +`, + want: portBundleAlternate, + }, + { + name: "custom port bundle", + input: `version: '3' +services: + devlake: + ports: + - 18080:8080 + grafana: + ports: + - 13002:3002 + config-ui: + ports: + - 14000:4000 +`, + want: portBundleCustom, + }, + { + name: "mixed custom and unrelated ports", + input: `version: '3' +services: + mysql: + ports: + - 3306:3306 + devlake: + ports: + - 9090:8080 +`, + want: portBundleCustom, + }, + { + name: "partial default bundle (has at least one default port)", + input: `version: '3' +services: + devlake: + ports: + - 8080:8080 +`, + want: portBundleDefault, + }, + { + name: "partial alternate bundle (has at least one alternate port)", + input: `version: '3' +services: + devlake: + ports: + - 8085:8080 +`, + want: portBundleAlternate, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Create temp file + tmpDir := t.TempDir() + composePath := filepath.Join(tmpDir, "docker-compose.yml") + if err := os.WriteFile(composePath, []byte(tt.input), 0644); err != nil { + t.Fatalf("Failed to write test compose file: %v", err) + } + + got := detectPortBundle(composePath) + if got != tt.want { + t.Errorf("detectPortBundle() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestDetectPortBundle_FileNotFound(t *testing.T) { + got := detectPortBundle("/nonexistent/path/docker-compose.yml") + if got != portBundleDefault { + t.Errorf("detectPortBundle() for nonexistent file = %v, want %v (default)", got, portBundleDefault) + } +} + +func TestExtractServicePorts_MissingFile(t *testing.T) { + // Should return empty map for non-existent file + ports := extractServicePorts("/nonexistent/docker-compose.yml", "devlake") + if len(ports) != 0 { + t.Errorf("extractServicePorts() for nonexistent file returned %v, want empty map", ports) + } +} + +func TestFindComposeFile(t *testing.T) { + // Test that findComposeFile checks filenames in the correct order + tests := []struct { + name string + filesToCreate []string + wantFilename string // Just the filename, not full path + }{ + { + name: "compose.yaml has highest precedence", + filesToCreate: []string{"compose.yaml", "compose.yml", "docker-compose.yaml", "docker-compose.yml"}, + wantFilename: "compose.yaml", + }, + { + name: "compose.yml has second precedence", + filesToCreate: []string{"compose.yml", "docker-compose.yaml", "docker-compose.yml"}, + wantFilename: "compose.yml", + }, + { + name: "docker-compose.yaml has third precedence", + filesToCreate: []string{"docker-compose.yaml", "docker-compose.yml"}, + wantFilename: "docker-compose.yaml", + }, + { + name: "docker-compose.yml has fourth precedence", + filesToCreate: []string{"docker-compose.yml"}, + wantFilename: "docker-compose.yml", + }, + { + name: "docker-compose-dev.yml is fallback for introspection", + filesToCreate: []string{"docker-compose-dev.yml"}, + wantFilename: "docker-compose-dev.yml", + }, + { + name: "returns empty string when no compose file found", + filesToCreate: []string{}, + wantFilename: "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tmpDir := t.TempDir() + + // Create test files + for _, filename := range tt.filesToCreate { + filePath := filepath.Join(tmpDir, filename) + if err := os.WriteFile(filePath, []byte("version: '3'\n"), 0644); err != nil { + t.Fatalf("Failed to create test file %s: %v", filename, err) + } + } + + got := findComposeFile(tmpDir) + var gotFilename string + if got != "" { + gotFilename = filepath.Base(got) + } + + if gotFilename != tt.wantFilename { + t.Errorf("findComposeFile() returned %q, want %q", gotFilename, tt.wantFilename) + } + }) + } +} + +func TestNewDeployLocalCmd(t *testing.T) { + cmd := newDeployLocalCmd() + + if cmd.Use != "local" { + t.Errorf("expected Use 'local', got %q", cmd.Use) + } + if cmd.Short != "Deploy DevLake locally via Docker Compose" { + t.Errorf("unexpected Short: %q", cmd.Short) + } + if !strings.Contains(cmd.Long, "alternate ports (8085/3004/4004)") { + t.Errorf("expected Long help to mention alternate port fallback, got: %q", cmd.Long) + } + + for _, flag := range []string{"dir", "version", "source", "repo-url", "start"} { + if cmd.Flags().Lookup(flag) == nil { + t.Errorf("expected --%s flag to be registered", flag) + } + } + + if got := cmd.Flags().Lookup("start").DefValue; got != "true" { + t.Errorf("expected --start default true, got %q", got) + } +} + +func TestInferCompanionURLs_FallbackFromBackendURL(t *testing.T) { + tests := []struct { + name string + backendURL string + wantGrafana string + wantConfigUI string + }{ + { + name: "default bundle", + backendURL: "http://localhost:8080", + wantGrafana: "http://localhost:3002", + wantConfigUI: "http://localhost:4000", + }, + { + name: "alternate bundle", + backendURL: "http://localhost:8085", + wantGrafana: "http://localhost:3004", + wantConfigUI: "http://localhost:4004", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + grafanaURL, configUIURL := inferCompanionURLs(tt.backendURL, "/nonexistent/docker-compose.yml") + if grafanaURL != tt.wantGrafana { + t.Errorf("grafanaURL = %q, want %q", grafanaURL, tt.wantGrafana) + } + if configUIURL != tt.wantConfigUI { + t.Errorf("configUIURL = %q, want %q", configUIURL, tt.wantConfigUI) + } + }) + } +} diff --git a/cmd/start.go b/cmd/start.go index 0aecde1..cef3ca8 100644 --- a/cmd/start.go +++ b/cmd/start.go @@ -116,9 +116,17 @@ func detectStartMode() string { if _, err := os.Stat(".devlake-local.json"); err == nil { return "local" } - // Fall back to docker-compose.yml in cwd. - if _, err := os.Stat("docker-compose.yml"); err == nil { - return "local" + // Fall back to Docker Compose default filenames in precedence order + composeFiles := []string{ + "compose.yaml", + "compose.yml", + "docker-compose.yaml", + "docker-compose.yml", + } + for _, filename := range composeFiles { + if _, err := os.Stat(filename); err == nil { + return "local" + } } return "" } diff --git a/cmd/stop.go b/cmd/stop.go index 7fdc99c..ec33754 100644 --- a/cmd/stop.go +++ b/cmd/stop.go @@ -99,9 +99,17 @@ func detectStopMode() string { if _, err := os.Stat(".devlake-local.json"); err == nil { return "local" } - // Fall back to docker-compose.yml in cwd. - if _, err := os.Stat("docker-compose.yml"); err == nil { - return "local" + // Fall back to Docker Compose default filenames in precedence order + composeFiles := []string{ + "compose.yaml", + "compose.yml", + "docker-compose.yaml", + "docker-compose.yml", + } + for _, filename := range composeFiles { + if _, err := os.Stat(filename); err == nil { + return "local" + } } return "" } diff --git a/docs/deploy.md b/docs/deploy.md index 32e4b81..85ab807 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -4,7 +4,7 @@ Deploy DevLake locally via Docker Compose or to Azure via Bicep. ## deploy local -Downloads the official Apache DevLake Docker Compose files, generates an `ENCRYPTION_SECRET`, and prepares the directory for `docker compose up`. +Sets up DevLake Docker Compose files and starts containers (by default). Supports official Apache releases, forked repositories, or custom compose configurations. ### Usage @@ -18,31 +18,59 @@ gh devlake deploy local [flags] |------|---------|-------------| | `--dir` | `.` | Target directory for Docker Compose files | | `--version` | `latest` | DevLake release version (e.g., `v1.0.2`) | +| `--source` | *(interactive if omitted)* | Image source: `official`, `fork`, or `custom` | +| `--repo-url` | | Repository URL to clone (for `fork` source) | +| `--start` | `true` | Start containers after setup | ### What It Does +Depending on `--source`: + +**official** (default): 1. Fetches the latest release tag from GitHub (or uses `--version`) 2. Downloads `docker-compose.yml` and `env.example` from the Apache DevLake release 3. Renames `env.example` → `.env` 4. Generates and injects a cryptographic `ENCRYPTION_SECRET` into `.env` 5. Checks that Docker is available +6. Starts containers (unless `--start=false`) + +**fork**: +1. Clones the repository specified by `--repo-url` +2. Builds DevLake images from source +3. Generates `.env` with `ENCRYPTION_SECRET` +4. Checks that Docker is available +5. Starts containers (unless `--start=false`) + +**custom**: +1. Uses an existing `docker-compose.yml` file in the target directory +2. Generates or updates `.env` with `ENCRYPTION_SECRET` if needed +3. Checks that Docker is available +4. Starts containers (unless `--start=false`) + + **Note**: In custom mode, the CLI currently expects `docker-compose.yml`. Development-specific compose files like `docker-compose-dev.yml` are only used for introspection (port detection/UX output) and are not modified or started automatically; if you only have a dev compose file, either rename it to `docker-compose.yml` or start containers manually with `docker compose -f docker-compose-dev.yml up -d`. ### After Running +Containers start automatically by default. Service endpoints are available immediately (wait ~2–3 minutes for all services to initialize). + +If you staged files without starting containers (`--start=false`), manually start them with: + ```bash cd docker compose up -d ``` -Wait ~2–3 minutes for all services to start. - ### Service Endpoints (local) | Service | URL | Default Credentials | |---------|-----|---------------------| -| Backend API | http://localhost:8080 | — | -| Config UI | http://localhost:4000 | — | -| Grafana | http://localhost:3002 | admin / admin | +| Backend API | http://localhost:8080 or http://localhost:8085 | — | +| Config UI | http://localhost:4000 or http://localhost:4004 | — | +| Grafana | http://localhost:3002 or http://localhost:3004 | admin / admin | + +**Port Fallback**: When deploying with `--source official` or `--source fork`, the CLI automatically recovers from port conflicts by retrying with alternate ports (`8085/3004/4004`). As part of this recovery, it updates the port mappings in the detected compose file on disk (creating a backup at `.bak` first) so that future `docker compose up` runs use the fallback ports. The CLI can auto-modify any of the standard compose filenames (`compose.yaml`, `compose.yml`, `docker-compose.yaml`, `docker-compose.yml`). Custom deployments (`--source custom`) with these filenames also benefit from automatic port fallback; other custom compose files (e.g., `docker-compose-dev.yml`) require manual port conflict resolution. + +To revert fallback changes, restore from the backup file (e.g., `docker-compose.yml.bak` or `compose.yml.bak`), edit your compose file to change the mapped ports back to your preferred values (e.g., `8080/3002/4000`), or re-run the deployment in a clean directory. ### Examples @@ -53,17 +81,22 @@ gh devlake deploy local # Deploy a specific version to ./devlake gh devlake deploy local --version v1.0.2 --dir ./devlake -# Then start the services -cd devlake -docker compose up -d +# Stage files without starting containers +gh devlake deploy local --start=false ``` ### Notes - If `.env` already exists in the target directory, it is backed up to `.env.bak` before being replaced. -- `docker compose up` is NOT run automatically — this lets you inspect or customize `.env` first. - To tear down: `gh devlake cleanup --local` or `docker compose down` from the target directory. +#### Deployment Resilience + +The CLI includes bounded recovery for common Docker errors: + +- **Port conflicts**: When deploying with `--source official` or `--source fork`, the CLI detects port conflicts (patterns: `port is already allocated`, `bind for`, `ports are not available`, `address already in use`, `failed programming external connectivity`) and automatically retries with alternate ports (`8085/3004/4004`). Recovery is bounded to a single retry. +- **Custom deployments**: Port conflicts in `--source custom` deployments require manual resolution — the CLI will identify the conflicting container and suggest remediation commands. + --- ## deploy azure @@ -89,13 +122,15 @@ gh devlake deploy azure [flags] ### What It Does -1. Checks Azure CLI login (runs `az login` if needed) +1. Checks Azure CLI login (runs `az login` if needed — **bounded recovery**) 2. Creates the resource group (saves partial state immediately for safe cleanup) 3. Generates MySQL password and encryption secret via Key Vault 4. Optionally builds Docker images and pushes to Azure Container Registry (when `--official` is not set) -5. Deploys infrastructure via Bicep templates (Container Instances + MySQL + Key Vault) -6. Waits for the backend to respond, then triggers DB migration -7. Saves `.devlake-azure.json` state file with endpoints, resource names, and subscription info +5. Checks for stopped MySQL servers and starts them (**bounded recovery**) +6. Checks for soft-deleted Key Vaults and purges them before deployment (**bounded recovery**) +7. Deploys infrastructure via Bicep templates (Container Instances + MySQL + Key Vault) +8. Waits for the backend to respond, then triggers DB migration +9. Saves `.devlake-azure.json` state file with endpoints, resource names, and subscription info ### Cost Estimate @@ -128,6 +163,17 @@ gh devlake deploy azure - Service endpoints are printed at the end of a successful deployment and saved to `.devlake-azure.json`. - The Bicep templates are embedded in the binary — no external template files needed. +#### Azure Deployment Resilience + +The CLI includes bounded recovery for known Azure failure modes: + +- **Missing authentication**: Automatically runs `az login` when not logged in (single attempt). +- **Stopped MySQL servers**: Detects stopped MySQL Flexible Servers and starts them before deployment (single attempt, non-fatal). +- **Soft-deleted Key Vaults**: Detects and purges soft-deleted Key Vaults that conflict with the deployment (single attempt). +- **State checkpointing**: Partial state file is written immediately after Resource Group creation to enable cleanup even when deployment fails mid-flight. + +All recovery actions are bounded to a single retry and report clear detection → repair → outcome messages. + ### Tear Down ```bash diff --git a/docs/start.md b/docs/start.md index 0e3c8f0..64f8ce3 100644 --- a/docs/start.md +++ b/docs/start.md @@ -26,7 +26,7 @@ Without `--local` or `--azure`, the command checks: 1. `--state-file` path (if provided) 2. `.devlake-azure.json` → Azure mode 3. `.devlake-local.json` → Local mode -4. `docker-compose.yml` in current directory → Local mode +4. Standard compose file in current directory → Local mode (searches for `compose.yaml`, `compose.yml`, `docker-compose.yaml`, `docker-compose.yml` in that order) If no deployment is detected, an error is returned — use `gh devlake deploy` to create a new deployment. diff --git a/docs/state-files.md b/docs/state-files.md index aadd38f..1faec58 100644 --- a/docs/state-files.md +++ b/docs/state-files.md @@ -37,11 +37,11 @@ The CLI finds the DevLake API endpoint using this priority: |----------|--------| | 1 | `--url` flag (explicit) | | 2 | State file in the current directory (`.devlake-azure.json` → `.devlake-local.json`) | -| 3 | Well-known local ports (`http://localhost:8080`) | +| 3 | Well-known local ports (`http://localhost:8080` or `http://localhost:8085`) | ## Location -State files are written to the **current working directory** when the command runs. Run your commands from the same directory (typically the one where you ran `deploy local` or `deploy azure`), or use `--url` to bypass state-based discovery. +State files are written to the **current working directory** when the command runs. Run your commands from the same directory (typically the one where you ran `deploy local` or `deploy azure`), or use `--url` to bypass state-based discovery when DevLake is instead reachable on the well-known local endpoints (`http://localhost:8080` or `http://localhost:8085`). ## Cleanup diff --git a/docs/status.md b/docs/status.md index 757c63d..fcf3137 100644 --- a/docs/status.md +++ b/docs/status.md @@ -59,7 +59,7 @@ gh devlake status [--url ] | āš ļø (code) | Unexpected HTTP status | | āŒ | Connection refused or timeout | -Grafana is checked at `/api/health`. Backend and Config UI are checked at their root URL. +Backend is checked at `/ping`, Grafana at `/api/health`, and Config UI at its root URL. When auto-discovery lands on `http://localhost:8080`, companion URLs infer to Grafana → `http://localhost:3002`, Config UI → `http://localhost:4000`. When it lands on `http://localhost:8085`, companion URLs infer to Grafana → `http://localhost:3004`, Config UI → `http://localhost:4004`. **Connections** — loaded from the state file. Shows plugin name, connection ID, display name, and org. @@ -76,6 +76,13 @@ If no state file is found but DevLake responds at a well-known port: Run 'gh devlake configure full' to set up connections. ``` +Or, when the alternate local bundle is active: + +``` + āœ… DevLake reachable at http://localhost:8085 + Run 'gh devlake configure full' to set up connections. +``` + ### No state file, DevLake unreachable ``` @@ -101,7 +108,7 @@ If no state file is found but DevLake responds at a well-known port: ## Examples ```bash -# Auto-discover from state file or localhost +# Auto-discover from state file or localhost (8080 or 8085) gh devlake status # Target a specific instance diff --git a/docs/stop.md b/docs/stop.md index 6ce73d8..9f8c53a 100644 --- a/docs/stop.md +++ b/docs/stop.md @@ -25,7 +25,7 @@ Without `--local` or `--azure`, the command checks: 1. `--state-file` path (if provided) 2. `.devlake-azure.json` → Azure mode 3. `.devlake-local.json` → Local mode -4. `docker-compose.yml` in current directory → Local mode +4. Standard compose file in current directory → Local mode (searches for `compose.yaml`, `compose.yml`, `docker-compose.yaml`, `docker-compose.yml` in that order) If no deployment is detected, an error is returned — use `gh devlake deploy` to create a new deployment.