Switch from callbacks to webhooks for repos (#2466)

* Agent task to move from callback to webhooks for repo syncing

* Move from callback to webhook for repos

* Add docs for local development

* Docs: UI as gateway for local dev; github app

* Not require code in callback

* Add retry logic to installation link fetching

Author: ZIJ

agent-tasks/github-webhook-repo-sync-plan.md

@@ -0,0 +1,43 @@
+# GitHub repo sync via webhooks
+
+## Goals
+- Move repo sync away from the OAuth callback and drive updates from GitHub webhooks.
+- Keep Digger’s repo list accurate when repos are added/removed from the app scope.
+- On uninstall, soft-delete repos (and their installation records) so they disappear from the UI/API.
+
+## Current behavior (source of truth today)
+- OAuth callback (`backend/controllers/github_callback.go`) validates the install, links/creates org, then lists all repos via `Apps.ListRepos`, soft-deletes existing `github_app_installations` and repos for the org, and recreates them via `GithubRepoAdded` + `createOrGetDiggerRepoForGithubRepo`.
+- Webhook handler (`backend/controllers/github.go`) only uses `installation` events with action `deleted` to mark installation links inactive and set `github_app_installations` status deleted for the repos in the payload. It does not touch `repos`. There is no handling for `installation_repositories` add/remove.
+- Runtime lookups (`GetGithubService` / `GetGithubClient`) require an active record in `github_app_installations` for the repo.
+
+## Target design
+- Keep OAuth callback minimal: verify installation, create/link org, store the install id/app id, but do **not** list or mutate repos. It should return immediately and rely on webhooks for repo population.
+- Webhook-driven reconciliation:
+  - `installation` event (`created`, `unsuspended`, `new_permissions_accepted`): ensure installation link exists/active; reconcile repos using the payload’s `installation.repositories` list as authoritative. If the link is missing, log an error and return (no auto-create).
+    - Soft-delete existing `github_app_installations` for that installation id, and soft-delete repos for the linked org (scoped to that installation) before re-adding.
+    - Upsert each repo: mark/install via `GithubRepoAdded` and create/restore the Digger repo record (store app id, installation id, default branch, clone URL when available).
+  - `installation_repositories` event: incrementally apply scope changes.
+    - For `repositories_added`: fetch repo details (to get default branch + clone URL), then call `GithubRepoAdded` and create/restore the repo record.
+    - For `repositories_removed`: mark `GithubRepoRemoved`, soft-delete the repo **and its projects**, and handle absence gracefully.
+  - `installation` event (`deleted`): mark installation link inactive, mark installation records deleted, and soft-delete repos **and projects** for that installation’s org so they no longer appear in APIs/UI.
+- Shared helpers:
+  - `syncReposForInstallation(installationId, appId, reposPayload)` to wrap the add/remove logic and reuse between `installation` and `installation_repositories` handlers.
+  - `softDeleteRepoAndProjects(orgId, repoFullName)` to encapsulate repo + project soft-deletion.
+- Observability: structured logs per action, and possibly a metric for sync success/failure per installation.
+
+## Migration plan
+1) Add webhook handling for `installation_repositories` in `GithubAppWebHook` switch and wire to a new handler.
+2) Extend `installation` handling to cover `created`/`unsuspended` (not just `deleted`) and call `syncReposForInstallation`.
+3) Update uninstall handling to also soft-delete repos and projects.
+4) Strip repo enumeration/deletion from the OAuth callback; leave only installation/org linking.
+5) Add tests using existing payload fixtures (`installationRepositoriesAddedPayload`, `installationRepositoriesDeletedPayload`, `installationCreatedEvent`) to verify DB state changes (installation records + repos soft-delete/restore).
+6) Backfill existing installations: one-off job/command or admin endpoint to resync repos via `Apps.ListRepos` and `syncReposForInstallation` to align data after deploying (manual trigger, no cron yet).
+
+## Testing / validation
+- Unit tests for add/remove/uninstall flows verifying:
+  - `github_app_installations` status transitions.
+  - Repos are created/restored with correct installation/app ids.
+  - Repos and projects are soft-deleted on removal/uninstall.
+
+## Open questions
+- None right now (decided: log missing-link errors only; soft-delete repos and projects on removal/uninstall; add manual resync endpoint, no cron yet).

backend/bootstrap/main.go

@@ -242,6 +242,7 @@ func Bootstrap(templates embed.FS, diggerController controllers.DiggerController
 
 		githubApiGroup := apiGroup.Group("/github")
 		githubApiGroup.POST("/link", controllers.LinkGithubInstallationToOrgApi)
+		githubApiGroup.POST("/resync", controllers.ResyncGithubInstallationApi)
 
 		vcsApiGroup := apiGroup.Group("/connections")
 		vcsApiGroup.GET("/:id", controllers.GetVCSConnection)

backend/controllers/github.go

@@ -73,6 +73,24 @@ func (d DiggerController) GithubAppWebHook(c *gin.Context) {
 				c.String(http.StatusAccepted, "Failed to handle webhook event.")
 				return
 			}
+		} else if *event.Action == "created" || *event.Action == "unsuspended" || *event.Action == "new_permissions_accepted" {
+			if err := handleInstallationUpsertEvent(c.Request.Context(), gh, event, appId64); err != nil {
+				slog.Error("Failed to handle installation upsert event", "error", err)
+				c.String(http.StatusAccepted, "Failed to handle webhook event.")
+				return
+			}
+		}
+	case *github.InstallationRepositoriesEvent:
+		slog.Info("Processing InstallationRepositoriesEvent",
+			"action", event.GetAction(),
+			"installationId", event.Installation.GetID(),
+			"added", len(event.RepositoriesAdded),
+			"removed", len(event.RepositoriesRemoved),
+		)
+		if err := handleInstallationRepositoriesEvent(c.Request.Context(), gh, event, appId64); err != nil {
+			slog.Error("Failed to handle installation repositories event", "error", err)
+			c.String(http.StatusAccepted, "Failed to handle webhook event.")
+			return
 		}
 	case *github.PushEvent:
 		slog.Info("Processing PushEvent",

backend/controllers/github_api.go

@@ -8,7 +8,10 @@ import (
 
 	"github.com/diggerhq/digger/backend/middleware"
 	"github.com/diggerhq/digger/backend/models"
+	"github.com/diggerhq/digger/backend/utils"
+	ci_github "github.com/diggerhq/digger/libs/ci/github"
 	"github.com/gin-gonic/gin"
+	"github.com/google/go-github/v61/github"
 	"gorm.io/gorm"
 )
 
@@ -85,3 +88,82 @@ func LinkGithubInstallationToOrgApi(c *gin.Context) {
 	c.JSON(http.StatusOK, gin.H{"status": "Successfully created Github installation link"})
 	return
 }
+
+func ResyncGithubInstallationApi(c *gin.Context) {
+	type ResyncInstallationRequest struct {
+		InstallationId string `json:"installation_id"`
+	}
+
+	var request ResyncInstallationRequest
+	if err := c.BindJSON(&request); err != nil {
+		slog.Error("Error binding JSON for resync", "error", err)
+		c.JSON(http.StatusBadRequest, gin.H{"status": "Invalid request format"})
+		return
+	}
+
+	installationId, err := strconv.ParseInt(request.InstallationId, 10, 64)
+	if err != nil {
+		slog.Error("Failed to convert InstallationId to int64", "installationId", request.InstallationId, "error", err)
+		c.JSON(http.StatusBadRequest, gin.H{"status": "installationID should be a valid integer"})
+		return
+	}
+
+	link, err := models.DB.GetGithubAppInstallationLink(installationId)
+	if err != nil {
+		slog.Error("Could not get installation link for resync", "installationId", installationId, "error", err)
+		c.JSON(http.StatusInternalServerError, gin.H{"status": "Could not get installation link"})
+		return
+	}
+	if link == nil {
+		slog.Warn("Installation link not found for resync", "installationId", installationId)
+		c.JSON(http.StatusNotFound, gin.H{"status": "Installation link not found"})
+		return
+	}
+
+	var installationRecord models.GithubAppInstallation
+	if err := models.DB.GormDB.Where("github_installation_id = ?", installationId).Order("updated_at desc").First(&installationRecord).Error; err != nil {
+		if errors.Is(err, gorm.ErrRecordNotFound) {
+			slog.Warn("No installation records found for resync", "installationId", installationId)
+			c.JSON(http.StatusNotFound, gin.H{"status": "No installation records found"})
+			return
+		}
+		slog.Error("Failed to fetch installation record for resync", "installationId", installationId, "error", err)
+		c.JSON(http.StatusInternalServerError, gin.H{"status": "Could not fetch installation records"})
+		return
+	}
+
+	appId := installationRecord.GithubAppId
+	ghProvider := utils.DiggerGithubRealClientProvider{}
+
+	client, _, err := ghProvider.Get(appId, installationId)
+	if err != nil {
+		slog.Error("Failed to create GitHub client for resync", "installationId", installationId, "appId", appId, "error", err)
+		c.JSON(http.StatusInternalServerError, gin.H{"status": "Failed to create GitHub client"})
+		return
+	}
+
+	repos, err := ci_github.ListGithubRepos(client)
+	if err != nil {
+		slog.Error("Failed to list repos for resync", "installationId", installationId, "error", err)
+		c.JSON(http.StatusInternalServerError, gin.H{"status": "Failed to list repos for resync"})
+		return
+	}
+
+	installationPayload := &github.Installation{
+		ID:    github.Int64(installationId),
+		AppID: github.Int64(appId),
+	}
+	resyncEvent := &github.InstallationEvent{
+		Installation: installationPayload,
+		Repositories: repos,
+	}
+
+	if err := handleInstallationUpsertEvent(c.Request.Context(), ghProvider, resyncEvent, appId); err != nil {
+		slog.Error("Resync failed", "installationId", installationId, "error", err)
+		c.JSON(http.StatusInternalServerError, gin.H{"status": "Resync failed"})
+		return
+	}
+
+	slog.Info("Resync completed", "installationId", installationId, "repoCount", len(repos))
+	c.JSON(http.StatusOK, gin.H{"status": "Resync completed", "repoCount": len(repos)})
+}