feat(github): connect commit SHAs in traces to GitHub via a GitHub App

.gitignore

@@ -43,3 +43,6 @@ playwright/.cache/
 /apps/ingest/apps/ingest/.wal
 /apps/ingest/target
 /apps/ingest/.wal
+
+# local-only Drizzle Studio config pointing at miniflare D1
+drizzle.config.local.ts

apps/api/alchemy.run.ts

@@ -1,6 +1,6 @@
 import path from "node:path"
 import alchemy from "alchemy"
-import { D1Database, KVNamespace, Worker, Workflow } from "alchemy/cloudflare"
+import { D1Database, KVNamespace, Queue, Worker, Workflow } from "alchemy/cloudflare"
 import type { MapleDomains, MapleStage } from "@maple/infra/cloudflare"
 import { resolveD1Name, resolveDeploymentEnvironment, resolveWorkerName } from "@maple/infra/cloudflare"
 
@@ -62,6 +62,19 @@ export const createMapleApi = async ({ stage, domains }: CreateMapleApiOptions)
 		className: "AiTriageWorkflow",
 	})
 
+	// Vendor-agnostic VCS sync queue (commit backfill + webhook deltas). The same
+	// `api` worker is both producer (binding) and consumer (eventSources). Local
+	// dev is wired separately in wrangler.jsonc so miniflare runs it in-process.
+	const vcsSyncDlq = await Queue("vcs-sync-dlq", {
+		name: resolveWorkerName("vcs-sync-dlq", stage),
+		adopt: true,
+	})
+	const vcsSyncQueue = await Queue("vcs-sync", {
+		name: resolveWorkerName("vcs-sync", stage),
+		adopt: true,
+		dlq: vcsSyncDlq,
+	})
+
 	const worker = await Worker("api", {
 		name: resolveWorkerName("api", stage),
 		cwd: import.meta.dirname,
@@ -71,9 +84,22 @@ export const createMapleApi = async ({ stage, domains }: CreateMapleApiOptions)
 		url: true,
 		adopt: true,
 		routes: domains.api ? [{ pattern: `${domains.api}/*`, adopt: true }] : undefined,
+		eventSources: [
+			{
+				queue: vcsSyncQueue,
+				settings: {
+					batchSize: 10,
+					maxConcurrency: 2,
+					maxRetries: 3,
+					maxWaitTimeMs: 5000,
+					deadLetterQueue: vcsSyncDlq,
+				},
+			},
+		],
 		bindings: {
 			MAPLE_DB: mapleDb,
 			MCP_SESSIONS: mcpSessions,
+			VCS_SYNC_QUEUE: vcsSyncQueue,
 			CLICKHOUSE_SCHEMA_APPLY_WORKFLOW: schemaApplyWorkflow,
 			AI_TRIAGE_WORKFLOW: aiTriageWorkflow,
 			TINYBIRD_HOST: requireEnv("TINYBIRD_HOST"),
@@ -112,6 +138,12 @@ export const createMapleApi = async ({ stage, domains }: CreateMapleApiOptions)
 			...optionalPlain("HAZEL_OAUTH_CLIENT_ID"),
 			...optionalSecret("HAZEL_OAUTH_CLIENT_SECRET"),
 			...optionalPlain("HAZEL_OAUTH_SCOPES"),
+			...optionalPlain("GITHUB_APP_ID"),
+			...optionalSecret("GITHUB_APP_PRIVATE_KEY"),
+			...optionalPlain("GITHUB_APP_CLIENT_ID"),
+			...optionalSecret("GITHUB_APP_CLIENT_SECRET"),
+			...optionalSecret("GITHUB_APP_WEBHOOK_SECRET"),
+			...optionalPlain("GITHUB_API_BASE_URL"),
 		},
 	})
 

apps/api/src/app.ts

@@ -25,6 +25,7 @@ import { HttpOrgClickHouseSettingsLive } from "./routes/org-clickhouse-settings.
 import { HttpOrganizationsLive } from "./routes/organizations.http"
 import { PrometheusScrapeProxyRouter } from "./routes/prometheus-scrape-proxy.http"
 import { ScraperInternalRouter } from "./routes/scraper-internal.http"
+import { VcsWebhookRouter } from "./routes/vcs-webhook.http"
 import { HttpQueryEngineLive } from "./routes/query-engine.http"
 import { HttpRecommendationIssuesLive } from "./routes/recommendation-issues.http"
 import { HttpScrapeTargetsLive } from "./routes/scrape-targets.http"
@@ -59,6 +60,14 @@ import { RawSqlChartService } from "@maple/query-engine/runtime"
 import { PlanetScaleDiscoveryService } from "./services/PlanetScaleDiscoveryService"
 import { ScrapeTargetsService } from "./services/ScrapeTargetsService"
 import { WarehouseQueryService } from "./lib/WarehouseQueryService"
+import { OAuthStateRepository } from "./services/OAuthStateRepository"
+import { GithubAppClient } from "./services/github/GithubAppClient"
+import { GithubConnectService } from "./services/github/GithubConnectService"
+import { GithubHttp } from "./services/github/GithubHttp"
+import { GithubProvider } from "./services/github/GithubProvider"
+import { VcsProviderRegistry } from "./services/vcs/VcsProviderRegistry"
+import { VcsRepository } from "./services/vcs/VcsRepository"
+import { VcsSyncQueue } from "./services/vcs/VcsSyncQueue"
 
 export const HealthRouter = HttpRouter.use((router) =>
 	router.add("GET", "/health", HttpServerResponse.text("OK")),
@@ -152,6 +161,24 @@ export const DigestServiceLive = DigestService.layer.pipe(
 	Layer.provideMerge(Layer.mergeAll(InfraLive, WarehouseQueryServiceLive, EmailServiceLive)),
 )
 
+// Vendor-agnostic VCS services for the fetch path: the webhook router needs the
+// provider registry + sync queue; the repo + state repo are ready for the
+// Step-2 settings endpoints. The sync orchestrator (VcsSyncService) lives only
+// in the queue-consumer runtime (vcs-sync-runtime.ts), not here. Database /
+// WorkerEnvironment are satisfied at worker scope (like CoreServicesLive).
+const GithubAppClientLive = GithubAppClient.layer.pipe(Layer.provide(GithubHttp.layer))
+const GithubProviderLive = GithubProvider.layer.pipe(Layer.provide(GithubAppClientLive))
+
+const VcsDataLive = Layer.mergeAll(VcsRepository.layer, OAuthStateRepository.layer, VcsSyncQueue.layer)
+
+export const VcsServicesLive = Layer.mergeAll(
+	VcsDataLive,
+	VcsProviderRegistry.layer.pipe(Layer.provide(GithubProviderLive)),
+	// The dashboard connect flow: needs the repo + state repo + sync queue
+	// (VcsDataLive) plus the GitHub App client (App-JWT installation lookup).
+	GithubConnectService.layer.pipe(Layer.provide(Layer.mergeAll(VcsDataLive, GithubAppClientLive))),
+).pipe(Layer.provideMerge(InfraLive))
+
 export const MainLive = Layer.mergeAll(
 	CoreServicesLive,
 	WarehouseQueryServiceLive,
@@ -163,6 +190,7 @@ export const MainLive = Layer.mergeAll(
 	RecommendationIssueServiceLive,
 	DigestServiceLive,
 	DemoServiceLive,
+	VcsServicesLive,
 	RawSqlChartService.layer,
 )
 
@@ -203,6 +231,7 @@ export const AllRoutes = Layer.mergeAll(
 	OAuthDiscoveryRouter,
 	PrometheusScrapeProxyRouter,
 	ScraperInternalRouter,
+	VcsWebhookRouter,
 	McpLive,
 	HealthRouter,
 	McpGetFallback,

apps/api/src/lib/Env.ts

@@ -37,6 +37,13 @@ export interface EnvShape {
 	readonly HAZEL_OAUTH_CLIENT_ID: Option.Option<string>
 	readonly HAZEL_OAUTH_CLIENT_SECRET: Option.Option<Redacted.Redacted<string>>
 	readonly HAZEL_OAUTH_SCOPES: string
+	readonly GITHUB_APP_ID: Option.Option<string>
+	readonly GITHUB_APP_SLUG: Option.Option<string>
+	readonly GITHUB_APP_PRIVATE_KEY: Option.Option<Redacted.Redacted<string>>
+	readonly GITHUB_APP_CLIENT_ID: Option.Option<string>
+	readonly GITHUB_APP_CLIENT_SECRET: Option.Option<Redacted.Redacted<string>>
+	readonly GITHUB_APP_WEBHOOK_SECRET: Option.Option<Redacted.Redacted<string>>
+	readonly GITHUB_API_BASE_URL: string
 }
 
 const stringWithDefault = (key: string, fallback: string) =>
@@ -95,6 +102,13 @@ const envConfig = Config.all({
 		"HAZEL_OAUTH_SCOPES",
 		"openid email profile organizations:read channels:read channel-webhooks:write",
 	),
+	GITHUB_APP_ID: optionalString("GITHUB_APP_ID"),
+	GITHUB_APP_SLUG: optionalString("GITHUB_APP_SLUG"),
+	GITHUB_APP_PRIVATE_KEY: optionalRedacted("GITHUB_APP_PRIVATE_KEY"),
+	GITHUB_APP_CLIENT_ID: optionalString("GITHUB_APP_CLIENT_ID"),
+	GITHUB_APP_CLIENT_SECRET: optionalRedacted("GITHUB_APP_CLIENT_SECRET"),
+	GITHUB_APP_WEBHOOK_SECRET: optionalRedacted("GITHUB_APP_WEBHOOK_SECRET"),
+	GITHUB_API_BASE_URL: stringWithDefault("GITHUB_API_BASE_URL", "https://api.github.com"),
 })
 
 const makeEnv = Effect.gen(function* () {