From 935b2c06652ad1077bca2959a4f71570776a0a50 Mon Sep 17 00:00:00 2001 From: tintinthong Date: Mon, 23 Feb 2026 22:46:03 +0800 Subject: [PATCH 1/3] wip --- docs/README.md | 1 + docs/submissions.md | 12 + packages/base/command.gts | 11 + packages/base/matrix-event.gts | 1 + packages/bot-runner/README.md | 1 + .../lib/create-listing-pr-handler.ts | 104 ++++++ packages/bot-runner/lib/github.ts | 155 +++++++++ packages/bot-runner/lib/timeline-handler.ts | 117 ++++--- packages/bot-runner/main.ts | 4 + packages/bot-runner/package.json | 2 + packages/bot-runner/tests/bot-runner-test.ts | 280 +++++++++++++--- .../a6d41a6d-9b63-4b37-b1f0-54b5e8c3d901.json | 64 ++++ .../SubmissionCard/submission-demo.json | 22 ++ packages/catalog-realm/index.json | 5 + .../submission-card/submission-card.gts | 34 ++ .../841febdf-5029-4752-9111-f5eea0eb1d75.json | 28 ++ .../84328985-e5f8-4e5b-bc23-a5f56efd1c32.json | 28 ++ .../caaa9e8b-bc59-46b2-8af0-66539e8a6d9e.json | 28 ++ .../ecad6650-615d-451f-a259-32f833ad5221.json | 28 ++ .../bot-requests/create-listing-pr-request.ts | 11 +- .../bot-requests/send-bot-trigger-event.ts | 5 + .../host/app/commands/create-submission.ts | 298 ++++++++++++++++++ packages/host/app/commands/index.ts | 6 + .../create-listing-pr-request-test.gts | 95 ++++++ .../commands/send-bot-trigger-event-test.gts | 5 +- .../matrix/scripts/setup-submission-bot.ts | 4 +- packages/runtime-common/bot-trigger.ts | 5 + packages/runtime-common/github-submissions.ts | 82 +++++ packages/runtime-common/index.ts | 1 + .../tests/github-submissions-test.ts | 95 ++++++ pnpm-lock.yaml | 6 + 31 files changed, 1448 insertions(+), 90 deletions(-) create mode 100644 docs/submissions.md create mode 100644 packages/bot-runner/lib/create-listing-pr-handler.ts create mode 100644 packages/bot-runner/lib/github.ts create mode 100644 packages/catalog-realm/CardListing/a6d41a6d-9b63-4b37-b1f0-54b5e8c3d901.json create mode 100644 packages/catalog-realm/SubmissionCard/submission-demo.json create mode 100644 packages/catalog-realm/submission-card/submission-card.gts create mode 100644 packages/experiments-realm/SubmissionCard/841febdf-5029-4752-9111-f5eea0eb1d75.json create mode 100644 packages/experiments-realm/SubmissionCard/84328985-e5f8-4e5b-bc23-a5f56efd1c32.json create mode 100644 packages/experiments-realm/SubmissionCard/caaa9e8b-bc59-46b2-8af0-66539e8a6d9e.json create mode 100644 packages/experiments-realm/SubmissionCard/ecad6650-615d-451f-a259-32f833ad5221.json create mode 100644 packages/host/app/commands/create-submission.ts create mode 100644 packages/host/tests/integration/commands/create-listing-pr-request-test.gts create mode 100644 packages/runtime-common/github-submissions.ts create mode 100644 packages/runtime-common/tests/github-submissions-test.ts diff --git a/docs/README.md b/docs/README.md index 8f4bc1d7735..3b589fbd827 100644 --- a/docs/README.md +++ b/docs/README.md @@ -22,3 +22,4 @@ The following are important concepts: ## Operations - From-scratch indexing timeout: set `FROM_SCRATCH_JOB_TIMEOUT_SEC` (seconds) to control the from-scratch indexing job timeout and the queue worker cap; default is 2400. +- [Submissions Flow](submissions.md): Short command-level flow for Matrix event -> PR creation -> submission card creation. diff --git a/docs/submissions.md b/docs/submissions.md new file mode 100644 index 00000000000..5c30af8096d --- /dev/null +++ b/docs/submissions.md @@ -0,0 +1,12 @@ +# Submissions Flow + +Submission flow is intentionally simple: + +1. Host issues a Matrix bot-trigger event via `@cardstack/boxel-host/commands/create-listing-pr-request`, which calls `@cardstack/boxel-host/commands/send-bot-trigger-event` with `type: 'pr-listing-create'`. +2. `bot-runner` handles `pr-listing-create` and opens the GitHub PR in `packages/bot-runner/lib/create-listing-pr-handler.ts` (`openCreateListingPR`). +3. `bot-runner` runs `@cardstack/boxel-host/commands/create-submission` to create a `SubmissionCard` in the submissions realm. + + +``` +http://localhost:4200/command-runner/%40cardstack%2Fboxel-host%2Fcommands%2Fcreate-submission%2Fdefault/%7B%22realm%22%3A%22http%3A%2F%2Flocalhost%3A4201%2Fexperiments%2F%22%2C%22roomId%22%3A%22!JTWMmANZcCwUHMIyaD%3Alocalhost%22%2C%22listingId%22%3A%22http%3A%2F%2Flocalhost%3A4201%2Fcatalog%2FAppListing%2F95cbe2c7-9b60-4afd-8a3c-1382b610e316%22%2C%22listingName%22%3A%22Blog%20App%22%7D/1 +``` diff --git a/packages/base/command.gts b/packages/base/command.gts index 6d51529e052..d701016661c 100644 --- a/packages/base/command.gts +++ b/packages/base/command.gts @@ -379,6 +379,17 @@ export class CreateListingPRResult extends CardDef { @field prNumber = contains(NumberField); } +export class CreateSubmissionInput extends CardDef { + @field roomId = contains(StringField); + @field realm = contains(RealmField); + @field listingId = contains(StringField); +} + +export class CreateSubmissionResult extends CardDef { + @field listing = linksTo(CardDef); + @field filesWithContent = containsMany(JsonField); +} + export class CreateListingPRRequestInput extends CardDef { @field roomId = contains(StringField); @field realm = contains(RealmField); diff --git a/packages/base/matrix-event.gts b/packages/base/matrix-event.gts index a731c984452..057f7cf5e59 100644 --- a/packages/base/matrix-event.gts +++ b/packages/base/matrix-event.gts @@ -290,6 +290,7 @@ export interface BotTriggerContent { type: string; realm: string; input: unknown; + userId: string; } export interface BotTriggerEvent extends BaseMatrixEvent { diff --git a/packages/bot-runner/README.md b/packages/bot-runner/README.md index 585b0e79b8a..e403eb2f20f 100644 --- a/packages/bot-runner/README.md +++ b/packages/bot-runner/README.md @@ -19,6 +19,7 @@ Environment variables: - `MATRIX_URL` (default: `http://localhost:8008`) - `SUBMISSION_BOT_USERNAME` (default: `submissionbot`) - `SUBMISSION_BOT_PASSWORD` (default: `password`) +- `SUBMISSION_BOT_GITHUB_TOKEN` (required for `pr-listing-create` workaround flow that opens PRs from bot-runner) - `LOG_LEVELS` (default: `*=info`) - `SENTRY_DSN` (optional) - `SENTRY_ENVIRONMENT` (optional, default: `development`) diff --git a/packages/bot-runner/lib/create-listing-pr-handler.ts b/packages/bot-runner/lib/create-listing-pr-handler.ts new file mode 100644 index 00000000000..73fa7b6a6f8 --- /dev/null +++ b/packages/bot-runner/lib/create-listing-pr-handler.ts @@ -0,0 +1,104 @@ +import { logger, toBranchName } from '@cardstack/runtime-common'; +import type { BotTriggerContent } from 'https://cardstack.com/base/matrix-event'; +import type { GitHubClient } from './github'; + +const log = logger('bot-runner:create-listing-pr'); + +const DEFAULT_REPO = 'cardstack/boxel-catalog'; +const DEFAULT_BASE_BRANCH = 'main'; + +export type BotTriggerEventContent = BotTriggerContent; + +export type CreateListingPRHandler = (args: { + eventContent: BotTriggerEventContent; + runAs: string; + githubClient: GitHubClient; +}) => Promise; + +export const openCreateListingPR: CreateListingPRHandler = async ({ + eventContent, + runAs, + githubClient, +}) => { + if (eventContent.type !== 'pr-listing-create') { + return; + } + + if (!eventContent.input || typeof eventContent.input !== 'object') { + log.warn('pr-listing-create trigger is missing input payload'); + return; + } + + let input = eventContent.input as Record; + let roomId = typeof input.roomId === 'string' ? input.roomId.trim() : ''; + let listingName = + typeof input.listingName === 'string' ? input.listingName.trim() : ''; + let listingDisplayName = listingName || 'UntitledListing'; + let title = + typeof input.title === 'string' && input.title.trim() + ? input.title.trim() + : `Add listing: ${listingDisplayName}`; + let headBranch = toBranchName(roomId, listingDisplayName); + + if (!headBranch) { + throw new Error('pr-listing-create trigger must include a valid branch'); + } + + if (!title) { + log.error('No title for the listing'); + return; + } + + let repo = DEFAULT_REPO; + + let [owner, repoName] = repo.split('/'); + if (!owner || !repoName) { + throw new Error(`Invalid repo format: ${repo}. Expected "owner/repo"`); + } + + // TODO: create the head branch before attempting to open the pull request. + let head = 'test-submissions'; //TODO: pls remove this temporary + // let head = headBranch; + + try { + let result = await githubClient.openPullRequest( + { + owner, + repo: repoName, + title, + head, + base: DEFAULT_BASE_BRANCH, + }, + { + label: runAs, + }, + ); + + log.info('opened PR from pr-listing-create trigger', { + runAs, + repo, + prUrl: result.html_url, + }); + } catch (error) { + let message = error instanceof Error ? error.message : String(error); + if (message.includes('A pull request already exists')) { + log.info('PR already exists for submission branch', { + runAs, + repo, + head, + error: message, + }); + return; + } + + log.error('failed to open PR from pr-listing-create trigger', { + runAs, + repo, + head, + error: message, + }); + throw error; + } + + return; +}; diff --git a/packages/bot-runner/lib/github.ts b/packages/bot-runner/lib/github.ts new file mode 100644 index 00000000000..7f114eeca42 --- /dev/null +++ b/packages/bot-runner/lib/github.ts @@ -0,0 +1,155 @@ +import { Octokit } from '@octokit/rest'; + +type CreatePullRequest = Octokit['rest']['pulls']['create']; +type RequestReviewers = Octokit['rest']['pulls']['requestReviewers']; +type AddLabels = Octokit['rest']['issues']['addLabels']; + +export interface OpenPullRequestParams { + owner: string; + repo: string; + title: string; + head: string; + base: string; + body?: string; +} + +export interface OpenPullRequestResult { + number: number; + html_url: string; +} + +export interface CreateBranchParams { + owner: string; + repo: string; + branch: string; + fromBranch: string; +} + +export interface CreateBranchResult { + ref: string; + sha: string; +} + +const HARDCODED_REVIEWER = 'tintinthong'; +const HARDCODED_PR_HEAD_BRANCH = 'test-submissions'; + +export interface OpenPullRequestOptions { + label: string; +} + +export interface GitHubClient { + openPullRequest( + params: OpenPullRequestParams, + options: OpenPullRequestOptions, + ): Promise; + createBranch(params: CreateBranchParams): Promise; +} + +export class OctokitGitHubClient implements GitHubClient { + private octokit: Octokit | undefined; + + constructor(private token: string | undefined) {} + + async openPullRequest( + params: OpenPullRequestParams, + options: OpenPullRequestOptions, + ): Promise { + let octokit = this.getClient(); + let label = options.label?.trim(); + if (!label) { + throw new Error('label is required'); + } + try { + // TODO: remove temporary hardcoded reviewer/head overrides once + // submission branch creation is fully wired through bot-runner. + let prParams: Parameters[0] = { + ...params, + head: HARDCODED_PR_HEAD_BRANCH, + }; + let response = await octokit.rest.pulls.create(prParams); + await octokit.rest.pulls.requestReviewers({ + owner: prParams.owner, + repo: prParams.repo, + pull_number: response.data.number, + reviewers: [HARDCODED_REVIEWER], + } as Parameters[0]); + await octokit.rest.issues.addLabels({ + owner: prParams.owner, + repo: prParams.repo, + issue_number: response.data.number, + labels: [label], + } as Parameters[0]); + return { + number: response.data.number, + html_url: response.data.html_url, + }; + } catch (error: any) { + throw toGitHubError('open pull request', error); + } + } + + async createBranch(params: CreateBranchParams): Promise { + let octokit = this.getClient(); + let fromBranch = normalizeBranchName(params.fromBranch); + let branch = normalizeBranchName(params.branch); + + if (!fromBranch) { + throw new Error('fromBranch is required'); + } + + if (!branch) { + throw new Error('branch is required'); + } + + try { + let sourceRef = await octokit.rest.git.getRef({ + owner: params.owner, + repo: params.repo, + ref: `heads/${fromBranch}`, + }); + let createdRef = await octokit.rest.git.createRef({ + owner: params.owner, + repo: params.repo, + ref: `refs/heads/${branch}`, + sha: sourceRef.data.object.sha, + }); + + return { + ref: createdRef.data.ref, + sha: createdRef.data.object.sha, + }; + } catch (error: any) { + throw toGitHubError('create branch', error); + } + } + + private getClient(): Octokit { + if (!this.token) { + throw new Error('SUBMISSION_BOT_GITHUB_TOKEN is not set'); + } + if (!this.octokit) { + this.octokit = new Octokit({ auth: this.token }); + } + return this.octokit; + } +} + +export function createGitHubClientFromEnv(): GitHubClient { + return new OctokitGitHubClient(process.env.SUBMISSION_BOT_GITHUB_TOKEN); +} + +function normalizeBranchName(branch: string): string { + return branch + .trim() + .replace(/^refs\/heads\//, '') + .replace(/^origin\//, ''); +} + +function toGitHubError(action: string, error: any): Error { + let status = typeof error?.status === 'number' ? String(error.status) : 'unknown'; + let payload = + error?.response?.data !== undefined ? error.response.data : error?.message; + return new Error( + `Failed to ${action} (${status}): ${JSON.stringify(payload)}`, + ); +} diff --git a/packages/bot-runner/lib/timeline-handler.ts b/packages/bot-runner/lib/timeline-handler.ts index 7549f9483fe..f6a2f84ee18 100644 --- a/packages/bot-runner/lib/timeline-handler.ts +++ b/packages/bot-runner/lib/timeline-handler.ts @@ -8,6 +8,11 @@ import { } from '@cardstack/runtime-common'; import { enqueueRunCommandJob } from '@cardstack/runtime-common/jobs/run-command'; import * as Sentry from '@sentry/node'; +import { + openCreateListingPR, + type BotTriggerEventContent, +} from './create-listing-pr-handler'; +import type { GitHubClient } from './github'; import type { DBAdapter, PgPrimitive, @@ -26,12 +31,16 @@ export interface TimelineHandlerOptions { authUserId: string; dbAdapter: DBAdapter; queuePublisher: QueuePublisher; + githubClient: GitHubClient; + startTime: number; } export function onTimelineEvent({ authUserId, dbAdapter, queuePublisher, + githubClient, + startTime, }: TimelineHandlerOptions) { return async function handleTimelineEvent( event: MatrixEvent, @@ -46,6 +55,10 @@ export function onTimelineEvent({ if (!isBotTriggerEvent(rawEvent)) { return; } + let eventTimestamp = rawEvent.origin_server_ts; + if (eventTimestamp == null || eventTimestamp < startTime) { + return; + } let eventContent = rawEvent.content; log.debug('event content', eventContent); let senderUsername = @@ -76,8 +89,7 @@ export function onTimelineEvent({ if (Number.isNaN(createdAt)) { continue; } - let eventTimestamp = event.event.origin_server_ts; - if (eventTimestamp == null || eventTimestamp < createdAt) { + if (eventTimestamp < createdAt) { continue; } log.debug( @@ -94,6 +106,7 @@ export function onTimelineEvent({ runAs: senderUsername, eventContent, allowedCommands, + githubClient, }); } for (let registration of registrations) { @@ -101,8 +114,7 @@ export function onTimelineEvent({ if (Number.isNaN(createdAt)) { continue; } - let eventTimestamp = event.event.origin_server_ts; - if (eventTimestamp == null || eventTimestamp < createdAt) { + if (eventTimestamp < createdAt) { continue; } // TODO: filter out events we want to handle based on the registration (e.g. command messages, system events) @@ -120,6 +132,7 @@ export function onTimelineEvent({ runAs: senderUsername, eventContent, allowedCommands, + githubClient, }); } } catch (error) { @@ -135,54 +148,74 @@ async function maybeEnqueueCommand({ runAs, eventContent, allowedCommands, + githubClient, }: { dbAdapter: DBAdapter; queuePublisher: QueuePublisher; runAs: string; - eventContent: { type?: unknown; input?: unknown; realm?: unknown }; + eventContent: BotTriggerEventContent; allowedCommands: { type: string; command: string }[]; -}) { - if ( - !allowedCommands.length || - typeof eventContent.type !== 'string' || - !allowedCommands.some((entry) => entry.type === eventContent.type) - ) { - return; - } + githubClient: GitHubClient; +}): Promise { + try { + if ( + !allowedCommands.length || + typeof eventContent.type !== 'string' || + !allowedCommands.some((entry) => entry.type === eventContent.type) + ) { + return; + } - if (!eventContent?.input || typeof eventContent.input !== 'object') { - return; - } + if (eventContent.type === 'pr-listing-create') { + // Temporary workaround: handle PR creation directly until this flow is moved to a proper command path. + await openCreateListingPR({ + eventContent, + runAs, + githubClient, + }); + } - let input = eventContent.input as Record; - let realmURL = - typeof eventContent.realm === 'string' ? eventContent.realm : undefined; - let commandRegistration = allowedCommands.find( - (entry) => entry.type === eventContent.type, - ); - let command = commandRegistration?.command?.trim(); - let commandInput: Record | null = input; - - if (!realmURL || !command) { - log.warn( - 'bot trigger missing required input for command (need realmURL and command)', - { realmURL, command }, + if (!eventContent?.input || typeof eventContent.input !== 'object') { + return; + } + + let input = eventContent.input as Record; + let realmURL = + typeof eventContent.realm === 'string' ? eventContent.realm : undefined; + let commandRegistration = allowedCommands.find( + (entry) => entry.type === eventContent.type, ); - return; - } + let command = commandRegistration?.command?.trim(); + let commandInput: Record | null = input; + + if (!realmURL || !command) { + log.warn( + 'bot trigger missing required input for command (need realmURL and command)', + { realmURL, command }, + ); + return; + } - await enqueueRunCommandJob( - { - realmURL, - realmUsername: runAs, + await enqueueRunCommandJob( + { + realmURL, + realmUsername: runAs, + runAs, + command, + commandInput, + }, + queuePublisher, + dbAdapter, + userInitiatedPriority, + ); + } catch (error) { + log.error('error in maybeEnqueueCommand', { runAs, - command, - commandInput, - }, - queuePublisher, - dbAdapter, - userInitiatedPriority, - ); + eventType: eventContent.type, + error, + }); + throw error; + } } function getRoomCreator(room: Room | undefined): string | undefined { diff --git a/packages/bot-runner/main.ts b/packages/bot-runner/main.ts index 254cbe8e9f8..bb2fe0e6244 100644 --- a/packages/bot-runner/main.ts +++ b/packages/bot-runner/main.ts @@ -6,6 +6,7 @@ import { logger } from '@cardstack/runtime-common'; import * as Sentry from '@sentry/node'; import { onMembershipEvent } from './lib/membership-handler'; import { onTimelineEvent } from './lib/timeline-handler'; +import { createGitHubClientFromEnv } from './lib/github'; const log = logger('bot-runner'); const startTime = Date.now(); @@ -34,6 +35,7 @@ const botPassword = process.env.SUBMISSION_BOT_PASSWORD || 'password'; let dbAdapter = new PgAdapter(); let queuePublisher = new PgQueuePublisher(dbAdapter); + let githubClient = createGitHubClientFromEnv(); const shutdown = async () => { log.info('shutting down bot runner...'); @@ -63,6 +65,8 @@ const botPassword = process.env.SUBMISSION_BOT_PASSWORD || 'password'; authUserId: auth.user_id, dbAdapter, queuePublisher, + githubClient, + startTime, }); client.on(RoomEvent.Timeline, async (event, room, toStartOfTimeline) => { await handleTimelineEvent(event, room, toStartOfTimeline); diff --git a/packages/bot-runner/package.json b/packages/bot-runner/package.json index cc14b22209b..d1ddb70bb3b 100644 --- a/packages/bot-runner/package.json +++ b/packages/bot-runner/package.json @@ -3,6 +3,8 @@ "dependencies": { "@cardstack/postgres": "workspace:*", "@cardstack/runtime-common": "workspace:*", + "@octokit/rest": "catalog:", + "@octokit/types": "16.0.0", "@sentry/node": "catalog:", "matrix-js-sdk": "catalog:", "ts-node": "^10.9.2", diff --git a/packages/bot-runner/tests/bot-runner-test.ts b/packages/bot-runner/tests/bot-runner-test.ts index ab031d9f4c5..5ccb6238f47 100644 --- a/packages/bot-runner/tests/bot-runner-test.ts +++ b/packages/bot-runner/tests/bot-runner-test.ts @@ -11,22 +11,26 @@ import type { Room, RoomMember, } from 'matrix-js-sdk'; +import type { GitHubClient } from '../lib/github'; import { onMembershipEvent } from '../lib/membership-handler'; import { onTimelineEvent } from '../lib/timeline-handler'; function makeBotTriggerEvent( sender: string | null | undefined, originServerTs: number, + contentType = 'show-card', ) { const BOT_TRIGGER_EVENT_TYPE = 'app.boxel.bot-trigger'; + let userId = sender ?? '@alice:localhost'; return { event: { origin_server_ts: originServerTs, type: BOT_TRIGGER_EVENT_TYPE, content: { - type: 'create-listing-pr', + type: contentType, input: {}, realm: 'http://localhost:4201/test/', + userId, }, }, getSender: () => sender, @@ -91,73 +95,267 @@ module('membership handler', () => { }); module('timeline handler', () => { - let currentRows: Record[] = []; - let registrationsHook: - | ((sql: string, opts?: ExecuteOptions) => void) - | undefined; let dbAdapter: DBAdapter; let queuePublisher: QueuePublisher; - let handleTimelineEvent: ReturnType; + let githubClient: GitHubClient; + let publishedJobs: unknown[] = []; + let senderRegistrations: Record[] = []; + let submissionBotRegistrations: Record[] = []; + let commandsByRegistrationId = new Map[]>(); dbAdapter = { kind: 'pg', isClosed: false, - execute: async (_sql: string, opts?: ExecuteOptions) => { - registrationsHook?.(_sql, opts); - return currentRows; + execute: async (sql: string, opts?: ExecuteOptions) => { + if (sql.includes('FROM bot_registrations br')) { + let username = opts?.bind?.[0]; + if (username === '@alice:localhost') { + return senderRegistrations; + } + if (username === '@submissionbot:localhost') { + return submissionBotRegistrations; + } + return []; + } + + if (sql.includes('FROM bot_commands WHERE bot_id =')) { + let registrationId = opts?.bind?.[0]; + if (typeof registrationId !== 'string') { + return []; + } + return commandsByRegistrationId.get(registrationId) ?? []; + } + + return []; }, close: async () => {}, getColumnNames: async () => [], } as DBAdapter; queuePublisher = { - publish: async () => ({ id: 1, done: Promise.resolve(undefined) }) as any, + publish: async (job: unknown) => { + publishedJobs.push(job); + return { id: 1, done: Promise.resolve(undefined) } as any; + }, destroy: async () => {}, }; + githubClient = { + openPullRequest: async () => ({ + number: 1, + html_url: 'https://example.com/pr/1', + }), + createBranch: async () => ({ + ref: 'refs/heads/room-branch', + sha: 'abc123', + }), + }; - handleTimelineEvent = onTimelineEvent({ - authUserId: '@submissionbot:localhost', - dbAdapter, - queuePublisher, + test('enqueues command when event matches', async (assert) => { + senderRegistrations = []; + submissionBotRegistrations = [ + { + id: 'bot-registration-1', + created_at: new Date(0) as unknown as PgPrimitive, + username: '@submissionbot:localhost', + }, + ]; + commandsByRegistrationId = new Map([ + [ + 'bot-registration-1', + [ + { + command_filter: { + type: 'matrix-event', + event_type: 'app.boxel.bot-trigger', + content_type: 'show-card', + }, + command: '@cardstack/boxel-host/commands/show-card/default', + }, + ], + ], + ]); + publishedJobs = []; + + let handleTimelineEvent = onTimelineEvent({ + authUserId: '@submissionbot:localhost', + dbAdapter, + queuePublisher, + githubClient, + startTime: 0, + }); + + await handleTimelineEvent( + makeBotTriggerEvent('@alice:localhost', 1000), + makeRoom('join'), + false, + ); + + assert.strictEqual(publishedJobs.length, 1, 'enqueues run-command job'); + assert.deepEqual( + publishedJobs[0], + { + jobType: 'run-command', + concurrencyGroup: 'command:http://localhost:4201/test/', + timeout: 60, + priority: 10, + args: { + realmURL: 'http://localhost:4201/test/', + realmUsername: '@alice:localhost', + runAs: '@alice:localhost', + command: '@cardstack/boxel-host/commands/show-card/default', + commandInput: {}, + }, + }, + 'enqueues expected command payload', + ); }); - function mockGetRegistrations( - onRows: (rows: Record[]) => void, - ) { - registrationsHook = (sql) => { - if ( - sql !== - 'SELECT br.id, br.username, br.created_at FROM bot_registrations br WHERE br.username = $1' - ) { - return; - } - onRows(currentRows); - }; - } + test('does not enqueue command when event type is pr-listing-create', async (assert) => { + senderRegistrations = []; + submissionBotRegistrations = [ + { + id: 'bot-registration-2', + created_at: new Date(0) as unknown as PgPrimitive, + username: '@submissionbot:localhost', + }, + ]; + commandsByRegistrationId = new Map([ + [ + 'bot-registration-2', + [ + { + command_filter: { + type: 'matrix-event', + event_type: 'app.boxel.bot-trigger', + content_type: 'pr-listing-create', + }, + command: '@cardstack/boxel-host/commands/create-listing-pr/default', + }, + ], + ], + ]); + publishedJobs = []; - test('loads registrations for sender and ignores if none', async (assert) => { - assert.expect(1); - currentRows = []; - mockGetRegistrations(() => {}); + let handleTimelineEvent = onTimelineEvent({ + authUserId: '@submissionbot:localhost', + dbAdapter, + queuePublisher, + githubClient, + startTime: 0, + }); await handleTimelineEvent( - makeBotTriggerEvent('@alice:localhost', 1000), + makeBotTriggerEvent('@alice:localhost', 1000, 'pr-listing-create'), makeRoom('join'), false, ); - assert.deepEqual(currentRows, [], 'loads registrations'); + assert.strictEqual( + publishedJobs.length, + 0, + 'does not enqueue run-command job for pr-listing-create', + ); }); - test('filters events older than registration created_at', async (assert) => { - assert.expect(1); - currentRows = [ + test('does not enqueue command for pr-listing-create across submission bot and sender registration paths', async (assert) => { + senderRegistrations = [ { - id: '1', - created_at: new Date(2000).toISOString(), + id: 'sender-registration-1', + created_at: new Date(0) as unknown as PgPrimitive, username: '@alice:localhost', }, ]; + submissionBotRegistrations = [ + { + id: 'bot-registration-3', + created_at: new Date(0) as unknown as PgPrimitive, + username: '@submissionbot:localhost', + }, + ]; + commandsByRegistrationId = new Map([ + [ + 'bot-registration-3', + [ + { + command_filter: { + type: 'matrix-event', + event_type: 'app.boxel.bot-trigger', + content_type: 'pr-listing-create', + }, + command: '@cardstack/boxel-host/commands/create-listing-pr/default', + }, + ], + ], + [ + 'sender-registration-1', + [ + { + command_filter: { + type: 'matrix-event', + event_type: 'app.boxel.bot-trigger', + content_type: 'pr-listing-create', + }, + command: '@cardstack/boxel-host/commands/create-listing-pr/default', + }, + ], + ], + ]); + publishedJobs = []; + + let handleTimelineEvent = onTimelineEvent({ + authUserId: '@submissionbot:localhost', + dbAdapter, + queuePublisher, + githubClient, + startTime: 0, + }); + + await handleTimelineEvent( + makeBotTriggerEvent('@alice:localhost', 1000, 'pr-listing-create'), + makeRoom('join'), + false, + ); + + assert.strictEqual( + publishedJobs.length, + 0, + 'does not enqueue run-command job for pr-listing-create', + ); + }); + + test('ignores timeline events older than startTime', async (assert) => { + senderRegistrations = []; + submissionBotRegistrations = [ + { + id: 'bot-registration-4', + created_at: new Date(0) as unknown as PgPrimitive, + username: '@submissionbot:localhost', + }, + ]; + commandsByRegistrationId = new Map([ + [ + 'bot-registration-4', + [ + { + command_filter: { + type: 'matrix-event', + event_type: 'app.boxel.bot-trigger', + content_type: 'show-card', + }, + command: '@cardstack/boxel-host/commands/show-card/default', + }, + ], + ], + ]); + publishedJobs = []; + + let handleTimelineEvent = onTimelineEvent({ + authUserId: '@submissionbot:localhost', + dbAdapter, + queuePublisher, + githubClient, + startTime: 2000, + }); await handleTimelineEvent( makeBotTriggerEvent('@alice:localhost', 1000), @@ -165,6 +363,10 @@ module('timeline handler', () => { false, ); - assert.ok(true, 'loads registrations'); + assert.strictEqual( + publishedJobs.length, + 0, + 'does not handle events that are older than startTime', + ); }); }); diff --git a/packages/catalog-realm/CardListing/a6d41a6d-9b63-4b37-b1f0-54b5e8c3d901.json b/packages/catalog-realm/CardListing/a6d41a6d-9b63-4b37-b1f0-54b5e8c3d901.json new file mode 100644 index 00000000000..6cd05f96ee0 --- /dev/null +++ b/packages/catalog-realm/CardListing/a6d41a6d-9b63-4b37-b1f0-54b5e8c3d901.json @@ -0,0 +1,64 @@ +{ + "data": { + "meta": { + "adoptsFrom": { + "name": "CardListing", + "module": "../catalog-app/listing/listing" + } + }, + "type": "card", + "attributes": { + "name": "Submission Card", + "images": [], + "summary": "A simple card for tracking submission status and the files included in a submission payload.", + "cardInfo": { + "notes": null, + "name": null, + "summary": null, + "cardThumbnailURL": null + } + }, + "relationships": { + "tags": { + "links": { + "self": null + } + }, + "skills": { + "links": { + "self": null + } + }, + "license": { + "links": { + "self": null + } + }, + "specs.0": { + "links": { + "self": "../Spec/0c1aed91-ae63-4b87-bcf0-3505520c7169" + } + }, + "publisher": { + "links": { + "self": null + } + }, + "examples.0": { + "links": { + "self": "../SubmissionCard/submission-demo" + } + }, + "categories.0": { + "links": { + "self": "../Category/software-development" + } + }, + "cardInfo.theme": { + "links": { + "self": null + } + } + } + } +} diff --git a/packages/catalog-realm/SubmissionCard/submission-demo.json b/packages/catalog-realm/SubmissionCard/submission-demo.json new file mode 100644 index 00000000000..52d87bfbc2e --- /dev/null +++ b/packages/catalog-realm/SubmissionCard/submission-demo.json @@ -0,0 +1,22 @@ +{ + "data": { + "type": "card", + "attributes": { + "roomId": "!XezEDqUlIJcNdsuaFB:localhost", + "branchName": "room-IVhlekVEcVVsSUpjTmRzdWFGQjpsb2NhbGhvc3Q/some-sample-listing" + }, + "relationships": { + "listing": { + "links": { + "self": "../CardListing/a6d41a6d-9b63-4b37-b1f0-54b5e8c3d901" + } + } + }, + "meta": { + "adoptsFrom": { + "module": "../submission-card/submission-card", + "name": "SubmissionCard" + } + } + } +} diff --git a/packages/catalog-realm/index.json b/packages/catalog-realm/index.json index 8f35198c363..df2018bd789 100644 --- a/packages/catalog-realm/index.json +++ b/packages/catalog-realm/index.json @@ -56,6 +56,11 @@ "self": "./AppListing/1d17b731-7108-42db-b4fb-93643288553c" } }, + "new.8": { + "links": { + "self": "./CardListing/a6d41a6d-9b63-4b37-b1f0-54b5e8c3d901" + } + }, "featured.0": { "links": { "self": "./AppListing/4c9a74c6-b8d4-47fc-ade7-d7e7b2999b6b" diff --git a/packages/catalog-realm/submission-card/submission-card.gts b/packages/catalog-realm/submission-card/submission-card.gts new file mode 100644 index 00000000000..3c7a0d23239 --- /dev/null +++ b/packages/catalog-realm/submission-card/submission-card.gts @@ -0,0 +1,34 @@ +import { + CardDef, + contains, + field, + linksTo, +} from 'https://cardstack.com/base/card-api'; +import StringField from 'https://cardstack.com/base/string'; +import { Listing } from '../catalog-app/listing/listing'; + +const GITHUB_BRANCH_URL_PREFIX = + 'https://github.com/cardstack/boxel-catalog/tree/'; + +function encodeBranchName(branchName: string): string { + return branchName + .split('/') + .map((segment) => encodeURIComponent(segment)) + .join('/'); +} + +export class SubmissionCard extends CardDef { + static displayName = 'SubmissionCard'; + + @field roomId = contains(StringField); + @field branchName = contains(StringField); + @field githubURL = contains(StringField, { + computeVia: function (this: SubmissionCard) { + if (!this.branchName) { + return undefined; + } + return `${GITHUB_BRANCH_URL_PREFIX}${encodeBranchName(this.branchName)}`; + }, + }); + @field listing = linksTo(() => Listing); +} diff --git a/packages/experiments-realm/SubmissionCard/841febdf-5029-4752-9111-f5eea0eb1d75.json b/packages/experiments-realm/SubmissionCard/841febdf-5029-4752-9111-f5eea0eb1d75.json new file mode 100644 index 00000000000..068d15e13fd --- /dev/null +++ b/packages/experiments-realm/SubmissionCard/841febdf-5029-4752-9111-f5eea0eb1d75.json @@ -0,0 +1,28 @@ +{ + "data": { + "type": "card", + "attributes": { + "roomId": "!lHFnOfNQmbeLdPgGEm:localhost", + "branchName": "room-IWxIRm5PZk5RbWJlTGRQZ0dFbTpsb2NhbGhvc3Q/blog-app", + "cardInfo": { + "name": null, + "summary": null, + "cardThumbnailURL": null, + "notes": null + } + }, + "relationships": { + "listing": { + "links": { + "self": "http://localhost:4201/catalog/AppListing/95cbe2c7-9b60-4afd-8a3c-1382b610e316" + } + } + }, + "meta": { + "adoptsFrom": { + "module": "http://localhost:4201/catalog/submission-card/submission-card", + "name": "SubmissionCard" + } + } + } +} \ No newline at end of file diff --git a/packages/experiments-realm/SubmissionCard/84328985-e5f8-4e5b-bc23-a5f56efd1c32.json b/packages/experiments-realm/SubmissionCard/84328985-e5f8-4e5b-bc23-a5f56efd1c32.json new file mode 100644 index 00000000000..69388fda48c --- /dev/null +++ b/packages/experiments-realm/SubmissionCard/84328985-e5f8-4e5b-bc23-a5f56efd1c32.json @@ -0,0 +1,28 @@ +{ + "data": { + "type": "card", + "attributes": { + "roomId": "!JTWMmANZcCwUHMIyaD:localhost", + "branchName": "room-IUpUV01tQU5aY0N3VUhNSXlhRDpsb2NhbGhvc3Q/blog-app", + "cardInfo": { + "name": null, + "summary": null, + "cardThumbnailURL": null, + "notes": null + } + }, + "relationships": { + "listing": { + "links": { + "self": "http://localhost:4201/catalog/AppListing/95cbe2c7-9b60-4afd-8a3c-1382b610e316" + } + } + }, + "meta": { + "adoptsFrom": { + "module": "http://localhost:4201/catalog/submission-card/submission-card", + "name": "SubmissionCard" + } + } + } +} \ No newline at end of file diff --git a/packages/experiments-realm/SubmissionCard/caaa9e8b-bc59-46b2-8af0-66539e8a6d9e.json b/packages/experiments-realm/SubmissionCard/caaa9e8b-bc59-46b2-8af0-66539e8a6d9e.json new file mode 100644 index 00000000000..69388fda48c --- /dev/null +++ b/packages/experiments-realm/SubmissionCard/caaa9e8b-bc59-46b2-8af0-66539e8a6d9e.json @@ -0,0 +1,28 @@ +{ + "data": { + "type": "card", + "attributes": { + "roomId": "!JTWMmANZcCwUHMIyaD:localhost", + "branchName": "room-IUpUV01tQU5aY0N3VUhNSXlhRDpsb2NhbGhvc3Q/blog-app", + "cardInfo": { + "name": null, + "summary": null, + "cardThumbnailURL": null, + "notes": null + } + }, + "relationships": { + "listing": { + "links": { + "self": "http://localhost:4201/catalog/AppListing/95cbe2c7-9b60-4afd-8a3c-1382b610e316" + } + } + }, + "meta": { + "adoptsFrom": { + "module": "http://localhost:4201/catalog/submission-card/submission-card", + "name": "SubmissionCard" + } + } + } +} \ No newline at end of file diff --git a/packages/experiments-realm/SubmissionCard/ecad6650-615d-451f-a259-32f833ad5221.json b/packages/experiments-realm/SubmissionCard/ecad6650-615d-451f-a259-32f833ad5221.json new file mode 100644 index 00000000000..90982dae1ef --- /dev/null +++ b/packages/experiments-realm/SubmissionCard/ecad6650-615d-451f-a259-32f833ad5221.json @@ -0,0 +1,28 @@ +{ + "data": { + "type": "card", + "attributes": { + "roomId": "!WjQugTdxaCsZZPdRGB:localhost", + "branchName": "room-IVdqUXVnVGR4YUNzWlpQZFJHQjpsb2NhbGhvc3Q/blog-app", + "cardInfo": { + "name": null, + "summary": null, + "cardThumbnailURL": null, + "notes": null + } + }, + "relationships": { + "listing": { + "links": { + "self": "http://localhost:4201/catalog/AppListing/95cbe2c7-9b60-4afd-8a3c-1382b610e316" + } + } + }, + "meta": { + "adoptsFrom": { + "module": "http://localhost:4201/catalog/submission-card/submission-card", + "name": "SubmissionCard" + } + } + } +} \ No newline at end of file diff --git a/packages/host/app/commands/bot-requests/create-listing-pr-request.ts b/packages/host/app/commands/bot-requests/create-listing-pr-request.ts index 432f55c0b65..73750e9fd61 100644 --- a/packages/host/app/commands/bot-requests/create-listing-pr-request.ts +++ b/packages/host/app/commands/bot-requests/create-listing-pr-request.ts @@ -39,12 +39,12 @@ export default class CreateListingPRRequestCommand extends HostBaseCommand< let { realm, listingId } = input; let roomId = input.roomId; let listingName: string | undefined; + let listing = await this.store.get(listingId); + if (listing && isCardInstance(listing)) { + listingName = listing.name ?? listing.id; + } if (!roomId) { - let listing = await this.store.get(listingId); - if (listing && isCardInstance(listing)) { - listingName = listing.name ?? listing.id; - } let useAiAssistantCommand = new UseAiAssistantCommand( this.commandContext, ); @@ -64,11 +64,12 @@ export default class CreateListingPRRequestCommand extends HostBaseCommand< await new SendBotTriggerEventCommand(this.commandContext).execute({ roomId, realm, - type: 'create-listing-pr', + type: 'pr-listing-create', input: { roomId, realm, listingId, + ...(listingName ? { listingName } : {}), }, }); } diff --git a/packages/host/app/commands/bot-requests/send-bot-trigger-event.ts b/packages/host/app/commands/bot-requests/send-bot-trigger-event.ts index 2601f59b7e0..213befd6bb8 100644 --- a/packages/host/app/commands/bot-requests/send-bot-trigger-event.ts +++ b/packages/host/app/commands/bot-requests/send-bot-trigger-event.ts @@ -28,6 +28,10 @@ export default class SendBotTriggerEventCommand extends HostBaseCommand< input: BaseCommandModule.SendBotTriggerEventInput, ): Promise { await this.matrixService.ready; + let userId = this.matrixService.userId; + if (!userId) { + throw new Error('userId is required to send bot trigger events'); + } const botTriggerEventType = 'app.boxel.bot-trigger'; let event = { @@ -36,6 +40,7 @@ export default class SendBotTriggerEventCommand extends HostBaseCommand< type: input.type, input: input.input, realm: input.realm, + userId, }, } as BotTriggerEvent; diff --git a/packages/host/app/commands/create-submission.ts b/packages/host/app/commands/create-submission.ts new file mode 100644 index 00000000000..2ac39daf32e --- /dev/null +++ b/packages/host/app/commands/create-submission.ts @@ -0,0 +1,298 @@ +import { service } from '@ember/service'; + +import { + RealmPaths, + PlanBuilder, + planInstanceInstall, + planModuleInstall, + toBranchName, + extractRelationshipIds, + isCardInstance, + logger, + type LooseSingleCardDocument, + type ListingPathResolver, + type Relationship, +} from '@cardstack/runtime-common'; +import type { CopyInstanceMeta } from '@cardstack/runtime-common/catalog'; +import type { CopyModuleMeta } from '@cardstack/runtime-common/catalog'; + +import ENV from '@cardstack/host/config/environment'; + +import type * as BaseCommandModule from 'https://cardstack.com/base/command'; + +import HostBaseCommand from '../lib/host-base-command'; + +import type CardService from '../services/card-service'; +import type StoreService from '../services/store'; +import type { Listing } from '@cardstack/catalog/listing/listing'; + +const log = logger('commands:create-submission'); +const SUBMISSION_CARD_MODULE = ENV.resolvedCatalogRealmURL + ? new URL('submission-card/submission-card', ENV.resolvedCatalogRealmURL).href + : undefined; + +interface FileWithContent { + path: string; + content: string; +} + +export default class CreateSubmissionCommand extends HostBaseCommand< + typeof BaseCommandModule.CreateSubmissionInput, + typeof BaseCommandModule.CreateSubmissionResult +> { + @service declare private cardService: CardService; + @service declare private store: StoreService; + + description = 'Prepare submission data for a catalog listing'; + + async getInputType() { + let commandModule = await this.loadCommandModule(); + const { CreateSubmissionInput } = commandModule; + return CreateSubmissionInput; + } + + async getResultType() { + let commandModule = await this.loadCommandModule(); + const { CreateSubmissionResult } = commandModule; + return CreateSubmissionResult; + } + + requireInputFields = ['roomId', 'realm', 'listingId']; + + protected async run( + input: BaseCommandModule.CreateSubmissionInput, + ): Promise { + let { listingId, realm, roomId } = input; + let realmUrl = new RealmPaths(new URL(realm)).url; + + if (!listingId) { + throw new Error('Missing listingId for CreateSubmission'); + } + + // Listing type is from catalog; base command cannot express that type + const listing = (await this.store.get(listingId)) as Listing; + if (!listing) { + throw new Error(`Listing not found: ${listingId}`); + } + if (!listing.name) { + throw new Error('Missing listing.name for CreateSubmission'); + } + let branchName = toBranchName(roomId, listing.name); + + // Expand examples to include related instances + let examplesToSnapshot = listing.examples; + if (listing.examples?.length) { + examplesToSnapshot = await this.expandInstances(listing.examples); + } + + // Build the file plan from the listing + const builder = new PlanBuilder(realmUrl, listing); + + builder + .addIf(listing.specs?.length > 0, (resolver: ListingPathResolver) => + planModuleInstall(listing.specs ?? [], resolver), + ) + .addIf(listing.specs?.length > 0, (resolver: ListingPathResolver) => + planInstanceInstall(listing.specs ?? [], resolver), + ) + .addIf(examplesToSnapshot?.length > 0, (resolver: ListingPathResolver) => + planInstanceInstall(examplesToSnapshot ?? [], resolver), + ) + .addIf(listing.skills?.length > 0, (resolver: ListingPathResolver) => + planInstanceInstall(listing.skills ?? [], resolver), + ); + + const plan = builder.build(); + + let filesWithContent = await this.collectAndFetchFiles( + listing, + plan, + realmUrl, + ); + + log.debug(`Prepared submission with ${filesWithContent.length} files`); + + await this.createSubmissionCard({ + listing, + branchName, + roomId, + realmURL: realmUrl, + }); + + const commandModule = await this.loadCommandModule(); + const { CreateSubmissionResult } = commandModule; + return new CreateSubmissionResult({ + listing, + filesWithContent, + }); + } + + private async collectAndFetchFiles( + listing: Listing, + plan: ReturnType, + realmUrl: string, + ): Promise { + const toRepoRelativePath = (fullUrl: string, extension: string): string => { + let url = fullUrl; + if (url.startsWith(realmUrl)) { + url = url.slice(realmUrl.length); + } + if (url.startsWith('/')) { + url = url.slice(1); + } + if (!url.endsWith(extension)) { + url = url + extension; + } + return url; + }; + + const filesWithContent: FileWithContent[] = []; + const seenPaths = new Set(); + + // Add the listing instance JSON + if (listing.id) { + const path = toRepoRelativePath(listing.id, '.json'); + if (!seenPaths.has(path)) { + seenPaths.add(path); + const response = await fetch(`${listing.id}.json`); + if (response.ok) { + filesWithContent.push({ path, content: await response.text() }); + } + } + } + + // Add module files (.gts) + for (const moduleMeta of plan.modulesToInstall as CopyModuleMeta[]) { + if (!moduleMeta?.sourceModule) { + log.warn('Skipping module with missing sourceModule', moduleMeta); + continue; + } + const path = toRepoRelativePath(moduleMeta.sourceModule, '.gts'); + if (!seenPaths.has(path)) { + seenPaths.add(path); + const res = await this.cardService.getSource( + new URL(moduleMeta.sourceModule), + ); + filesWithContent.push({ path, content: res.content }); + } + } + + // Add instance files (.json) + for (const copyMeta of plan.instancesCopy as CopyInstanceMeta[]) { + if (!copyMeta?.sourceCard?.id) { + log.warn('Skipping instance with missing sourceCard', copyMeta); + continue; + } + const sourceCardId = copyMeta.sourceCard.id; + const path = toRepoRelativePath(sourceCardId, '.json'); + if (!seenPaths.has(path)) { + seenPaths.add(path); + const response = await fetch(`${sourceCardId}.json`); + if (response.ok) { + filesWithContent.push({ path, content: await response.text() }); + } + } + } + + return filesWithContent; + } + + private async createSubmissionCard({ + listing, + branchName, + roomId, + realmURL, + }: { + listing: Listing; + branchName: string; + roomId: string; + realmURL: string; + }): Promise { + if (!listing.id) { + throw new Error('Missing listing.id for submission card creation'); + } + if (!SUBMISSION_CARD_MODULE) { + throw new Error('Catalog realm URL is not configured'); + } + let doc: LooseSingleCardDocument = { + data: { + type: 'card', + relationships: { + listing: { + links: { + self: listing.id, + }, + }, + }, + attributes: { + roomId, + branchName, + }, + meta: { + adoptsFrom: { + module: SUBMISSION_CARD_MODULE, + name: 'SubmissionCard', + }, + }, + }, + }; + + let createdCardId = await this.store.create(doc, { + realm: realmURL, + }); + if (typeof createdCardId !== 'string') { + throw new Error( + `unable to create submission card: ${JSON.stringify(createdCardId, null, 2)}`, + ); + } + } + + // Walk relationships by fetching linked cards and enqueueing their ids. + private async expandInstances(instances: any[]): Promise { + const instancesById = new Map(); + const visited = new Set(); + const queue: string[] = instances + .map((instance) => instance.id) + .filter((id): id is string => typeof id === 'string'); + + while (queue.length > 0) { + const id = queue.shift(); + if (!id || visited.has(id)) { + continue; + } + visited.add(id); + + const cachedInstance = this.store.peek(id); + let relationships: Record = {}; + let baseUrl = id; + const instance = isCardInstance(cachedInstance) + ? cachedInstance + : await this.store.get(id); + if (!isCardInstance(instance)) { + throw new Error(`Expected card instance for ${id}`); + } + instancesById.set(instance.id ?? id, instance); + const serialized = await this.cardService.serializeCard(instance, { + omitQueryFields: true, + }); + if (serialized.data.id) { + baseUrl = serialized.data.id; + } + relationships = serialized.data.relationships ?? {}; + + for (const rel of Object.values(relationships)) { + const rels = Array.isArray(rel) ? rel : [rel]; + for (const relationship of rels) { + const relatedIds = extractRelationshipIds(relationship, baseUrl); + for (const relatedId of relatedIds) { + if (!visited.has(relatedId)) { + queue.push(relatedId); + } + } + } + } + } + + return [...instancesById.values()]; + } +} diff --git a/packages/host/app/commands/index.ts b/packages/host/app/commands/index.ts index 35afdc96d36..2bc681cebae 100644 --- a/packages/host/app/commands/index.ts +++ b/packages/host/app/commands/index.ts @@ -15,6 +15,7 @@ import * as CopySourceCommandModule from './copy-source'; import * as CreateAIAssistantRoomCommandModule from './create-ai-assistant-room'; import * as CreateListingPRCommandModule from './create-listing-pr'; import * as CreateSpecCommandModule from './create-specs'; +import * as CreateSubmissionCommandModule from './create-submission'; import * as GenerateExampleCardsCommandModule from './generate-example-cards'; import * as GenerateReadmeSpecCommandModule from './generate-readme-spec'; import * as GenerateThemeExampleCommandModule from './generate-theme-example'; @@ -160,6 +161,10 @@ export function shimHostCommands(virtualNetwork: VirtualNetwork) { '@cardstack/boxel-host/commands/create-listing-pr', CreateListingPRCommandModule, ); + virtualNetwork.shimModule( + '@cardstack/boxel-host/commands/create-submission', + CreateSubmissionCommandModule, + ); virtualNetwork.shimModule( '@cardstack/boxel-host/commands/create-listing-pr-request', CreateListingPRRequestCommandModule, @@ -356,6 +361,7 @@ export const HostCommandClasses: (typeof HostBaseCommand)[] = [ ListingInstallCommandModule.default, ListingRemixCommandModule.default, CreateListingPRCommandModule.default, + CreateSubmissionCommandModule.default, CreateListingPRRequestCommandModule.default, ListingUpdateSpecsCommandModule.default, ListingUseCommandModule.default, diff --git a/packages/host/tests/integration/commands/create-listing-pr-request-test.gts b/packages/host/tests/integration/commands/create-listing-pr-request-test.gts new file mode 100644 index 00000000000..79275897a4f --- /dev/null +++ b/packages/host/tests/integration/commands/create-listing-pr-request-test.gts @@ -0,0 +1,95 @@ +import { getOwner } from '@ember/owner'; +import type { RenderingTestContext } from '@ember/test-helpers'; + +import { getService } from '@universal-ember/test-support'; +import { module, test } from 'qunit'; + +import { isBotTriggerEvent } from '@cardstack/runtime-common'; + +import CreateListingPRRequestCommand from '@cardstack/host/commands/bot-requests/create-listing-pr-request'; +import RealmService from '@cardstack/host/services/realm'; + +import { + setupIntegrationTestRealm, + setupLocalIndexing, + testRealmInfo, + testRealmURL, +} from '../../helpers'; +import { + CardDef, + contains, + field, + setupBaseRealm, + StringField, +} from '../../helpers/base-realm'; +import { setupMockMatrix } from '../../helpers/mock-matrix'; +import { setupRenderingTest } from '../../helpers/setup'; + +class StubRealmService extends RealmService { + get defaultReadableRealm() { + return { + path: testRealmURL, + info: testRealmInfo, + }; + } +} + +module('Integration | commands | create-listing-pr-request', function (hooks) { + setupRenderingTest(hooks); + setupLocalIndexing(hooks); + setupBaseRealm(hooks); + + let mockMatrixUtils = setupMockMatrix(hooks, { + loggedInAs: '@testuser:localhost', + activeRealms: [testRealmURL], + }); + + let { createAndJoinRoom, getRoomEvents } = mockMatrixUtils; + + hooks.beforeEach(function (this: RenderingTestContext) { + getOwner(this)!.register('service:realm', StubRealmService); + }); + + hooks.beforeEach(async function () { + class Listing extends CardDef { + @field name = contains(StringField); + } + + await setupIntegrationTestRealm({ + mockMatrixUtils, + contents: { + 'listing.gts': { Listing }, + 'Listing/test-listing.json': new Listing({ name: 'Some Listing' }), + }, + }); + }); + + test('sends listingName in pr-listing-create trigger input', async function (assert) { + let roomId = createAndJoinRoom({ + sender: '@testuser:localhost', + name: 'room-test', + }); + let commandService = getService('command-service'); + + let command = new CreateListingPRRequestCommand( + commandService.commandContext, + ); + await command.execute({ + roomId, + realm: testRealmURL, + listingId: `${testRealmURL}Listing/test-listing`, + }); + + let event = getRoomEvents(roomId).pop()!; + assert.ok(isBotTriggerEvent(event)); + assert.strictEqual(event.content.type, 'pr-listing-create'); + assert.strictEqual(event.content.realm, testRealmURL); + assert.strictEqual(event.content.userId, '@testuser:localhost'); + assert.deepEqual(event.content.input, { + roomId, + realm: testRealmURL, + listingId: `${testRealmURL}Listing/test-listing`, + listingName: 'Some Listing', + }); + }); +}); diff --git a/packages/host/tests/integration/commands/send-bot-trigger-event-test.gts b/packages/host/tests/integration/commands/send-bot-trigger-event-test.gts index f0283c4f7e7..af399b202e3 100644 --- a/packages/host/tests/integration/commands/send-bot-trigger-event-test.gts +++ b/packages/host/tests/integration/commands/send-bot-trigger-event-test.gts @@ -60,15 +60,16 @@ module('Integration | commands | send-bot-trigger-event', function (hooks) { let command = new SendBotTriggerEventCommand(commandService.commandContext); await command.execute({ roomId, - type: 'create-listing-pr', + type: 'pr-listing-create', realm: testRealmURL, input: { listingId: 'catalog/listing-1' }, }); let event = getRoomEvents(roomId).pop()!; assert.ok(isBotTriggerEvent(event)); - assert.strictEqual(event.content.type, 'create-listing-pr'); + assert.strictEqual(event.content.type, 'pr-listing-create'); assert.strictEqual(event.content.realm, testRealmURL); + assert.strictEqual(event.content.userId, '@testuser:localhost'); assert.deepEqual(event.content.input, { listingId: 'catalog/listing-1' }); }); }); diff --git a/packages/matrix/scripts/setup-submission-bot.ts b/packages/matrix/scripts/setup-submission-bot.ts index e651621633e..4ff9eba50a2 100644 --- a/packages/matrix/scripts/setup-submission-bot.ts +++ b/packages/matrix/scripts/setup-submission-bot.ts @@ -5,11 +5,11 @@ const realmServerURL = process.env.REALM_SERVER_URL || 'http://localhost:4201'; const botCommands = [ { name: 'create-listing-pr', - commandURL: '@cardstack/boxel-host/commands/create-listing-pr/default', + commandURL: '@cardstack/boxel-host/commands/create-submission/default', filter: { type: 'matrix-event', event_type: 'app.boxel.bot-trigger', - content_type: 'create-listing-pr', + content_type: 'pr-listing-create', }, }, { diff --git a/packages/runtime-common/bot-trigger.ts b/packages/runtime-common/bot-trigger.ts index 41ebf0b4fd0..3fd49a50a1f 100644 --- a/packages/runtime-common/bot-trigger.ts +++ b/packages/runtime-common/bot-trigger.ts @@ -19,6 +19,7 @@ export function isBotTriggerEvent(value: unknown): value is BotTriggerEvent { type?: string; input?: Record; realm?: string; + userId?: string; }; if (typeof content.type !== 'string') { return false; @@ -28,5 +29,9 @@ export function isBotTriggerEvent(value: unknown): value is BotTriggerEvent { return false; } + if (typeof content.userId !== 'string') { + return false; + } + return 'input' in content; } diff --git a/packages/runtime-common/github-submissions.ts b/packages/runtime-common/github-submissions.ts new file mode 100644 index 00000000000..cb59c6b515f --- /dev/null +++ b/packages/runtime-common/github-submissions.ts @@ -0,0 +1,82 @@ +const ROOM_ID_PREFIX = 'room-'; + +function toBase64Url(input: string): string { + let base64 = + typeof btoa === 'function' + ? btoa(input) + : Buffer.from(input, 'utf8').toString('base64'); + return base64.replace(/\+/g, '-').replace(/\//g, '_').replace(/=+$/g, ''); +} + +function fromBase64Url(input: string): string { + let base64 = input.replace(/-/g, '+').replace(/_/g, '/'); + let padding = base64.length % 4; + if (padding !== 0) { + base64 += '='.repeat(4 - padding); + } + return typeof atob === 'function' + ? atob(base64) + : Buffer.from(base64, 'base64').toString('utf8'); +} + +function matrixRoomIdToBranchName(matrixRoomId: string): string { + if (!matrixRoomId) { + throw new Error('matrixRoomId is required'); + } + return `${ROOM_ID_PREFIX}${toBase64Url(matrixRoomId)}`; +} + +function toKebabSlug(value: string): string { + return value + .trim() + .replace(/([a-z0-9])([A-Z])/g, '$1-$2') + .toLowerCase() + .replace(/[^a-z0-9]+/g, '-') + .replace(/^-+|-+$/g, ''); +} + +export function toBranchName( + matrixRoomId: string, + listingName: string, +): string { + if (!listingName) { + throw new Error('listingName is required'); + } + let roomPrefix = matrixRoomIdToBranchName(matrixRoomId); + let listingSlug = toKebabSlug(listingName); + return listingSlug ? `${roomPrefix}/${listingSlug}` : roomPrefix; +} + +export interface SubmissionMeta { + matrixRoomId: string; + listingName?: string; +} + +export function branchNameToSubmissionMeta(branchName: string): SubmissionMeta { + let [roomSegment, ...listingParts] = branchName.split('/'); + let matrixRoomId = decodeMatrixRoomIdFromBranchSegment(roomSegment); + let listingName = listingParts.join('/'); + return listingName ? { matrixRoomId, listingName } : { matrixRoomId }; +} + +function decodeMatrixRoomIdFromBranchSegment(branchName: string): string { + if (!branchName) { + throw new Error('branchName is required'); + } + let roomSegment = branchName.split('/')[0]; + if (!roomSegment.startsWith(ROOM_ID_PREFIX)) { + throw new Error('branchName does not include a matrix room id prefix'); + } + let encoded = roomSegment.slice(ROOM_ID_PREFIX.length); + if (!encoded) { + throw new Error('branchName has no encoded matrix room id'); + } + if (!/^[A-Za-z0-9_-]+$/.test(encoded)) { + throw new Error('branchName has an invalid encoded matrix room id'); + } + try { + return fromBase64Url(encoded); + } catch (error) { + throw new Error('branchName has an invalid encoded matrix room id'); + } +} diff --git a/packages/runtime-common/index.ts b/packages/runtime-common/index.ts index 0c07a6ce9ff..5a0ffc7644f 100644 --- a/packages/runtime-common/index.ts +++ b/packages/runtime-common/index.ts @@ -243,6 +243,7 @@ export * from './prerendered-html-format'; export * from './query-field-utils'; export * from './relationship-utils'; export * from './formats'; +export * from './github-submissions'; export { getCreatedTime } from './file-meta'; export { mergeRelationships } from './merge-relationships'; export { makeLogDefinitions, logger } from './log'; diff --git a/packages/runtime-common/tests/github-submissions-test.ts b/packages/runtime-common/tests/github-submissions-test.ts new file mode 100644 index 00000000000..35b1dd4c50d --- /dev/null +++ b/packages/runtime-common/tests/github-submissions-test.ts @@ -0,0 +1,95 @@ +import { + branchNameToSubmissionMeta, + toBranchName, +} from '../github-submissions'; +import type { SharedTests } from '../helpers'; + +const tests = Object.freeze({ + 'it converts a branch name back to a matrix room id': async (assert) => { + let branchName = 'room-IUZFZGlxWXRoRW52WU51QmlnbTpib3hlbC5haQ'; + assert.deepEqual(branchNameToSubmissionMeta(branchName), { + matrixRoomId: '!FEdiqYthEnvYNuBigm:boxel.ai', + }); + }, + + 'it extracts the room id from a branch name with a suffix': async ( + assert, + ) => { + let roomId = '!NVeTCArRGAqTnFzcPU:stack.cards'; + let roomPrefix = toBranchName(roomId, 'seed').split('/')[0]; + let branchName = `${roomPrefix}/my-feature`; + assert.strictEqual( + branchNameToSubmissionMeta(branchName).matrixRoomId, + roomId, + ); + }, + + 'it builds a branch name from a room id and listing name': async (assert) => { + let roomId = '!XezEDqUlIJcNdsuaFB:localhost'; + assert.strictEqual( + toBranchName(roomId, 'SomeSampleListing'), + 'room-IVhlekVEcVVsSUpjTmRzdWFGQjpsb2NhbGhvc3Q/some-sample-listing', + ); + }, + + 'it normalizes listing names with spaces and punctuation': async (assert) => { + let roomId = '!XezEDqUlIJcNdsuaFB:localhost'; + let roomPrefix = toBranchName(roomId, 'seed').split('/')[0]; + assert.strictEqual( + toBranchName(roomId, ' My New Listing '), + `${roomPrefix}/my-new-listing`, + ); + assert.strictEqual( + toBranchName(roomId, 'Weird---Name!!'), + `${roomPrefix}/weird-name`, + ); + }, + + 'it normalizes camelcase listing names': async (assert) => { + let roomId = '!XezEDqUlIJcNdsuaFB:localhost'; + let roomPrefix = toBranchName(roomId, 'seed').split('/')[0]; + assert.strictEqual( + toBranchName(roomId, 'CamelCaseListing'), + `${roomPrefix}/camel-case-listing`, + ); + }, + + 'it drops the listing segment when the slug is empty': async (assert) => { + let roomId = '!XezEDqUlIJcNdsuaFB:localhost'; + let roomPrefix = toBranchName(roomId, 'seed').split('/')[0]; + assert.strictEqual(toBranchName(roomId, ' '), roomPrefix); + }, + + 'it rejects missing listing names when building branch names': async ( + assert, + ) => { + let roomId = '!XezEDqUlIJcNdsuaFB:localhost'; + assert.throws(() => toBranchName(roomId, ''), /listingName is required/); + }, + + 'it can invert a branch name into room id and listing name': async ( + assert, + ) => { + let roomId = '!XezEDqUlIJcNdsuaFB:localhost'; + let branchName = toBranchName(roomId, 'SomeSampleListing'); + let result = branchNameToSubmissionMeta(branchName); + assert.strictEqual(result.matrixRoomId, roomId); + assert.strictEqual(result.listingName, 'some-sample-listing'); + }, + + 'it rejects branch names without the prefix': async (assert) => { + assert.throws( + () => branchNameToSubmissionMeta('feature/room-abc'), + /matrix room id prefix/, + ); + }, + + 'it rejects branch names with an invalid encoded room id': async (assert) => { + assert.throws( + () => branchNameToSubmissionMeta('room-@@@'), + /invalid encoded matrix room id/, + ); + }, +} as SharedTests>); + +export default tests; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 3bbb8cb603a..66eaa71912a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -972,6 +972,12 @@ importers: '@cardstack/runtime-common': specifier: workspace:* version: link:../runtime-common + '@octokit/rest': + specifier: 'catalog:' + version: 22.0.1 + '@octokit/types': + specifier: 16.0.0 + version: 16.0.0 '@sentry/node': specifier: 'catalog:' version: 8.55.0 From d54d5fc00153b8e67364bfcfccf39080d13912dc Mon Sep 17 00:00:00 2001 From: tintinthong Date: Tue, 24 Feb 2026 01:36:39 +0800 Subject: [PATCH 2/3] doNotWaitForPersist so no timeout --- packages/host/app/commands/create-submission.ts | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/packages/host/app/commands/create-submission.ts b/packages/host/app/commands/create-submission.ts index 2ac39daf32e..7cbda5a444d 100644 --- a/packages/host/app/commands/create-submission.ts +++ b/packages/host/app/commands/create-submission.ts @@ -51,12 +51,6 @@ export default class CreateSubmissionCommand extends HostBaseCommand< return CreateSubmissionInput; } - async getResultType() { - let commandModule = await this.loadCommandModule(); - const { CreateSubmissionResult } = commandModule; - return CreateSubmissionResult; - } - requireInputFields = ['roomId', 'realm', 'listingId']; protected async run( @@ -123,7 +117,6 @@ export default class CreateSubmissionCommand extends HostBaseCommand< const { CreateSubmissionResult } = commandModule; return new CreateSubmissionResult({ listing, - filesWithContent, }); } @@ -237,14 +230,10 @@ export default class CreateSubmissionCommand extends HostBaseCommand< }, }; - let createdCardId = await this.store.create(doc, { + await this.store.add(doc, { realm: realmURL, + doNotWaitForPersist: true, }); - if (typeof createdCardId !== 'string') { - throw new Error( - `unable to create submission card: ${JSON.stringify(createdCardId, null, 2)}`, - ); - } } // Walk relationships by fetching linked cards and enqueueing their ids. From 49fabaa3266cc0e9b36708bee5159d0d39d311f9 Mon Sep 17 00:00:00 2001 From: tintinthong Date: Tue, 24 Feb 2026 22:37:48 +0800 Subject: [PATCH 3/3] get submission bot to open a branch with contents returned from a command --- packages/base/command.gts | 1 + .../lib/create-listing-pr-handler.ts | 224 ++++++++++++++++-- packages/bot-runner/lib/github.ts | 137 ++++++++++- packages/bot-runner/lib/timeline-handler.ts | 45 +++- packages/bot-runner/tests/bot-runner-test.ts | 3 + .../submission-card/submission-card.gts | 8 + .../host/app/commands/create-submission.ts | 37 ++- 7 files changed, 405 insertions(+), 50 deletions(-) diff --git a/packages/base/command.gts b/packages/base/command.gts index d701016661c..88f1cb9ef2f 100644 --- a/packages/base/command.gts +++ b/packages/base/command.gts @@ -387,6 +387,7 @@ export class CreateSubmissionInput extends CardDef { export class CreateSubmissionResult extends CardDef { @field listing = linksTo(CardDef); + @field submission = linksTo(CardDef); @field filesWithContent = containsMany(JsonField); } diff --git a/packages/bot-runner/lib/create-listing-pr-handler.ts b/packages/bot-runner/lib/create-listing-pr-handler.ts index 73fa7b6a6f8..e137a064ffc 100644 --- a/packages/bot-runner/lib/create-listing-pr-handler.ts +++ b/packages/bot-runner/lib/create-listing-pr-handler.ts @@ -1,5 +1,6 @@ import { logger, toBranchName } from '@cardstack/runtime-common'; import type { BotTriggerContent } from 'https://cardstack.com/base/matrix-event'; +import { createHash } from 'node:crypto'; import type { GitHubClient } from './github'; const log = logger('bot-runner:create-listing-pr'); @@ -13,20 +14,28 @@ export type CreateListingPRHandler = (args: { eventContent: BotTriggerEventContent; runAs: string; githubClient: GitHubClient; + runCommandResult?: { cardResultString?: string | null } | null; }) => Promise; -export const openCreateListingPR: CreateListingPRHandler = async ({ - eventContent, - runAs, - githubClient, -}) => { +interface CreateListingPRContext { + owner: string; + repoName: string; + repo: string; + head: string; + title: string; + listingDisplayName: string; +} + +function getCreateListingPRContext( + eventContent: BotTriggerEventContent, +): CreateListingPRContext | null { if (eventContent.type !== 'pr-listing-create') { - return; + return null; } if (!eventContent.input || typeof eventContent.input !== 'object') { log.warn('pr-listing-create trigger is missing input payload'); - return; + return null; } let input = eventContent.input as Record; @@ -46,7 +55,7 @@ export const openCreateListingPR: CreateListingPRHandler = async ({ if (!title) { log.error('No title for the listing'); - return; + return null; } let repo = DEFAULT_REPO; @@ -56,23 +65,86 @@ export const openCreateListingPR: CreateListingPRHandler = async ({ throw new Error(`Invalid repo format: ${repo}. Expected "owner/repo"`); } - // TODO: create the head branch before attempting to open the pull request. - let head = 'test-submissions'; //TODO: pls remove this temporary - // let head = headBranch; + return { + owner, + repoName, + repo, + head: headBranch, + title, + listingDisplayName, + }; +} +export async function ensureCreateListingBranch(args: { + eventContent: BotTriggerEventContent; + githubClient: GitHubClient; +}): Promise { + let context = getCreateListingPRContext(args.eventContent); + if (!context) { + return; + } try { - let result = await githubClient.openPullRequest( - { - owner, - repo: repoName, - title, - head, - base: DEFAULT_BASE_BRANCH, - }, - { - label: runAs, - }, - ); + await args.githubClient.createBranch({ + owner: context.owner, + repo: context.repoName, + branch: context.head, + fromBranch: DEFAULT_BASE_BRANCH, + }); + } catch (error) { + let message = error instanceof Error ? error.message : String(error); + if (!message.includes('Reference already exists')) { + throw error; + } + } +} + +export async function addContentsToCommit(args: { + eventContent: BotTriggerEventContent; + githubClient: GitHubClient; + runCommandResult?: { cardResultString?: string | null } | null; +}): Promise { + let context = getCreateListingPRContext(args.eventContent); + if (!context) { + return; + } + let branchWrite = await getContentsFromRealm( + args.runCommandResult?.cardResultString, + ); + if (branchWrite.files.length === 0) { + return; + } + await args.githubClient.writeFilesToBranch({ + owner: context.owner, + repo: context.repoName, + branch: context.head, + files: branchWrite.files, + message: `chore: add submission output [boxel-content-hash:${branchWrite.hash}]`, + }); +} + +export const openCreateListingPR: CreateListingPRHandler = async ({ + eventContent, + runAs, + githubClient, +}) => { + let context = getCreateListingPRContext(eventContent); + if (!context) { + return; + } + let { owner, repoName, repo, head, title, listingDisplayName } = context; + + try { + let prParams = { + owner, + repo: repoName, + title, + head, + base: DEFAULT_BASE_BRANCH, + }; + let prOptions = { + label: runAs, + }; + let result = await githubClient.openPullRequest(prParams, prOptions); log.info('opened PR from pr-listing-create trigger', { runAs, @@ -81,6 +153,17 @@ export const openCreateListingPR: CreateListingPRHandler = async ({ }); } catch (error) { let message = error instanceof Error ? error.message : String(error); + if (message.includes('No commits between')) { + log.info('cannot open PR because branch has no commits beyond base', { + runAs, + repo, + head, + listingDisplayName, + error: message, + }); + return; + } + if (message.includes('A pull request already exists')) { log.info('PR already exists for submission branch', { runAs, @@ -102,3 +185,98 @@ export const openCreateListingPR: CreateListingPRHandler = async ({ return; }; + +async function getContentsFromRealm(cardResultString?: string | null): Promise<{ + files: { path: string; content: string }[]; + hash: string; +}> { + if (!cardResultString || !cardResultString.trim()) { + return { files: [], hash: hashFiles([]) }; + } + + let parsed = parseJSONLike(cardResultString); + if (parsed === undefined) { + return { files: [], hash: hashFiles([]) }; + } + + let files = extractFileContents(parsed); + return { files, hash: hashFiles(files) }; +} + +function parseJSONLike(value: string): unknown | undefined { + let current: unknown = value; + for (let i = 0; i < 3; i++) { + if (typeof current !== 'string') { + return current; + } + let text = current.trim(); + if (!text) { + return undefined; + } + try { + current = JSON.parse(text); + continue; + } catch { + try { + current = JSON.parse(decodeURIComponent(text)); + continue; + } catch { + return undefined; + } + } + } + return current; +} + +function extractFileContents( + doc: unknown, +): { path: string; content: string }[] { + if (!doc || typeof doc !== 'object') { + return []; + } + let root = doc as Record; + let attributes = (root.data as Record | undefined) + ?.attributes as Record | undefined; + let items = attributes?.allFileContents; + if (!Array.isArray(items)) { + items = attributes?.filesWithContent; + } + if (!Array.isArray(items)) { + return []; + } + let dedupe = new Map(); + for (let item of items) { + let normalized = + typeof item === 'string' ? parseJSONLike(item) : (item as unknown); + if (!normalized || typeof normalized !== 'object') { + continue; + } + let record = normalized as Record; + let path = + typeof record.filename === 'string' + ? record.filename.trim() + : typeof record.path === 'string' + ? record.path.trim() + : ''; + let content = + typeof record.contents === 'string' + ? record.contents + : typeof record.content === 'string' + ? record.content + : ''; + if (!path) { + continue; + } + dedupe.set(path, { path, content }); + } + return [...dedupe.values()]; +} + +function hashFiles(files: { path: string; content: string }[]): string { + let normalized = files + .slice() + .sort((a, b) => a.path.localeCompare(b.path)) + .map((file) => `${file.path}\n${file.content}`) + .join('\n---\n'); + return createHash('sha256').update(normalized).digest('hex').slice(0, 12); +} diff --git a/packages/bot-runner/lib/github.ts b/packages/bot-runner/lib/github.ts index 7f114eeca42..6f625b029e1 100644 --- a/packages/bot-runner/lib/github.ts +++ b/packages/bot-runner/lib/github.ts @@ -30,8 +30,32 @@ export interface CreateBranchResult { sha: string; } +export interface WriteFileToBranchParams { + owner: string; + repo: string; + branch: string; + path: string; + content: string; + message: string; +} + +export interface WriteFileToBranchResult { + commitSha: string; +} + +export interface WriteFilesToBranchParams { + owner: string; + repo: string; + branch: string; + files: { path: string; content: string }[]; + message: string; +} + +export interface WriteFilesToBranchResult { + commitSha: string; +} + const HARDCODED_REVIEWER = 'tintinthong'; -const HARDCODED_PR_HEAD_BRANCH = 'test-submissions'; export interface OpenPullRequestOptions { label: string; @@ -43,6 +67,12 @@ export interface GitHubClient { options: OpenPullRequestOptions, ): Promise; createBranch(params: CreateBranchParams): Promise; + writeFileToBranch( + params: WriteFileToBranchParams, + ): Promise; + writeFilesToBranch( + params: WriteFilesToBranchParams, + ): Promise; } export class OctokitGitHubClient implements GitHubClient { @@ -60,11 +90,8 @@ export class OctokitGitHubClient implements GitHubClient { throw new Error('label is required'); } try { - // TODO: remove temporary hardcoded reviewer/head overrides once - // submission branch creation is fully wired through bot-runner. let prParams: Parameters[0] = { ...params, - head: HARDCODED_PR_HEAD_BRANCH, }; let response = await octokit.rest.pulls.create(prParams); await octokit.rest.pulls.requestReviewers({ @@ -123,6 +150,105 @@ export class OctokitGitHubClient implements GitHubClient { } } + async writeFileToBranch( + params: WriteFileToBranchParams, + ): Promise { + return this.writeFilesToBranch({ + owner: params.owner, + repo: params.repo, + branch: params.branch, + files: [{ path: params.path, content: params.content }], + message: params.message, + }); + } + + async writeFilesToBranch( + params: WriteFilesToBranchParams, + ): Promise { + let octokit = this.getClient(); + let branch = normalizeBranchName(params.branch); + + if (!branch) { + throw new Error('branch is required'); + } + if (!params.message?.trim()) { + throw new Error('message is required'); + } + if (!Array.isArray(params.files) || params.files.length === 0) { + throw new Error('files are required'); + } + + let normalizedFiles = params.files + .map((file) => ({ + path: file.path?.trim(), + content: file.content ?? '', + })) + .filter((file) => !!file.path) as { path: string; content: string }[]; + if (normalizedFiles.length === 0) { + throw new Error('at least one file path is required'); + } + + try { + let branchRef = await octokit.rest.git.getRef({ + owner: params.owner, + repo: params.repo, + ref: `heads/${branch}`, + }); + let parentCommitSha = branchRef.data.object.sha; + + let parentCommit = await octokit.rest.git.getCommit({ + owner: params.owner, + repo: params.repo, + commit_sha: parentCommitSha, + }); + let baseTreeSha = parentCommit.data.tree.sha; + + let tree = await Promise.all( + normalizedFiles.map(async (file) => { + let blob = await octokit.rest.git.createBlob({ + owner: params.owner, + repo: params.repo, + content: file.content, + encoding: 'utf-8', + }); + return { + path: file.path, + mode: '100644' as const, + type: 'blob' as const, + sha: blob.data.sha, + }; + }), + ); + + let createdTree = await octokit.rest.git.createTree({ + owner: params.owner, + repo: params.repo, + base_tree: baseTreeSha, + tree, + }); + + let createdCommit = await octokit.rest.git.createCommit({ + owner: params.owner, + repo: params.repo, + message: params.message.trim(), + tree: createdTree.data.sha, + parents: [parentCommitSha], + }); + + await octokit.rest.git.updateRef({ + owner: params.owner, + repo: params.repo, + ref: `heads/${branch}`, + sha: createdCommit.data.sha, + force: false, + }); + + return { commitSha: createdCommit.data.sha }; + } catch (error: any) { + throw toGitHubError('write files to branch', error); + } + } + private getClient(): Octokit { if (!this.token) { throw new Error('SUBMISSION_BOT_GITHUB_TOKEN is not set'); @@ -146,7 +272,8 @@ function normalizeBranchName(branch: string): string { } function toGitHubError(action: string, error: any): Error { - let status = typeof error?.status === 'number' ? String(error.status) : 'unknown'; + let status = + typeof error?.status === 'number' ? String(error.status) : 'unknown'; let payload = error?.response?.data !== undefined ? error.response.data : error?.message; return new Error( diff --git a/packages/bot-runner/lib/timeline-handler.ts b/packages/bot-runner/lib/timeline-handler.ts index f6a2f84ee18..af295db67ff 100644 --- a/packages/bot-runner/lib/timeline-handler.ts +++ b/packages/bot-runner/lib/timeline-handler.ts @@ -9,7 +9,9 @@ import { import { enqueueRunCommandJob } from '@cardstack/runtime-common/jobs/run-command'; import * as Sentry from '@sentry/node'; import { + addContentsToCommit, openCreateListingPR, + ensureCreateListingBranch, type BotTriggerEventContent, } from './create-listing-pr-handler'; import type { GitHubClient } from './github'; @@ -17,6 +19,7 @@ import type { DBAdapter, PgPrimitive, QueuePublisher, + RunCommandResponse, } from '@cardstack/runtime-common'; import type { MatrixEvent, Room } from 'matrix-js-sdk'; @@ -156,7 +159,7 @@ async function maybeEnqueueCommand({ eventContent: BotTriggerEventContent; allowedCommands: { type: string; command: string }[]; githubClient: GitHubClient; -}): Promise { +}): Promise { try { if ( !allowedCommands.length || @@ -166,15 +169,6 @@ async function maybeEnqueueCommand({ return; } - if (eventContent.type === 'pr-listing-create') { - // Temporary workaround: handle PR creation directly until this flow is moved to a proper command path. - await openCreateListingPR({ - eventContent, - runAs, - githubClient, - }); - } - if (!eventContent?.input || typeof eventContent.input !== 'object') { return; } @@ -196,7 +190,35 @@ async function maybeEnqueueCommand({ return; } - await enqueueRunCommandJob( + if (eventContent.type === 'pr-listing-create') { + await ensureCreateListingBranch({ eventContent, githubClient }); + let job = await enqueueRunCommandJob( + { + realmURL, + realmUsername: runAs, + runAs, + command, + commandInput, + }, + queuePublisher, + dbAdapter, + userInitiatedPriority, + ); + let result: RunCommandResponse = await job.done; + await addContentsToCommit({ + eventContent, + githubClient, + runCommandResult: result, + }); + await openCreateListingPR({ + eventContent, + runAs, + githubClient, + }); + return result; + } + + let job = await enqueueRunCommandJob( { realmURL, realmUsername: runAs, @@ -208,6 +230,7 @@ async function maybeEnqueueCommand({ dbAdapter, userInitiatedPriority, ); + return await job.done; } catch (error) { log.error('error in maybeEnqueueCommand', { runAs, diff --git a/packages/bot-runner/tests/bot-runner-test.ts b/packages/bot-runner/tests/bot-runner-test.ts index 5ccb6238f47..e82ec6ce8cf 100644 --- a/packages/bot-runner/tests/bot-runner-test.ts +++ b/packages/bot-runner/tests/bot-runner-test.ts @@ -148,6 +148,9 @@ module('timeline handler', () => { ref: 'refs/heads/room-branch', sha: 'abc123', }), + createEmptyCommit: async () => ({ + sha: 'def456', + }), }; test('enqueues command when event matches', async (assert) => { diff --git a/packages/catalog-realm/submission-card/submission-card.gts b/packages/catalog-realm/submission-card/submission-card.gts index 3c7a0d23239..cab3a1e4955 100644 --- a/packages/catalog-realm/submission-card/submission-card.gts +++ b/packages/catalog-realm/submission-card/submission-card.gts @@ -1,6 +1,8 @@ import { CardDef, + FieldDef, contains, + containsMany, field, linksTo, } from 'https://cardstack.com/base/card-api'; @@ -17,6 +19,11 @@ function encodeBranchName(branchName: string): string { .join('/'); } +export class FileContentField extends FieldDef { + @field filename = contains(StringField); + @field contents = contains(StringField); +} + export class SubmissionCard extends CardDef { static displayName = 'SubmissionCard'; @@ -31,4 +38,5 @@ export class SubmissionCard extends CardDef { }, }); @field listing = linksTo(() => Listing); + @field allFileContents = containsMany(FileContentField); } diff --git a/packages/host/app/commands/create-submission.ts b/packages/host/app/commands/create-submission.ts index 7cbda5a444d..ddb51135ed2 100644 --- a/packages/host/app/commands/create-submission.ts +++ b/packages/host/app/commands/create-submission.ts @@ -18,6 +18,7 @@ import type { CopyModuleMeta } from '@cardstack/runtime-common/catalog'; import ENV from '@cardstack/host/config/environment'; +import type { CardDef } from 'https://cardstack.com/base/card-api'; import type * as BaseCommandModule from 'https://cardstack.com/base/command'; import HostBaseCommand from '../lib/host-base-command'; @@ -106,17 +107,20 @@ export default class CreateSubmissionCommand extends HostBaseCommand< log.debug(`Prepared submission with ${filesWithContent.length} files`); - await this.createSubmissionCard({ + let submission = await this.createSubmissionCard({ listing, branchName, roomId, realmURL: realmUrl, + filesWithContent, }); const commandModule = await this.loadCommandModule(); const { CreateSubmissionResult } = commandModule; return new CreateSubmissionResult({ listing, + submission, + filesWithContent, }); } @@ -126,17 +130,22 @@ export default class CreateSubmissionCommand extends HostBaseCommand< realmUrl: string, ): Promise { const toRepoRelativePath = (fullUrl: string, extension: string): string => { - let url = fullUrl; - if (url.startsWith(realmUrl)) { - url = url.slice(realmUrl.length); + let path = fullUrl; + if (path.startsWith(realmUrl)) { + path = path.slice(realmUrl.length); } - if (url.startsWith('/')) { - url = url.slice(1); + try { + path = decodeURIComponent(new URL(path).pathname); + } catch { + // keep non-URL input as-is } - if (!url.endsWith(extension)) { - url = url + extension; + if (path.startsWith('/')) { + path = path.slice(1); } - return url; + if (!path.endsWith(extension)) { + path = path + extension; + } + return path; }; const filesWithContent: FileWithContent[] = []; @@ -195,12 +204,14 @@ export default class CreateSubmissionCommand extends HostBaseCommand< branchName, roomId, realmURL, + filesWithContent, }: { listing: Listing; branchName: string; roomId: string; realmURL: string; - }): Promise { + filesWithContent: FileWithContent[]; + }): Promise { if (!listing.id) { throw new Error('Missing listing.id for submission card creation'); } @@ -220,6 +231,10 @@ export default class CreateSubmissionCommand extends HostBaseCommand< attributes: { roomId, branchName, + allFileContents: filesWithContent.map((file) => ({ + filename: file.path, + contents: file.content, + })), }, meta: { adoptsFrom: { @@ -230,7 +245,7 @@ export default class CreateSubmissionCommand extends HostBaseCommand< }, }; - await this.store.add(doc, { + return await this.store.add(doc, { realm: realmURL, doNotWaitForPersist: true, });