From 5060a73dfa8b64775cad545085cd43cc3fa5d59a Mon Sep 17 00:00:00 2001 From: Daniel Naab Date: Tue, 7 Oct 2025 09:14:51 -0500 Subject: [PATCH 01/17] Implement simple Postgres-based async task runner for PDF imports. We may want a more sophisticated method later on. --- .../20251007000000_create_form_jobs.mjs | 58 ++++ packages/database/src/clients/kysely/types.ts | 18 ++ packages/forms/src/blueprint.ts | 19 ++ packages/forms/src/builder/parse-form.test.ts | 4 +- .../forms/src/context/browser/form-repo.ts | 31 +- packages/forms/src/context/index.ts | 1 + packages/forms/src/index.ts | 4 + .../src/patterns/phone-number/phone-number.ts | 40 ++- packages/forms/src/repository/add-document.ts | 4 +- .../forms/src/repository/form-jobs.test.ts | 273 ++++++++++++++++++ packages/forms/src/repository/form-jobs.ts | 248 ++++++++++++++++ packages/forms/src/repository/get-document.ts | 5 +- packages/forms/src/repository/index.ts | 26 ++ .../src/repository/jobs/complete-form-job.ts | 32 ++ .../src/repository/jobs/create-form-job.ts | 49 ++++ .../src/repository/jobs/fail-form-job.ts | 32 ++ .../src/repository/jobs/get-form-jobs.ts | 31 ++ .../repository/jobs/get-latest-form-job.ts | 40 +++ packages/forms/src/repository/jobs/types.ts | 69 +++++ packages/forms/src/repository/types.ts | 10 + .../forms/src/services/get-form-status.ts | 83 ++++++ packages/forms/src/services/index.ts | 3 + .../src/services/initialize-form.test.ts | 227 ++++++++++++++- .../forms/src/services/initialize-form.ts | 175 +++++++++-- packages/server/src/lib/api-client.ts | 18 +- .../server/src/pages/api/forms/[id]/status.ts | 20 ++ 26 files changed, 1463 insertions(+), 57 deletions(-) create mode 100644 packages/database/migrations/20251007000000_create_form_jobs.mjs create mode 100644 packages/forms/src/repository/form-jobs.test.ts create mode 100644 packages/forms/src/repository/form-jobs.ts create mode 100644 packages/forms/src/repository/jobs/complete-form-job.ts create mode 100644 packages/forms/src/repository/jobs/create-form-job.ts create mode 100644 packages/forms/src/repository/jobs/fail-form-job.ts create mode 100644 packages/forms/src/repository/jobs/get-form-jobs.ts create mode 100644 packages/forms/src/repository/jobs/get-latest-form-job.ts create mode 100644 packages/forms/src/repository/jobs/types.ts create mode 100644 packages/forms/src/services/get-form-status.ts create mode 100644 packages/server/src/pages/api/forms/[id]/status.ts diff --git a/packages/database/migrations/20251007000000_create_form_jobs.mjs b/packages/database/migrations/20251007000000_create_form_jobs.mjs new file mode 100644 index 00000000..df23e6ae --- /dev/null +++ b/packages/database/migrations/20251007000000_create_form_jobs.mjs @@ -0,0 +1,58 @@ +/** + * @param { import("knex").Knex } knex + * @returns { Promise } + */ +export async function up(knex) { + await knex.schema.createTable('form_jobs', table => { + table.uuid('id').primary(); + + // Foreign key to forms (CASCADE delete: jobs belong to forms) + table + .uuid('form_id') + .notNullable() + .references('id') + .inTable('forms') + .onDelete('CASCADE'); + + // Job type - extensible for future operations + table.text('job_type').notNullable(); + // Values: 'import-pdf', 'validate-schema', 'publish', 'export', etc. + + // Job status - represents operation state + table.text('status').notNullable(); + // Values: 'pending', 'processing', 'completed', 'failed' + + // Timing information + table.timestamp('created_at').notNullable().defaultTo(knex.fn.now()); + table.timestamp('started_at').nullable(); + table.timestamp('completed_at').nullable(); + + // Error tracking + table.text('error_message').nullable(); + table.text('error_stack').nullable(); + + // Job metadata (input parameters, varies by job_type) + // For 'import-pdf': { documentId, fileName, userId } + // For 'publish': { targetEnvironment, publisherId } + table.text('metadata').nullable(); + + // Job result (output data, varies by job_type) + // For 'import-pdf': { patternsAdded: 5, fieldsExtracted: 12 } + // For 'validate': { errorsFound: 2, warningsFound: 5 } + table.text('result').nullable(); + + // Indexes for common queries + table.index('form_id', 'idx_form_jobs_form_id'); + table.index('status', 'idx_form_jobs_status'); + table.index(['form_id', 'job_type'], 'idx_form_jobs_form_type'); + table.index('created_at', 'idx_form_jobs_created'); + }); +} + +/** + * @param { import("knex").Knex } knex + * @returns { Promise } + */ +export async function down(knex) { + await knex.schema.dropTableIfExists('form_jobs'); +} diff --git a/packages/database/src/clients/kysely/types.ts b/packages/database/src/clients/kysely/types.ts index 7590feb7..75ff9eb7 100644 --- a/packages/database/src/clients/kysely/types.ts +++ b/packages/database/src/clients/kysely/types.ts @@ -16,6 +16,7 @@ export interface Database { forms: FormsTable; form_sessions: FormSessionsTable; form_documents: FormDocumentsTable; + form_jobs: FormJobsTable; llm_request_cache: LlmRequestCacheTable; } export type DatabaseClient = Kysely; @@ -74,6 +75,23 @@ export type FormDocumentsTableSelectable = Selectable; export type FormDocumentsTableInsertable = Insertable; export type FormDocumentsTableUpdateable = Updateable; +interface FormJobsTable { + id: string; + form_id: string; + job_type: string; + status: string; + created_at: Generated; + started_at: Date | null; + completed_at: Date | null; + error_message: string | null; + error_stack: string | null; + metadata: string | null; + result: string | null; +} +export type FormJobsTableSelectable = Selectable; +export type FormJobsTableInsertable = Insertable; +export type FormJobsTableUpdateable = Updateable; + interface LlmRequestCacheTable { id: Generated; cache_key: string; diff --git a/packages/forms/src/blueprint.ts b/packages/forms/src/blueprint.ts index b9c61887..5de67c96 100644 --- a/packages/forms/src/blueprint.ts +++ b/packages/forms/src/blueprint.ts @@ -710,14 +710,33 @@ export const addFormOutput = ( /** * Updates the summary of a given form with the provided summary details. + * Also updates the form-summary pattern to keep them in sync. */ export const updateFormSummary = ( form: Blueprint, summary: FormSummary ): Blueprint => { + // Find the form-summary pattern and update it + const formSummaryPatternEntry = Object.entries(form.patterns).find( + ([_, pattern]) => pattern.type === 'form-summary' + ); + + const updatedPatterns = { ...form.patterns }; + if (formSummaryPatternEntry) { + const [patternId, pattern] = formSummaryPatternEntry; + updatedPatterns[patternId] = { + ...pattern, + data: { + ...pattern.data, + ...summary, + }, + }; + } + return { ...form, summary, + patterns: updatedPatterns, }; }; diff --git a/packages/forms/src/builder/parse-form.test.ts b/packages/forms/src/builder/parse-form.test.ts index 6748256a..72a40704 100644 --- a/packages/forms/src/builder/parse-form.test.ts +++ b/packages/forms/src/builder/parse-form.test.ts @@ -139,11 +139,11 @@ describe('parseFormString', () => { '[\n' + ' {\n' + ' "code": "custom",\n' + - ' "message": "Invalid pattern",\n' + ' "path": [\n' + ' "patterns",\n' + ' "invalidPattern"\n' + - ' ]\n' + + ' ],\n' + + ' "message": "Invalid pattern"\n' + ' }\n' + ']', }); diff --git a/packages/forms/src/context/browser/form-repo.ts b/packages/forms/src/context/browser/form-repo.ts index 84fa8bef..c3bc138a 100644 --- a/packages/forms/src/context/browser/form-repo.ts +++ b/packages/forms/src/context/browser/form-repo.ts @@ -144,7 +144,7 @@ export class BrowserFormRepository implements FormRepository { addDocument(document: { fileName: string; data: Uint8Array; - extract: { parsedPdf: ParsedPdf; fields: DocumentFieldMap }; + extract?: { parsedPdf: ParsedPdf; fields: DocumentFieldMap }; }) { const documentId = crypto.randomUUID(); const data = uint8ArrayToBase64(document.data); @@ -155,7 +155,7 @@ export class BrowserFormRepository implements FormRepository { type: 'pdf', file_name: document.fileName, data, - extract: JSON.stringify(document.extract), + extract: document.extract ? JSON.stringify(document.extract) : '', }) ); return Promise.resolve( @@ -183,6 +183,33 @@ export class BrowserFormRepository implements FormRepository { data: base64ToUint8Array(json.data), }); } + + // Job methods are not supported in browser context + createFormJob(): Promise { + return Promise.resolve( + failure('Job management is not supported in browser context') + ); + } + + completeFormJob(): Promise { + return Promise.resolve( + failure('Job management is not supported in browser context') + ); + } + + failFormJob(): Promise { + return Promise.resolve( + failure('Job management is not supported in browser context') + ); + } + + getLatestFormJob(): Promise { + return Promise.resolve(success(null)); + } + + getFormJobs(): Promise { + return Promise.resolve(success([])); + } } /** diff --git a/packages/forms/src/context/index.ts b/packages/forms/src/context/index.ts index 92bb4ebf..a0e52b57 100644 --- a/packages/forms/src/context/index.ts +++ b/packages/forms/src/context/index.ts @@ -23,6 +23,7 @@ export type FormServiceContext = { repository: FormRepository; config: FormConfig; isUserLoggedIn: () => boolean; + getUserId?: () => string; parser: PdfParser; }; diff --git a/packages/forms/src/index.ts b/packages/forms/src/index.ts index 9312eb9c..86aeceb2 100644 --- a/packages/forms/src/index.ts +++ b/packages/forms/src/index.ts @@ -11,6 +11,10 @@ export * from './types.js'; export * from './util/base64.js'; export * from './patterns/address/jurisdictions.js'; export { type FormService, createFormService } from './services/index.js'; +export { + type FormStatusResponse, + type GetFormStatusError, +} from './services/get-form-status.js'; export { defaultFormConfig, attachmentFileTypeOptions, diff --git a/packages/forms/src/patterns/phone-number/phone-number.ts b/packages/forms/src/patterns/phone-number/phone-number.ts index f01824ad..4e2f948e 100644 --- a/packages/forms/src/patterns/phone-number/phone-number.ts +++ b/packages/forms/src/patterns/phone-number/phone-number.ts @@ -22,21 +22,37 @@ export type PhoneNumberPatternOutput = z.infer< export const createPhoneSchema = (data: PhoneNumberPattern['data']) => { const phoneSchema = z .string() - .regex(/^(\d{3}-\d{3}-\d{4}|\d{10})$/, { - message: 'Invalid phone number format', + .superRefine((value, ctx) => { + // Allow empty string if not required + if (value === '' && !data.required) { + return; + } + + // Validate format + if (!/^(\d{3}-\d{3}-\d{4}|\d{10})$/.test(value)) { + ctx.addIssue({ + code: 'custom', + message: 'Invalid phone number format', + }); + return; + } + + // Validate length + const digits = value.replace(/[^\d]/g, ''); + if (digits.length !== 10) { + ctx.addIssue({ + code: 'custom', + message: 'Invalid phone number format', + }); + } }) .transform(value => { + if (value === '') { + return value; + } const digits = value.replace(/[^\d]/g, ''); return `${digits.slice(0, 3)}-${digits.slice(3, 6)}-${digits.slice(6)}`; - }) - .refine(value => { - const digits = value.replace(/[^\d]/g, ''); - return digits.length === 10; - }, 'Phone number must contain exactly 10 digits'); - - if (!data.required) { - return z.union([z.literal(''), phoneSchema]); - } + }); return phoneSchema; }; @@ -64,7 +80,7 @@ export const phoneNumberConfig: PatternConfig< return []; }, - createPrompt(_, session, pattern, options) { + createPrompt(_, session, pattern) { const sessionValue = getFormSessionValue(session, pattern.id); const sessionError = getFormSessionError(session, pattern.id); diff --git a/packages/forms/src/repository/add-document.ts b/packages/forms/src/repository/add-document.ts index 16ac29e1..ec1af5ec 100644 --- a/packages/forms/src/repository/add-document.ts +++ b/packages/forms/src/repository/add-document.ts @@ -9,7 +9,7 @@ export type AddDocument = ( document: { fileName: string; data: Uint8Array; - extract: { + extract?: { parsedPdf: ParsedPdf; fields: DocumentFieldMap; }; @@ -30,7 +30,7 @@ export const addDocument: AddDocument = async (ctx, document) => { type: 'pdf', file_name: document.fileName, data: Buffer.from(document.data), - extract: JSON.stringify(document.extract), + extract: document.extract ? JSON.stringify(document.extract) : '', }) .execute() .then(() => diff --git a/packages/forms/src/repository/form-jobs.test.ts b/packages/forms/src/repository/form-jobs.test.ts new file mode 100644 index 00000000..8a02f9d4 --- /dev/null +++ b/packages/forms/src/repository/form-jobs.test.ts @@ -0,0 +1,273 @@ +import { beforeAll, expect, it, vi } from 'vitest'; + +import { + type DbTestContext, + describeDatabase, +} from '@flexion/forms-database/testing'; +import { createFormJob } from './jobs/create-form-job.js'; +import { completeFormJob } from './jobs/complete-form-job.js'; +import { failFormJob } from './jobs/fail-form-job.js'; +import { getLatestFormJob } from './jobs/get-latest-form-job.js'; +import { getFormJobs } from './jobs/get-form-jobs.js'; +import { addForm } from './add-form.js'; +import { defaultFormConfig } from '../patterns/index.js'; + +describeDatabase('form jobs repository', getDb => { + const today = new Date(2000, 1, 1); + + beforeAll(async () => { + vi.setSystemTime(today); + }); + + it('creates a job in processing state', async ({ db }) => { + const ctx = { db: db.ctx, formConfig: defaultFormConfig }; + + // Create a form first + const formResult = await addForm(ctx, testForm); + if (!formResult.success) { + expect.fail('addForm failed'); + } + + // Create a job + const jobResult = await createFormJob(ctx, { + formId: formResult.data.id, + jobType: 'import-pdf', + metadata: { + documentId: 'doc-1', + fileName: 'test.pdf', + userId: 'user-1', + }, + }); + + if (!jobResult.success) { + console.error('Job creation failed:', jobResult.error); + expect.fail(`createFormJob failed: ${jobResult.error}`); + } + expect(jobResult.success).toBe(true); + + expect(jobResult.data.status).toBe('processing'); + expect(jobResult.data.jobType).toBe('import-pdf'); + expect(jobResult.data.formId).toBe(formResult.data.id); + expect(jobResult.data.metadata).toEqual({ + documentId: 'doc-1', + fileName: 'test.pdf', + userId: 'user-1', + }); + }); + + it('completes a job with result data', async ({ db }) => { + const ctx = { db: db.ctx, formConfig: defaultFormConfig }; + + // Create a form + const formResult = await addForm(ctx, testForm); + if (!formResult.success) { + expect.fail('addForm failed'); + } + + // Create a job + const jobResult = await createFormJob(ctx, { + formId: formResult.data.id, + jobType: 'import-pdf', + }); + if (!jobResult.success) { + expect.fail('createFormJob failed'); + } + + // Complete the job + const completeResult = await completeFormJob(ctx, jobResult.data.id, { + patternsAdded: 5, + fieldsExtracted: 12, + documentId: 'doc-1', + }); + + expect(completeResult.success).toBe(true); + + // Verify the job was updated + const latestResult = await getLatestFormJob( + ctx, + formResult.data.id, + 'import-pdf' + ); + if (!latestResult.success || !latestResult.data) { + expect.fail('getLatestFormJob failed'); + } + + expect(latestResult.data.status).toBe('completed'); + expect(latestResult.data.result).toEqual({ + patternsAdded: 5, + fieldsExtracted: 12, + documentId: 'doc-1', + }); + expect(latestResult.data.completedAt).toBeDefined(); + }); + + it('fails a job with error message', async ({ db }) => { + const ctx = { db: db.ctx, formConfig: defaultFormConfig }; + + // Create a form + const formResult = await addForm(ctx, testForm); + if (!formResult.success) { + expect.fail('addForm failed'); + } + + // Create a job + const jobResult = await createFormJob(ctx, { + formId: formResult.data.id, + jobType: 'import-pdf', + }); + if (!jobResult.success) { + expect.fail('createFormJob failed'); + } + + // Fail the job + const failResult = await failFormJob(ctx, jobResult.data.id, { + message: 'PDF parsing failed', + stack: 'Error stack trace', + }); + + expect(failResult.success).toBe(true); + + // Verify the job was updated + const latestResult = await getLatestFormJob( + ctx, + formResult.data.id, + 'import-pdf' + ); + if (!latestResult.success || !latestResult.data) { + expect.fail('getLatestFormJob failed'); + } + + expect(latestResult.data.status).toBe('failed'); + expect(latestResult.data.errorMessage).toBe('PDF parsing failed'); + expect(latestResult.data.errorStack).toBe('Error stack trace'); + expect(latestResult.data.completedAt).toBeDefined(); + }); + + it('returns null when no job exists', async ({ db }) => { + const ctx = { db: db.ctx, formConfig: defaultFormConfig }; + + // Create a form + const formResult = await addForm(ctx, testForm); + if (!formResult.success) { + expect.fail('addForm failed'); + } + + // Try to get a job that doesn't exist + const latestResult = await getLatestFormJob( + ctx, + formResult.data.id, + 'import-pdf' + ); + + expect(latestResult.success).toBe(true); + if (!latestResult.success) { + expect.fail('getLatestFormJob failed'); + } + expect(latestResult.data).toBeNull(); + }); + + it('returns job history in chronological order', async ({ + db, + }) => { + const ctx = { db: db.ctx, formConfig: defaultFormConfig }; + + // Create a form + const formResult = await addForm(ctx, testForm); + if (!formResult.success) { + expect.fail('addForm failed'); + } + + // Create 3 jobs with different timestamps + vi.setSystemTime(new Date(2000, 1, 1, 0, 0, 0)); + const job1 = await createFormJob(ctx, { + formId: formResult.data.id, + jobType: 'import-pdf', + }); + expect(job1.success).toBe(true); + + vi.setSystemTime(new Date(2000, 1, 1, 0, 0, 1)); + const job2 = await createFormJob(ctx, { + formId: formResult.data.id, + jobType: 'import-pdf', + }); + expect(job2.success).toBe(true); + + vi.setSystemTime(new Date(2000, 1, 1, 0, 0, 2)); + const job3 = await createFormJob(ctx, { + formId: formResult.data.id, + jobType: 'validate-schema', + metadata: { + validatorVersion: '1.0', + userId: 'user-1', + }, + }); + expect(job3.success).toBe(true); + + // Get all jobs + const historyResult = await getFormJobs(ctx, formResult.data.id); + if (!historyResult.success) { + expect.fail('getFormJobs failed'); + } + + expect(historyResult.data.length).toBe(3); + + // Should be ordered newest first + expect(historyResult.data[0].createdAt.getTime()).toBeGreaterThan( + historyResult.data[1].createdAt.getTime() + ); + expect(historyResult.data[1].createdAt.getTime()).toBeGreaterThan( + historyResult.data[2].createdAt.getTime() + ); + + // Verify the last one is the validate-schema job + expect(historyResult.data[0].jobType).toBe('validate-schema'); + }); + + it('gets only the latest job of a specific type', async ({ + db, + }) => { + const ctx = { db: db.ctx, formConfig: defaultFormConfig }; + + // Create a form + const formResult = await addForm(ctx, testForm); + if (!formResult.success) { + expect.fail('addForm failed'); + } + + // Create 2 import-pdf jobs + vi.setSystemTime(new Date(2000, 1, 1, 0, 0, 0)); + await createFormJob(ctx, { + formId: formResult.data.id, + jobType: 'import-pdf', + }); + + vi.setSystemTime(new Date(2000, 1, 1, 0, 0, 1)); + const job2 = await createFormJob(ctx, { + formId: formResult.data.id, + jobType: 'import-pdf', + }); + if (!job2.success) { + expect.fail('createFormJob failed'); + } + + // Get latest import-pdf job + const latestResult = await getLatestFormJob( + ctx, + formResult.data.id, + 'import-pdf' + ); + if (!latestResult.success || !latestResult.data) { + expect.fail('getLatestFormJob failed'); + } + + // Should be the second job + expect(latestResult.data.id).toBe(job2.data.id); + }); +}); + +const testForm = { + summary: { title: 'Test form', description: 'Test description' }, + root: 'root', + patterns: { root: { type: 'sequence', id: 'root', data: { patterns: [] } } }, + outputs: [], +}; diff --git a/packages/forms/src/repository/form-jobs.ts b/packages/forms/src/repository/form-jobs.ts new file mode 100644 index 00000000..f7790de6 --- /dev/null +++ b/packages/forms/src/repository/form-jobs.ts @@ -0,0 +1,248 @@ +import { type Result, success, failure } from '@flexion/forms-common'; +import type { FormRepositoryContext } from './index.js'; + +// Type-safe job metadata by job type +export type JobMetadata = { + 'import-pdf': { + documentId: string; + fileName: string; + userId: string; + }; + 'validate-schema': { + validatorVersion: string; + userId: string; + }; + publish: { + targetEnvironment: 'staging' | 'production'; + publisherId: string; + }; +}; + +// Type-safe job results by job type +export type JobResult = { + 'import-pdf': { + patternsAdded: number; + fieldsExtracted: number; + documentId: string; + }; + 'validate-schema': { + errorsFound: number; + warningsFound: number; + issues: Array<{ field: string; message: string }>; + }; + publish: { + publishedAt: string; + url: string; + }; +}; + +export type JobType = keyof JobMetadata; +export type JobStatus = 'pending' | 'processing' | 'completed' | 'failed'; + +export type FormJob = { + id: string; + formId: string; + jobType: T; + status: JobStatus; + createdAt: Date; + startedAt?: Date; + completedAt?: Date; + errorMessage?: string; + errorStack?: string; + metadata?: JobMetadata[T]; + result?: JobResult[T]; +}; + +/** + * Create a new job record in 'processing' state. + * Call this before starting async work. + */ +export type CreateFormJob = ( + ctx: FormRepositoryContext, + params: { + formId: string; + jobType: T; + metadata?: JobMetadata[T]; + } +) => Promise, string>>; + +export const createFormJob: CreateFormJob = async (ctx, params) => { + const uuid = crypto.randomUUID(); + const db = await ctx.db.getKysely(); + + try { + const now = new Date(); + await db + .insertInto('form_jobs') + .values({ + id: uuid, + form_id: params.formId, + job_type: params.jobType, + status: 'processing', + created_at: now.toISOString() as any, + started_at: now.toISOString() as any, + metadata: params.metadata ? JSON.stringify(params.metadata) : null, + }) + .execute(); + + return success({ + id: uuid, + formId: params.formId, + jobType: params.jobType, + status: 'processing' as JobStatus, + createdAt: new Date(), + startedAt: new Date(), + metadata: params.metadata, + }) as any; + } catch (err) { + return failure(`Failed to create job: ${(err as Error).message}`); + } +}; + +/** + * Mark a job as completed with result data. + */ +export type CompleteFormJob = ( + ctx: FormRepositoryContext, + jobId: string, + result?: JobResult[T] +) => Promise>; + +export const completeFormJob: CompleteFormJob = async (ctx, jobId, result) => { + const db = await ctx.db.getKysely(); + + try { + await db + .updateTable('form_jobs') + .set({ + status: 'completed', + completed_at: new Date().toISOString() as any, + result: result ? JSON.stringify(result) : null, + }) + .where('id', '=', jobId) + .execute(); + + return success(undefined); + } catch (err) { + return failure(`Failed to complete job: ${(err as Error).message}`); + } +}; + +/** + * Mark a job as failed with error information. + */ +export type FailFormJob = ( + ctx: FormRepositoryContext, + jobId: string, + error: { message: string; stack?: string } +) => Promise>; + +export const failFormJob: FailFormJob = async (ctx, jobId, error) => { + const db = await ctx.db.getKysely(); + + try { + await db + .updateTable('form_jobs') + .set({ + status: 'failed', + completed_at: new Date().toISOString() as any, + error_message: error.message, + error_stack: error.stack || null, + }) + .where('id', '=', jobId) + .execute(); + + return success(undefined); + } catch (err) { + return failure(`Failed to fail job: ${(err as Error).message}`); + } +}; + +/** + * Get the latest job for a form of a specific type. + * Useful for status polling: "What's the current import-pdf job status?" + */ +export type GetLatestFormJob = ( + ctx: FormRepositoryContext, + formId: string, + jobType: T +) => Promise | null, string>>; + +export const getLatestFormJob: GetLatestFormJob = async ( + ctx, + formId, + jobType +) => { + const db = await ctx.db.getKysely(); + + try { + const row = await db + .selectFrom('form_jobs') + .selectAll() + .where('form_id', '=', formId) + .where('job_type', '=', jobType) + .orderBy('created_at', 'desc') + .limit(1) + .executeTakeFirst(); + + if (!row) { + return success(null); + } + + return success({ + id: row.id, + formId: row.form_id, + jobType: row.job_type as JobType, + status: row.status as JobStatus, + createdAt: new Date(row.created_at), + startedAt: row.started_at ? new Date(row.started_at) : undefined, + completedAt: row.completed_at ? new Date(row.completed_at) : undefined, + errorMessage: row.error_message || undefined, + errorStack: row.error_stack || undefined, + metadata: row.metadata ? JSON.parse(row.metadata) : undefined, + result: row.result ? JSON.parse(row.result) : undefined, + }) as any; + } catch (err) { + return failure(`Failed to get latest job: ${(err as Error).message}`); + } +}; + +/** + * Get all jobs for a form (full history). + * Useful for debugging and audit logs. + */ +export type GetFormJobs = ( + ctx: FormRepositoryContext, + formId: string +) => Promise>; + +export const getFormJobs: GetFormJobs = async (ctx, formId) => { + const db = await ctx.db.getKysely(); + + try { + const rows = await db + .selectFrom('form_jobs') + .selectAll() + .where('form_id', '=', formId) + .orderBy('created_at', 'desc') + .execute(); + + const jobs = rows.map(row => ({ + id: row.id, + formId: row.form_id, + jobType: row.job_type as JobType, + status: row.status as JobStatus, + createdAt: new Date(row.created_at), + startedAt: row.started_at ? new Date(row.started_at) : undefined, + completedAt: row.completed_at ? new Date(row.completed_at) : undefined, + errorMessage: row.error_message || undefined, + errorStack: row.error_stack || undefined, + metadata: row.metadata ? JSON.parse(row.metadata) : undefined, + result: row.result ? JSON.parse(row.result) : undefined, + })); + + return success(jobs as FormJob[]); + } catch (err) { + return failure(`Failed to get jobs: ${(err as Error).message}`); + } +}; diff --git a/packages/forms/src/repository/get-document.ts b/packages/forms/src/repository/get-document.ts index dd13a3b9..63d199c8 100644 --- a/packages/forms/src/repository/get-document.ts +++ b/packages/forms/src/repository/get-document.ts @@ -29,8 +29,11 @@ export const getDocument: GetDocument = async (ctx, id) => { .where('id', '=', id) .executeTakeFirstOrThrow() .then(data => { + // Handle documents without extract (stored during async processing initialization) const extract: { parsedPdf: ParsedPdf; fields: DocumentFieldMap } = - JSON.parse(data.extract); + data.extract && data.extract !== '' + ? JSON.parse(data.extract) + : { parsedPdf: {} as ParsedPdf, fields: {} }; return success({ id: data.id, data: data.data, diff --git a/packages/forms/src/repository/index.ts b/packages/forms/src/repository/index.ts index b09c4511..1ce0bc99 100644 --- a/packages/forms/src/repository/index.ts +++ b/packages/forms/src/repository/index.ts @@ -16,6 +16,25 @@ import { type UpsertFormSession, upsertFormSession, } from './upsert-form-session.js'; +import { + type CreateFormJob, + createFormJob, +} from './jobs/create-form-job.js'; +import { + type CompleteFormJob, + completeFormJob, +} from './jobs/complete-form-job.js'; +import { type FailFormJob, failFormJob } from './jobs/fail-form-job.js'; +import { + type GetLatestFormJob, + getLatestFormJob, +} from './jobs/get-latest-form-job.js'; +import { type GetFormJobs, getFormJobs } from './jobs/get-form-jobs.js'; +import { + type FormJob, + type JobType, + type JobStatus, +} from './jobs/types.js'; export type { FormRepository }; @@ -24,6 +43,8 @@ export type FormRepositoryContext = { formConfig: FormConfig; }; +export { type FormJob, type JobType, type JobStatus } from './jobs/types.js'; + export const createFormsRepository = ( ctx: FormRepositoryContext ): FormRepository => @@ -37,4 +58,9 @@ export const createFormsRepository = ( getForm, saveForm, upsertFormSession, + createFormJob, + completeFormJob, + failFormJob, + getLatestFormJob, + getFormJobs, }); diff --git a/packages/forms/src/repository/jobs/complete-form-job.ts b/packages/forms/src/repository/jobs/complete-form-job.ts new file mode 100644 index 00000000..f90874c1 --- /dev/null +++ b/packages/forms/src/repository/jobs/complete-form-job.ts @@ -0,0 +1,32 @@ +import { type Result, success, failure } from '@flexion/forms-common'; +import type { FormRepositoryContext } from '../index.js'; +import { type JobResult, type JobType } from './types.js'; + +/** + * Mark a job as completed with result data. + */ +export type CompleteFormJob = ( + ctx: FormRepositoryContext, + jobId: string, + result?: JobResult[T] +) => Promise>; + +export const completeFormJob: CompleteFormJob = async (ctx, jobId, result) => { + const db = await ctx.db.getKysely(); + + try { + await db + .updateTable('form_jobs') + .set({ + status: 'completed', + completed_at: new Date().toISOString() as any, + result: result ? JSON.stringify(result) : null, + }) + .where('id', '=', jobId) + .execute(); + + return success(undefined); + } catch (err) { + return failure(`Failed to complete job: ${(err as Error).message}`); + } +}; diff --git a/packages/forms/src/repository/jobs/create-form-job.ts b/packages/forms/src/repository/jobs/create-form-job.ts new file mode 100644 index 00000000..1cc6ddf7 --- /dev/null +++ b/packages/forms/src/repository/jobs/create-form-job.ts @@ -0,0 +1,49 @@ +import { type Result, success, failure } from '@flexion/forms-common'; +import type { FormRepositoryContext } from '../index.js'; +import { type FormJob, type JobMetadata, type JobStatus, type JobType } from './types.js'; + +/** + * Create a new job record in 'processing' state. + * Call this before starting async work. + */ +export type CreateFormJob = ( + ctx: FormRepositoryContext, + params: { + formId: string; + jobType: T; + metadata?: JobMetadata[T]; + } +) => Promise, string>>; + +export const createFormJob: CreateFormJob = async (ctx, params) => { + const uuid = crypto.randomUUID(); + const db = await ctx.db.getKysely(); + + try { + const now = new Date(); + await db + .insertInto('form_jobs') + .values({ + id: uuid, + form_id: params.formId, + job_type: params.jobType, + status: 'processing', + created_at: now.toISOString() as any, + started_at: now.toISOString() as any, + metadata: params.metadata ? JSON.stringify(params.metadata) : null, + }) + .execute(); + + return success({ + id: uuid, + formId: params.formId, + jobType: params.jobType, + status: 'processing' as JobStatus, + createdAt: now, + startedAt: now, + metadata: params.metadata, + }) as any; + } catch (err) { + return failure(`Failed to create job: ${(err as Error).message}`); + } +}; diff --git a/packages/forms/src/repository/jobs/fail-form-job.ts b/packages/forms/src/repository/jobs/fail-form-job.ts new file mode 100644 index 00000000..10577d1f --- /dev/null +++ b/packages/forms/src/repository/jobs/fail-form-job.ts @@ -0,0 +1,32 @@ +import { type Result, success, failure } from '@flexion/forms-common'; +import type { FormRepositoryContext } from '../index.js'; + +/** + * Mark a job as failed with error information. + */ +export type FailFormJob = ( + ctx: FormRepositoryContext, + jobId: string, + error: { message: string; stack?: string } +) => Promise>; + +export const failFormJob: FailFormJob = async (ctx, jobId, error) => { + const db = await ctx.db.getKysely(); + + try { + await db + .updateTable('form_jobs') + .set({ + status: 'failed', + completed_at: new Date().toISOString() as any, + error_message: error.message, + error_stack: error.stack || null, + }) + .where('id', '=', jobId) + .execute(); + + return success(undefined); + } catch (err) { + return failure(`Failed to fail job: ${(err as Error).message}`); + } +}; diff --git a/packages/forms/src/repository/jobs/get-form-jobs.ts b/packages/forms/src/repository/jobs/get-form-jobs.ts new file mode 100644 index 00000000..6effe49f --- /dev/null +++ b/packages/forms/src/repository/jobs/get-form-jobs.ts @@ -0,0 +1,31 @@ +import { type Result, success, failure } from '@flexion/forms-common'; +import type { FormRepositoryContext } from '../index.js'; +import { type FormJob, rowToFormJob } from './types.js'; + +/** + * Get all jobs for a form (full history). + * Useful for debugging and audit logs. + */ +export type GetFormJobs = ( + ctx: FormRepositoryContext, + formId: string +) => Promise>; + +export const getFormJobs: GetFormJobs = async (ctx, formId) => { + const db = await ctx.db.getKysely(); + + try { + const rows = await db + .selectFrom('form_jobs') + .selectAll() + .where('form_id', '=', formId) + .orderBy('created_at', 'desc') + .execute(); + + const jobs = rows.map(rowToFormJob); + + return success(jobs as FormJob[]); + } catch (err) { + return failure(`Failed to get jobs: ${(err as Error).message}`); + } +}; diff --git a/packages/forms/src/repository/jobs/get-latest-form-job.ts b/packages/forms/src/repository/jobs/get-latest-form-job.ts new file mode 100644 index 00000000..651a3a94 --- /dev/null +++ b/packages/forms/src/repository/jobs/get-latest-form-job.ts @@ -0,0 +1,40 @@ +import { type Result, success, failure } from '@flexion/forms-common'; +import type { FormRepositoryContext } from '../index.js'; +import { type FormJob, type JobType, rowToFormJob } from './types.js'; + +/** + * Get the latest job for a form of a specific type. + * Useful for status polling: "What's the current import-pdf job status?" + */ +export type GetLatestFormJob = ( + ctx: FormRepositoryContext, + formId: string, + jobType: T +) => Promise | null, string>>; + +export const getLatestFormJob: GetLatestFormJob = async ( + ctx, + formId, + jobType +) => { + const db = await ctx.db.getKysely(); + + try { + const row = await db + .selectFrom('form_jobs') + .selectAll() + .where('form_id', '=', formId) + .where('job_type', '=', jobType) + .orderBy('created_at', 'desc') + .limit(1) + .executeTakeFirst(); + + if (!row) { + return success(null); + } + + return success(rowToFormJob(row)) as any; + } catch (err) { + return failure(`Failed to get latest job: ${(err as Error).message}`); + } +}; diff --git a/packages/forms/src/repository/jobs/types.ts b/packages/forms/src/repository/jobs/types.ts new file mode 100644 index 00000000..9af83d49 --- /dev/null +++ b/packages/forms/src/repository/jobs/types.ts @@ -0,0 +1,69 @@ +// Type-safe job metadata by job type +export type JobMetadata = { + 'import-pdf': { + documentId: string; + fileName: string; + userId: string; + }; + 'validate-schema': { + validatorVersion: string; + userId: string; + }; + publish: { + targetEnvironment: 'staging' | 'production'; + publisherId: string; + }; +}; + +// Type-safe job results by job type +export type JobResult = { + 'import-pdf': { + patternsAdded: number; + fieldsExtracted: number; + documentId: string; + }; + 'validate-schema': { + errorsFound: number; + warningsFound: number; + issues: Array<{ field: string; message: string }>; + }; + publish: { + publishedAt: string; + url: string; + }; +}; + +export type JobType = keyof JobMetadata; +export type JobStatus = 'pending' | 'processing' | 'completed' | 'failed'; + +export type FormJob = { + id: string; + formId: string; + jobType: T; + status: JobStatus; + createdAt: Date; + startedAt?: Date; + completedAt?: Date; + errorMessage?: string; + errorStack?: string; + metadata?: JobMetadata[T]; + result?: JobResult[T]; +}; + +/** + * Helper to convert database row to FormJob object. + * Handles date conversions and JSON parsing. + */ +export const rowToFormJob = (row: any): FormJob => ({ + id: row.id, + formId: row.form_id, + jobType: row.job_type as JobType, + status: row.status as JobStatus, + createdAt: new Date(row.created_at), + startedAt: row.started_at ? new Date(row.started_at) : undefined, + completedAt: row.completed_at ? new Date(row.completed_at) : undefined, + errorMessage: row.error_message || undefined, + errorStack: row.error_stack || undefined, + metadata: row.metadata ? JSON.parse(row.metadata) : undefined, + result: row.result ? JSON.parse(row.result) : undefined, +}); diff --git a/packages/forms/src/repository/types.ts b/packages/forms/src/repository/types.ts index 07142ed0..9291a6b8 100644 --- a/packages/forms/src/repository/types.ts +++ b/packages/forms/src/repository/types.ts @@ -9,6 +9,11 @@ import type { GetFormList } from './get-form-list.js'; import type { GetFormSession } from './get-form-session.js'; import type { SaveForm } from './save-form.js'; import type { UpsertFormSession } from './upsert-form-session.js'; +import type { CreateFormJob } from './jobs/create-form-job.js'; +import type { CompleteFormJob } from './jobs/complete-form-job.js'; +import type { FailFormJob } from './jobs/fail-form-job.js'; +import type { GetLatestFormJob } from './jobs/get-latest-form-job.js'; +import type { GetFormJobs } from './jobs/get-form-jobs.js'; /** * Interface for the forms repository. @@ -24,4 +29,9 @@ export interface FormRepository { getFormList: ServiceMethod; saveForm: ServiceMethod; upsertFormSession: ServiceMethod; + createFormJob: ServiceMethod; + completeFormJob: ServiceMethod; + failFormJob: ServiceMethod; + getLatestFormJob: ServiceMethod; + getFormJobs: ServiceMethod; } diff --git a/packages/forms/src/services/get-form-status.ts b/packages/forms/src/services/get-form-status.ts new file mode 100644 index 00000000..d30821b3 --- /dev/null +++ b/packages/forms/src/services/get-form-status.ts @@ -0,0 +1,83 @@ +import { type Result, success, failure } from '@flexion/forms-common'; +import type { InternalFormServiceContext } from '../context/index.js'; +import type { JobStatus } from '../repository/jobs/types.js'; + +export type FormStatusResponse = { + formId: string; + formStatus: 'draft' | 'ready'; // Usability status + latestJob?: { + id: string; + jobType: string; + status: JobStatus; + createdAt: string; + completedAt?: string; + errorMessage?: string; + }; +}; + +export type GetFormStatusError = { + status: number; + message: string; +}; + +export type GetFormStatus = ( + ctx: InternalFormServiceContext, + formId: string +) => Promise>; + +/** + * Get form status and latest import-pdf job status. + * Used by frontend polling to check processing progress. + */ +export const getFormStatus: GetFormStatus = async (ctx, formId) => { + // Get form to check if it exists + const formResult = await ctx.repository.getForm(formId); + if (!formResult.success) { + return failure({ + status: 404, + message: 'Form not found', + }); + } + + const form = formResult.data; + if (!form) { + return failure({ + status: 404, + message: 'Form not found', + }); + } + + // Check if form has patterns (content) + const hasContent = Object.keys(form.patterns).length > 1; // >1 because root pattern always exists + const formStatus = hasContent ? 'ready' : 'draft'; + + // Get latest import-pdf job (if any) + const latestJobResult = await ctx.repository.getLatestFormJob( + formId, + 'import-pdf' + ); + + if (!latestJobResult.success) { + return failure({ + status: 500, + message: 'Failed to get job status', + }); + } + + const job = latestJobResult.data; + + return success({ + formId, + formStatus, + latestJob: job + ? { + id: job.id, + jobType: job.jobType, + status: job.status, + createdAt: job.createdAt.toISOString(), + completedAt: job.completedAt?.toISOString(), + errorMessage: job.errorMessage, + } + : undefined, + }); +}; diff --git a/packages/forms/src/services/index.ts b/packages/forms/src/services/index.ts index 161b0e2d..7c35be75 100644 --- a/packages/forms/src/services/index.ts +++ b/packages/forms/src/services/index.ts @@ -11,6 +11,7 @@ import { type DeleteForm, deleteForm } from './delete-form.js'; import { type GetForm, getForm } from './get-form.js'; import { type GetFormList, getFormList } from './get-form-list.js'; import { type GetFormSession, getFormSession } from './get-form-session.js'; +import { type GetFormStatus, getFormStatus } from './get-form-status.js'; import { type InitializeForm, initializeForm } from './initialize-form.js'; import { type SaveForm, saveForm } from './save-form.js'; import { type SubmitForm, submitForm } from './submit-form.js'; @@ -34,6 +35,7 @@ export const createFormService = (ctx: FormServiceContext): FormService => { getForm, getFormList, getFormSession, + getFormStatus, initializeForm, saveForm, submitForm, @@ -46,6 +48,7 @@ export type FormService = { getForm: ServiceMethod; getFormList: ServiceMethod; getFormSession: ServiceMethod; + getFormStatus: ServiceMethod; initializeForm: ServiceMethod; saveForm: ServiceMethod; submitForm: ServiceMethod; diff --git a/packages/forms/src/services/initialize-form.test.ts b/packages/forms/src/services/initialize-form.test.ts index f9ee4695..57339191 100644 --- a/packages/forms/src/services/initialize-form.test.ts +++ b/packages/forms/src/services/initialize-form.test.ts @@ -1,4 +1,4 @@ -import { describe, expect, it } from 'vitest'; +import { describe, expect, it, vi } from 'vitest'; import { createTestFormServiceContext } from '../testing.js'; @@ -31,25 +31,242 @@ describe('initializeForm', () => { data: { timestamp: expect.any(String), id: expect.any(String), + status: 'ready', }, }); }); - it.skip('initializes successfully with document when user is logged in', async () => { - // TODO: Update this test to use a fake parser that returns expected structure + it('initializes and returns immediately with document (async processing)', async () => { + const mockParsedPdf = { + title: 'Parsed Form Title', + description: 'This form was parsed from a PDF', + root: 'page1', + patterns: { + page1: { + type: 'page', + id: 'page1', + data: { + title: 'Page 1', + patterns: ['paragraph1', 'input1'], + }, + }, + paragraph1: { + type: 'paragraph', + id: 'paragraph1', + data: { + text: 'Welcome to the parsed form', + }, + }, + input1: { + type: 'input', + id: 'input1', + data: { + label: 'First Name', + required: true, + }, + }, + }, + outputs: { + firstName: { type: 'text' as const, name: 'firstName', value: '' }, + }, + errors: [], + }; + + const mockFields = { + firstName: { + type: 'TextField' as const, + name: 'firstName', + label: 'First Name', + value: '', + required: true, + }, + }; + const ctx = await createTestFormServiceContext({ isUserLoggedIn: () => true, }); + + // Mock parsePdf to avoid needing a real PDF + ctx.parsePdf = async () => ({ + parsedPdf: mockParsedPdf as any, + fields: mockFields, + }); + + // Base64 encoded "This is test PDF data" + const testPdfData = 'VGhpcyBpcyB0ZXN0IFBERiBkYXRh'; + const result = await initializeForm(ctx, { - summary, - document: { fileName: 'test.pdf', data: 'VGhpcyBpcyBub3QgYSBQREYu' }, + document: { fileName: 'test.pdf', data: testPdfData }, }); + + // Should return immediately with processing status expect(result).toEqual({ success: true, data: { timestamp: expect.any(String), id: expect.any(String), + jobId: expect.any(String), + status: 'processing', }, }); + + if (!result.success) return; + + const { id: formId, jobId } = result.data; + + // Wait for async processing to complete + await vi.waitFor( + async () => { + const job = await ctx.repository.getLatestFormJob(formId, 'import-pdf'); + expect(job.success).toBe(true); + if (!job.success) throw new Error('Job not found'); + expect(job.data?.status).toBe('completed'); + }, + { timeout: 5000 } + ); + + // Verify job was completed successfully + const jobResult = await ctx.repository.getLatestFormJob(formId, 'import-pdf'); + expect(jobResult.success).toBe(true); + if (!jobResult.success) return; + + const job = jobResult.data!; + expect(job.status).toBe('completed'); + expect(job.completedAt).toBeDefined(); + expect(job.result).toEqual({ + patternsAdded: expect.any(Number), + fieldsExtracted: expect.any(Number), + documentId: expect.any(String), + }); + + // Verify form was updated with parsed content + const formResult = await ctx.repository.getForm(formId); + expect(formResult.success).toBe(true); + if (!formResult.success) return; + + const form = formResult.data!; + + // Form should have been updated with parsed title + expect(form.summary.title).toBe('Parsed Form Title'); + expect(form.summary.description).toBe('This form was parsed from a PDF'); + + // Form should have patterns added (more than just root) + expect(Object.keys(form.patterns).length).toBeGreaterThan(1); + }); + + it('handles async processing errors gracefully', async () => { + const ctx = await createTestFormServiceContext({ + isUserLoggedIn: () => true, + }); + + // Mock parsePdf to throw an error + ctx.parsePdf = async () => { + throw new Error('Parser failed to process PDF'); + }; + + const testPdfData = 'VGhpcyBpcyB0ZXN0IFBERiBkYXRh'; + + const result = await initializeForm(ctx, { + document: { fileName: 'failing-test.pdf', data: testPdfData }, + }); + + // Should still return successfully (processing happens async) + expect(result.success).toBe(true); + if (!result.success) return; + + const { id: formId } = result.data; + + // Wait for async processing to fail + await vi.waitFor( + async () => { + const job = await ctx.repository.getLatestFormJob(formId, 'import-pdf'); + expect(job.success).toBe(true); + if (!job.success) throw new Error('Job not found'); + expect(job.data?.status).toBe('failed'); + }, + { timeout: 5000 } + ); + + // Verify job failed with error message + const jobResult = await ctx.repository.getLatestFormJob(formId, 'import-pdf'); + expect(jobResult.success).toBe(true); + if (!jobResult.success) return; + + const job = jobResult.data!; + expect(job.status).toBe('failed'); + expect(job.errorMessage).toContain('Parser failed to process PDF'); + expect(job.completedAt).toBeDefined(); + expect(job.result).toBeUndefined(); + }); + + it('validates document data is valid base64', async () => { + const ctx = await createTestFormServiceContext({ + isUserLoggedIn: () => true, + }); + + const result = await initializeForm(ctx, { + document: { fileName: 'test.pdf', data: 'not-valid-base64!!!' }, + }); + + expect(result).toEqual({ + success: false, + error: { + status: 400, + message: 'Invalid options', + }, + }); + }); + + it('uses filename as title when no summary provided', async () => { + const mockParsedPdf = { + title: 'Override Title', + description: 'Description from PDF', + root: 'root', + patterns: { + root: { + type: 'page', + id: 'root', + data: { title: 'Page 1', patterns: [] }, + }, + }, + outputs: {}, + errors: [], + }; + + const ctx = await createTestFormServiceContext({ + isUserLoggedIn: () => true, + }); + + // Mock parsePdf + ctx.parsePdf = async () => ({ + parsedPdf: mockParsedPdf as any, + fields: {}, + }); + + const result = await initializeForm(ctx, { + document: { fileName: 'my-form.pdf', data: 'VGVzdA==' }, + }); + + expect(result.success).toBe(true); + if (!result.success) return; + + const { id: formId } = result.data; + + // Wait for processing + await vi.waitFor( + async () => { + const job = await ctx.repository.getLatestFormJob(formId, 'import-pdf'); + if (!job.success) throw new Error('Job not found'); + expect(job.data?.status).toBe('completed'); + }, + { timeout: 5000 } + ); + + // Check that the parsed title was used (not the filename) + const formResult = await ctx.repository.getForm(formId); + expect(formResult.success).toBe(true); + if (!formResult.success) return; + + expect(formResult.data!.summary.title).toBe('Override Title'); }); }); diff --git a/packages/forms/src/services/initialize-form.ts b/packages/forms/src/services/initialize-form.ts index 544f97f8..cd5d35e7 100644 --- a/packages/forms/src/services/initialize-form.ts +++ b/packages/forms/src/services/initialize-form.ts @@ -14,6 +14,8 @@ type InitializeFormError = { type InitializeFormResult = { timestamp: string; id: string; + jobId?: string; + status: 'ready' | 'processing'; }; export type InitializeForm = ( @@ -50,8 +52,9 @@ const optionSchema = z.object({ }); /** - * Asynchronously initializes a new form based on the provided context and options. Handles schema validation, - * document import (parses uploaded PDF), builds a Blueprint, and saves to the repository. + * Asynchronously initializes a new form based on the provided context and options. + * If a document is provided, creates the form immediately and processes the PDF asynchronously. + * Otherwise, creates a form with the provided summary. */ export const initializeForm: InitializeForm = async (ctx, opts) => { if (!ctx.isUserLoggedIn()) { @@ -71,56 +74,164 @@ export const initializeForm: InitializeForm = async (ctx, opts) => { } const { document, summary } = parseResult.data; + // Create empty blueprint const builder = new BlueprintBuilder(ctx.config); - if (document !== undefined) { - const parsePdfResult = await ctx - .parsePdf(document.data) - .then(result => success(result)) - .catch(err => - failure({ - status: 400, - message: `Failed to parse PDF: ${err.message}`, - }) - ); - if (!parsePdfResult.success) { - return parsePdfResult; - } - const { parsedPdf } = parsePdfResult.data; + if (summary) { + builder.setFormSummary(summary); + } else if (document) { builder.setFormSummary({ - title: parsedPdf.title || document.fileName, - description: parsedPdf.description, + title: document.fileName, + description: '', }); + } + // Step 1: Create form in database (empty, draft state) + const formResult = await ctx.repository.addForm(builder.form); + if (!formResult.success) { + console.error('Failed to add form:', formResult.error); + return failure({ + status: 500, + message: formResult.error, + }); + } + + const formId = formResult.data.id; + + // Step 2: If document provided, store it and initiate async processing + if (document !== undefined) { const fileName = document.fileName.split('/').pop() || 'my-form.pdf'; + + // Store document (without extract, will be filled by job processing) const addDocumentResult = await ctx.repository.addDocument({ fileName, data: document.data, - extract: parsePdfResult.data, + extract: undefined, }); + if (!addDocumentResult.success) { return failure({ status: 500, message: `Failed to add document: ${addDocumentResult.error}`, }); } + + const documentId = addDocumentResult.data.id; + + // Create job record (status: 'processing') + const jobResult = await ctx.repository.createFormJob({ + formId, + jobType: 'import-pdf', + metadata: { + documentId, + fileName, + userId: ctx.getUserId?.() || 'system', + }, + }); + + if (!jobResult.success) { + return failure({ + status: 500, + message: 'Failed to create processing job', + }); + } + + const job = jobResult.data; + + // Step 3: Fire async processing (don't await!) + processFormDocumentAsync(ctx, formId, job.id, documentId).catch(err => { + console.error('Async form processing failed:', err); + // Error already logged to database by processFormDocumentAsync + }); + + // Step 4: Return immediately + return success({ + id: formId, + timestamp: formResult.data.timestamp, + jobId: job.id, + status: 'processing', + }); + } + + // No document, form is ready immediately + return success({ + id: formId, + timestamp: formResult.data.timestamp, + status: 'ready', + }); +}; + +/** + * Async function that processes PDF and updates form + job. + * Runs in background, not awaited by HTTP request. + */ +async function processFormDocumentAsync( + ctx: InternalFormServiceContext, + formId: string, + jobId: string, + documentId: string +): Promise { + try { + // Get document data + const documentResult = await ctx.repository.getDocument(documentId); + if (!documentResult.success) { + await ctx.repository.failFormJob(jobId, { + message: `Document not found: ${documentResult.error}`, + }); + return; + } + + // Parse PDF via Bedrock + const parsePdfResult = await ctx.parsePdf(documentResult.data.data); + const { parsedPdf, fields } = parsePdfResult; + + // Get current form + const formResult = await ctx.repository.getForm(formId); + if (!formResult.success || !formResult.data) { + await ctx.repository.failFormJob(jobId, { + message: 'Form not found', + }); + return; + } + + // Build updated form with parsed patterns + const builder = new BlueprintBuilder(ctx.config, formResult.data); + + // Update summary from parsed PDF + builder.setFormSummary({ + title: parsedPdf.title || documentResult.data.path || 'Untitled', + description: parsedPdf.description || '', + }); + + // Add document reference await builder.addDocumentRef({ - id: addDocumentResult.data.id, + id: documentId, extract: parsedPdf, }); - } - if (summary) { - builder.setFormSummary(summary); - } + // Save updated form + const saveResult = await ctx.repository.saveForm(formId, builder.form); + if (!saveResult.success) { + await ctx.repository.failFormJob(jobId, { + message: `Failed to save form: ${saveResult.error}`, + }); + return; + } - const result = await ctx.repository.addForm(builder.form); - if (!result.success) { - console.error('Failed to add form:', result.error); - return failure({ - status: 500, - message: result.error, + // Mark job as completed + await ctx.repository.completeFormJob(jobId, { + patternsAdded: Object.keys(builder.form.patterns).length, + fieldsExtracted: Object.keys(fields).length, + documentId, + }); + + console.log(`Form ${formId} processed successfully`); + } catch (err) { + // Catch any unexpected errors + console.error('Unexpected error in processFormDocumentAsync:', err); + await ctx.repository.failFormJob(jobId, { + message: (err as Error).message, + stack: (err as Error).stack, }); } - return result; -}; +} diff --git a/packages/server/src/lib/api-client.ts b/packages/server/src/lib/api-client.ts index f31da251..fbdb9f26 100644 --- a/packages/server/src/lib/api-client.ts +++ b/packages/server/src/lib/api-client.ts @@ -6,6 +6,8 @@ import { type Blueprint, type FormService, type FormSummary, + type FormStatusResponse, + type GetFormStatusError, } from '@flexion/forms-core'; import { type FormServiceContext } from '@flexion/forms-core/context'; @@ -37,7 +39,12 @@ export class FormServiceClient implements FormService { } ): Promise< Result< - { timestamp: string; id: string }, + { + timestamp: string; + id: string; + jobId?: string; + status: 'ready' | 'processing'; + }, { status: number; message: string } > > { @@ -136,6 +143,15 @@ export class FormServiceClient implements FormService { throw new Error('Not implemented'); } + async getFormStatus( + formId: string + ): Promise> { + const response = await fetch( + `${this.ctx.baseUrl}api/forms/${formId}/status` + ); + return await response.json(); + } + getContext() { return {} as unknown as FormServiceContext; } diff --git a/packages/server/src/pages/api/forms/[id]/status.ts b/packages/server/src/pages/api/forms/[id]/status.ts new file mode 100644 index 00000000..a86e2645 --- /dev/null +++ b/packages/server/src/pages/api/forms/[id]/status.ts @@ -0,0 +1,20 @@ +import type { APIRoute } from 'astro'; +import { getServerContext } from '../../../../config/astro.js'; + +export const GET: APIRoute = async context => { + const ctx = await getServerContext(context); + const formId = context.params.id; + + if (!formId) { + return new Response('Form ID is required', { status: 400 }); + } + + const result = await ctx.formService.getFormStatus(formId); + + return new Response(JSON.stringify(result), { + headers: { + 'Content-Type': 'application/json', + }, + status: result.success ? 200 : result.error.status, + }); +}; From e107161f195ced75e21eb8aacd7b9e2d6f16fc0d Mon Sep 17 00:00:00 2001 From: Daniel Naab Date: Tue, 7 Oct 2025 09:17:22 -0500 Subject: [PATCH 02/17] Remove old "experiments" directory --- .../src/experiments/document-assembler.tsx | 52 ------------------- 1 file changed, 52 deletions(-) delete mode 100644 packages/design/src/experiments/document-assembler.tsx diff --git a/packages/design/src/experiments/document-assembler.tsx b/packages/design/src/experiments/document-assembler.tsx deleted file mode 100644 index 55118680..00000000 --- a/packages/design/src/experiments/document-assembler.tsx +++ /dev/null @@ -1,52 +0,0 @@ -import React, { useState } from 'react'; - -import { generateDummyPDF } from '@flexion/forms-core'; - -export const downloadPdfBytes = (bytes: Uint8Array) => { - const base64 = btoa(String.fromCharCode(...bytes)); - const element = document.createElement('a'); - element.setAttribute( - 'href', - 'data:application/pdf;base64,' + encodeURIComponent(base64) - ); - element.setAttribute('download', 'sample-document.pdf'); - element.style.display = 'none'; - document.body.appendChild(element); - element.click(); - document.body.removeChild(element); -}; - -const generatePDF = async () => { - const timestamp = new Date().toISOString(); - const pdfBytes = await generateDummyPDF({ timestamp }); - downloadPdfBytes(pdfBytes); -}; - -const previewPDF = async (setPreviewPdfUrl: (url: string) => void) => { - const timestamp = new Date().toISOString(); - const pdfBytes = await generateDummyPDF({ timestamp }); - const pdfBlob = new Blob([pdfBytes], { type: 'application/pdf' }); - setPreviewPdfUrl(URL.createObjectURL(pdfBlob)); -}; - -export const DocumentAssembler = () => { - const [previewPdfUrl, setPreviewPdfUrl] = useState(); - return ( -
- - -
- {previewPdfUrl ? ( - - ) : null} -
-
- ); -}; From 11d179c48d7d06af3123b9ae2383e078df440803 Mon Sep 17 00:00:00 2001 From: Daniel Naab Date: Tue, 7 Oct 2025 09:20:42 -0500 Subject: [PATCH 03/17] Fix type error on StoryObj usage --- .../FormManager/FormManagerLayout/FormManagerLayout.stories.tsx | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/design/src/FormManager/FormManagerLayout/FormManagerLayout.stories.tsx b/packages/design/src/FormManager/FormManagerLayout/FormManagerLayout.stories.tsx index 80c7ef0d..dbd93b80 100644 --- a/packages/design/src/FormManager/FormManagerLayout/FormManagerLayout.stories.tsx +++ b/packages/design/src/FormManager/FormManagerLayout/FormManagerLayout.stories.tsx @@ -43,7 +43,6 @@ export const Edit = { step: NavPage.edit, next: '#', back: '#', - preview: '#', }, } satisfies StoryObj; @@ -52,6 +51,5 @@ export const Publish = { step: NavPage.publish, next: '#', back: '#', - preview: '#', }, } satisfies StoryObj; From 578825811b02c298dad76e78458527a3dd1e113b Mon Sep 17 00:00:00 2001 From: Daniel Naab Date: Tue, 7 Oct 2025 14:49:51 -0500 Subject: [PATCH 04/17] Address Zod 3 -> 4 compat issue --- packages/forms/src/util/zod.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/packages/forms/src/util/zod.ts b/packages/forms/src/util/zod.ts index 99e4c17b..c1885cef 100644 --- a/packages/forms/src/util/zod.ts +++ b/packages/forms/src/util/zod.ts @@ -62,9 +62,15 @@ export const convertZodErrorToFormErrors = ( zodError.issues.forEach((error: z.ZodIssue) => { const path = error.path.join('.'); if (error.code === 'too_small' && error.minimum === 1) { + // Replace default Zod message with consistent custom message + const message = + error.message === 'String must contain at least 1 character(s)' || + error.message.startsWith('Too small') + ? 'String must contain at least 1 character(s)' + : error.message; formErrors[path] = { type: 'required', - message: error.message, + message, }; } else { formErrors[path] = { From f7380822b5ed97f12549f8b317dfd5c60d2a3cbc Mon Sep 17 00:00:00 2001 From: Daniel Naab Date: Tue, 7 Oct 2025 14:50:55 -0500 Subject: [PATCH 05/17] Formatting --- packages/forms/src/repository/index.ts | 11 ++--------- packages/forms/src/repository/jobs/create-form-job.ts | 7 ++++++- packages/forms/src/services/initialize-form.test.ts | 10 ++++++++-- 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/packages/forms/src/repository/index.ts b/packages/forms/src/repository/index.ts index 1ce0bc99..e70bc403 100644 --- a/packages/forms/src/repository/index.ts +++ b/packages/forms/src/repository/index.ts @@ -16,10 +16,7 @@ import { type UpsertFormSession, upsertFormSession, } from './upsert-form-session.js'; -import { - type CreateFormJob, - createFormJob, -} from './jobs/create-form-job.js'; +import { type CreateFormJob, createFormJob } from './jobs/create-form-job.js'; import { type CompleteFormJob, completeFormJob, @@ -30,11 +27,7 @@ import { getLatestFormJob, } from './jobs/get-latest-form-job.js'; import { type GetFormJobs, getFormJobs } from './jobs/get-form-jobs.js'; -import { - type FormJob, - type JobType, - type JobStatus, -} from './jobs/types.js'; +import { type FormJob, type JobType, type JobStatus } from './jobs/types.js'; export type { FormRepository }; diff --git a/packages/forms/src/repository/jobs/create-form-job.ts b/packages/forms/src/repository/jobs/create-form-job.ts index 1cc6ddf7..dc36602d 100644 --- a/packages/forms/src/repository/jobs/create-form-job.ts +++ b/packages/forms/src/repository/jobs/create-form-job.ts @@ -1,6 +1,11 @@ import { type Result, success, failure } from '@flexion/forms-common'; import type { FormRepositoryContext } from '../index.js'; -import { type FormJob, type JobMetadata, type JobStatus, type JobType } from './types.js'; +import { + type FormJob, + type JobMetadata, + type JobStatus, + type JobType, +} from './types.js'; /** * Create a new job record in 'processing' state. diff --git a/packages/forms/src/services/initialize-form.test.ts b/packages/forms/src/services/initialize-form.test.ts index 57339191..efedf88a 100644 --- a/packages/forms/src/services/initialize-form.test.ts +++ b/packages/forms/src/services/initialize-form.test.ts @@ -126,7 +126,10 @@ describe('initializeForm', () => { ); // Verify job was completed successfully - const jobResult = await ctx.repository.getLatestFormJob(formId, 'import-pdf'); + const jobResult = await ctx.repository.getLatestFormJob( + formId, + 'import-pdf' + ); expect(jobResult.success).toBe(true); if (!jobResult.success) return; @@ -188,7 +191,10 @@ describe('initializeForm', () => { ); // Verify job failed with error message - const jobResult = await ctx.repository.getLatestFormJob(formId, 'import-pdf'); + const jobResult = await ctx.repository.getLatestFormJob( + formId, + 'import-pdf' + ); expect(jobResult.success).toBe(true); if (!jobResult.success) return; From 09bd235288cd77a2ad71cbd606ae9fdf7d44f084 Mon Sep 17 00:00:00 2001 From: Daniel Naab Date: Wed, 8 Oct 2025 17:54:01 -0500 Subject: [PATCH 06/17] Add cache backend tests; fix sqlite3 date issue on database cache backend. --- packages/database/src/clients/kysely/types.ts | 8 +- packages/database/src/schema.ts | 8 -- .../src/llm/cache/backends/database.test.ts | 75 +++++++++++++++++ .../src/llm/cache/{ => backends}/database.ts | 8 +- .../src/llm/cache/backends/filesystem.test.ts | 81 +++++++++++++++++++ .../llm/cache/{ => backends}/filesystem.ts | 2 +- .../src/llm/cache/{ => backends}/noop.ts | 2 +- packages/forms/src/llm/cache/cache-spec.ts | 54 +++++++++++++ packages/forms/src/llm/cache/hash.ts | 12 ++- packages/forms/src/llm/cache/index.ts | 6 +- packages/forms/src/llm/cache/types.ts | 4 +- packages/forms/src/llm/services/context.ts | 6 +- 12 files changed, 238 insertions(+), 28 deletions(-) delete mode 100644 packages/database/src/schema.ts create mode 100644 packages/forms/src/llm/cache/backends/database.test.ts rename packages/forms/src/llm/cache/{ => backends}/database.ts (89%) create mode 100644 packages/forms/src/llm/cache/backends/filesystem.test.ts rename packages/forms/src/llm/cache/{ => backends}/filesystem.ts (97%) rename packages/forms/src/llm/cache/{ => backends}/noop.ts (87%) create mode 100644 packages/forms/src/llm/cache/cache-spec.ts diff --git a/packages/database/src/clients/kysely/types.ts b/packages/database/src/clients/kysely/types.ts index 75ff9eb7..bbf37af1 100644 --- a/packages/database/src/clients/kysely/types.ts +++ b/packages/database/src/clients/kysely/types.ts @@ -17,7 +17,7 @@ export interface Database { form_sessions: FormSessionsTable; form_documents: FormDocumentsTable; form_jobs: FormJobsTable; - llm_request_cache: LlmRequestCacheTable; + llm_request_cache: LlmRequestCacheTable; } export type DatabaseClient = Kysely; @@ -92,12 +92,12 @@ export type FormJobsTableSelectable = Selectable; export type FormJobsTableInsertable = Insertable; export type FormJobsTableUpdateable = Updateable; -interface LlmRequestCacheTable { +interface LlmRequestCacheTable { id: Generated; cache_key: string; response_data: string; - created_at: Generated; - accessed_at: Date; + created_at: DbDate; + accessed_at: DbDate; access_count: number; } export type LlmRequestCacheTableSelectable = Selectable; diff --git a/packages/database/src/schema.ts b/packages/database/src/schema.ts deleted file mode 100644 index 319fe17a..00000000 --- a/packages/database/src/schema.ts +++ /dev/null @@ -1,8 +0,0 @@ -export type SessionSchema = { - id: string; - user_id: string; - session_token: string; - expires_at: number; - created_at: number; - updated_at: number; -}; diff --git a/packages/forms/src/llm/cache/backends/database.test.ts b/packages/forms/src/llm/cache/backends/database.test.ts new file mode 100644 index 00000000..e10facad --- /dev/null +++ b/packages/forms/src/llm/cache/backends/database.test.ts @@ -0,0 +1,75 @@ +import { expect, it } from 'vitest'; +import { + type DbTestContext, + describeDatabase, +} from '@flexion/forms-database/testing'; +import { DatabaseCache } from './database.js'; +import { testCacheSpecification } from '../cache-spec.js'; + +describeDatabase('DatabaseCache', () => { + it('passes shared cache specification', async ({ db }) => { + const cache = new DatabaseCache(db.ctx); + await testCacheSpecification(cache); + }); + + it( + 'tracks access statistics on cache hits', + async ({ db }) => { + const cache = new DatabaseCache(db.ctx); + await cache.set('key', 'value'); + + const kysely = await db.ctx.getKysely(); + const initial = await kysely + .selectFrom('llm_request_cache') + .select('access_count') + .where('cache_key', '=', 'key') + .executeTakeFirst(); + + expect(initial?.access_count).toBe(1); + + // Trigger cache hit (stats update is async) + await cache.get('key'); + await new Promise(resolve => setTimeout(resolve, 100)); + + const updated = await kysely + .selectFrom('llm_request_cache') + .select('access_count') + .where('cache_key', '=', 'key') + .executeTakeFirst(); + + expect(updated?.access_count).toBe(2); + }, + 10000 + ); + + it( + 'handles concurrent writes to same key', + async ({ db }) => { + const cache = new DatabaseCache(db.ctx); + const key = 'concurrent'; + + // SQLite: sequential writes; Postgres: concurrent writes + if (db.engine === 'sqlite') { + for (let i = 0; i < 5; i++) { + await cache.set(key, `v${i}`); + } + } else { + await Promise.all( + Array.from({ length: 5 }, (_, i) => cache.set(key, `v${i}`)) + ); + } + + // Should have exactly one entry + const kysely = await db.ctx.getKysely(); + const count = await kysely + .selectFrom('llm_request_cache') + .select(kysely.fn.count('id').as('count')) + .where('cache_key', '=', key) + .executeTakeFirst(); + + expect(Number(count?.count)).toBe(1); + expect(await cache.get(key)).toMatch(/^v[0-4]$/); + }, + 10000 + ); +}); diff --git a/packages/forms/src/llm/cache/database.ts b/packages/forms/src/llm/cache/backends/database.ts similarity index 89% rename from packages/forms/src/llm/cache/database.ts rename to packages/forms/src/llm/cache/backends/database.ts index da1e55ef..96e03170 100644 --- a/packages/forms/src/llm/cache/database.ts +++ b/packages/forms/src/llm/cache/backends/database.ts @@ -1,7 +1,7 @@ import { sql } from 'kysely'; -import { type DatabaseContext } from '@flexion/forms-database'; -import type { AiRequestCache } from './types.js'; +import { type DatabaseContext, dateValue } from '@flexion/forms-database'; +import type { AiRequestCache } from '../types.js'; /** * Database-backed cache for production use. @@ -34,7 +34,7 @@ export class DatabaseCache implements AiRequestCache { async set(key: string, value: T): Promise { const kysely = await this.db.getKysely(); - const now = new Date(); + const now = dateValue(this.db.engine, new Date()); await kysely .insertInto('llm_request_cache') @@ -69,7 +69,7 @@ export class DatabaseCache implements AiRequestCache { await kysely .updateTable('llm_request_cache') .set({ - accessed_at: new Date(), + accessed_at: dateValue(this.db.engine, new Date()), access_count: sql`access_count + 1`, }) .where('cache_key', '=', key) diff --git a/packages/forms/src/llm/cache/backends/filesystem.test.ts b/packages/forms/src/llm/cache/backends/filesystem.test.ts new file mode 100644 index 00000000..682e8c22 --- /dev/null +++ b/packages/forms/src/llm/cache/backends/filesystem.test.ts @@ -0,0 +1,81 @@ +import fs from 'fs/promises'; +import os from 'os'; +import path from 'path'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { FilesystemCache } from './filesystem.js'; +import { testCacheSpecification } from '../cache-spec.js'; + +describe('FilesystemCache', () => { + let tempDir: string; + let cache: FilesystemCache; + + beforeEach(async () => { + tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'cache-test-')); + cache = new FilesystemCache(tempDir); + }); + + afterEach(async () => { + await fs.rm(tempDir, { recursive: true, force: true }); + }); + + it('passes shared cache specification', async () => { + await testCacheSpecification(cache); + }); + + it('organizes cache files by prefix', async () => { + await cache.set('abcdef123', 'test-value'); + + const filePath = path.join(tempDir, 'ab', 'abcdef123.json'); + const exists = await fs + .access(filePath) + .then(() => true) + .catch(() => false); + + expect(exists).toBe(true); + }); + + it('stores JSON in minified format by default', async () => { + const key = 'minified'; + const value = { foo: 'bar', nested: { value: 123 } }; + + await cache.set(key, value); + + const filePath = path.join(tempDir, key.slice(0, 2), `${key}.json`); + const content = await fs.readFile(filePath, 'utf-8'); + + expect(content).not.toContain('\n '); + expect(JSON.parse(content)).toEqual(value); + }); + + it('stores JSON in pretty format when enabled', async () => { + const prettyCache = new FilesystemCache(tempDir, { pretty: true }); + const key = 'pretty'; + const value = { foo: 'bar' }; + + await prettyCache.set(key, value); + + const filePath = path.join(tempDir, key.slice(0, 2), `${key}.json`); + const content = await fs.readFile(filePath, 'utf-8'); + + expect(content).toContain('\n '); + expect(JSON.parse(content)).toEqual(value); + }); + + it('clear succeeds even if directory does not exist', async () => { + const nonExistentCache = new FilesystemCache( + path.join(tempDir, 'non-existent') + ); + + await expect(nonExistentCache.clear()).resolves.toBeUndefined(); + }); + + it('throws error for corrupt cache files', async () => { + const key = 'corrupt'; + const filePath = path.join(tempDir, key.slice(0, 2), `${key}.json`); + + await fs.mkdir(path.dirname(filePath), { recursive: true }); + await fs.writeFile(filePath, 'invalid json', 'utf-8'); + + await expect(cache.get(key)).rejects.toThrow(SyntaxError); + }); +}); diff --git a/packages/forms/src/llm/cache/filesystem.ts b/packages/forms/src/llm/cache/backends/filesystem.ts similarity index 97% rename from packages/forms/src/llm/cache/filesystem.ts rename to packages/forms/src/llm/cache/backends/filesystem.ts index 8eb2bba7..d152999d 100644 --- a/packages/forms/src/llm/cache/filesystem.ts +++ b/packages/forms/src/llm/cache/backends/filesystem.ts @@ -1,6 +1,6 @@ import fs from 'fs/promises'; import path from 'path'; -import type { AiRequestCache } from './types.js'; +import type { AiRequestCache } from '../types.js'; /** * Options for configuring filesystem cache behavior. diff --git a/packages/forms/src/llm/cache/noop.ts b/packages/forms/src/llm/cache/backends/noop.ts similarity index 87% rename from packages/forms/src/llm/cache/noop.ts rename to packages/forms/src/llm/cache/backends/noop.ts index d255e592..5d5a0b6c 100644 --- a/packages/forms/src/llm/cache/noop.ts +++ b/packages/forms/src/llm/cache/backends/noop.ts @@ -1,4 +1,4 @@ -import type { AiRequestCache } from './types.js'; +import type { AiRequestCache } from '../types.js'; /** * No-operation cache that never stores or retrieves values. diff --git a/packages/forms/src/llm/cache/cache-spec.ts b/packages/forms/src/llm/cache/cache-spec.ts new file mode 100644 index 00000000..cbcee8a0 --- /dev/null +++ b/packages/forms/src/llm/cache/cache-spec.ts @@ -0,0 +1,54 @@ +import { expect } from 'vitest'; +import type { AiRequestCache } from './types.js'; + +/** + * Shared specification for all cache implementations. + * Tests core functionality: get, set, clear, and basic edge cases. + */ +export async function testCacheSpecification(cache: AiRequestCache) { + // Non-existent key + expect(await cache.get('non-existent')).toBeNull(); + + // Basic string value + await cache.set('key1', 'value1'); + expect(await cache.get('key1')).toBe('value1'); + + // Complex object (tests JSON serialization) + const complexValue = { + text: 'Hello', + nested: { array: [1, 2, 3], nullValue: null, bool: true }, + }; + await cache.set('complex', complexValue); + expect(await cache.get('complex')).toEqual( + complexValue + ); + + // Overwriting existing value + await cache.set('key1', 'updated'); + expect(await cache.get('key1')).toBe('updated'); + + // Multiple independent keys + await cache.set('a', 'value-a'); + await cache.set('b', 'value-b'); + expect(await cache.get('a')).toBe('value-a'); + expect(await cache.get('b')).toBe('value-b'); + + // Edge cases: empty string, null, zero, booleans + await cache.set('empty', ''); + await cache.set('null', null); + await cache.set('zero', 0); + await cache.set('true', true); + await cache.set('false', false); + + expect(await cache.get('empty')).toBe(''); + expect(await cache.get('null')).toBeNull(); + expect(await cache.get('zero')).toBe(0); + expect(await cache.get('true')).toBe(true); + expect(await cache.get('false')).toBe(false); + + // Clear removes all entries + await cache.clear(); + expect(await cache.get('key1')).toBeNull(); + expect(await cache.get('a')).toBeNull(); + expect(await cache.get('complex')).toBeNull(); +} diff --git a/packages/forms/src/llm/cache/hash.ts b/packages/forms/src/llm/cache/hash.ts index 42d3f295..92d12d6e 100644 --- a/packages/forms/src/llm/cache/hash.ts +++ b/packages/forms/src/llm/cache/hash.ts @@ -73,8 +73,16 @@ const sha256 = async (data: string): Promise => { /** * Computes SHA-256 hash of a buffer using Web Crypto API. */ -const sha256Buffer = async (data: Uint8Array): Promise => { - const hashBuffer = await globalThis.crypto.subtle.digest('SHA-256', data); +const sha256Buffer = async ( + data: Uint8Array | ArrayBuffer | ArrayBufferLike +): Promise => { + // Create a new Uint8Array to ensure proper ArrayBuffer backing + const bytes = data instanceof Uint8Array ? data : new Uint8Array(data); + const normalizedData = new Uint8Array(bytes); + const hashBuffer = await globalThis.crypto.subtle.digest( + 'SHA-256', + normalizedData + ); return Array.from(new Uint8Array(hashBuffer)) .map(b => b.toString(16).padStart(2, '0')) .join(''); diff --git a/packages/forms/src/llm/cache/index.ts b/packages/forms/src/llm/cache/index.ts index 039cb934..d356c18f 100644 --- a/packages/forms/src/llm/cache/index.ts +++ b/packages/forms/src/llm/cache/index.ts @@ -1,5 +1,5 @@ export type { AiRequestCache, CacheMetadata } from './types.js'; export { computeObjectCacheKey } from './hash.js'; -export { DatabaseCache } from './database.js'; -export { FilesystemCache } from './filesystem.js'; -export { NoOpCache } from './noop.js'; +export { DatabaseCache } from './backends/database.js'; +export { FilesystemCache } from './backends/filesystem.js'; +export { NoOpCache } from './backends/noop.js'; diff --git a/packages/forms/src/llm/cache/types.ts b/packages/forms/src/llm/cache/types.ts index 21b13a2c..b9f0f567 100644 --- a/packages/forms/src/llm/cache/types.ts +++ b/packages/forms/src/llm/cache/types.ts @@ -19,9 +19,9 @@ export interface AiRequestCache { /** * Clears all cached entries. - * Optional - primarily for testing. + * Primarily intended for testing. */ - clear?(): Promise; + clear(): Promise; } /** diff --git a/packages/forms/src/llm/services/context.ts b/packages/forms/src/llm/services/context.ts index 726c4b56..9274bb2f 100644 --- a/packages/forms/src/llm/services/context.ts +++ b/packages/forms/src/llm/services/context.ts @@ -1,8 +1,8 @@ import type { DatabaseContext } from '@flexion/forms-database'; import type { AiRequestCache } from '../cache/types.js'; -import { DatabaseCache } from '../cache/database.js'; -import { FilesystemCache } from '../cache/filesystem.js'; -import { NoOpCache } from '../cache/noop.js'; +import { DatabaseCache } from '../cache/backends/database.js'; +import { FilesystemCache } from '../cache/backends/filesystem.js'; +import { NoOpCache } from '../cache/backends/noop.js'; /** * Context for LLM operations. From ca2a5f4ac8cc4f9f3cd6b089ddfb0a979bf65aa4 Mon Sep 17 00:00:00 2001 From: Daniel Naab Date: Wed, 8 Oct 2025 21:54:55 -0500 Subject: [PATCH 07/17] Add status indicators for form processing states --- .../src/AvailableFormList/FormStatusBadge.tsx | 71 +++++++++ .../formStatusBadge.module.css | 68 ++++++++ .../design/src/AvailableFormList/index.tsx | 134 +++++++++++++--- .../design/src/FormManager/FormManager.tsx | 133 ++++++++++++---- .../FormProcessingIndicator.stories.tsx | 146 ++++++++++++++++++ .../FormProcessingIndicator.tsx | 142 +++++++++++++++++ .../formProcessingIndicator.module.css | 120 ++++++++++++++ .../FormProcessingIndicator/index.ts | 1 + packages/design/src/FormManager/hooks.ts | 3 + .../hooks/useBlueprintWithStatus.ts | 91 +++++++++++ .../src/FormManager/hooks/useFormStatus.ts | 111 +++++++++++++ packages/forms/src/index.ts | 1 + .../forms/src/repository/get-form-list.ts | 81 +++++++--- packages/forms/src/services/get-form-list.ts | 10 +- packages/server/src/styles.css | 1 + 15 files changed, 1033 insertions(+), 80 deletions(-) create mode 100644 packages/design/src/AvailableFormList/FormStatusBadge.tsx create mode 100644 packages/design/src/AvailableFormList/formStatusBadge.module.css create mode 100644 packages/design/src/FormManager/FormProcessingIndicator/FormProcessingIndicator.stories.tsx create mode 100644 packages/design/src/FormManager/FormProcessingIndicator/FormProcessingIndicator.tsx create mode 100644 packages/design/src/FormManager/FormProcessingIndicator/formProcessingIndicator.module.css create mode 100644 packages/design/src/FormManager/FormProcessingIndicator/index.ts create mode 100644 packages/design/src/FormManager/hooks/useBlueprintWithStatus.ts create mode 100644 packages/design/src/FormManager/hooks/useFormStatus.ts diff --git a/packages/design/src/AvailableFormList/FormStatusBadge.tsx b/packages/design/src/AvailableFormList/FormStatusBadge.tsx new file mode 100644 index 00000000..b9844a72 --- /dev/null +++ b/packages/design/src/AvailableFormList/FormStatusBadge.tsx @@ -0,0 +1,71 @@ +import React from 'react'; +import type { FormListItem } from '@flexion/forms-core'; +import styles from './formStatusBadge.module.css'; + +type FormStatusBadgeProps = { + form: FormListItem; +}; + +export const FormStatusBadge: React.FC = ({ form }) => { + const { latestJob } = form; + + // No job or completed job - no badge needed + if (!latestJob || latestJob.status === 'completed') { + return null; + } + + const isProcessing = latestJob.status === 'processing'; + const isFailed = latestJob.status === 'failed'; + + if (isProcessing) { + return ( + + + Processing + + ); + } + + if (isFailed) { + return ( + + + Import failed + + ); + } + + return null; +}; diff --git a/packages/design/src/AvailableFormList/formStatusBadge.module.css b/packages/design/src/AvailableFormList/formStatusBadge.module.css new file mode 100644 index 00000000..8ff3dcf3 --- /dev/null +++ b/packages/design/src/AvailableFormList/formStatusBadge.module.css @@ -0,0 +1,68 @@ +/* Badge container */ +.badge { + display: inline-flex; + align-items: center; + gap: 0.375rem; + padding: 0.25rem 0.5rem; + border-radius: 0.25rem; + font-size: 0.875rem; /* 14px */ + font-weight: 600; + line-height: 1.2; + white-space: nowrap; +} + +/* Processing badge */ +.badgeProcessing { + background-color: #e7f6f8; /* USWDS info-lighter */ + color: #00687d; /* USWDS info-darker */ +} + +/* Failed badge */ +.badgeFailed { + background-color: #f4e3db; /* USWDS error-lighter */ + color: #b50909; /* USWDS red-warm-60v */ +} + +/* Spinner animation */ +.spinner { + width: 1rem; + height: 1rem; + animation: rotate 1s linear infinite; + flex-shrink: 0; +} + +.spinnerCircle { + stroke: currentColor; + stroke-dasharray: 40, 60; + stroke-dashoffset: 0; + animation: dash 1.5s ease-in-out infinite; + stroke-linecap: round; +} + +@keyframes rotate { + 100% { + transform: rotate(360deg); + } +} + +@keyframes dash { + 0% { + stroke-dasharray: 1, 100; + stroke-dashoffset: 0; + } + 50% { + stroke-dasharray: 40, 100; + stroke-dashoffset: -20; + } + 100% { + stroke-dasharray: 40, 100; + stroke-dashoffset: -60; + } +} + +/* Error icon */ +.errorIcon { + width: 1rem; + height: 1rem; + flex-shrink: 0; +} diff --git a/packages/design/src/AvailableFormList/index.tsx b/packages/design/src/AvailableFormList/index.tsx index 5a438eea..565f3eeb 100644 --- a/packages/design/src/AvailableFormList/index.tsx +++ b/packages/design/src/AvailableFormList/index.tsx @@ -1,15 +1,11 @@ -import React, { useEffect, useState } from 'react'; +import React, { useEffect, useState, useRef } from 'react'; import { Link, useLocation } from 'react-router-dom'; -import { type FormService } from '@flexion/forms-core'; +import { type FormService, type FormListItem } from '@flexion/forms-core'; import * as AppRoutes from '../FormManager/routes.js'; +import { FormStatusBadge } from './FormStatusBadge.js'; -type FormDetails = { - id: string; - title: string; - description: string; -}; export type UrlForForm = (id: string) => string | null; export type UrlForFormManager = UrlForForm; @@ -22,8 +18,9 @@ export default function AvailableFormList({ urlForForm: UrlForForm; urlForFormManager: UrlForFormManager; }) { - const [forms, setForms] = useState([]); + const [forms, setForms] = useState([]); const location = useLocation(); + const pollIntervalRef = useRef(null); const loadForms = React.useCallback(() => { formService.getFormList().then(result => { @@ -37,6 +34,32 @@ export default function AvailableFormList({ loadForms(); }, [location.pathname, location.hash, location.key, loadForms]); + // Poll if any forms are processing + useEffect(() => { + const hasProcessingForms = forms.some( + form => form.latestJob?.status === 'processing' + ); + + // Clear any existing interval + if (pollIntervalRef.current) { + clearInterval(pollIntervalRef.current); + pollIntervalRef.current = null; + } + + // Start polling if needed + if (hasProcessingForms) { + pollIntervalRef.current = setInterval(() => { + loadForms(); + }, 3000); // Poll every 3 seconds + } + + return () => { + if (pollIntervalRef.current) { + clearInterval(pollIntervalRef.current); + } + }; + }, [forms, loadForms]); + return ( <>
@@ -74,7 +97,7 @@ const FormList = ({ urlForForm, urlForFormManager, }: { - forms: FormDetails[]; + forms: FormListItem[]; urlForForm: UrlForForm; urlForFormManager: UrlForFormManager; }) => { @@ -104,19 +127,12 @@ const FormList = ({ ) : ( forms.map((form, index) => ( - - - {form.title} - - {form.description} - - - - + )) )} @@ -124,20 +140,87 @@ const FormList = ({ ); }; +const FormRow = ({ + form, + urlForForm, + urlForFormManager, +}: { + form: FormListItem; + urlForForm: UrlForForm; + urlForFormManager: UrlForFormManager; +}) => { + const [showError, setShowError] = React.useState(false); + const isFailed = form.latestJob?.status === 'failed'; + + return ( + <> + + +
+ {form.title} + +
+ + {form.description} + + + + + {isFailed && form.latestJob && ( + + +
+
+

+ Import error:{' '} + {form.latestJob.errorMessage || + 'An error occurred while processing this form.'} + {' '} + +

+ {showError && form.latestJob.errorMessage && ( +
+ Technical details + 
+                      {form.latestJob.errorMessage}
+
+
+ )} +
+
+ + + )} + + ); +}; + const FormActions = ({ form, urlForForm, urlForFormManager, }: { - form: FormDetails; + form: FormListItem; urlForForm: UrlForForm; urlForFormManager: UrlForFormManager; }) => { const formUrl = urlForForm(form.id); + const isProcessing = form.latestJob?.status === 'processing'; + const isFailed = form.latestJob?.status === 'failed'; return ( ; } - const form = useBlueprint(formId, context.formService); - if (form === null) { + const { form, status, isLoading, isProcessing } = useBlueprintWithStatus( + formId, + context.formService + ); + + if (isProcessing && status) { + return ( + + + + + + ); + } + + if (isLoading || form === null) { return
Loading...
; } + return ( formId is undefined; } - const form = useBlueprint(formId, context.formService); - if (form === null) { + const { form, status, isLoading, isProcessing } = useBlueprintWithStatus( + formId, + context.formService + ); + + if (isProcessing && status) { + return ( + + + + + + ); + } + + if (isLoading || form === null) { return
Loading...
; } + return ( formId is undefined; } - const form = useBlueprint(formId, context.formService); - if (form === null) { + const { form, status, isLoading, isProcessing } = useBlueprintWithStatus( + formId, + context.formService + ); + + if (isProcessing && status) { + return ( + + + + + + ); + } + + if (isLoading || form === null) { return
Loading...
; } + return ( formId is undefined; } - const form = useBlueprint(formId, context.formService); - if (form === null) { + const { form, status, isLoading, isProcessing } = useBlueprintWithStatus( + formId, + context.formService + ); + + if (isProcessing && status) { + return ( + + + + + + ); + } + + if (isLoading || form === null) { return
Loading...
; } + const session = createFormSession(form); return ( { - if (formId === undefined) { - console.error('formId is undefined'); - return null; - } - const [form, setForm] = useState(null); - useEffect(() => { - formService.getForm(formId).then(result => { - if (result.success) { - setForm(result.data); - } else { - console.error('Error loading form', result.error); - } - }); - }, []); - return form; -}; diff --git a/packages/design/src/FormManager/FormProcessingIndicator/FormProcessingIndicator.stories.tsx b/packages/design/src/FormManager/FormProcessingIndicator/FormProcessingIndicator.stories.tsx new file mode 100644 index 00000000..12ccd89e --- /dev/null +++ b/packages/design/src/FormManager/FormProcessingIndicator/FormProcessingIndicator.stories.tsx @@ -0,0 +1,146 @@ +import type { Meta, StoryObj } from '@storybook/react'; +import { FormProcessingIndicator } from './FormProcessingIndicator.js'; + +const meta = { + title: 'FormManager/FormProcessingIndicator', + component: FormProcessingIndicator, + parameters: { + layout: 'fullscreen', + }, + tags: ['autodocs'], +} satisfies Meta; + +export default meta; +type Story = StoryObj; + +/** + * The processing indicator shown immediately after PDF upload. + * Shows an animated spinner, progress steps, and elapsed time. + */ +export const Processing: Story = { + args: { + formId: 'test-form-123', + status: { + formId: 'test-form-123', + formStatus: 'draft', + latestJob: { + id: 'job-123', + jobType: 'import-pdf', + status: 'processing', + createdAt: new Date(Date.now() - 15000).toISOString(), // 15 seconds ago + }, + }, + }, +}; + +/** + * Processing state after 45 seconds (approaching the upper end of expected time). + */ +export const ProcessingLongRunning: Story = { + args: { + formId: 'test-form-456', + status: { + formId: 'test-form-456', + formStatus: 'draft', + latestJob: { + id: 'job-456', + jobType: 'import-pdf', + status: 'processing', + createdAt: new Date(Date.now() - 45000).toISOString(), // 45 seconds ago + }, + }, + }, +}; + +/** + * Processing state after more than a minute (exceeding expected time). + */ +export const ProcessingVeryLongRunning: Story = { + args: { + formId: 'test-form-789', + status: { + formId: 'test-form-789', + formStatus: 'draft', + latestJob: { + id: 'job-789', + jobType: 'import-pdf', + status: 'processing', + createdAt: new Date(Date.now() - 90000).toISOString(), // 90 seconds ago + }, + }, + }, +}; + +/** + * Failed processing state with generic error message. + */ +export const Failed: Story = { + args: { + formId: 'test-form-error', + status: { + formId: 'test-form-error', + formStatus: 'draft', + latestJob: { + id: 'job-error', + jobType: 'import-pdf', + status: 'failed', + createdAt: new Date(Date.now() - 30000).toISOString(), + completedAt: new Date(Date.now() - 5000).toISOString(), + }, + }, + }, +}; + +/** + * Failed processing state with specific error message. + */ +export const FailedWithMessage: Story = { + args: { + formId: 'test-form-error-msg', + status: { + formId: 'test-form-error-msg', + formStatus: 'draft', + latestJob: { + id: 'job-error-msg', + jobType: 'import-pdf', + status: 'failed', + createdAt: new Date(Date.now() - 30000).toISOString(), + completedAt: new Date(Date.now() - 5000).toISOString(), + errorMessage: 'Unable to extract text from PDF. The document may be image-based or corrupted.', + }, + }, + }, +}; + +/** + * Completed state - component should not render anything. + */ +export const Completed: Story = { + args: { + formId: 'test-form-complete', + status: { + formId: 'test-form-complete', + formStatus: 'ready', + latestJob: { + id: 'job-complete', + jobType: 'import-pdf', + status: 'completed', + createdAt: new Date(Date.now() - 60000).toISOString(), + completedAt: new Date(Date.now() - 5000).toISOString(), + }, + }, + }, +}; + +/** + * No job state - component should not render anything. + */ +export const NoJob: Story = { + args: { + formId: 'test-form-no-job', + status: { + formId: 'test-form-no-job', + formStatus: 'draft', + }, + }, +}; diff --git a/packages/design/src/FormManager/FormProcessingIndicator/FormProcessingIndicator.tsx b/packages/design/src/FormManager/FormProcessingIndicator/FormProcessingIndicator.tsx new file mode 100644 index 00000000..093626ff --- /dev/null +++ b/packages/design/src/FormManager/FormProcessingIndicator/FormProcessingIndicator.tsx @@ -0,0 +1,142 @@ +import React, { useState, useEffect } from 'react'; +import type { FormStatusResponse } from '@flexion/forms-core'; +import styles from './formProcessingIndicator.module.css'; + +type FormProcessingIndicatorProps = { + formId: string; + status: FormStatusResponse; +}; + +export const FormProcessingIndicator: React.FC< + FormProcessingIndicatorProps +> = ({ formId, status }) => { + const { latestJob } = status; + const [elapsedSeconds, setElapsedSeconds] = useState(0); + + // Don't show anything if no job or job is completed + if (!latestJob || latestJob.status === 'completed') { + return null; + } + + const isProcessing = latestJob.status === 'processing'; + const isFailed = latestJob.status === 'failed'; + + // Calculate elapsed time for processing jobs + useEffect(() => { + if (!isProcessing || !latestJob.createdAt) { + return; + } + + const startTime = new Date(latestJob.createdAt).getTime(); + + const updateElapsed = () => { + const now = Date.now(); + const elapsed = Math.floor((now - startTime) / 1000); + setElapsedSeconds(elapsed); + }; + + // Update immediately + updateElapsed(); + + // Then update every second + const interval = setInterval(updateElapsed, 1000); + + return () => clearInterval(interval); + }, [isProcessing, latestJob.createdAt]); + + const formatElapsedTime = (seconds: number): string => { + if (seconds < 60) { + return `${seconds} second${seconds !== 1 ? 's' : ''}`; + } + const minutes = Math.floor(seconds / 60); + const remainingSeconds = seconds % 60; + return `${minutes} minute${minutes !== 1 ? 's' : ''}, ${remainingSeconds} second${remainingSeconds !== 1 ? 's' : ''}`; + }; + + return ( + + ); +}; diff --git a/packages/design/src/FormManager/FormProcessingIndicator/formProcessingIndicator.module.css b/packages/design/src/FormManager/FormProcessingIndicator/formProcessingIndicator.module.css new file mode 100644 index 00000000..5aadcd9f --- /dev/null +++ b/packages/design/src/FormManager/FormProcessingIndicator/formProcessingIndicator.module.css @@ -0,0 +1,120 @@ +/* Alert container styling */ +.processingAlert { + border-left-width: 0.5rem; + box-shadow: 0 0.25rem 0.5rem rgba(0, 0, 0, 0.1); +} + +/* Main content layout */ +.processingContent { + display: flex; + align-items: flex-start; + gap: 1.5rem; + padding: 0.5rem 0; +} + +/* Spinner container and animation */ +.spinnerContainer { + flex-shrink: 0; + width: 3.5rem; + height: 3.5rem; + position: relative; +} + +.spinner { + width: 100%; + height: 100%; + animation: rotate 2s linear infinite; +} + +.spinnerTrack { + stroke: #dfe1e2; /* USWDS gray-cool-10 */ + opacity: 0.3; +} + +.spinnerProgress { + stroke: #005ea2; /* USWDS primary blue */ + stroke-linecap: round; + stroke-dasharray: 80, 200; + stroke-dashoffset: 0; + animation: dash 1.5s ease-in-out infinite; + transform-origin: center; +} + +@keyframes rotate { + 100% { + transform: rotate(360deg); + } +} + +@keyframes dash { + 0% { + stroke-dasharray: 1, 200; + stroke-dashoffset: 0; + } + 50% { + stroke-dasharray: 100, 200; + stroke-dashoffset: -15; + } + 100% { + stroke-dasharray: 100, 200; + stroke-dashoffset: -125; + } +} + +/* Text content */ +.processingText { + flex: 1; + min-width: 0; +} + +.heading { + margin-top: 0; + margin-bottom: 1rem; + color: #1b1b1b; /* USWDS ink */ +} + +/* Description text */ +.description { + margin: 0 0 1.5rem 0; + font-size: 0.9375rem; /* 15px - USWDS body size */ + line-height: 1.5; + color: #1b1b1b; /* USWDS ink */ +} + +/* Time information */ +.timeInfo { + padding: 1rem; + background-color: #f0f0f0; /* USWDS gray-cool-5 */ + border-radius: 0.25rem; + border-left: 4px solid #005ea2; +} + +.elapsedTime { + margin: 0 0 0.5rem 0; + font-size: 0.9375rem; + color: #1b1b1b; +} + +.elapsedTime strong { + font-weight: 700; +} + +.warningText { + margin: 0.5rem 0 0 0; + font-size: 0.875rem; /* 14px - USWDS small text */ + color: #b50909; /* USWDS red-warm-60v */ + font-weight: 600; +} + +/* Responsive adjustments */ +@media (max-width: 640px) { + .processingContent { + flex-direction: column; + gap: 1rem; + } + + .spinnerContainer { + width: 3rem; + height: 3rem; + } +} diff --git a/packages/design/src/FormManager/FormProcessingIndicator/index.ts b/packages/design/src/FormManager/FormProcessingIndicator/index.ts new file mode 100644 index 00000000..ad1e2f3b --- /dev/null +++ b/packages/design/src/FormManager/FormProcessingIndicator/index.ts @@ -0,0 +1 @@ +export { FormProcessingIndicator } from './FormProcessingIndicator.js'; diff --git a/packages/design/src/FormManager/hooks.ts b/packages/design/src/FormManager/hooks.ts index 34bae99b..f6b53096 100644 --- a/packages/design/src/FormManager/hooks.ts +++ b/packages/design/src/FormManager/hooks.ts @@ -18,3 +18,6 @@ export const useRouteParams = (): { pathname: location.pathname, }; }; + +export { useFormStatus } from './hooks/useFormStatus.js'; +export { useBlueprintWithStatus } from './hooks/useBlueprintWithStatus.js'; diff --git a/packages/design/src/FormManager/hooks/useBlueprintWithStatus.ts b/packages/design/src/FormManager/hooks/useBlueprintWithStatus.ts new file mode 100644 index 00000000..be389c95 --- /dev/null +++ b/packages/design/src/FormManager/hooks/useBlueprintWithStatus.ts @@ -0,0 +1,91 @@ +import { useState, useEffect, useRef } from 'react'; +import type { Blueprint, FormService, FormStatusResponse } from '@flexion/forms-core'; +import { useFormStatus } from './useFormStatus.js'; + +type UseBlueprintWithStatusResult = { + form: Blueprint | null; + status: FormStatusResponse | null; + isLoading: boolean; + isProcessing: boolean; + error: string | null; +}; + +/** + * Combined hook that loads a form blueprint and monitors its processing status. + * Returns both the form data and status information, automatically polling + * when the form is being processed. + */ +export const useBlueprintWithStatus = ( + formId: string | undefined, + formService: FormService +): UseBlueprintWithStatusResult => { + const [form, setForm] = useState(null); + const [formError, setFormError] = useState(null); + const [formLoading, setFormLoading] = useState(true); + const previousStatusRef = useRef(null); + + const { status, isLoading: statusLoading, error: statusError } = useFormStatus( + formId, + formService + ); + + const currentStatus = status?.latestJob?.status || null; + + // Load the form initially + useEffect(() => { + if (!formId) { + setFormLoading(false); + return; + } + + setFormLoading(true); + formService.getForm(formId).then(result => { + if (result.success) { + setForm(result.data); + setFormError(null); + } else { + console.error('Error loading form', result.error); + setFormError('Failed to load form'); + } + setFormLoading(false); + }); + }, [formId]); + + // Reload form when processing completes (transition from 'processing' to 'completed') + useEffect(() => { + const previousStatus = previousStatusRef.current; + const hasTransitionedToCompleted = + previousStatus === 'processing' && currentStatus === 'completed'; + + if (hasTransitionedToCompleted && formId) { + console.log('Processing completed, reloading form...'); + setFormLoading(true); + formService.getForm(formId).then(result => { + if (result.success) { + console.log('Form reloaded successfully with patterns:', Object.keys(result.data.patterns).length); + setForm(result.data); + setFormError(null); + } else { + console.error('Failed to reload form:', result.error); + setFormError('Failed to load form'); + } + setFormLoading(false); + }); + } + + // Update the previous status ref + previousStatusRef.current = currentStatus; + }, [currentStatus, formId]); + + const isProcessing = currentStatus === 'processing'; + const isLoading = formLoading || statusLoading; + const error = formError || statusError; + + return { + form, + status, + isLoading, + isProcessing, + error, + }; +}; diff --git a/packages/design/src/FormManager/hooks/useFormStatus.ts b/packages/design/src/FormManager/hooks/useFormStatus.ts new file mode 100644 index 00000000..6375d036 --- /dev/null +++ b/packages/design/src/FormManager/hooks/useFormStatus.ts @@ -0,0 +1,111 @@ +import { useState, useEffect, useRef } from 'react'; +import type { FormService, FormStatusResponse } from '@flexion/forms-core'; + +type UseFormStatusResult = { + status: FormStatusResponse | null; + isLoading: boolean; + error: string | null; + refetch: () => void; +}; + +/** + * Hook to poll form status. Automatically refreshes when form is processing. + * Stops polling when processing is complete or failed. + */ +export const useFormStatus = ( + formId: string | undefined, + formService: FormService, + options?: { + pollInterval?: number; // milliseconds, defaults to 2000 (2 seconds) + enabled?: boolean; // whether to poll, defaults to true + } +): UseFormStatusResult => { + const [status, setStatus] = useState(null); + const [isLoading, setIsLoading] = useState(true); + const [error, setError] = useState(null); + const pollIntervalRef = useRef(null); + const mountedRef = useRef(true); + + const pollInterval = options?.pollInterval ?? 2000; + const enabled = options?.enabled ?? true; + + const fetchStatus = async () => { + if (!formId || !enabled) { + return; + } + + try { + const result = await formService.getFormStatus(formId); + + if (!mountedRef.current) { + return; + } + + if (result.success) { + setStatus(result.data); + setError(null); + } else { + setError(result.error.message); + } + } catch (err) { + if (!mountedRef.current) { + return; + } + setError((err as Error).message); + } finally { + if (mountedRef.current) { + setIsLoading(false); + } + } + }; + + const shouldPoll = (currentStatus: FormStatusResponse | null): boolean => { + if (!currentStatus?.latestJob) { + return false; + } + return currentStatus.latestJob.status === 'processing'; + }; + + useEffect(() => { + mountedRef.current = true; + + // Initial fetch + fetchStatus(); + + return () => { + mountedRef.current = false; + if (pollIntervalRef.current) { + clearInterval(pollIntervalRef.current); + } + }; + }, [formId, enabled]); + + // Set up polling based on status + useEffect(() => { + // Clear any existing interval + if (pollIntervalRef.current) { + clearInterval(pollIntervalRef.current); + pollIntervalRef.current = null; + } + + // Only poll if we should + if (shouldPoll(status)) { + pollIntervalRef.current = setInterval(() => { + fetchStatus(); + }, pollInterval); + } + + return () => { + if (pollIntervalRef.current) { + clearInterval(pollIntervalRef.current); + } + }; + }, [status, pollInterval, enabled]); + + return { + status, + isLoading, + error, + refetch: fetchStatus, + }; +}; diff --git a/packages/forms/src/index.ts b/packages/forms/src/index.ts index 86aeceb2..2ab3088b 100644 --- a/packages/forms/src/index.ts +++ b/packages/forms/src/index.ts @@ -15,6 +15,7 @@ export { type FormStatusResponse, type GetFormStatusError, } from './services/get-form-status.js'; +export { type FormListItem } from './services/get-form-list.js'; export { defaultFormConfig, attachmentFileTypeOptions, diff --git a/packages/forms/src/repository/get-form-list.ts b/packages/forms/src/repository/get-form-list.ts index 77822733..4f7d1aa5 100644 --- a/packages/forms/src/repository/get-form-list.ts +++ b/packages/forms/src/repository/get-form-list.ts @@ -1,27 +1,70 @@ import type { FormRepositoryContext } from '.'; +import type { JobStatus } from './jobs/types.js'; -export type GetFormList = (ctx: FormRepositoryContext) => Promise< - | { - id: string; - title: string; - description: string; - }[] - | null ->; +export type FormListItem = { + id: string; + title: string; + description: string; + latestJob?: { + id: string; + jobType: string; + status: JobStatus; + createdAt: Date; + completedAt?: Date; + errorMessage?: string; + }; +}; + +export type GetFormList = ( + ctx: FormRepositoryContext +) => Promise; /** - * Retrieves a list of forms from the database. + * Retrieves a list of forms from the database with their latest job status. */ export const getFormList: GetFormList = async ctx => { const db = await ctx.db.getKysely(); - const rows = await db.selectFrom('forms').select(['id', 'data']).execute(); - - return rows.map(row => { - const form = JSON.parse(row.data); - return { - id: row.id, - title: form.summary.title, - description: form.summary.description, - }; - }); + + // Get all forms + const forms = await db.selectFrom('forms').select(['id', 'data']).execute(); + + // For each form, get the latest import-pdf job + const formList = await Promise.all( + forms.map(async row => { + const form = JSON.parse(row.data); + + // Get latest import-pdf job for this form + const latestJob = await db + .selectFrom('form_jobs') + .selectAll() + .where('form_id', '=', row.id) + .where('job_type', '=', 'import-pdf') + .orderBy('created_at', 'desc') + .limit(1) + .executeTakeFirst(); + + const formListItem: FormListItem = { + id: row.id, + title: form.summary.title, + description: form.summary.description, + }; + + if (latestJob) { + formListItem.latestJob = { + id: latestJob.id, + jobType: latestJob.job_type, + status: latestJob.status as JobStatus, + createdAt: new Date(latestJob.created_at), + completedAt: latestJob.completed_at + ? new Date(latestJob.completed_at) + : undefined, + errorMessage: latestJob.error_message || undefined, + }; + } + + return formListItem; + }) + ); + + return formList; }; diff --git a/packages/forms/src/services/get-form-list.ts b/packages/forms/src/services/get-form-list.ts index eaaf1935..0626524d 100644 --- a/packages/forms/src/services/get-form-list.ts +++ b/packages/forms/src/services/get-form-list.ts @@ -1,12 +1,10 @@ import { type Result, failure, success } from '@flexion/forms-common'; import { type FormServiceContext } from '../context/index.js'; +import type { FormListItem as RepositoryFormListItem } from '../repository/get-form-list.js'; + +export type FormListItem = RepositoryFormListItem; -export type FormListItem = { - id: string; - title: string; - description: string; -}; type FormListError = { status: number; message: string; @@ -24,7 +22,7 @@ export const getFormList: GetFormList = async ctx => { if (!ctx.isUserLoggedIn()) { return failure({ status: 401, - message: 'You must be logged in to delete a form', + message: 'You must be logged in to get form list', }); } const forms = await ctx.repository.getFormList(); diff --git a/packages/server/src/styles.css b/packages/server/src/styles.css index 26f6dc29..5809a82e 100644 --- a/packages/server/src/styles.css +++ b/packages/server/src/styles.css @@ -1 +1,2 @@ @import '@flexion/forms-design/static/uswds/styles/styles.css'; +@import '@flexion/forms-design/dist/assets/forms-design.css'; From e2c613e2918ec665cfeed97233f74c24f6dc24c2 Mon Sep 17 00:00:00 2001 From: Daniel Naab Date: Wed, 8 Oct 2025 22:53:35 -0500 Subject: [PATCH 08/17] Add conditional exports so we can avoid transpilation in dev mode. --- CLAUDE.md | 45 ++- apps/sandbox/tsconfig.json | 3 +- apps/server-doj/tsconfig.json | 3 +- apps/spotlight/astro.config.mjs | 5 + apps/spotlight/tsconfig.json | 3 +- packages/auth/package.json | 11 + packages/common/package.json | 11 + packages/database/package.json | 12 + packages/design/.eslintrc.cjs | 5 + packages/design/package.json | 12 + .../design/src/AvailableFormList/index.tsx | 12 +- .../FormProcessingIndicator.tsx | 5 +- packages/forms/package.json | 12 + packages/server/astro.config.mjs | 5 + packages/server/tsconfig.json | 3 +- pnpm-lock.yaml | 314 ++++++++++-------- 16 files changed, 310 insertions(+), 151 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 461cd593..5920dab4 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -114,6 +114,45 @@ common โ†’ (no dependencies) - **Testing**: Vitest (unit/integration), Playwright (E2E), @vitest/browser (Storybook) - **Language**: TypeScript throughout +### Development Workflow with Conditional Exports + +This monorepo uses **conditional exports** for zero-build development workflow: + +**In Development:** +- Library packages (common, database, forms-core, auth, design) are consumed directly from TypeScript source files +- No build step required when editing library code - changes are immediately reflected in consuming apps +- Hot module replacement (HMR) works instantly across package boundaries +- Consumer apps (server, spotlight, etc.) are configured with `customConditions: ["development"]` in tsconfig.json +- Vite/Astro resolve the `development` export condition to use `./src/**/*.ts` files + +**In Production:** +- Library packages are built and published from `dist/` folders +- Production builds use optimized, transpiled artifacts +- The `development` export condition is not used + +**How it Works:** +Each library package.json has exports like: +```json +{ + "exports": { + ".": { + "development": { + "types": "./src/index.ts", + "import": "./src/index.ts" + }, + "types": "./dist/index.d.ts", + "import": "./dist/index.js" + } + } +} +``` + +**Build Scripts:** +- **Important:** The `design` package requires CSS/SASS compilation: + - First time setup: Run `pnpm --filter @flexion/forms-design build:styles` (one-time) + - Or use `pnpm dev` which includes `dev:styles` (gulp watch) for the design package +- Production builds (`pnpm build`) are still required before publishing + ### Pattern System Patterns are the platform's primary building blocks. Each pattern has: @@ -137,5 +176,9 @@ Use `describeDatabase` helper for testing database routines against both SQLite - Node version is specified in `.nvmrc` - use `nvm install` to ensure correct version - Requires Docker or Podman for running tests (PostgreSQL container) - Playwright version must match exactly (1.51.1) across local and CI environments -- Build is required before running `pnpm dev` +- **Development**: + - No TypeScript build required - packages are consumed from source via conditional exports + - CSS/styles must be built once: `pnpm --filter @flexion/forms-design build:styles` + - Or run `pnpm dev` which includes style watching +- **Production/Publishing**: Run `pnpm build` to create optimized artifacts before publishing - Pre-commit hook runs `pnpm format` automatically diff --git a/apps/sandbox/tsconfig.json b/apps/sandbox/tsconfig.json index 11cf6557..09185aaf 100644 --- a/apps/sandbox/tsconfig.json +++ b/apps/sandbox/tsconfig.json @@ -4,7 +4,8 @@ "module": "NodeNext", "moduleResolution": "NodeNext", "outDir": "./dist", - "emitDeclarationOnly": true + "emitDeclarationOnly": true, + "customConditions": ["development"] }, "include": ["./src"], "references": [] diff --git a/apps/server-doj/tsconfig.json b/apps/server-doj/tsconfig.json index 11cf6557..09185aaf 100644 --- a/apps/server-doj/tsconfig.json +++ b/apps/server-doj/tsconfig.json @@ -4,7 +4,8 @@ "module": "NodeNext", "moduleResolution": "NodeNext", "outDir": "./dist", - "emitDeclarationOnly": true + "emitDeclarationOnly": true, + "customConditions": ["development"] }, "include": ["./src"], "references": [] diff --git a/apps/spotlight/astro.config.mjs b/apps/spotlight/astro.config.mjs index 01f796f2..50895063 100644 --- a/apps/spotlight/astro.config.mjs +++ b/apps/spotlight/astro.config.mjs @@ -21,6 +21,11 @@ export default defineConfig({ define: { 'import.meta.env.GITHUB': JSON.stringify(githubRepository), }, + resolve: { + conditions: process.env.NODE_ENV === 'production' + ? ['production', 'import', 'module', 'browser', 'default'] + : ['development', 'import', 'module', 'browser', 'default'], + }, }, }); diff --git a/apps/spotlight/tsconfig.json b/apps/spotlight/tsconfig.json index 7ea57e6d..70c020ab 100644 --- a/apps/spotlight/tsconfig.json +++ b/apps/spotlight/tsconfig.json @@ -6,7 +6,8 @@ "module": "NodeNext", "moduleResolution": "NodeNext", "jsx": "react", - "resolveJsonModule": true + "resolveJsonModule": true, + "customConditions": ["development"] }, "include": ["src/**/*.ts", "src/**/*.tsx", "src/**/*.astro"], "exclude": [".astro", "dist", "node_modules"] diff --git a/packages/auth/package.json b/packages/auth/package.json index d363d0f7..f8fa8e4d 100644 --- a/packages/auth/package.json +++ b/packages/auth/package.json @@ -6,6 +6,17 @@ "license": "CC0", "main": "dist/index.js", "types": "dist/index.d.js", + "exports": { + ".": { + "development": { + "types": "./src/index.ts", + "import": "./src/index.ts" + }, + "types": "./dist/index.d.ts", + "import": "./dist/index.js", + "default": "./dist/index.js" + } + }, "publishConfig": { "registry": "https://npm.pkg.github.com" }, diff --git a/packages/common/package.json b/packages/common/package.json index 48246d6f..b8c44fdd 100644 --- a/packages/common/package.json +++ b/packages/common/package.json @@ -6,6 +6,17 @@ "license": "CC0", "main": "dist/index.js", "types": "dist/index.d.ts", + "exports": { + ".": { + "development": { + "types": "./src/index.ts", + "import": "./src/index.ts" + }, + "types": "./dist/index.d.ts", + "import": "./dist/index.js", + "default": "./dist/index.js" + } + }, "publishConfig": { "registry": "https://npm.pkg.github.com" }, diff --git a/packages/database/package.json b/packages/database/package.json index 38ca8d0a..7379b76d 100644 --- a/packages/database/package.json +++ b/packages/database/package.json @@ -12,16 +12,28 @@ }, "exports": { ".": { + "development": { + "types": "./src/index.ts", + "import": "./src/index.ts" + }, "types": "./dist/types/index.d.ts", "import": "./dist/esm/index.js", "require": "./dist/cjs/index.js" }, "./context": { + "development": { + "types": "./src/context/index.ts", + "import": "./src/context/index.ts" + }, "types": "./dist/types/context/index.d.ts", "import": "./dist/esm/context.js", "require": "./dist/cjs/context.js" }, "./testing": { + "development": { + "types": "./src/testing.ts", + "import": "./src/testing.ts" + }, "types": "./dist/types/testing.d.ts", "import": "./dist/esm/testing.js", "require": "./dist/cjs/testing.js" diff --git a/packages/design/.eslintrc.cjs b/packages/design/.eslintrc.cjs index 0d5fa822..3ae371ac 100644 --- a/packages/design/.eslintrc.cjs +++ b/packages/design/.eslintrc.cjs @@ -28,4 +28,9 @@ module.exports = { rules: { 'react/prop-types': 'off', }, + settings: { + react: { + version: 'detect', + }, + }, }; diff --git a/packages/design/package.json b/packages/design/package.json index c25e1efa..223ddd2f 100644 --- a/packages/design/package.json +++ b/packages/design/package.json @@ -4,6 +4,18 @@ "main": "dist/index.js", "types": "dist/index.d.ts", "type": "module", + "exports": { + ".": { + "development": { + "types": "./src/index.ts", + "import": "./src/index.ts" + }, + "types": "./dist/index.d.ts", + "import": "./dist/index.js", + "default": "./dist/index.js" + }, + "./*": "./*" + }, "publishConfig": { "registry": "https://npm.pkg.github.com" }, diff --git a/packages/design/src/AvailableFormList/index.tsx b/packages/design/src/AvailableFormList/index.tsx index 565f3eeb..9a60d15d 100644 --- a/packages/design/src/AvailableFormList/index.tsx +++ b/packages/design/src/AvailableFormList/index.tsx @@ -150,13 +150,15 @@ const FormRow = ({ urlForFormManager: UrlForFormManager; }) => { const [showError, setShowError] = React.useState(false); - const isFailed = form.latestJob?.status === 'failed'; return ( <> -
+
{form.title}
@@ -170,7 +172,7 @@ const FormRow = ({ /> - {isFailed && form.latestJob && ( + {form.latestJob?.status === 'failed' && form.latestJob && (
@@ -178,8 +180,7 @@ const FormRow = ({

Import error:{' '} {form.latestJob.errorMessage || - 'An error occurred while processing this form.'} - {' '} + 'An error occurred while processing this form.'}{' '}