From 5d7f7e900e6a2652f5e9e43e01cb3110e968c26c Mon Sep 17 00:00:00 2001 From: Vikhyath Mondreti Date: Wed, 24 Jun 2026 00:04:32 -0700 Subject: [PATCH 01/15] improvement(pi): minor improvements to docs (#5192) --- apps/docs/content/docs/en/workflows/blocks/meta.json | 4 ++-- apps/docs/content/docs/en/workflows/blocks/pi.mdx | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/apps/docs/content/docs/en/workflows/blocks/meta.json b/apps/docs/content/docs/en/workflows/blocks/meta.json index 567a0c3417f..8b5cd58012a 100644 --- a/apps/docs/content/docs/en/workflows/blocks/meta.json +++ b/apps/docs/content/docs/en/workflows/blocks/meta.json @@ -2,7 +2,6 @@ "title": "Core Blocks", "pages": [ "agent", - "pi", "api", "function", "condition", @@ -17,6 +16,7 @@ "human-in-the-loop", "variables", "wait", - "credential" + "credential", + "pi" ] } diff --git a/apps/docs/content/docs/en/workflows/blocks/pi.mdx b/apps/docs/content/docs/en/workflows/blocks/pi.mdx index f98095bdb1a..2bee2901742 100644 --- a/apps/docs/content/docs/en/workflows/blocks/pi.mdx +++ b/apps/docs/content/docs/en/workflows/blocks/pi.mdx @@ -25,7 +25,7 @@ Pick the mode with the **Mode** dropdown. The fields below it change to match. Cloud runs entirely inside a disposable sandbox, so it never touches your machine. It clones the repo, lets the agent work with full read/shell/edit/git, pushes a branch, and opens a PR you review and merge. - Requires sandbox execution to be enabled (the Cloud option only appears when it is). -- Requires **your own provider API key (BYOK)** — the model key is handed to the sandbox, so Sim never injects a hosted key there. +- Requires **your own provider API key (BYOK)** — the model key is handed to the sandbox. - Needs a **GitHub token** with permission to clone, push, and open a PR (see [Setup](#setup-cloud)). - The deliverable is a **pull request** — nothing is committed to your default branch directly. @@ -118,7 +118,7 @@ The one case neither layer can rescue is a *first* prompt that already exceeds t ## Setup -### Cloud +### Cloud [#setup-cloud] Cloud runs in a sandbox image with the Pi CLI and git baked in. @@ -128,7 +128,7 @@ Cloud runs in a sandbox image with the Pi CLI and git baked in. - *Fine-grained:* select the repo, then **Contents: Read and write** + **Pull requests: Read and write**. - *Classic:* the **`repo`** scope. For org repos, authorize the token for SSO. -### Local +### Local [#setup-local] 1. **Enable SSH** on the target machine (on macOS: System Settings → General → Sharing → Remote Login). 2. **Expose it on a public host.** Sim blocks `localhost`/LAN, so use a TCP tunnel — for example `ngrok tcp 22`, which gives a `host:port` to put in **Host** and **Port**. From b212a5d37d05ae05cd2ad35404ab7390e96fdacb Mon Sep 17 00:00:00 2001 From: Waleed Date: Wed, 24 Jun 2026 00:12:20 -0700 Subject: [PATCH 02/15] improvement(mistral): update OCR pricing to OCR 4 rate ($4/1,000 pages) (#5193) * improvement(mistral): update OCR pricing to OCR 4 rate ($4/1,000 pages) * docs(mistral): document mistral-ocr-latest alias resolves to OCR 4 --- apps/sim/tools/mistral/parser.ts | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/apps/sim/tools/mistral/parser.ts b/apps/sim/tools/mistral/parser.ts index c0e7ee5e11d..948e2753f92 100644 --- a/apps/sim/tools/mistral/parser.ts +++ b/apps/sim/tools/mistral/parser.ts @@ -12,6 +12,19 @@ import type { ToolConfig } from '@/tools/types' const logger = createLogger('MistralParserTool') +/** + * Mistral OCR 4 standard pricing, in USD per page ($4 per 1,000 pages). + * + * This tool calls the synchronous `/v1/ocr` endpoint with the `mistral-ocr-latest` + * alias, which Mistral repointed to OCR 4 (`mistral-ocr-4-0`) on 2026-06-23, so the + * standard (non-batch) OCR 4 rate applies. Document AI / annotation pages are priced + * separately, but this tool does not submit annotation requests. + * + * @see https://mistral.ai/news/ocr-4/ + * @see https://docs.mistral.ai/getting-started/changelog + */ +const MISTRAL_OCR_COST_PER_PAGE = 0.004 + const MISTRAL_OCR_HOSTING = { envKeyPrefix: 'MISTRAL_API_KEY', apiKeyParam: 'apiKey', @@ -19,9 +32,6 @@ const MISTRAL_OCR_HOSTING = { pricing: { type: 'custom' as const, getCost: (_params: unknown, output: Record) => { - // Mistral OCR 3 standard pricing: $2 per 1,000 pages ($0.002/page). - // Annotated pages are priced separately at $3 per 1,000 annotated pages, but this tool does - // not submit annotation requests. Source: https://docs.mistral.ai/models/ocr-3-25-12 const rawUsageInfo = output.usage_info as { pages_processed?: number } | undefined const transformedUsageInfo = ( output.metadata as { usageInfo?: { pagesProcessed?: number } } | undefined @@ -33,7 +43,7 @@ const MISTRAL_OCR_HOSTING = { 'Mistral OCR response missing pages_processed in usage_info or metadata.usageInfo.pagesProcessed' ) } - const cost = pagesProcessed * 0.002 + const cost = pagesProcessed * MISTRAL_OCR_COST_PER_PAGE return { cost, metadata: { pagesProcessed } } }, }, From 86dc04d789640d852ec87d6eeb538490df398b41 Mon Sep 17 00:00:00 2001 From: Waleed Date: Wed, 24 Jun 2026 10:44:57 -0700 Subject: [PATCH 03/15] perf(workspace): server-prefetch home, knowledge, tables, and files list pages (#5196) --- .../workspace/[workspaceId]/files/page.tsx | 18 +- .../workspace/[workspaceId]/files/prefetch.ts | 43 +++++ .../app/workspace/[workspaceId]/home/page.tsx | 20 ++- .../workspace/[workspaceId]/home/prefetch.ts | 48 +++++ .../[workspaceId]/knowledge/page.tsx | 20 ++- .../[workspaceId]/knowledge/prefetch.ts | 28 +++ .../lib/prefetch-internal-fetch.ts | 25 +++ .../[workspaceId]/lib/prefetch.test.ts | 169 ++++++++++++++++++ .../workspace/[workspaceId]/tables/page.tsx | 18 +- .../[workspaceId]/tables/prefetch.ts | 26 +++ apps/sim/hooks/queries/folders.ts | 10 +- 11 files changed, 411 insertions(+), 14 deletions(-) create mode 100644 apps/sim/app/workspace/[workspaceId]/files/prefetch.ts create mode 100644 apps/sim/app/workspace/[workspaceId]/home/prefetch.ts create mode 100644 apps/sim/app/workspace/[workspaceId]/knowledge/prefetch.ts create mode 100644 apps/sim/app/workspace/[workspaceId]/lib/prefetch-internal-fetch.ts create mode 100644 apps/sim/app/workspace/[workspaceId]/lib/prefetch.test.ts create mode 100644 apps/sim/app/workspace/[workspaceId]/tables/prefetch.ts diff --git a/apps/sim/app/workspace/[workspaceId]/files/page.tsx b/apps/sim/app/workspace/[workspaceId]/files/page.tsx index ab21f2f3b72..2ed876e4ba0 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/page.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/page.tsx @@ -1,5 +1,8 @@ import { Suspense } from 'react' +import { dehydrate, HydrationBoundary } from '@tanstack/react-query' import type { Metadata } from 'next' +import { getQueryClient } from '@/app/_shell/providers/get-query-client' +import { prefetchFilesBrowser } from '@/app/workspace/[workspaceId]/files/prefetch' import { Files } from './files' import FilesLoading from './loading' @@ -15,10 +18,17 @@ export const metadata: Metadata = { * table headers) so a suspend never shows a blank frame; the route-level * `loading.tsx` covers the navigation/chunk-load transition the same way. */ -export default function FilesPage() { +export default async function FilesPage({ params }: { params: Promise<{ workspaceId: string }> }) { + const { workspaceId } = await params + + const queryClient = getQueryClient() + await prefetchFilesBrowser(queryClient, workspaceId) + return ( - }> - - + + }> + + + ) } diff --git a/apps/sim/app/workspace/[workspaceId]/files/prefetch.ts b/apps/sim/app/workspace/[workspaceId]/files/prefetch.ts new file mode 100644 index 00000000000..8780aa537fa --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/files/prefetch.ts @@ -0,0 +1,43 @@ +import type { QueryClient } from '@tanstack/react-query' +import type { WorkspaceFileFolderApi } from '@/lib/api/contracts/workspace-file-folders' +import type { ListWorkspaceFilesResponse } from '@/lib/api/contracts/workspace-files' +import { prefetchInternalJson } from '@/app/workspace/[workspaceId]/lib/prefetch-internal-fetch' +import { workspaceFileFolderKeys } from '@/hooks/queries/workspace-file-folders' +import { workspaceFilesKeys } from '@/hooks/queries/workspace-files' + +/** + * Prefetches the Files browser's two lists — workspace files and file folders — + * under the same query keys their client hooks (`useWorkspaceFiles`, + * `useWorkspaceFileFolders`) use (scope `active`), so the browser paints + * populated on first render. + * + * Both payloads carry `Date` fields, so they go through their routes and cache + * the serialized wire shape — see {@link prefetchInternalJson}. + */ +export async function prefetchFilesBrowser( + queryClient: QueryClient, + workspaceId: string +): Promise { + await Promise.all([ + queryClient.prefetchQuery({ + queryKey: workspaceFilesKeys.list(workspaceId, 'active'), + queryFn: async () => { + const data = await prefetchInternalJson( + `/api/workspaces/${workspaceId}/files?scope=active` + ) + return data.success ? data.files : [] + }, + staleTime: 30 * 1000, + }), + queryClient.prefetchQuery({ + queryKey: workspaceFileFolderKeys.list(workspaceId, 'active'), + queryFn: async () => { + const data = await prefetchInternalJson<{ folders?: WorkspaceFileFolderApi[] }>( + `/api/workspaces/${workspaceId}/files/folders?scope=active` + ) + return data.folders ?? [] + }, + staleTime: 30 * 1000, + }), + ]) +} diff --git a/apps/sim/app/workspace/[workspaceId]/home/page.tsx b/apps/sim/app/workspace/[workspaceId]/home/page.tsx index e29acc640ee..13595d65398 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/page.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/page.tsx @@ -1,6 +1,9 @@ import { Suspense } from 'react' +import { dehydrate, HydrationBoundary } from '@tanstack/react-query' import type { Metadata } from 'next' import { getSession } from '@/lib/auth' +import { getQueryClient } from '@/app/_shell/providers/get-query-client' +import { prefetchHomeLists } from '@/app/workspace/[workspaceId]/home/prefetch' import { Home } from './home' import { HomeFallback } from './home-fallback' @@ -8,11 +11,20 @@ export const metadata: Metadata = { title: 'New chat', } -export default async function HomePage() { +export default async function HomePage({ params }: { params: Promise<{ workspaceId: string }> }) { + const { workspaceId } = await params + + const queryClient = getQueryClient() + const listsPrefetch = prefetchHomeLists(queryClient, workspaceId) + const session = await getSession() + await listsPrefetch + return ( - }> - - + + }> + + + ) } diff --git a/apps/sim/app/workspace/[workspaceId]/home/prefetch.ts b/apps/sim/app/workspace/[workspaceId]/home/prefetch.ts new file mode 100644 index 00000000000..956a9e95555 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/home/prefetch.ts @@ -0,0 +1,48 @@ +import type { QueryClient } from '@tanstack/react-query' +import type { FolderApi } from '@/lib/api/contracts' +import type { ListWorkspaceFilesResponse } from '@/lib/api/contracts/workspace-files' +import { prefetchInternalJson } from '@/app/workspace/[workspaceId]/lib/prefetch-internal-fetch' +import { FOLDER_LIST_STALE_TIME, mapFolder } from '@/hooks/queries/folders' +import { folderKeys } from '@/hooks/queries/utils/folder-keys' +import { workspaceFilesKeys } from '@/hooks/queries/workspace-files' + +/** + * Prefetches the home page's secondary lists — folders and workspace files — + * under the same query keys their client hooks (`useFolders`, + * `useWorkspaceFiles`) use, so the home view paints populated on first render. + * + * The workflow list (`workflowKeys.list(ws, 'active')`) is already hydrated by + * the workspace sidebar prefetch and is intentionally not repeated here. + * + * Folders are fetched through the route and mapped with the same `mapFolder` + * the hook applies, matching its cached shape (string dates → `Date`). Files + * carry `Date` fields, so they go through the route and cache the serialized + * wire shape — see {@link prefetchInternalJson}. + */ +export async function prefetchHomeLists( + queryClient: QueryClient, + workspaceId: string +): Promise { + await Promise.all([ + queryClient.prefetchQuery({ + queryKey: folderKeys.list(workspaceId, 'active'), + queryFn: async () => { + const { folders } = await prefetchInternalJson<{ folders?: FolderApi[] }>( + `/api/folders?workspaceId=${workspaceId}&scope=active` + ) + return (folders ?? []).map(mapFolder) + }, + staleTime: FOLDER_LIST_STALE_TIME, + }), + queryClient.prefetchQuery({ + queryKey: workspaceFilesKeys.list(workspaceId, 'active'), + queryFn: async () => { + const data = await prefetchInternalJson( + `/api/workspaces/${workspaceId}/files?scope=active` + ) + return data.success ? data.files : [] + }, + staleTime: 30 * 1000, + }), + ]) +} diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/page.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/page.tsx index be3743be659..6243dc42035 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/page.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/page.tsx @@ -1,8 +1,26 @@ +import { dehydrate, HydrationBoundary } from '@tanstack/react-query' import type { Metadata } from 'next' +import { getQueryClient } from '@/app/_shell/providers/get-query-client' +import { prefetchKnowledgeBases } from '@/app/workspace/[workspaceId]/knowledge/prefetch' import { Knowledge } from './knowledge' export const metadata: Metadata = { title: 'Knowledge Base', } -export default Knowledge +export default async function KnowledgePage({ + params, +}: { + params: Promise<{ workspaceId: string }> +}) { + const { workspaceId } = await params + + const queryClient = getQueryClient() + await prefetchKnowledgeBases(queryClient, workspaceId) + + return ( + + + + ) +} diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/prefetch.ts b/apps/sim/app/workspace/[workspaceId]/knowledge/prefetch.ts new file mode 100644 index 00000000000..0e8e5578840 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/prefetch.ts @@ -0,0 +1,28 @@ +import type { QueryClient } from '@tanstack/react-query' +import type { KnowledgeBaseData } from '@/lib/api/contracts/knowledge' +import { prefetchInternalJson } from '@/app/workspace/[workspaceId]/lib/prefetch-internal-fetch' +import { knowledgeKeys } from '@/hooks/queries/kb/knowledge' + +/** + * Prefetches the workspace's knowledge-bases list under the same query key the + * client `useKnowledgeBasesQuery` hook uses (scope `active`), so the list paints + * populated on first render. + * + * The list carries `Date` fields, so it goes through the `/api/knowledge` route + * and caches the serialized wire shape — see {@link prefetchInternalJson}. + */ +export async function prefetchKnowledgeBases( + queryClient: QueryClient, + workspaceId: string +): Promise { + await queryClient.prefetchQuery({ + queryKey: knowledgeKeys.list(workspaceId, 'active'), + queryFn: async () => { + const result = await prefetchInternalJson<{ data: KnowledgeBaseData[] }>( + `/api/knowledge?workspaceId=${workspaceId}&scope=active` + ) + return result.data + }, + staleTime: 60 * 1000, + }) +} diff --git a/apps/sim/app/workspace/[workspaceId]/lib/prefetch-internal-fetch.ts b/apps/sim/app/workspace/[workspaceId]/lib/prefetch-internal-fetch.ts new file mode 100644 index 00000000000..e48f6064c17 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/lib/prefetch-internal-fetch.ts @@ -0,0 +1,25 @@ +import { headers } from 'next/headers' +import { getInternalApiBaseUrl } from '@/lib/core/utils/urls' + +/** + * Server-side GET against an internal `/api` route, forwarding the incoming + * request's cookie so the route authenticates as the current user. + * + * List prefetches go through the route (rather than the data layer) when the + * payload carries `Date` fields: `NextResponse.json` serializes them to the + * string wire shape the client caches via `requestJson`, so the + * server-hydrated entry byte-matches the client-fetched one through + * dehydration. Calling the data layer directly would cache raw `Date` objects + * and drift from that wire shape. Mirrors the settings/subscription prefetch. + */ +export async function prefetchInternalJson(path: string): Promise { + const cookie = (await headers()).get('cookie') + // boundary-raw-fetch: server-side RSC prefetch forwarding the session cookie to an internal API route; requestJson is client-only and cannot run here + const response = await fetch(`${getInternalApiBaseUrl()}${path}`, { + headers: cookie ? { cookie } : {}, + }) + if (!response.ok) { + throw new Error(`Prefetch failed for ${path}: ${response.status}`) + } + return response.json() as Promise +} diff --git a/apps/sim/app/workspace/[workspaceId]/lib/prefetch.test.ts b/apps/sim/app/workspace/[workspaceId]/lib/prefetch.test.ts new file mode 100644 index 00000000000..d031c0648ef --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/lib/prefetch.test.ts @@ -0,0 +1,169 @@ +/** + * @vitest-environment node + */ +import { QueryClient } from '@tanstack/react-query' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockPrefetchInternalJson } = vi.hoisted(() => ({ + mockPrefetchInternalJson: vi.fn(), +})) + +vi.mock('@/app/workspace/[workspaceId]/lib/prefetch-internal-fetch', () => ({ + prefetchInternalJson: mockPrefetchInternalJson, +})) + +vi.mock('@/components/emcn', () => ({ + toast: { success: vi.fn(), error: vi.fn() }, +})) + +import { prefetchFilesBrowser } from '@/app/workspace/[workspaceId]/files/prefetch' +import { prefetchHomeLists } from '@/app/workspace/[workspaceId]/home/prefetch' +import { prefetchKnowledgeBases } from '@/app/workspace/[workspaceId]/knowledge/prefetch' +import { prefetchTables } from '@/app/workspace/[workspaceId]/tables/prefetch' +import { knowledgeKeys } from '@/hooks/queries/kb/knowledge' +import { tableKeys } from '@/hooks/queries/tables' +import { folderKeys } from '@/hooks/queries/utils/folder-keys' +import { workspaceFileFolderKeys } from '@/hooks/queries/workspace-file-folders' +import { workspaceFilesKeys } from '@/hooks/queries/workspace-files' + +const WORKSPACE_ID = 'ws-123' + +function makeClient() { + return new QueryClient({ defaultOptions: { queries: { retry: false } } }) +} + +describe('workspace list prefetches', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + describe('prefetchKnowledgeBases', () => { + it('primes the exact key useKnowledgeBasesQuery reads and unwraps data', async () => { + const bases = [{ id: 'kb-1' }] + mockPrefetchInternalJson.mockResolvedValue({ data: bases }) + const client = makeClient() + + await prefetchKnowledgeBases(client, WORKSPACE_ID) + + expect(mockPrefetchInternalJson).toHaveBeenCalledWith( + `/api/knowledge?workspaceId=${WORKSPACE_ID}&scope=active` + ) + expect(client.getQueryData(knowledgeKeys.list(WORKSPACE_ID, 'active'))).toEqual(bases) + }) + }) + + describe('prefetchTables', () => { + it('primes the exact key useTablesList reads and unwraps data.tables', async () => { + const tables = [{ id: 't-1' }] + mockPrefetchInternalJson.mockResolvedValue({ data: { tables } }) + const client = makeClient() + + await prefetchTables(client, WORKSPACE_ID) + + expect(mockPrefetchInternalJson).toHaveBeenCalledWith( + `/api/table?workspaceId=${WORKSPACE_ID}&scope=active` + ) + expect(client.getQueryData(tableKeys.list(WORKSPACE_ID, 'active'))).toEqual(tables) + }) + }) + + describe('prefetchFilesBrowser', () => { + it('primes both file + folder keys the client hooks read', async () => { + const files = [{ id: 'f-1' }] + const folders = [{ id: 'folder-1' }] + mockPrefetchInternalJson.mockImplementation(async (path: string) => + path.includes('/folders') ? { folders } : { success: true, files } + ) + const client = makeClient() + + await prefetchFilesBrowser(client, WORKSPACE_ID) + + expect(mockPrefetchInternalJson).toHaveBeenCalledWith( + `/api/workspaces/${WORKSPACE_ID}/files?scope=active` + ) + expect(mockPrefetchInternalJson).toHaveBeenCalledWith( + `/api/workspaces/${WORKSPACE_ID}/files/folders?scope=active` + ) + expect(client.getQueryData(workspaceFilesKeys.list(WORKSPACE_ID, 'active'))).toEqual(files) + expect(client.getQueryData(workspaceFileFolderKeys.list(WORKSPACE_ID, 'active'))).toEqual( + folders + ) + }) + + it('caches an empty file list when the route reports failure', async () => { + mockPrefetchInternalJson.mockImplementation(async (path: string) => + path.includes('/folders') ? { folders: [] } : { success: false, files: [] } + ) + const client = makeClient() + + await prefetchFilesBrowser(client, WORKSPACE_ID) + + expect(client.getQueryData(workspaceFilesKeys.list(WORKSPACE_ID, 'active'))).toEqual([]) + }) + }) + + describe('prefetchHomeLists', () => { + it('primes folder + file keys, mapping folder rows to the client shape', async () => { + const folderRow = { + id: 'folder-1', + name: 'Docs', + userId: 'u-1', + workspaceId: WORKSPACE_ID, + parentId: null, + color: null, + isExpanded: true, + locked: false, + sortOrder: 0, + createdAt: '2026-01-01T00:00:00.000Z', + updatedAt: '2026-01-02T00:00:00.000Z', + archivedAt: null, + } + const files = [{ id: 'f-1' }] + mockPrefetchInternalJson.mockImplementation(async (path: string) => + path.startsWith('/api/folders') ? { folders: [folderRow] } : { success: true, files } + ) + const client = makeClient() + + await prefetchHomeLists(client, WORKSPACE_ID) + + expect(mockPrefetchInternalJson).toHaveBeenCalledWith( + `/api/folders?workspaceId=${WORKSPACE_ID}&scope=active` + ) + const cachedFolders = client.getQueryData(folderKeys.list(WORKSPACE_ID, 'active')) as Array<{ + id: string + color: string + createdAt: Date + }> + expect(cachedFolders).toHaveLength(1) + expect(cachedFolders[0].color).toBe('#6B7280') + expect(cachedFolders[0].createdAt).toBeInstanceOf(Date) + expect(client.getQueryData(workspaceFilesKeys.list(WORKSPACE_ID, 'active'))).toEqual(files) + }) + }) + + describe('graceful failure', () => { + it.each([ + [ + 'prefetchKnowledgeBases', + prefetchKnowledgeBases, + knowledgeKeys.list(WORKSPACE_ID, 'active'), + ], + ['prefetchTables', prefetchTables, tableKeys.list(WORKSPACE_ID, 'active')], + ['prefetchHomeLists', prefetchHomeLists, folderKeys.list(WORKSPACE_ID, 'active')], + [ + 'prefetchFilesBrowser', + prefetchFilesBrowser, + workspaceFilesKeys.list(WORKSPACE_ID, 'active'), + ], + ] as const)( + '%s does not throw when the fetcher rejects (page still renders, client refetches)', + async (_name, prefetch, queryKey) => { + mockPrefetchInternalJson.mockRejectedValue(new Error('500')) + const client = makeClient() + + await expect(prefetch(client, WORKSPACE_ID)).resolves.toBeUndefined() + expect(client.getQueryData(queryKey)).toBeUndefined() + } + ) + }) +}) diff --git a/apps/sim/app/workspace/[workspaceId]/tables/page.tsx b/apps/sim/app/workspace/[workspaceId]/tables/page.tsx index 15a016a25a2..0e9390a5d95 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/page.tsx +++ b/apps/sim/app/workspace/[workspaceId]/tables/page.tsx @@ -1,6 +1,9 @@ import { Suspense } from 'react' +import { dehydrate, HydrationBoundary } from '@tanstack/react-query' import type { Metadata } from 'next' +import { getQueryClient } from '@/app/_shell/providers/get-query-client' import TablesLoading from '@/app/workspace/[workspaceId]/tables/loading' +import { prefetchTables } from '@/app/workspace/[workspaceId]/tables/prefetch' import { Tables } from './tables' export const metadata: Metadata = { @@ -13,10 +16,17 @@ export const metadata: Metadata = { * fallback renders the real chrome so a suspend never shows a blank frame; the * route-level `loading.tsx` covers the navigation/chunk-load transition. */ -export default function TablesPage() { +export default async function TablesPage({ params }: { params: Promise<{ workspaceId: string }> }) { + const { workspaceId } = await params + + const queryClient = getQueryClient() + await prefetchTables(queryClient, workspaceId) + return ( - }> - - + + }> + + + ) } diff --git a/apps/sim/app/workspace/[workspaceId]/tables/prefetch.ts b/apps/sim/app/workspace/[workspaceId]/tables/prefetch.ts new file mode 100644 index 00000000000..60d6a79a735 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/tables/prefetch.ts @@ -0,0 +1,26 @@ +import type { QueryClient } from '@tanstack/react-query' +import type { TableDefinition } from '@/lib/table' +import { prefetchInternalJson } from '@/app/workspace/[workspaceId]/lib/prefetch-internal-fetch' +import { tableKeys } from '@/hooks/queries/tables' + +/** + * Prefetches the workspace's tables list under the same query key the client + * `useTablesList` hook uses (scope `active`), so the list paints populated on + * first render. + * + * Table definitions carry `Date` fields, so the list goes through the + * `/api/table` route and caches the serialized wire shape — see + * {@link prefetchInternalJson}. + */ +export async function prefetchTables(queryClient: QueryClient, workspaceId: string): Promise { + await queryClient.prefetchQuery({ + queryKey: tableKeys.list(workspaceId, 'active'), + queryFn: async () => { + const response = await prefetchInternalJson<{ data: { tables: TableDefinition[] } }>( + `/api/table?workspaceId=${workspaceId}&scope=active` + ) + return response.data.tables + }, + staleTime: 30 * 1000, + }) +} diff --git a/apps/sim/hooks/queries/folders.ts b/apps/sim/hooks/queries/folders.ts index f0d8913cd05..34b695374fd 100644 --- a/apps/sim/hooks/queries/folders.ts +++ b/apps/sim/hooks/queries/folders.ts @@ -25,7 +25,15 @@ import type { WorkflowFolder } from '@/stores/folders/types' const logger = createLogger('FolderQueries') -function mapFolder(folder: FolderApi): WorkflowFolder { +export const FOLDER_LIST_STALE_TIME = 60 * 1000 + +/** + * Maps a wire folder row to the client `WorkflowFolder` shape (string dates → + * `Date`, color default). Exported so the server-side home prefetch produces + * the exact cached value `useFolders` stores, keeping the hydrated entry in + * sync with a client fetch. + */ +export function mapFolder(folder: FolderApi): WorkflowFolder { return { id: folder.id, name: folder.name, From b52fcc094e060dce715549fcefc7215542f31b88 Mon Sep 17 00:00:00 2001 From: Waleed Date: Wed, 24 Jun 2026 10:45:33 -0700 Subject: [PATCH 04/15] refactor(stores): model execution and workflow-diff state as status enums (#5197) --- apps/sim/hooks/use-undo-redo.ts | 21 +- apps/sim/stores/execution/store.test.ts | 192 +++++++++++++++-- apps/sim/stores/execution/store.ts | 31 ++- apps/sim/stores/execution/types.ts | 77 ++++++- apps/sim/stores/workflow-diff/store.test.ts | 228 ++++++++++++++++++++ apps/sim/stores/workflow-diff/store.ts | 37 ++-- apps/sim/stores/workflow-diff/types.ts | 50 +++++ apps/sim/vitest.setup.ts | 3 + 8 files changed, 569 insertions(+), 70 deletions(-) create mode 100644 apps/sim/stores/workflow-diff/store.test.ts diff --git a/apps/sim/hooks/use-undo-redo.ts b/apps/sim/hooks/use-undo-redo.ts index 025c087de0e..22dfdaa6e0e 100644 --- a/apps/sim/hooks/use-undo-redo.ts +++ b/apps/sim/hooks/use-undo-redo.ts @@ -41,6 +41,7 @@ import { type UpdateParentOperation, useUndoRedoStore, } from '@/stores/undo-redo' +import { deriveDiffFlags } from '@/stores/workflow-diff/types' import { useWorkflowRegistry } from '@/stores/workflows/registry/store' import { useSubBlockStore } from '@/stores/workflows/subblock/store' import { useWorkflowStore } from '@/stores/workflows/workflow/store' @@ -1234,9 +1235,7 @@ export function useUndoRedo() { // Restore diff state with baseline (local UI only) diffStore._batchedStateUpdate({ - hasActiveDiff: true, - isShowingDiff: true, - isDiffReady: true, + ...deriveDiffFlags('showing'), baselineWorkflow: originalBaseline || null, baselineWorkflowId: activeWorkflowId, diffAnalysis: diffAnalysis, @@ -1285,9 +1284,7 @@ export function useUndoRedo() { // Restore diff state with baseline (local UI only) const diffStore = useWorkflowDiffStore.getState() diffStore._batchedStateUpdate({ - hasActiveDiff: true, - isShowingDiff: true, - isDiffReady: true, + ...deriveDiffFlags('showing'), baselineWorkflow: baselineSnapshot || null, baselineWorkflowId: activeWorkflowId, diffAnalysis: diffAnalysis, @@ -1805,9 +1802,7 @@ export function useUndoRedo() { // Restore diff state with original baseline (local UI only) diffStore._batchedStateUpdate({ - hasActiveDiff: true, - isShowingDiff: true, - isDiffReady: true, + ...deriveDiffFlags('showing'), baselineWorkflow: baselineSnapshot, baselineWorkflowId: activeWorkflowId, diffAnalysis: diffAnalysis, @@ -1834,9 +1829,7 @@ export function useUndoRedo() { // Clear diff state FIRST to prevent flash of colors (local UI only) // Use setState directly to ensure synchronous clearing useWorkflowDiffStore.setState({ - hasActiveDiff: false, - isShowingDiff: false, - isDiffReady: false, + ...deriveDiffFlags('none'), baselineWorkflow: null, baselineWorkflowId: null, diffAnalysis: null, @@ -1886,9 +1879,7 @@ export function useUndoRedo() { // Clear diff state FIRST to prevent flash of colors (local UI only) // Use setState directly to ensure synchronous clearing useWorkflowDiffStore.setState({ - hasActiveDiff: false, - isShowingDiff: false, - isDiffReady: false, + ...deriveDiffFlags('none'), baselineWorkflow: null, baselineWorkflowId: null, diffAnalysis: null, diff --git a/apps/sim/stores/execution/store.test.ts b/apps/sim/stores/execution/store.test.ts index 6888894b80b..c0b11fc87ce 100644 --- a/apps/sim/stores/execution/store.test.ts +++ b/apps/sim/stores/execution/store.test.ts @@ -1,4 +1,6 @@ /** + * @vitest-environment node + * * Tests for the per-workflow execution store. * * These tests cover: @@ -7,12 +9,19 @@ * - Execution lifecycle (start/stop clears run path) * - Block and edge run status tracking * - Active block management - * - Debug state management + * - The {@link ExecutionStatus} enum and its derived `isExecuting` / + * `isDebugging` booleans (exhaustive status → flag mapping + transitions) * - Execution snapshot management * - Store reset * - Immutability guarantees * * @remarks + * The store under test transitively imports the workflow registry store, + * which drags in the block registry and emcn icon CSS. To keep this a true + * unit test that loads under the node environment, the registry store is + * mocked to a minimal stub (the store actions never touch it — only the + * convenience hooks do, which are not exercised here). + * * Most tests use `it.concurrent` with unique workflow IDs per test. * Because the store isolates state by workflow ID, concurrent tests * do not interfere with each other. The `reset` and `immutability` @@ -21,17 +30,30 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' +vi.mock('@/stores/workflows/registry/store', () => ({ + useWorkflowRegistry: Object.assign( + vi.fn(() => null), + { getState: vi.fn(() => ({ activeWorkflowId: null })) } + ), +})) + vi.unmock('@/stores/execution/store') vi.unmock('@/stores/execution/types') import { useExecutionStore } from '@/stores/execution/store' -import { defaultWorkflowExecutionState, initialState } from '@/stores/execution/types' +import { + defaultWorkflowExecutionState, + deriveExecutionFlags, + type ExecutionStatus, + initialState, +} from '@/stores/execution/types' describe('useExecutionStore', () => { describe('getWorkflowExecution', () => { it.concurrent('should return default state for an unknown workflow', () => { const state = useExecutionStore.getState().getWorkflowExecution('wf-get-default') + expect(state.status).toBe('idle') expect(state.isExecuting).toBe(false) expect(state.isDebugging).toBe(false) expect(state.activeBlockIds.size).toBe(0) @@ -63,22 +85,35 @@ describe('useExecutionStore', () => { }) }) + describe('deriveExecutionFlags', () => { + it.concurrent('maps every status to the documented legacy booleans', () => { + const cases: Array<[ExecutionStatus, boolean, boolean]> = [ + ['idle', false, false], + ['running', true, false], + ['debugging', true, true], + ] + for (const [status, isExecuting, isDebugging] of cases) { + expect(deriveExecutionFlags(status)).toEqual({ isExecuting, isDebugging }) + } + }) + }) + describe('setIsExecuting', () => { - it.concurrent('should set isExecuting to true', () => { + it.concurrent('should set isExecuting to true (status running)', () => { useExecutionStore.getState().setIsExecuting('wf-exec-true', true) - expect(useExecutionStore.getState().getWorkflowExecution('wf-exec-true').isExecuting).toBe( - true - ) + const state = useExecutionStore.getState().getWorkflowExecution('wf-exec-true') + expect(state.isExecuting).toBe(true) + expect(state.status).toBe('running') }) - it.concurrent('should set isExecuting to false', () => { + it.concurrent('should set isExecuting to false (status idle)', () => { useExecutionStore.getState().setIsExecuting('wf-exec-false', true) useExecutionStore.getState().setIsExecuting('wf-exec-false', false) - expect(useExecutionStore.getState().getWorkflowExecution('wf-exec-false').isExecuting).toBe( - false - ) + const state = useExecutionStore.getState().getWorkflowExecution('wf-exec-false') + expect(state.isExecuting).toBe(false) + expect(state.status).toBe('idle') }) it.concurrent('should clear lastRunPath and lastRunEdges when starting execution', () => { @@ -107,6 +142,131 @@ describe('useExecutionStore', () => { expect(state.isExecuting).toBe(false) expect(state.lastRunPath.get('block-1')).toBe('success') }) + + it.concurrent('starting a debug run then setIsExecuting(true) clears the run path', () => { + const wf = 'wf-exec-debug-start-clears' + useExecutionStore.getState().setIsExecuting(wf, true) + useExecutionStore.getState().setIsDebugging(wf, true) + useExecutionStore.getState().setBlockRunStatus(wf, 'block-1', 'success') + + useExecutionStore.getState().setIsExecuting(wf, true) + + const state = useExecutionStore.getState().getWorkflowExecution(wf) + expect(state.status).toBe('debugging') + expect(state.isExecuting).toBe(true) + expect(state.isDebugging).toBe(true) + expect(state.lastRunPath.size).toBe(0) + expect(state.lastRunEdges.size).toBe(0) + }) + }) + + describe('setIsDebugging', () => { + it.concurrent('should toggle debug mode', () => { + const wf = 'wf-debug-toggle' + useExecutionStore.getState().setIsDebugging(wf, true) + + expect(useExecutionStore.getState().getWorkflowExecution(wf).isDebugging).toBe(true) + expect(useExecutionStore.getState().getWorkflowExecution(wf).isExecuting).toBe(true) + expect(useExecutionStore.getState().getWorkflowExecution(wf).status).toBe('debugging') + + useExecutionStore.getState().setIsDebugging(wf, false) + expect(useExecutionStore.getState().getWorkflowExecution(wf).isDebugging).toBe(false) + expect(useExecutionStore.getState().getWorkflowExecution(wf).isExecuting).toBe(true) + expect(useExecutionStore.getState().getWorkflowExecution(wf).status).toBe('running') + }) + + it.concurrent('setIsDebugging(false) while idle is a no-op (stays idle)', () => { + const wf = 'wf-debug-false-idle' + useExecutionStore.getState().setIsDebugging(wf, false) + expect(useExecutionStore.getState().getWorkflowExecution(wf).status).toBe('idle') + expect(useExecutionStore.getState().getWorkflowExecution(wf).isExecuting).toBe(false) + }) + + it.concurrent('setIsDebugging(false) while running keeps running', () => { + const wf = 'wf-debug-false-running' + useExecutionStore.getState().setIsExecuting(wf, true) + useExecutionStore.getState().setIsDebugging(wf, false) + expect(useExecutionStore.getState().getWorkflowExecution(wf).status).toBe('running') + expect(useExecutionStore.getState().getWorkflowExecution(wf).isExecuting).toBe(true) + }) + + it.concurrent('does not clear the run path when entering debug mode', () => { + const wf = 'wf-debug-keeps-path' + useExecutionStore.getState().setBlockRunStatus(wf, 'block-1', 'success') + useExecutionStore.getState().setIsDebugging(wf, true) + expect(useExecutionStore.getState().getWorkflowExecution(wf).lastRunPath.get('block-1')).toBe( + 'success' + ) + }) + }) + + describe('status enum', () => { + it.concurrent('idle derives both flags false', () => { + const wf = 'wf-status-idle' + const state = useExecutionStore.getState().getWorkflowExecution(wf) + expect(state.status).toBe('idle') + expect(state.isExecuting).toBe(false) + expect(state.isDebugging).toBe(false) + }) + + it.concurrent('running derives isExecuting only', () => { + const wf = 'wf-status-running' + useExecutionStore.getState().setStatus(wf, 'running') + const state = useExecutionStore.getState().getWorkflowExecution(wf) + expect(state.status).toBe('running') + expect(state.isExecuting).toBe(true) + expect(state.isDebugging).toBe(false) + }) + + it.concurrent('debugging derives both flags true', () => { + const wf = 'wf-status-debugging' + useExecutionStore.getState().setStatus(wf, 'debugging') + const state = useExecutionStore.getState().getWorkflowExecution(wf) + expect(state.status).toBe('debugging') + expect(state.isExecuting).toBe(true) + expect(state.isDebugging).toBe(true) + }) + + it.concurrent('setStatus preserves the run path unless clearRunPath is passed', () => { + const wf = 'wf-status-path-rules' + useExecutionStore.getState().setStatus(wf, 'debugging') + useExecutionStore.getState().setBlockRunStatus(wf, 'block-1', 'success') + expect(useExecutionStore.getState().getWorkflowExecution(wf).lastRunPath.size).toBe(1) + + useExecutionStore.getState().setStatus(wf, 'running') + expect(useExecutionStore.getState().getWorkflowExecution(wf).lastRunPath.size).toBe(1) + + useExecutionStore.getState().setStatus(wf, 'running', { clearRunPath: true }) + expect(useExecutionStore.getState().getWorkflowExecution(wf).lastRunPath.size).toBe(0) + }) + + it.concurrent('the derived booleans always agree with the stored status', () => { + const wf = 'wf-status-no-drift' + for (const status of ['idle', 'running', 'debugging', 'idle'] as const) { + useExecutionStore.getState().setStatus(wf, status) + const state = useExecutionStore.getState().getWorkflowExecution(wf) + expect({ isExecuting: state.isExecuting, isDebugging: state.isDebugging }).toEqual( + deriveExecutionFlags(status) + ) + } + }) + + it.concurrent('setIsExecuting(true) preserves an active debug session', () => { + const wf = 'wf-status-debug-preserve' + useExecutionStore.getState().setStatus(wf, 'debugging') + useExecutionStore.getState().setIsExecuting(wf, true) + expect(useExecutionStore.getState().getWorkflowExecution(wf).status).toBe('debugging') + }) + + it.concurrent('setIsExecuting(false) returns to idle from any mode', () => { + const wf = 'wf-status-stop' + useExecutionStore.getState().setStatus(wf, 'debugging') + useExecutionStore.getState().setIsExecuting(wf, false) + const state = useExecutionStore.getState().getWorkflowExecution(wf) + expect(state.status).toBe('idle') + expect(state.isExecuting).toBe(false) + expect(state.isDebugging).toBe(false) + }) }) describe('setActiveBlocks', () => { @@ -151,18 +311,6 @@ describe('useExecutionStore', () => { }) }) - describe('setIsDebugging', () => { - it.concurrent('should toggle debug mode', () => { - const wf = 'wf-debug-toggle' - useExecutionStore.getState().setIsDebugging(wf, true) - - expect(useExecutionStore.getState().getWorkflowExecution(wf).isDebugging).toBe(true) - - useExecutionStore.getState().setIsDebugging(wf, false) - expect(useExecutionStore.getState().getWorkflowExecution(wf).isDebugging).toBe(false) - }) - }) - describe('setExecutor', () => { it.concurrent('should store and clear executor', () => { const wf = 'wf-executor' diff --git a/apps/sim/stores/execution/store.ts b/apps/sim/stores/execution/store.ts index 8d9e2827f34..5efa313f021 100644 --- a/apps/sim/stores/execution/store.ts +++ b/apps/sim/stores/execution/store.ts @@ -3,9 +3,11 @@ import { useWorkflowRegistry } from '@/stores/workflows/registry/store' import { type BlockRunStatus, defaultWorkflowExecutionState, + deriveExecutionFlags, type EdgeRunStatus, type ExecutionActions, type ExecutionState, + type ExecutionStatus, initialState, type WorkflowExecutionState, } from './types' @@ -78,9 +80,12 @@ export const useExecutionStore = create()((se }) }, - setIsExecuting: (workflowId, isExecuting) => { - const patch: Partial = { isExecuting } - if (isExecuting) { + setStatus: (workflowId, status, options) => { + const patch: Partial = { + status, + ...deriveExecutionFlags(status), + } + if (options?.clearRunPath) { patch.lastRunPath = new Map() patch.lastRunEdges = new Map() } @@ -89,10 +94,24 @@ export const useExecutionStore = create()((se }) }, + setIsExecuting: (workflowId, isExecuting) => { + const current = getOrCreate(get().workflowExecutions, workflowId) + const nextStatus: ExecutionStatus = isExecuting + ? current.status === 'debugging' + ? 'debugging' + : 'running' + : 'idle' + get().setStatus(workflowId, nextStatus, { clearRunPath: isExecuting }) + }, + setIsDebugging: (workflowId, isDebugging) => { - set({ - workflowExecutions: updatedMap(get().workflowExecutions, workflowId, { isDebugging }), - }) + const current = getOrCreate(get().workflowExecutions, workflowId) + const nextStatus: ExecutionStatus = isDebugging + ? 'debugging' + : current.status === 'debugging' + ? 'running' + : current.status + get().setStatus(workflowId, nextStatus) }, setExecutor: (workflowId, executor) => { diff --git a/apps/sim/stores/execution/types.ts b/apps/sim/stores/execution/types.ts index b36ea43a190..bb48c9d45a2 100644 --- a/apps/sim/stores/execution/types.ts +++ b/apps/sim/stores/execution/types.ts @@ -12,6 +12,22 @@ export type BlockRunStatus = 'success' | 'error' */ export type EdgeRunStatus = 'success' | 'error' +/** + * The mutually-exclusive execution mode of a single workflow. + * + * @remarks + * This is the single source of truth for whether a workflow is running. + * The legacy `isExecuting` / `isDebugging` booleans are derived from it + * via {@link deriveExecutionFlags} so illegal combinations — such as + * "debugging while not executing" — are unrepresentable. + * + * - `idle` — not running. + * - `running` — executing normally (derives `isExecuting`). + * - `debugging` — executing in step-by-step debug mode (derives both + * `isExecuting` and `isDebugging`). + */ +export type ExecutionStatus = 'idle' | 'running' | 'debugging' + /** * Execution state scoped to a single workflow. * @@ -19,9 +35,11 @@ export type EdgeRunStatus = 'success' | 'error' * do not interfere with one another. */ export interface WorkflowExecutionState { - /** Whether this workflow is currently executing */ + /** Mutually-exclusive execution mode; the source of truth for run state */ + status: ExecutionStatus + /** Derived from {@link status}: whether this workflow is currently executing */ isExecuting: boolean - /** Whether this workflow is in step-by-step debug mode */ + /** Derived from {@link status}: whether this workflow is in step-by-step debug mode */ isDebugging: boolean /** Block IDs that are currently running (pulsing in the UI) */ activeBlockIds: Set @@ -39,6 +57,24 @@ export interface WorkflowExecutionState { currentExecutionId: string | null } +/** + * Computes the legacy `isExecuting` / `isDebugging` booleans from a status. + * + * @remarks + * Keeping the derived booleans on the stored state object lets existing + * consumers keep reading `state.isExecuting` / `state.isDebugging` + * unchanged while {@link ExecutionStatus} remains the single source of truth. + */ +export function deriveExecutionFlags(status: ExecutionStatus): { + isExecuting: boolean + isDebugging: boolean +} { + return { + isExecuting: status !== 'idle', + isDebugging: status === 'debugging', + } +} + /** * Default values for a workflow that has never been executed. * @@ -48,8 +84,8 @@ export interface WorkflowExecutionState { * re-renders in Zustand selectors that use `Object.is` equality. */ export const defaultWorkflowExecutionState: WorkflowExecutionState = { - isExecuting: false, - isDebugging: false, + status: 'idle', + ...deriveExecutionFlags('idle'), activeBlockIds: new Set(), pendingBlocks: [], executor: null, @@ -83,9 +119,38 @@ export interface ExecutionActions { getWorkflowExecution: (workflowId: string) => WorkflowExecutionState /** Replaces the set of currently-executing block IDs for a workflow */ setActiveBlocks: (workflowId: string, blockIds: Set) => void - /** Marks a workflow as executing or idle. Starting clears the run path */ + /** + * Sets the {@link ExecutionStatus} for a workflow. + * + * @remarks + * Pass `{ clearRunPath: true }` to also reset `lastRunPath` / `lastRunEdges`. + * Run-path clearing is opt-in: it is owned by + * {@link ExecutionActions.setIsExecuting} (which clears on start), matching + * the legacy behavior where only starting execution wiped the run history. + */ + setStatus: ( + workflowId: string, + status: ExecutionStatus, + options?: { clearRunPath?: boolean } + ) => void + /** + * Marks a workflow as executing or idle. Starting (`true`) clears the run path. + * + * @remarks + * Translates to {@link ExecutionActions.setStatus}: `true` preserves an + * active debug session (`debugging`) and otherwise enters `running`, and + * always clears the run path; `false` returns to `idle` and preserves it. + */ setIsExecuting: (workflowId: string, isExecuting: boolean) => void - /** Toggles debug mode for a workflow */ + /** + * Toggles step-by-step debug mode for a workflow. + * + * @remarks + * Translates to {@link ExecutionActions.setStatus}: `true` enters + * `debugging` (which implies executing); `false` returns to `running` only + * when currently `debugging`, otherwise the status is preserved (e.g. calling + * it while `idle` is a no-op). + */ setIsDebugging: (workflowId: string, isDebugging: boolean) => void /** Sets the list of blocks pending execution during debug stepping */ setPendingBlocks: (workflowId: string, blockIds: string[]) => void diff --git a/apps/sim/stores/workflow-diff/store.test.ts b/apps/sim/stores/workflow-diff/store.test.ts new file mode 100644 index 00000000000..c9bca038048 --- /dev/null +++ b/apps/sim/stores/workflow-diff/store.test.ts @@ -0,0 +1,228 @@ +/** + * @vitest-environment node + * + * Tests for the workflow-diff store's status modeling. + * + * Focus: the {@link WorkflowDiffStatus} enum is the single source of truth and + * the legacy `hasActiveDiff` / `isShowingDiff` / `isDiffReady` booleans are + * derived from it, so contradictory combinations are unrepresentable. We assert + * the exhaustive status → boolean mapping and the status transitions driven by + * the tractable actions (`toggleDiffView`, `clearDiff`, `_batchedStateUpdate`). + * + * @remarks + * The store transitively imports the diff engine, serializer, socket + * operations, and the workflow/registry stores, all of which drag in the block + * registry and emcn icon CSS. Every such dependency is mocked so the suite + * loads under the node environment and exercises only the store + its types. + */ + +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { applyWorkflowStateToStores } = vi.hoisted(() => ({ + applyWorkflowStateToStores: vi.fn(), +})) + +vi.mock('@sim/logger', () => ({ + createLogger: () => ({ info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() }), +})) + +vi.mock('@/lib/workflows/diff', () => ({ + WorkflowDiffEngine: class { + clearDiff = vi.fn() + createDiffFromWorkflowState = vi.fn() + }, + stripWorkflowDiffMarkers: vi.fn((s) => s), +})) + +vi.mock('@/lib/workflows/operations/socket-operations', () => ({ + enqueueReplaceWorkflowState: vi.fn().mockResolvedValue(undefined), +})) + +vi.mock('@/lib/workflows/sanitization/validation', () => ({ + validateWorkflowState: vi.fn(() => ({ valid: true, errors: [], sanitizedState: null })), +})) + +vi.mock('@/serializer', () => ({ + Serializer: class { + serializeWorkflow = vi.fn() + deserializeWorkflow = vi.fn() + }, +})) + +vi.mock('@/stores/workflows/registry/store', () => ({ + useWorkflowRegistry: { getState: vi.fn(() => ({ activeWorkflowId: null })) }, +})) + +vi.mock('@/stores/workflows/utils', () => ({ + mergeSubblockState: vi.fn((blocks) => blocks), +})) + +vi.mock('@/stores/workflows/workflow/store', () => ({ + useWorkflowStore: { + getState: vi.fn(() => ({ + getWorkflowState: vi.fn(() => ({ blocks: {}, edges: [], loops: {}, parallels: {} })), + blocks: {}, + lastSaved: 0, + })), + setState: vi.fn(), + }, +})) + +vi.mock('@/stores/workflow-diff/utils', () => ({ + applyWorkflowStateToStores, + captureBaselineSnapshot: vi.fn(), + cloneWorkflowState: vi.fn((s) => s), + createBatchedUpdater: + (set: (u: Record) => void) => (updates: Record) => + set(updates), + getLatestUserMessageId: vi.fn().mockResolvedValue(null), + persistWorkflowStateToServer: vi.fn().mockResolvedValue(true), + WORKFLOW_DIFF_SETTLED_EVENT: 'workflow-diff-settled', +})) + +import { RESET_DIFF_STATE, useWorkflowDiffStore } from '@/stores/workflow-diff/store' +import { + deriveDiffFlags, + type WorkflowDiffState, + type WorkflowDiffStatus, +} from '@/stores/workflow-diff/types' + +function seedStatus(status: WorkflowDiffStatus): void { + useWorkflowDiffStore.setState(deriveDiffFlags(status)) +} + +describe('useWorkflowDiffStore status modeling', () => { + beforeEach(() => { + vi.clearAllMocks() + useWorkflowDiffStore.setState({ + ...RESET_DIFF_STATE, + pendingExternalUpdates: {}, + remoteUpdateVersions: {}, + reconcilingWorkflows: {}, + reconciliationErrors: {}, + } as Partial) + }) + + describe('deriveDiffFlags', () => { + it('maps every status to the documented legacy booleans', () => { + expect(deriveDiffFlags('none')).toEqual({ + status: 'none', + hasActiveDiff: false, + isShowingDiff: false, + isDiffReady: false, + }) + expect(deriveDiffFlags('staged')).toEqual({ + status: 'staged', + hasActiveDiff: true, + isShowingDiff: false, + isDiffReady: true, + }) + expect(deriveDiffFlags('showing')).toEqual({ + status: 'showing', + hasActiveDiff: true, + isShowingDiff: true, + isDiffReady: true, + }) + }) + + it('keeps hasActiveDiff and isDiffReady in lockstep (legacy invariant)', () => { + for (const status of ['none', 'staged', 'showing'] as const) { + const flags = deriveDiffFlags(status) + expect(flags.hasActiveDiff).toBe(flags.isDiffReady) + } + }) + }) + + describe('initial / reset state', () => { + it('starts in the none-derived state', () => { + const state = useWorkflowDiffStore.getState() + expect(state.status).toBe('none') + expect(state.hasActiveDiff).toBe(false) + expect(state.isShowingDiff).toBe(false) + expect(state.isDiffReady).toBe(false) + }) + + it('RESET_DIFF_STATE carries the none-derived flags and clears diff payload', () => { + expect(RESET_DIFF_STATE.status).toBe('none') + expect(RESET_DIFF_STATE.hasActiveDiff).toBe(false) + expect(RESET_DIFF_STATE.isShowingDiff).toBe(false) + expect(RESET_DIFF_STATE.isDiffReady).toBe(false) + expect(RESET_DIFF_STATE.baselineWorkflow).toBeNull() + expect(RESET_DIFF_STATE.diffAnalysis).toBeNull() + }) + }) + + describe('toggleDiffView', () => { + it('is a guarded no-op when there is no active diff', () => { + seedStatus('none') + useWorkflowDiffStore.getState().toggleDiffView() + expect(useWorkflowDiffStore.getState().status).toBe('none') + }) + + it('toggles showing → staged (hides the proposed changes)', () => { + seedStatus('showing') + useWorkflowDiffStore.getState().toggleDiffView() + + const state = useWorkflowDiffStore.getState() + expect(state.status).toBe('staged') + expect(state.hasActiveDiff).toBe(true) + expect(state.isDiffReady).toBe(true) + expect(state.isShowingDiff).toBe(false) + }) + + it('toggles staged → showing (reveals the proposed changes)', () => { + seedStatus('staged') + useWorkflowDiffStore.getState().toggleDiffView() + + const state = useWorkflowDiffStore.getState() + expect(state.status).toBe('showing') + expect(state.isShowingDiff).toBe(true) + }) + }) + + describe('clearDiff', () => { + it('returns the store to the none status', () => { + seedStatus('showing') + useWorkflowDiffStore.getState().clearDiff({ restoreBaseline: false }) + + const state = useWorkflowDiffStore.getState() + expect(state.status).toBe('none') + expect(state.hasActiveDiff).toBe(false) + expect(state.isShowingDiff).toBe(false) + expect(state.isDiffReady).toBe(false) + }) + }) + + describe('_batchedStateUpdate (undo/redo writer)', () => { + it('restores the showing status via deriveDiffFlags', () => { + seedStatus('none') + useWorkflowDiffStore.getState()._batchedStateUpdate({ + ...deriveDiffFlags('showing'), + baselineWorkflow: null, + baselineWorkflowId: 'wf-1', + }) + + const state = useWorkflowDiffStore.getState() + expect(state.status).toBe('showing') + expect(state.hasActiveDiff).toBe(true) + expect(state.isShowingDiff).toBe(true) + expect(state.isDiffReady).toBe(true) + }) + + it('the derived booleans always agree with the stored status', () => { + for (const status of ['none', 'staged', 'showing', 'none'] as const) { + seedStatus(status) + const state = useWorkflowDiffStore.getState() + expect({ + hasActiveDiff: state.hasActiveDiff, + isShowingDiff: state.isShowingDiff, + isDiffReady: state.isDiffReady, + }).toEqual({ + hasActiveDiff: deriveDiffFlags(status).hasActiveDiff, + isShowingDiff: deriveDiffFlags(status).isShowingDiff, + isDiffReady: deriveDiffFlags(status).isDiffReady, + }) + } + }) + }) +}) diff --git a/apps/sim/stores/workflow-diff/store.ts b/apps/sim/stores/workflow-diff/store.ts index 38d08ad3b97..ec46c2192a3 100644 --- a/apps/sim/stores/workflow-diff/store.ts +++ b/apps/sim/stores/workflow-diff/store.ts @@ -8,7 +8,7 @@ import { Serializer } from '@/serializer' import { useWorkflowRegistry } from '../workflows/registry/store' import { mergeSubblockState } from '../workflows/utils' import { useWorkflowStore } from '../workflows/workflow/store' -import type { WorkflowDiffActions, WorkflowDiffState } from './types' +import { deriveDiffFlags, type WorkflowDiffActions, type WorkflowDiffState } from './types' import { applyWorkflowStateToStores, captureBaselineSnapshot, @@ -21,17 +21,20 @@ import { const logger = createLogger('WorkflowDiffStore') const diffEngine = new WorkflowDiffEngine() -const RESET_DIFF_STATE = { - hasActiveDiff: false, - isShowingDiff: false, - isDiffReady: false, + +/** + * Canonical state patch that clears the diff overlay back to `none`: the + * none-derived flags plus a wipe of all diff payload fields. + */ +export const RESET_DIFF_STATE = { + ...deriveDiffFlags('none'), baselineWorkflow: null, baselineWorkflowId: null, diffAnalysis: null, diffMetadata: null, diffError: null, _triggerMessageId: null, -} +} as const /** * Detects when a diff contains no meaningful changes. @@ -70,9 +73,7 @@ export const useWorkflowDiffStore = create { - const { hasActiveDiff, isDiffReady, isShowingDiff } = get() - if (!hasActiveDiff) { - logger.warn('Cannot toggle diff view without active diff') - return - } - if (!isDiffReady) { - logger.warn('Cannot toggle diff view before diff is ready') + const { status } = get() + if (status === 'none') { + logger.warn('Cannot toggle diff view without an active, ready diff') return } - batchedUpdate({ isShowingDiff: !isShowingDiff }) + batchedUpdate(deriveDiffFlags(status === 'showing' ? 'staged' : 'showing')) }, acceptChanges: async (options) => { diff --git a/apps/sim/stores/workflow-diff/types.ts b/apps/sim/stores/workflow-diff/types.ts index 6c3ea3990a1..d4e05e68760 100644 --- a/apps/sim/stores/workflow-diff/types.ts +++ b/apps/sim/stores/workflow-diff/types.ts @@ -1,9 +1,31 @@ import type { DiffAnalysis, WorkflowDiff } from '@/lib/workflows/diff' import type { WorkflowState } from '../workflows/workflow/types' +/** + * The lifecycle stage of the workflow diff overlay. + * + * @remarks + * This is the single source of truth for the diff overlay. The legacy + * `hasActiveDiff` / `isShowingDiff` / `isDiffReady` booleans are derived from + * it via {@link deriveDiffFlags}, which makes contradictory combinations — + * such as "showing a diff that has no active diff" — unrepresentable. + * + * - `none` — no diff staged; the canvas shows the live workflow. + * - `staged` — a diff is staged and ready, but the canvas is showing the + * baseline (proposed changes hidden). + * - `showing` — a diff is staged and ready, and the canvas is showing the + * proposed changes with diff markers. + */ +export type WorkflowDiffStatus = 'none' | 'staged' | 'showing' + export interface WorkflowDiffState { + /** Lifecycle stage of the diff overlay; the source of truth for diff flags */ + status: WorkflowDiffStatus + /** Derived from {@link status}: a diff is staged (`staged` or `showing`) */ hasActiveDiff: boolean + /** Derived from {@link status}: the canvas is rendering the proposed changes */ isShowingDiff: boolean + /** Derived from {@link status}: a staged diff is ready to view/toggle */ isDiffReady: boolean baselineWorkflow: WorkflowState | null baselineWorkflowId: string | null @@ -48,3 +70,31 @@ export interface WorkflowDiffActions { setWorkflowReconciliationError: (workflowId: string, error: string | null) => void _batchedStateUpdate: (updates: Partial) => void } + +/** + * The {@link WorkflowDiffStatus} fields shared by `status` and its derived + * booleans. Spread this into a state patch so the source of truth and the + * legacy flags never drift apart. + */ +export type DiffStatusFlags = Pick< + WorkflowDiffState, + 'status' | 'hasActiveDiff' | 'isShowingDiff' | 'isDiffReady' +> + +/** + * Computes the legacy `hasActiveDiff` / `isShowingDiff` / `isDiffReady` + * booleans (plus the `status` itself) from a {@link WorkflowDiffStatus}. + * + * @remarks + * Keeping the derived booleans on the stored state lets existing consumers + * keep reading `state.hasActiveDiff` etc. unchanged while + * {@link WorkflowDiffStatus} remains the single source of truth. + */ +export function deriveDiffFlags(status: WorkflowDiffStatus): DiffStatusFlags { + return { + status, + hasActiveDiff: status !== 'none', + isShowingDiff: status === 'showing', + isDiffReady: status !== 'none', + } +} diff --git a/apps/sim/vitest.setup.ts b/apps/sim/vitest.setup.ts index 92e945dcca0..b9a9ae7d14a 100644 --- a/apps/sim/vitest.setup.ts +++ b/apps/sim/vitest.setup.ts @@ -41,6 +41,7 @@ vi.mock('@/stores/execution/store', () => ({ useExecutionStore: { getState: vi.fn().mockReturnValue({ getWorkflowExecution: vi.fn().mockReturnValue({ + status: 'idle', isExecuting: false, isDebugging: false, activeBlockIds: new Set(), @@ -50,6 +51,7 @@ vi.mock('@/stores/execution/store', () => ({ lastRunPath: new Map(), lastRunEdges: new Map(), }), + setStatus: vi.fn(), setIsExecuting: vi.fn(), setIsDebugging: vi.fn(), setPendingBlocks: vi.fn(), @@ -61,6 +63,7 @@ vi.mock('@/stores/execution/store', () => ({ }), }, useCurrentWorkflowExecution: vi.fn().mockReturnValue({ + status: 'idle', isExecuting: false, isDebugging: false, activeBlockIds: new Set(), From 038e8f0d84c659886a4b39f7f5c8f2f4bbfbc08e Mon Sep 17 00:00:00 2001 From: Waleed Date: Wed, 24 Jun 2026 10:46:18 -0700 Subject: [PATCH 05/15] refactor(sse): consolidate client SSE readers behind a single typed primitive (#5195) Replace four hand-rolled client SSE decode loops with two layered primitives in lib/core/utils/sse.ts: - readSSELines: the single byte-stream decode engine. Splits on \n, strips trailing \r, tolerates data: with/without a leading space, skips the [DONE] sentinel, honors an AbortSignal before each chunk and between events, and releases the reader lock only when it acquired it. - readSSEEvents: a thin JSON layer that parses each payload and routes unparseable lines to onParseError (default: skip). An SSESource union accepts a Response, a ReadableStream, or an already-acquired reader so callers that must stash the reader for external cancellation keep ownership of the lock. Migrates use-execution-stream, chat use-chat-streaming, home use-chat (via readSSELines for schema-validated decode), and the workflow chat panel. Legacy server/wand exports (encodeSSE, SSE_HEADERS, readSSEStream) are untouched. Behavior is preserved across abort, RAF batching, TTS, [DONE], delimiter tolerance, and reader-lock ownership. Tests in sse.test.ts pin the prior behavior: \n and \n\n framing, mid-chunk splits, [DONE], data: with/without leading space, \r\n stripping, sync/async early-stop, pre-aborted and mid-stream abort, lock release/non-release per source, lock release on a throwing handler, and Response/stream/reader sources. --- apps/sim/app/chat/hooks/use-chat-streaming.ts | 437 +++++++++--------- .../[workspaceId]/home/hooks/use-chat.ts | 98 ++-- .../w/[workflowId]/components/chat/chat.tsx | 82 ++-- apps/sim/hooks/use-execution-stream.test.ts | 42 ++ apps/sim/hooks/use-execution-stream.ts | 40 +- apps/sim/lib/core/utils/sse.test.ts | 321 ++++++++++++- apps/sim/lib/core/utils/sse.ts | 175 +++++++ 7 files changed, 821 insertions(+), 374 deletions(-) diff --git a/apps/sim/app/chat/hooks/use-chat-streaming.ts b/apps/sim/app/chat/hooks/use-chat-streaming.ts index f4c9fcb9b97..dd315dafe73 100644 --- a/apps/sim/app/chat/hooks/use-chat-streaming.ts +++ b/apps/sim/app/chat/hooks/use-chat-streaming.ts @@ -3,6 +3,7 @@ import { useRef, useState } from 'react' import { createLogger } from '@sim/logger' import { generateId } from '@sim/utils/id' +import { readSSEEvents } from '@/lib/core/utils/sse' import { isUserFileWithMetadata } from '@/lib/core/utils/user-file' import type { ChatFile, ChatMessage } from '@/app/chat/components/message/message' import { CHAT_ERROR_MESSAGES } from '@/app/chat/constants' @@ -125,14 +126,12 @@ export function useChatStreaming() { streamingOptions?.voiceSettings?.autoPlayResponses && streamingOptions?.audioStreamHandler - const reader = response.body?.getReader() - if (!reader) { + if (!response.body) { setIsLoading(false) setIsStreamingResponse(false) return } - const decoder = new TextDecoder() let accumulatedText = '' let lastAudioPosition = 0 @@ -192,264 +191,252 @@ export function useChatStreaming() { setIsLoading(false) + let terminated = false + try { - while (true) { - // Check if aborted - if (abortControllerRef.current === null) { - break + await readSSEEvents<{ + blockId?: string + chunk?: string + event?: string + error?: string + data?: { + success: boolean + error?: string | { message?: string } + output?: Record> } - - const { done, value } = await reader.read() - - if (done) { - flushUI() - // Stream any remaining text for TTS - if ( - shouldPlayAudio && - streamingOptions?.audioStreamHandler && - accumulatedText.length > lastAudioPosition - ) { - const remainingText = accumulatedText.substring(lastAudioPosition).trim() - if (remainingText) { - try { - await streamingOptions.audioStreamHandler(remainingText) - } catch (error) { - logger.error('TTS error for remaining text:', error) - } - } + }>(response.body, { + signal: abortControllerRef.current.signal, + onParseError: (_data, parseError) => { + logger.error('Error parsing stream data:', parseError) + }, + onEvent: async (json) => { + const { blockId, chunk: contentChunk, event: eventType } = json + + if (eventType === 'error' || json.event === 'error') { + const errorMessage = json.error || CHAT_ERROR_MESSAGES.GENERIC_ERROR + setMessages((prev) => + prev.map((msg) => + msg.id === messageId + ? { + ...msg, + content: errorMessage, + isStreaming: false, + type: 'assistant' as const, + } + : msg + ) + ) + setIsLoading(false) + terminated = true + return true } - break - } - const chunk = decoder.decode(value, { stream: true }) - const lines = chunk.split('\n\n') + if (eventType === 'final' && json.data) { + flushUI() + const finalData = json.data - for (const line of lines) { - if (line.startsWith('data: ')) { - const data = line.substring(6) + const outputConfigs = streamingOptions?.outputConfigs + const formattedOutputs: string[] = [] + let extractedFiles: ChatFile[] = [] - if (data === '[DONE]') { - continue - } + const formatValue = (value: any): string | null => { + if (value === null || value === undefined) { + return null + } - try { - const json = JSON.parse(data) - const { blockId, chunk: contentChunk, event: eventType } = json - - if (eventType === 'error' || json.event === 'error') { - const errorMessage = json.error || CHAT_ERROR_MESSAGES.GENERIC_ERROR - setMessages((prev) => - prev.map((msg) => - msg.id === messageId - ? { - ...msg, - content: errorMessage, - isStreaming: false, - type: 'assistant' as const, - } - : msg - ) - ) - setIsLoading(false) - return + if (isUserFileWithMetadata(value)) { + return null } - if (eventType === 'final' && json.data) { - flushUI() - const finalData = json.data as { - success: boolean - error?: string | { message?: string } - output?: Record> + if (Array.isArray(value) && value.length === 0) { + return null + } + + if (typeof value === 'string') { + return value + } + + if (typeof value === 'object') { + try { + return `\`\`\`json\n${JSON.stringify(value, null, 2)}\n\`\`\`` + } catch { + return String(value) } + } - const outputConfigs = streamingOptions?.outputConfigs - const formattedOutputs: string[] = [] - let extractedFiles: ChatFile[] = [] + return String(value) + } - const formatValue = (value: any): string | null => { - if (value === null || value === undefined) { - return null - } + const getOutputValue = (blockOutputs: Record, path?: string) => { + if (!path || path === 'content') { + if (blockOutputs.content !== undefined) return blockOutputs.content + if (blockOutputs.result !== undefined) return blockOutputs.result + return blockOutputs + } - if (isUserFileWithMetadata(value)) { - return null - } + if (blockOutputs[path] !== undefined) { + return blockOutputs[path] + } - if (Array.isArray(value) && value.length === 0) { - return null + if (path.includes('.')) { + return path.split('.').reduce((current, segment) => { + if (current && typeof current === 'object' && segment in current) { + return current[segment] } + return undefined + }, blockOutputs) + } - if (typeof value === 'string') { - return value - } + return undefined + } - if (typeof value === 'object') { - try { - return `\`\`\`json\n${JSON.stringify(value, null, 2)}\n\`\`\`` - } catch { - return String(value) - } - } + if (outputConfigs?.length && finalData.output) { + for (const config of outputConfigs) { + const blockOutputs = finalData.output[config.blockId] + if (!blockOutputs) continue + + const value = getOutputValue(blockOutputs, config.path) + + if (isUserFileWithMetadata(value)) { + extractedFiles.push({ + id: value.id, + name: value.name, + url: value.url, + key: value.key, + size: value.size, + type: value.type, + context: value.context, + }) + continue + } - return String(value) + const nestedFiles = extractFilesFromData(value) + if (nestedFiles.length > 0) { + extractedFiles = [...extractedFiles, ...nestedFiles] + continue } - const getOutputValue = (blockOutputs: Record, path?: string) => { - if (!path || path === 'content') { - if (blockOutputs.content !== undefined) return blockOutputs.content - if (blockOutputs.result !== undefined) return blockOutputs.result - return blockOutputs - } + const formatted = formatValue(value) + if (formatted) { + formattedOutputs.push(formatted) + } + } + } - if (blockOutputs[path] !== undefined) { - return blockOutputs[path] - } + let finalContent = accumulatedText - if (path.includes('.')) { - return path.split('.').reduce((current, segment) => { - if (current && typeof current === 'object' && segment in current) { - return current[segment] - } - return undefined - }, blockOutputs) - } + if (formattedOutputs.length > 0) { + const nonEmptyOutputs = formattedOutputs.filter((output) => output.trim()) + if (nonEmptyOutputs.length > 0) { + const combinedOutputs = nonEmptyOutputs.join('\n\n') + finalContent = finalContent + ? `${finalContent.trim()}\n\n${combinedOutputs}` + : combinedOutputs + } + } - return undefined + if (!finalContent && extractedFiles.length === 0) { + if (finalData.error) { + if (typeof finalData.error === 'string') { + finalContent = finalData.error + } else if (typeof finalData.error?.message === 'string') { + finalContent = finalData.error.message } + } else if (finalData.success && finalData.output) { + const fallbackOutput = Object.values(finalData.output) + .map((block) => formatValue(block)?.trim()) + .filter(Boolean)[0] + if (fallbackOutput) { + finalContent = fallbackOutput + } + } + } - if (outputConfigs?.length && finalData.output) { - for (const config of outputConfigs) { - const blockOutputs = finalData.output[config.blockId] - if (!blockOutputs) continue - - const value = getOutputValue(blockOutputs, config.path) - - if (isUserFileWithMetadata(value)) { - extractedFiles.push({ - id: value.id, - name: value.name, - url: value.url, - key: value.key, - size: value.size, - type: value.type, - context: value.context, - }) - continue - } - - const nestedFiles = extractFilesFromData(value) - if (nestedFiles.length > 0) { - extractedFiles = [...extractedFiles, ...nestedFiles] - continue + setMessages((prev) => + prev.map((msg) => + msg.id === messageId + ? { + ...msg, + isStreaming: false, + content: finalContent ?? msg.content, + files: extractedFiles.length > 0 ? extractedFiles : undefined, } + : msg + ) + ) - const formatted = formatValue(value) - if (formatted) { - formattedOutputs.push(formatted) - } - } - } + accumulatedTextRef.current = '' + lastStreamedPositionRef.current = 0 + lastDisplayedPositionRef.current = 0 + audioStreamingActiveRef.current = false - let finalContent = accumulatedText + terminated = true + return true + } - if (formattedOutputs.length > 0) { - const nonEmptyOutputs = formattedOutputs.filter((output) => output.trim()) - if (nonEmptyOutputs.length > 0) { - const combinedOutputs = nonEmptyOutputs.join('\n\n') - finalContent = finalContent - ? `${finalContent.trim()}\n\n${combinedOutputs}` - : combinedOutputs - } - } + if (blockId && contentChunk) { + if (!messageIdMap.has(blockId)) { + messageIdMap.set(blockId, messageId) + } - if (!finalContent && extractedFiles.length === 0) { - if (finalData.error) { - if (typeof finalData.error === 'string') { - finalContent = finalData.error - } else if (typeof finalData.error?.message === 'string') { - finalContent = finalData.error.message - } - } else if (finalData.success && finalData.output) { - const fallbackOutput = Object.values(finalData.output) - .map((block) => formatValue(block)?.trim()) - .filter(Boolean)[0] - if (fallbackOutput) { - finalContent = fallbackOutput - } - } + accumulatedText += contentChunk + accumulatedTextRef.current = accumulatedText + logger.debug('[useChatStreaming] Received chunk', { + blockId, + chunkLength: contentChunk.length, + totalLength: accumulatedText.length, + messageId, + chunk: contentChunk.substring(0, 20), + }) + uiDirty = true + scheduleUIFlush() + + if (shouldPlayAudio && streamingOptions?.audioStreamHandler) { + const newText = accumulatedText.substring(lastAudioPosition) + const sentenceEndings = ['. ', '! ', '? ', '.\n', '!\n', '?\n', '.', '!', '?'] + let sentenceEnd = -1 + + for (const ending of sentenceEndings) { + const index = newText.indexOf(ending) + if (index > 0) { + sentenceEnd = index + ending.length + break } - - setMessages((prev) => - prev.map((msg) => - msg.id === messageId - ? { - ...msg, - isStreaming: false, - content: finalContent ?? msg.content, - files: extractedFiles.length > 0 ? extractedFiles : undefined, - } - : msg - ) - ) - - accumulatedTextRef.current = '' - lastStreamedPositionRef.current = 0 - lastDisplayedPositionRef.current = 0 - audioStreamingActiveRef.current = false - - return } - if (blockId && contentChunk) { - if (!messageIdMap.has(blockId)) { - messageIdMap.set(blockId, messageId) - } - - accumulatedText += contentChunk - accumulatedTextRef.current = accumulatedText - logger.debug('[useChatStreaming] Received chunk', { - blockId, - chunkLength: contentChunk.length, - totalLength: accumulatedText.length, - messageId, - chunk: contentChunk.substring(0, 20), - }) - uiDirty = true - scheduleUIFlush() - - // Real-time TTS for voice mode - if (shouldPlayAudio && streamingOptions?.audioStreamHandler) { - const newText = accumulatedText.substring(lastAudioPosition) - const sentenceEndings = ['. ', '! ', '? ', '.\n', '!\n', '?\n', '.', '!', '?'] - let sentenceEnd = -1 - - for (const ending of sentenceEndings) { - const index = newText.indexOf(ending) - if (index > 0) { - sentenceEnd = index + ending.length - break - } - } - - if (sentenceEnd > 0) { - const sentence = newText.substring(0, sentenceEnd).trim() - if (sentence && sentence.length >= 3) { - try { - await streamingOptions.audioStreamHandler(sentence) - lastAudioPosition += sentenceEnd - } catch (error) { - logger.error('TTS error:', error) - } - } + if (sentenceEnd > 0) { + const sentence = newText.substring(0, sentenceEnd).trim() + if (sentence && sentence.length >= 3) { + try { + await streamingOptions.audioStreamHandler(sentence) + lastAudioPosition += sentenceEnd + } catch (error) { + logger.error('TTS error:', error) } } - } else if (blockId && eventType === 'end') { - setMessages((prev) => - prev.map((msg) => (msg.id === messageId ? { ...msg, isStreaming: false } : msg)) - ) } - } catch (parseError) { - logger.error('Error parsing stream data:', parseError) + } + } else if (blockId && eventType === 'end') { + setMessages((prev) => + prev.map((msg) => (msg.id === messageId ? { ...msg, isStreaming: false } : msg)) + ) + } + }, + }) + + if (!terminated) { + flushUI() + if ( + shouldPlayAudio && + streamingOptions?.audioStreamHandler && + accumulatedText.length > lastAudioPosition + ) { + const remainingText = accumulatedText.substring(lastAudioPosition).trim() + if (remainingText) { + try { + await streamingOptions.audioStreamHandler(remainingText) + } catch (error) { + logger.error('TTS error for remaining text:', error) } } } diff --git a/apps/sim/app/workspace/[workspaceId]/home/hooks/use-chat.ts b/apps/sim/app/workspace/[workspaceId]/home/hooks/use-chat.ts index b1ec30520af..e81ec0603f9 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/hooks/use-chat.ts +++ b/apps/sim/app/workspace/[workspaceId]/home/hooks/use-chat.ts @@ -63,6 +63,7 @@ import { } from '@/lib/copilot/tools/client/run-tool-execution' import { setCurrentChatTraceparent } from '@/lib/copilot/tools/client/trace-context' import { isWorkflowToolName } from '@/lib/copilot/tools/workflow-tools' +import { readSSELines } from '@/lib/core/utils/sse' import { getQueryClient } from '@/app/_shell/providers/get-query-client' import { useFilePreviewController } from '@/app/workspace/[workspaceId]/home/hooks/preview' import { @@ -1934,7 +1935,6 @@ export function useChat( shouldContinue?: () => boolean } ) => { - const decoder = new TextDecoder() const ctx = createStreamLoopContext({ workspaceId, queryClient, @@ -1987,71 +1987,47 @@ export function useChat( return { sawStreamError: false, sawComplete: false } } streamReaderRef.current = reader - let buffer = '' try { - const pendingLines: string[] = [] - - while (true) { - if (pendingLines.length === 0) { - // Don't read another chunk after `complete` has drained. - if (state.sawCompleteEvent) break - const { done, value } = await reader.read() - if (done) break - if (ops.isStale()) continue - - buffer += decoder.decode(value, { stream: true }) - const lines = buffer.split('\n') - buffer = lines.pop() || '' - pendingLines.push(...lines) - if (pendingLines.length === 0) { - continue + await readSSELines(reader, { + onData: (raw) => { + if (state.sawCompleteEvent) return true + if (ops.isStale()) return + + const parsedResult = parsePersistedStreamEventEnvelopeJson(raw) + if (!parsedResult.ok) { + const error = createStreamSchemaValidationError(parsedResult, 'Live SSE event.') + logger.error('Rejected chat SSE event due to client-side schema enforcement', { + reason: parsedResult.reason, + message: parsedResult.message, + errors: parsedResult.errors, + error: error.message, + }) + throw error } - } - - const line = pendingLines.shift() - if (line === undefined) { - continue - } - if (ops.isStale()) { - pendingLines.length = 0 - continue - } - if (!line.startsWith('data: ')) continue - const raw = line.slice(6) - - const parsedResult = parsePersistedStreamEventEnvelopeJson(raw) - if (!parsedResult.ok) { - const error = createStreamSchemaValidationError(parsedResult, 'Live SSE event.') - logger.error('Rejected chat SSE event due to client-side schema enforcement', { - reason: parsedResult.reason, - message: parsedResult.message, - errors: parsedResult.errors, - error: error.message, - }) - throw error - } - const parsed = parsedResult.event + const parsed = parsedResult.event - if (parsed.trace?.requestId && parsed.trace.requestId !== state.streamRequestId) { - state.streamRequestId = parsed.trace.requestId - streamRequestIdRef.current = state.streamRequestId - ops.flush() - } - if (parsed.stream?.streamId) { - streamIdRef.current = parsed.stream.streamId - } - const eventCursor = parsed.stream?.cursor ?? String(parsed.seq) - if (isAlreadyProcessedStreamCursor(eventCursor, lastCursorRef.current)) { - continue - } - if (eventCursor) { - lastCursorRef.current = eventCursor - } + if (parsed.trace?.requestId && parsed.trace.requestId !== state.streamRequestId) { + state.streamRequestId = parsed.trace.requestId + streamRequestIdRef.current = state.streamRequestId + ops.flush() + } + if (parsed.stream?.streamId) { + streamIdRef.current = parsed.stream.streamId + } + const eventCursor = parsed.stream?.cursor ?? String(parsed.seq) + if (isAlreadyProcessedStreamCursor(eventCursor, lastCursorRef.current)) { + return + } + if (eventCursor) { + lastCursorRef.current = eventCursor + } - logger.debug('SSE event received', parsed) - dispatchStreamEvent(ctx, parsed) - } + logger.debug('SSE event received', parsed) + dispatchStreamEvent(ctx, parsed) + if (state.sawCompleteEvent) return true + }, + }) } finally { if (state.sawStreamError && !state.sawCompleteEvent) { applyTurnTerminal(state.model, 'error') diff --git a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/chat/chat.tsx b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/chat/chat.tsx index 588ff6da5cc..d4caa19424f 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/chat/chat.tsx +++ b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/chat/chat.tsx @@ -25,6 +25,7 @@ import { extractPathFromOutputId, parseOutputContentSafely, } from '@/lib/core/utils/response-format' +import { readSSEEvents } from '@/lib/core/utils/sse' import { CHAT_ACCEPT_ATTRIBUTE } from '@/lib/uploads/utils/validation' import { normalizeInputFormatValue } from '@/lib/workflows/input-format' import { StartBlockPath, TriggerUtils } from '@/lib/workflows/triggers/triggers' @@ -520,12 +521,10 @@ export function Chat() { * @param responseMessageId - ID of the message to update with streamed content */ const processStreamingResponse = useCallback( - async (stream: ReadableStream, responseMessageId: string) => { + async (stream: ReadableStream, responseMessageId: string) => { const reader = stream.getReader() streamReaderRef.current = reader - const decoder = new TextDecoder() let accumulatedContent = '' - let buffer = '' const BATCH_MAX_MS = 50 let pendingChunks = '' @@ -563,63 +562,34 @@ export function Chat() { } try { - while (true) { - const { done, value } = await reader.read() - if (done) { - flushChunks() - finalizeMessageStream(responseMessageId) - break - } - - const chunk = decoder.decode(value, { stream: true }) - buffer += chunk - - const separatorIndex = buffer.lastIndexOf('\n\n') - if (separatorIndex === -1) { - continue - } - - const processable = buffer.slice(0, separatorIndex) - buffer = buffer.slice(separatorIndex + 2) - - const lines = processable.split('\n\n') - - for (const line of lines) { - if (!line.startsWith('data: ')) continue - - const data = line.substring(6) - if (data === '[DONE]') continue - - try { - const json = JSON.parse(data) - const { event, data: eventData, chunk: contentChunk } = json - - if (event === 'final' && eventData) { - const result = eventData as ExecutionResult - - if ('success' in result && !result.success) { - const errorMessage = result.error || 'Workflow execution failed' - flushChunks() - appendMessageContent( - responseMessageId, - `${accumulatedContent ? '\n\n' : ''}Error: ${errorMessage}` - ) - finalizeMessageStream(responseMessageId) - return - } + await readSSEEvents<{ event?: string; data?: ExecutionResult; chunk?: string }>(reader, { + onParseError: (_data, e) => { + logger.error('Error parsing stream data:', e) + }, + onEvent: (json) => { + const { event, data: eventData, chunk: contentChunk } = json + if (event === 'final' && eventData) { + if ('success' in eventData && !eventData.success) { + const errorMessage = eventData.error || 'Workflow execution failed' flushChunks() - finalizeMessageStream(responseMessageId) - } else if (contentChunk) { - accumulatedContent += contentChunk - pendingChunks += contentChunk - scheduleFlush() + appendMessageContent( + responseMessageId, + `${accumulatedContent ? '\n\n' : ''}Error: ${errorMessage}` + ) } - } catch (e) { - logger.error('Error parsing stream data:', e) + return true } - } - } + + if (contentChunk) { + accumulatedContent += contentChunk + pendingChunks += contentChunk + scheduleFlush() + } + }, + }) + flushChunks() + finalizeMessageStream(responseMessageId) } catch (error) { if ((error as Error)?.name !== 'AbortError') { logger.error('Error processing stream:', error) diff --git a/apps/sim/hooks/use-execution-stream.test.ts b/apps/sim/hooks/use-execution-stream.test.ts index da52635ff99..f38f028c805 100644 --- a/apps/sim/hooks/use-execution-stream.test.ts +++ b/apps/sim/hooks/use-execution-stream.test.ts @@ -84,4 +84,46 @@ describe('processSSEStream', () => { expect(onEventId).not.toHaveBeenCalled() }) + + it('releases the reader lock after the stream completes', async () => { + const stream = streamEvents([]) + const reader = stream.getReader() + expect(stream.locked).toBe(true) + + await processSSEStream(reader, {}, 'test') + + expect(stream.locked).toBe(false) + }) + + it('releases the reader lock even when a handler throws', async () => { + const event: ExecutionEvent = { + type: 'block:started', + eventId: 7, + timestamp: new Date().toISOString(), + executionId: 'exec-1', + workflowId: 'wf-1', + data: { + blockId: 'block-1', + blockName: 'Block 1', + blockType: 'function', + executionOrder: 1, + }, + } + const stream = streamEvents([event]) + const reader = stream.getReader() + + await expect( + processSSEStream( + reader, + { + onBlockStarted: () => { + throw new Error('boom') + }, + }, + 'test' + ) + ).rejects.toThrow('boom') + + expect(stream.locked).toBe(false) + }) }) diff --git a/apps/sim/hooks/use-execution-stream.ts b/apps/sim/hooks/use-execution-stream.ts index 24a6e0cad4a..ffe862c4710 100644 --- a/apps/sim/hooks/use-execution-stream.ts +++ b/apps/sim/hooks/use-execution-stream.ts @@ -1,6 +1,7 @@ import { useCallback } from 'react' import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' +import { readSSEEvents } from '@/lib/core/utils/sse' import type { BlockChildWorkflowStartedData, BlockCompletedData, @@ -82,36 +83,12 @@ export async function processSSEStream( callbacks: ExecutionStreamCallbacks, logPrefix: string ): Promise { - const decoder = new TextDecoder() - let buffer = '' - try { - while (true) { - const { done, value } = await reader.read() - - if (done) break - - buffer += decoder.decode(value, { stream: true }) - const lines = buffer.split('\n\n') - buffer = lines.pop() || '' - - for (const line of lines) { - if (!line.trim() || !line.startsWith('data: ')) continue - - const data = line.substring(6).trim() - if (data === '[DONE]') { - logger.info(`${logPrefix} stream completed`) - continue - } - - let event: ExecutionEvent - try { - event = JSON.parse(data) as ExecutionEvent - } catch (error) { - logger.error('Failed to parse SSE event:', error, { data }) - continue - } - + await readSSEEvents(reader, { + onParseError: (data, error) => { + logger.error('Failed to parse SSE event:', error, { data }) + }, + onEvent: async (event) => { try { switch (event.type) { case 'execution:started': @@ -168,8 +145,9 @@ export async function processSSEStream( error ) } - } - } + }, + }) + logger.debug(`${logPrefix} stream completed`) } finally { reader.releaseLock() } diff --git a/apps/sim/lib/core/utils/sse.test.ts b/apps/sim/lib/core/utils/sse.test.ts index 524c00b83d3..579e23fd607 100644 --- a/apps/sim/lib/core/utils/sse.test.ts +++ b/apps/sim/lib/core/utils/sse.test.ts @@ -2,7 +2,13 @@ * @vitest-environment node */ import { describe, expect, it, vi } from 'vitest' -import { encodeSSE, readSSEStream, SSE_HEADERS } from '@/lib/core/utils/sse' +import { + encodeSSE, + readSSEEvents, + readSSELines, + readSSEStream, + SSE_HEADERS, +} from '@/lib/core/utils/sse' function createStreamFromChunks(chunks: Uint8Array[]): ReadableStream { let index = 0 @@ -311,3 +317,316 @@ describe('readSSEStream', () => { }) }) }) + +function streamFromStringChunks(chunks: string[]): ReadableStream { + const encoder = new TextEncoder() + return createStreamFromChunks(chunks.map((c) => encoder.encode(c))) +} + +describe('readSSEEvents', () => { + it('parses `\\n\\n`-framed events', async () => { + const stream = streamFromStringChunks([ + 'data: {"n":1}\n\n', + 'data: {"n":2}\n\n', + 'data: {"n":3}\n\n', + ]) + const events: number[] = [] + await readSSEEvents<{ n: number }>(stream, { + onEvent: (e) => { + events.push(e.n) + }, + }) + expect(events).toEqual([1, 2, 3]) + }) + + it('parses `\\n`-framed events', async () => { + const stream = streamFromStringChunks(['data: {"n":1}\ndata: {"n":2}\ndata: {"n":3}\n']) + const events: number[] = [] + await readSSEEvents<{ n: number }>(stream, { + onEvent: (e) => { + events.push(e.n) + }, + }) + expect(events).toEqual([1, 2, 3]) + }) + + it('reassembles events split across chunk boundaries', async () => { + const stream = streamFromStringChunks(['data: {"ms', 'g":"hel', 'lo"}\n\n']) + const events: Array<{ msg: string }> = [] + await readSSEEvents<{ msg: string }>(stream, { + onEvent: (e) => { + events.push(e) + }, + }) + expect(events).toEqual([{ msg: 'hello' }]) + }) + + it('skips the [DONE] sentinel', async () => { + const stream = streamFromStringChunks(['data: {"n":1}\n\n', 'data: [DONE]\n\n']) + const events: number[] = [] + await readSSEEvents<{ n: number }>(stream, { + onEvent: (e) => { + events.push(e.n) + }, + }) + expect(events).toEqual([1]) + }) + + it('accepts `data:` with and without a leading space', async () => { + const stream = streamFromStringChunks(['data:{"n":1}\n\n', 'data: {"n":2}\n\n']) + const events: number[] = [] + await readSSEEvents<{ n: number }>(stream, { + onEvent: (e) => { + events.push(e.n) + }, + }) + expect(events).toEqual([1, 2]) + }) + + it('strips trailing carriage returns (\\r\\n framing)', async () => { + const stream = streamFromStringChunks(['data: {"n":1}\r\n\r\n', 'data: {"n":2}\r\n\r\n']) + const events: number[] = [] + await readSSEEvents<{ n: number }>(stream, { + onEvent: (e) => { + events.push(e.n) + }, + }) + expect(events).toEqual([1, 2]) + }) + + it('routes unparseable payloads to onParseError and continues', async () => { + const stream = streamFromStringChunks(['data: not-json\n\n', 'data: {"n":2}\n\n']) + const events: number[] = [] + const onParseError = vi.fn() + await readSSEEvents<{ n: number }>(stream, { + onEvent: (e) => { + events.push(e.n) + }, + onParseError, + }) + expect(events).toEqual([2]) + expect(onParseError).toHaveBeenCalledTimes(1) + expect(onParseError).toHaveBeenCalledWith('not-json', expect.any(Error)) + }) + + it('stops early when onEvent returns true', async () => { + const stream = streamFromStringChunks([ + 'data: {"n":1}\n\n', + 'data: {"n":2}\n\n', + 'data: {"n":3}\n\n', + ]) + const events: number[] = [] + await readSSEEvents<{ n: number }>(stream, { + onEvent: (e) => { + events.push(e.n) + return e.n === 2 + }, + }) + expect(events).toEqual([1, 2]) + }) + + it('does not process events once the signal is aborted', async () => { + const controller = new AbortController() + const stream = streamFromStringChunks(['data: {"n":1}\n\n', 'data: {"n":2}\n\n']) + const events: number[] = [] + await readSSEEvents<{ n: number }>(stream, { + signal: controller.signal, + onEvent: (e) => { + events.push(e.n) + controller.abort() + }, + }) + expect(events).toEqual([1]) + }) + + it('returns immediately when the signal is already aborted', async () => { + const controller = new AbortController() + controller.abort() + const stream = streamFromStringChunks(['data: {"n":1}\n\n']) + const onEvent = vi.fn() + await readSSEEvents(stream, { signal: controller.signal, onEvent }) + expect(onEvent).not.toHaveBeenCalled() + }) + + it('releases the reader lock for a stream source', async () => { + const stream = streamFromStringChunks(['data: {"n":1}\n\n']) + await readSSEEvents<{ n: number }>(stream, { onEvent: () => {} }) + expect(() => stream.getReader()).not.toThrow() + }) + + it('does not release the lock for a reader source', async () => { + const stream = streamFromStringChunks(['data: {"n":1}\n\n']) + const reader = stream.getReader() + await readSSEEvents<{ n: number }>(reader, { onEvent: () => {} }) + expect(() => stream.getReader()).toThrow() + reader.releaseLock() + }) + + it('accepts a Response source', async () => { + const response = new Response(streamFromStringChunks(['data: {"n":7}\n\n'])) + const events: number[] = [] + await readSSEEvents<{ n: number }>(response, { + onEvent: (e) => { + events.push(e.n) + }, + }) + expect(events).toEqual([7]) + }) + + it('silently skips unparseable payloads when no onParseError is provided', async () => { + const stream = streamFromStringChunks(['data: not-json\n\n', 'data: {"n":2}\n\n']) + const events: number[] = [] + await expect( + readSSEEvents<{ n: number }>(stream, { + onEvent: (e) => { + events.push(e.n) + }, + }) + ).resolves.toBeUndefined() + expect(events).toEqual([2]) + }) + + it('surfaces a fatal parse error when onParseError throws', async () => { + const stream = streamFromStringChunks(['data: not-json\n\n', 'data: {"n":2}\n\n']) + const events: number[] = [] + await expect( + readSSEEvents<{ n: number }>(stream, { + onEvent: (e) => { + events.push(e.n) + }, + onParseError: () => { + throw new Error('boom') + }, + }) + ).rejects.toThrow('boom') + expect(events).toEqual([]) + }) + + it('stops early when onEvent resolves true asynchronously', async () => { + const stream = streamFromStringChunks([ + 'data: {"n":1}\n\n', + 'data: {"n":2}\n\n', + 'data: {"n":3}\n\n', + ]) + const events: number[] = [] + await readSSEEvents<{ n: number }>(stream, { + onEvent: async (e) => { + events.push(e.n) + return e.n === 2 + }, + }) + expect(events).toEqual([1, 2]) + }) + + it('throws "No response body" for a Response without a body', async () => { + const response = new Response(null) + await expect(readSSEEvents(response, { onEvent: () => {} })).rejects.toThrow('No response body') + }) +}) + +describe('readSSELines', () => { + it('delivers raw (un-parsed) data payloads', async () => { + const stream = streamFromStringChunks(['data: raw-one\n\n', 'data: {"keep":"asString"}\n\n']) + const lines: string[] = [] + await readSSELines(stream, { + onData: (raw) => { + lines.push(raw) + }, + }) + expect(lines).toEqual(['raw-one', '{"keep":"asString"}']) + }) + + it('skips [DONE] and blank separator lines', async () => { + const stream = streamFromStringChunks(['data: a\n\ndata: b\n\ndata: [DONE]\n\n']) + const lines: string[] = [] + await readSSELines(stream, { + onData: (raw) => { + lines.push(raw) + }, + }) + expect(lines).toEqual(['a', 'b']) + }) + + it('preserves the raw payload verbatim (no JSON parsing)', async () => { + const stream = streamFromStringChunks(['data: {"unterminated\n\n', 'data:no-space\n\n']) + const lines: string[] = [] + await readSSELines(stream, { + onData: (raw) => { + lines.push(raw) + }, + }) + expect(lines).toEqual(['{"unterminated', 'no-space']) + }) + + it('strips a trailing carriage return from each line', async () => { + const stream = streamFromStringChunks(['data: one\r\n\r\ndata: two\r\n\r\n']) + const lines: string[] = [] + await readSSELines(stream, { + onData: (raw) => { + lines.push(raw) + }, + }) + expect(lines).toEqual(['one', 'two']) + }) + + it('stops early when onData returns true', async () => { + const stream = streamFromStringChunks(['data: a\n\ndata: b\n\ndata: c\n\n']) + const lines: string[] = [] + await readSSELines(stream, { + onData: (raw) => { + lines.push(raw) + return raw === 'b' + }, + }) + expect(lines).toEqual(['a', 'b']) + }) + + it('does not deliver any line when the signal is already aborted', async () => { + const controller = new AbortController() + controller.abort() + const stream = streamFromStringChunks(['data: a\n\n']) + const onData = vi.fn() + await readSSELines(stream, { signal: controller.signal, onData }) + expect(onData).not.toHaveBeenCalled() + }) + + it('stops between events in the same chunk once aborted mid-stream', async () => { + const controller = new AbortController() + const stream = streamFromStringChunks(['data: a\n\ndata: b\n\ndata: c\n\n']) + const lines: string[] = [] + await readSSELines(stream, { + signal: controller.signal, + onData: (raw) => { + lines.push(raw) + if (raw === 'a') controller.abort() + }, + }) + expect(lines).toEqual(['a']) + }) + + it('releases the lock for a stream source', async () => { + const stream = streamFromStringChunks(['data: a\n\n']) + await readSSELines(stream, { onData: () => {} }) + expect(() => stream.getReader()).not.toThrow() + }) + + it('does not release the lock for a reader source', async () => { + const stream = streamFromStringChunks(['data: a\n\n']) + const reader = stream.getReader() + await readSSELines(reader, { onData: () => {} }) + expect(() => stream.getReader()).toThrow() + reader.releaseLock() + }) + + it('releases the lock for a stream source even when onData throws', async () => { + const stream = streamFromStringChunks(['data: a\n\n']) + await expect( + readSSELines(stream, { + onData: () => { + throw new Error('handler failed') + }, + }) + ).rejects.toThrow('handler failed') + expect(() => stream.getReader()).not.toThrow() + }) +}) diff --git a/apps/sim/lib/core/utils/sse.ts b/apps/sim/lib/core/utils/sse.ts index 9d9d3f785a5..50c758f0013 100644 --- a/apps/sim/lib/core/utils/sse.ts +++ b/apps/sim/lib/core/utils/sse.ts @@ -20,6 +20,181 @@ export function encodeSSE(data: any): Uint8Array { return new TextEncoder().encode(`data: ${JSON.stringify(data)}\n\n`) } +/** + * The sentinel value servers emit to signal end-of-stream. Lines carrying this + * payload are skipped before reaching the consumer's `onEvent` callback. + */ +const DONE_SENTINEL = '[DONE]' + +/** + * A source the SSE reader can consume: a fetch `Response`, its `ReadableStream` + * body, or an already-acquired reader. Passing a `Response`/stream lets the + * primitive own `getReader()` and the reader lifecycle (lock release); passing a + * reader is supported for callers that must acquire it first (e.g. to stash it + * for external cancellation). + */ +export type SSESource = + | Response + | ReadableStream + | ReadableStreamDefaultReader + +/** + * The result of an SSE event/line callback. Only the literal `true` (returned + * synchronously or resolved from a `Promise`) stops processing and returns + * early — useful for terminal events. Any other value (including the + * `undefined` a handler that returns nothing produces) keeps processing. + * + * Typed as `unknown` rather than `boolean | void | Promise` so + * both sync and `async` handlers — including `async` handlers that return + * nothing (`Promise`) — stay assignable, without the confusing + * `void`-inside-a-`Promise` union that the precise type would require. + */ +export type SSEStopSignal = unknown + +/** + * Options for {@link readSSELines} — the low-level line engine that delivers the + * raw `data:` payload string (no JSON parsing). + */ +export interface ReadSSELinesOptions { + /** Invoked once per SSE `data:` line with the raw (un-parsed) payload string. */ + onData: (rawData: string) => SSEStopSignal + /** Aborts the read; checked before each chunk and between events. */ + signal?: AbortSignal +} + +/** + * Options for {@link readSSEEvents} — the JSON convenience layer over + * {@link readSSELines}. + */ +export interface ReadSSEEventsOptions { + /** + * Invoked once per parsed SSE `data:` event with the JSON-parsed payload. + * Return (or resolve) `true` to stop processing and return early. Callers + * narrow the typed payload. + */ + onEvent: (event: T) => SSEStopSignal + /** + * Invoked for a `data:` line whose payload is not valid JSON. Defaults to + * silently skipping the line. Throw from here to surface a fatal parse error. + */ + onParseError?: (rawData: string, error: unknown) => void + /** Aborts the read; checked before each chunk and between events. */ + signal?: AbortSignal +} + +/** + * Resolves an {@link SSESource} to a reader, reporting whether this call + * acquired the lock (and is therefore responsible for releasing it). + */ +function toReader(source: SSESource): { + reader: ReadableStreamDefaultReader + ownsLock: boolean +} { + if (source instanceof ReadableStream) { + return { reader: source.getReader(), ownsLock: true } + } + if (source instanceof Response) { + if (!source.body) throw new Error('No response body') + return { reader: source.body.getReader(), ownsLock: true } + } + return { reader: source, ownsLock: false } +} + +/** + * Strips an optional trailing carriage return from a single SSE line, so both + * `\n`- and `\r\n`-terminated framings parse identically. + */ +function stripCarriageReturn(line: string): string { + return line.endsWith('\r') ? line.slice(0, -1) : line +} + +/** + * The single client-side SSE decode engine. Reads a byte stream, decodes it + * incrementally, splits it into lines, and invokes `onData` once per `data:` + * line with its raw (un-parsed) payload string. + * + * It splits on `\n` and processes each `data:` line individually, which makes it + * tolerant of BOTH `\n`- and `\n\n`-separated framings (the blank separator + * lines between events are simply ignored). Trailing `\r` is stripped, a single + * optional space after `data:` is consumed, and the `[DONE]` sentinel is + * skipped. The reader's lock is always released on completion, abort, or error + * (only when this function acquired it). + * + * This is the low-level engine. Most callers want {@link readSSEEvents}, which + * adds JSON parsing on top. Reach for `readSSELines` only when the payload needs + * custom parsing (e.g. schema-validated decoding). + * + * @param source - A `Response`, `ReadableStream`, or stream reader. + * @param options - The `onData` callback plus an optional `signal`. + */ +export async function readSSELines(source: SSESource, options: ReadSSELinesOptions): Promise { + const { onData, signal } = options + const { reader, ownsLock } = toReader(source) + const decoder = new TextDecoder() + let buffer = '' + + try { + while (true) { + if (signal?.aborted) break + + const { done, value } = await reader.read() + if (done) break + + buffer += decoder.decode(value, { stream: true }) + const lines = buffer.split('\n') + buffer = lines.pop() ?? '' + + for (const rawLine of lines) { + if (signal?.aborted) return + + const line = stripCarriageReturn(rawLine) + if (!line.startsWith('data:')) continue + + let data = line.slice(5) + if (data.startsWith(' ')) data = data.slice(1) + if (data === DONE_SENTINEL) continue + + if ((await onData(data)) === true) return + } + } + } finally { + if (ownsLock) reader.releaseLock() + } +} + +/** + * The JSON convenience layer over {@link readSSELines}: invokes `onEvent` once + * per `data:` event with its JSON-parsed payload. Unparseable lines are passed + * to `onParseError` (default: silently skipped). All framing, `\r`, `[DONE]`, + * abort, and reader-lifecycle behavior is inherited from `readSSELines`. + * + * Higher-level concerns — UI batching, reconnect, error classification, event + * dispatch — belong in the caller's `onEvent`, not here. + * + * @typeParam T - The parsed event type the caller expects (defaults to `unknown`). + * @param source - A `Response`, `ReadableStream`, or stream reader. + * @param options - The `onEvent` callback plus optional `signal`/`onParseError`. + */ +export async function readSSEEvents( + source: SSESource, + options: ReadSSEEventsOptions +): Promise { + const { onEvent, onParseError, signal } = options + await readSSELines(source, { + signal, + onData: (data) => { + let parsed: T + try { + parsed = JSON.parse(data) as T + } catch (error) { + onParseError?.(data, error) + return + } + return onEvent(parsed) + }, + }) +} + /** * Options for reading SSE stream */ From 067f9e9dc99401896fc9837b0d038628fec26a51 Mon Sep 17 00:00:00 2001 From: Waleed Date: Wed, 24 Jun 2026 11:37:06 -0700 Subject: [PATCH 06/15] feat(gitlab): support self-managed GitLab host across tools, block, triggers, webhook, and connector (#5200) * feat(gitlab): support self-managed GitLab host across tools, block, triggers, webhook, and connector Add an optional `host` so the GitLab integration can target a self-managed instance (e.g. gitlab.example.com) instead of gitlab.com. Defaults to gitlab.com everywhere, so existing workflows, blocks, triggers, and stored webhooks are unchanged. - Shared host helper (normalizeGitLabHost/getGitLabApiBase) used by all 19 tools, the block, triggers, the webhook provider, and the connector - SSRF hardening: reject structurally unsafe hosts (userinfo `@`, whitespace, control chars, embedded path/query, empty labels) before the token-bearing request is built; allow self-managed hosts, ports, and IDN punycode - Route the webhook provider's previously-raw fetches through secureFetchWithValidation (DNS + private-IP rejection + IP pinning), matching the tool and connector paths - Add regression tests for the host validator * fix(gitlab): handle unsafe-host errors gracefully in webhook provider Address review feedback: - Validate the optional self-managed host up front in createSubscription and deleteSubscription so a structurally unsafe value surfaces as a clear error (create) or a graceful non-strict skip (delete) instead of an unhandled UnsafeGitLabHostError, mirroring the connector's handling - Document the layered SSRF defense: bare IP literals pass the structural host guard by design and are rejected at the fetch layer (validateUrlWithDNS); add a confirming test group making that intent explicit * fix(slack): drop assistant:write scope pending app review approval Requesting assistant:write before Slack approved it fails the OAuth/install flow for users. Remove it from both request paths until approval lands: - Remove from the user OAuth scope list (oauth.ts), matching the existing users:read.email TODO pattern - Remove the action_assistant capability from the bot manifest generator (capabilities.ts), leaving a TODO to restore it after approval The set_status/set_title/set_suggested_prompts tools remain and surface their existing graceful "reconnect with assistant:write" message until re-enabled. --- apps/sim/blocks/blocks/gitlab.ts | 11 +++ apps/sim/connectors/gitlab/gitlab.ts | 31 +++++--- apps/sim/lib/oauth/oauth.ts | 2 +- apps/sim/lib/webhooks/providers/gitlab.ts | 62 ++++++++++++---- apps/sim/tools/gitlab/cancel_pipeline.ts | 9 ++- apps/sim/tools/gitlab/create_issue.ts | 9 ++- apps/sim/tools/gitlab/create_issue_note.ts | 9 ++- apps/sim/tools/gitlab/create_merge_request.ts | 9 ++- .../tools/gitlab/create_merge_request_note.ts | 9 ++- apps/sim/tools/gitlab/create_pipeline.ts | 9 ++- apps/sim/tools/gitlab/delete_issue.ts | 9 ++- apps/sim/tools/gitlab/get_issue.ts | 9 ++- apps/sim/tools/gitlab/get_merge_request.ts | 9 ++- apps/sim/tools/gitlab/get_pipeline.ts | 9 ++- apps/sim/tools/gitlab/get_project.ts | 9 ++- apps/sim/tools/gitlab/list_issues.ts | 9 ++- apps/sim/tools/gitlab/list_merge_requests.ts | 9 ++- apps/sim/tools/gitlab/list_pipelines.ts | 9 ++- apps/sim/tools/gitlab/list_projects.ts | 9 ++- apps/sim/tools/gitlab/merge_merge_request.ts | 9 ++- apps/sim/tools/gitlab/retry_pipeline.ts | 9 ++- apps/sim/tools/gitlab/types.ts | 5 ++ apps/sim/tools/gitlab/update_issue.ts | 9 ++- apps/sim/tools/gitlab/update_merge_request.ts | 9 ++- apps/sim/tools/gitlab/utils.test.ts | 71 +++++++++++++++++++ apps/sim/tools/gitlab/utils.ts | 68 ++++++++++++++++++ apps/sim/triggers/gitlab/utils.ts | 9 +++ apps/sim/triggers/slack/capabilities.ts | 11 +-- 28 files changed, 388 insertions(+), 53 deletions(-) create mode 100644 apps/sim/tools/gitlab/utils.test.ts create mode 100644 apps/sim/tools/gitlab/utils.ts diff --git a/apps/sim/blocks/blocks/gitlab.ts b/apps/sim/blocks/blocks/gitlab.ts index 59de55f8c26..1cfe24a9e25 100644 --- a/apps/sim/blocks/blocks/gitlab.ts +++ b/apps/sim/blocks/blocks/gitlab.ts @@ -57,6 +57,15 @@ export const GitLabBlock: BlockConfig = { password: true, required: true, }, + // Self-managed GitLab host (defaults to gitlab.com) + { + id: 'host', + title: 'GitLab Host', + type: 'short-input', + placeholder: 'gitlab.com', + mode: 'advanced', + description: 'Self-managed GitLab host. Leave blank for gitlab.com.', + }, // Project ID (required for most operations) { id: 'projectId', @@ -474,6 +483,7 @@ Return ONLY the commit message - no explanations, no extra text.`, params: (params) => { const baseParams: Record = { accessToken: params.accessToken, + host: params.host?.trim() || undefined, } switch (params.operation) { @@ -709,6 +719,7 @@ Return ONLY the commit message - no explanations, no extra text.`, inputs: { operation: { type: 'string', description: 'Operation to perform' }, credential: { type: 'string', description: 'GitLab access token' }, + host: { type: 'string', description: 'Self-managed GitLab host (defaults to gitlab.com)' }, projectId: { type: 'string', description: 'Project ID or URL-encoded path' }, issueIid: { type: 'number', description: 'Issue internal ID' }, mergeRequestIid: { type: 'number', description: 'Merge request internal ID' }, diff --git a/apps/sim/connectors/gitlab/gitlab.ts b/apps/sim/connectors/gitlab/gitlab.ts index 18247214006..99586321f48 100644 --- a/apps/sim/connectors/gitlab/gitlab.ts +++ b/apps/sim/connectors/gitlab/gitlab.ts @@ -14,10 +14,10 @@ import { parseTagDate, sizeLimitSkipReason, } from '@/connectors/utils' +import { normalizeGitLabHost, UnsafeGitLabHostError } from '@/tools/gitlab/utils' const logger = createLogger('GitLabConnector') -const DEFAULT_HOST = 'gitlab.com' const PAGE_SIZE = 100 /** Max repository file size to index. Larger blobs are skipped. */ const MAX_FILE_SIZE = CONNECTOR_MAX_FILE_BYTES @@ -175,16 +175,16 @@ interface GitLabProject { } /** - * Normalizes the host config value: trims whitespace, strips any protocol - * prefix and trailing slashes, and falls back to gitlab.com when empty. + * Normalizes the host config value via the shared GitLab host normalizer: + * trims, strips any protocol prefix and trailing slashes, rejects structurally + * unsafe hosts (userinfo, whitespace, embedded path), and falls back to + * gitlab.com when empty. Shared with the GitLab tools and webhook provider so + * every surface resolves and validates hosts identically. + * + * @throws {UnsafeGitLabHostError} when a non-empty host is structurally unsafe. */ function normalizeHost(rawHost: unknown): string { - const host = typeof rawHost === 'string' ? rawHost.trim() : '' - if (!host) return DEFAULT_HOST - return host - .replace(/^https?:\/\//i, '') - .replace(/\/+$/, '') - .trim() + return normalizeGitLabHost(rawHost) } /** @@ -941,7 +941,18 @@ export const gitlabConnector: ConnectorConfig = { return { valid: false, error: 'Max items must be a positive number' } } - const host = normalizeHost(sourceConfig.host) + let host: string + try { + host = normalizeHost(sourceConfig.host) + } catch (error) { + if (error instanceof UnsafeGitLabHostError) { + return { + valid: false, + error: 'Host must be a valid GitLab domain (e.g. gitlab.example.com)', + } + } + throw error + } const apiBase = buildApiBase(host) const encodedProject = encodeProjectId(project) const choice = getContentTypeChoice(sourceConfig) diff --git a/apps/sim/lib/oauth/oauth.ts b/apps/sim/lib/oauth/oauth.ts index a9a71270042..8f6b17100fe 100644 --- a/apps/sim/lib/oauth/oauth.ts +++ b/apps/sim/lib/oauth/oauth.ts @@ -711,7 +711,7 @@ export const OAUTH_PROVIDERS: Record = { 'groups:write', 'chat:write', 'chat:write.public', - 'assistant:write', + // TODO: Add 'assistant:write' once Slack app review is approved 'im:write', 'im:read', 'users:read', diff --git a/apps/sim/lib/webhooks/providers/gitlab.ts b/apps/sim/lib/webhooks/providers/gitlab.ts index 3f6ffcbf12e..d812ee19d92 100644 --- a/apps/sim/lib/webhooks/providers/gitlab.ts +++ b/apps/sim/lib/webhooks/providers/gitlab.ts @@ -2,6 +2,7 @@ import { createLogger } from '@sim/logger' import { safeCompare } from '@sim/security/compare' import { generateId } from '@sim/utils/id' import { NextResponse } from 'next/server' +import { secureFetchWithValidation } from '@/lib/core/security/input-validation.server' import { getNotificationUrl, getProviderConfig } from '@/lib/webhooks/provider-subscription-utils' import type { AuthContext, @@ -13,17 +14,16 @@ import type { SubscriptionResult, WebhookProviderHandler, } from '@/lib/webhooks/providers/types' +import { getGitLabApiBase, UnsafeGitLabHostError } from '@/tools/gitlab/utils' const logger = createLogger('WebhookProvider:GitLab') -const GITLAB_API_BASE = 'https://gitlab.com/api/v4' - function asRecord(value: unknown): Record { return (value as Record) || {} } -function gitlabProjectHooksUrl(projectId: string): string { - return `${GITLAB_API_BASE}/projects/${encodeURIComponent(projectId)}/hooks` +function gitlabProjectHooksUrl(projectId: string, host: unknown): string { + return `${getGitLabApiBase(host)}/projects/${encodeURIComponent(projectId)}/hooks` } /** @@ -33,9 +33,10 @@ function gitlabProjectHooksUrl(projectId: string): string { async function cleanupGitLabHookByUrl( projectId: string, accessToken: string, - url: string + url: string, + host: unknown ): Promise { - const res = await fetch(gitlabProjectHooksUrl(projectId), { + const res = await secureFetchWithValidation(gitlabProjectHooksUrl(projectId, host), { headers: { 'PRIVATE-TOKEN': accessToken }, }).catch(() => null) if (!res || !res.ok) return @@ -47,7 +48,7 @@ async function cleanupGitLabHookByUrl( hooks .filter((hook) => hook.url === url && hook.id != null) .map((hook) => - fetch(`${gitlabProjectHooksUrl(projectId)}/${hook.id}`, { + secureFetchWithValidation(`${gitlabProjectHooksUrl(projectId, host)}/${hook.id}`, { method: 'DELETE', headers: { 'PRIVATE-TOKEN': accessToken }, }).catch(() => null) @@ -113,14 +114,28 @@ export const gitlabHandler: WebhookProviderHandler = { const accessToken = config.accessToken as string | undefined const projectId = config.projectId as string | undefined const triggerId = config.triggerId as string | undefined + const host = config.host as string | undefined if (!accessToken) throw new Error('GitLab Personal Access Token is required to create the webhook.') if (!projectId) throw new Error('GitLab Project ID is required to create the webhook.') + // Validate the optional self-managed host up front so a structurally unsafe + // value surfaces as a clear error instead of an unhandled UnsafeGitLabHostError. + try { + getGitLabApiBase(host) + } catch (error) { + if (error instanceof UnsafeGitLabHostError) { + throw new Error( + 'GitLab host is invalid. Provide a domain like gitlab.example.com (no protocol, path, or credentials).' + ) + } + throw error + } + const { getGitLabEventFlags } = await import('@/triggers/gitlab/utils') const secretToken = generateId() - const res = await fetch(gitlabProjectHooksUrl(projectId), { + const res = await secureFetchWithValidation(gitlabProjectHooksUrl(projectId, host), { method: 'POST', headers: { 'PRIVATE-TOKEN': accessToken, 'Content-Type': 'application/json' }, body: JSON.stringify({ @@ -150,7 +165,7 @@ export const gitlabHandler: WebhookProviderHandler = { if (created.id === undefined || created.id === null) { // The hook was created but we can't read its id — delete it by URL so it // is not orphaned in GitLab. - await cleanupGitLabHookByUrl(projectId, accessToken, getNotificationUrl(ctx.webhook)) + await cleanupGitLabHookByUrl(projectId, accessToken, getNotificationUrl(ctx.webhook), host) throw new Error('GitLab webhook created but no hook ID was returned.') } @@ -163,6 +178,7 @@ export const gitlabHandler: WebhookProviderHandler = { const accessToken = config.accessToken as string | undefined const projectId = config.projectId as string | undefined const externalId = config.externalId as string | undefined + const host = config.host as string | undefined if (!accessToken || !projectId || !externalId) { if (ctx.strict) throw new Error('Missing GitLab credentials or hook ID for webhook deletion.') @@ -172,10 +188,30 @@ export const gitlabHandler: WebhookProviderHandler = { return } - const res = await fetch(`${gitlabProjectHooksUrl(projectId)}/${externalId}`, { - method: 'DELETE', - headers: { 'PRIVATE-TOKEN': accessToken }, - }) + // A structurally unsafe host must not abort cleanup in non-strict mode — mirror + // the graceful skip used for missing credentials above. + try { + getGitLabApiBase(host) + } catch (error) { + if (error instanceof UnsafeGitLabHostError) { + if (ctx.strict) { + throw new Error('Cannot delete GitLab webhook: the configured host is invalid.') + } + logger.warn( + `[${ctx.requestId}] Skipping GitLab webhook cleanup — configured host is invalid` + ) + return + } + throw error + } + + const res = await secureFetchWithValidation( + `${gitlabProjectHooksUrl(projectId, host)}/${externalId}`, + { + method: 'DELETE', + headers: { 'PRIVATE-TOKEN': accessToken }, + } + ) if (!res.ok && res.status !== 404) { if (ctx.strict) throw new Error(`Failed to delete GitLab webhook: ${res.status}`) diff --git a/apps/sim/tools/gitlab/cancel_pipeline.ts b/apps/sim/tools/gitlab/cancel_pipeline.ts index 62b9e096b95..9707f758a0e 100644 --- a/apps/sim/tools/gitlab/cancel_pipeline.ts +++ b/apps/sim/tools/gitlab/cancel_pipeline.ts @@ -1,4 +1,5 @@ import type { GitLabCancelPipelineParams, GitLabCancelPipelineResponse } from '@/tools/gitlab/types' +import { getGitLabApiBase } from '@/tools/gitlab/utils' import type { ToolConfig } from '@/tools/types' export const gitlabCancelPipelineTool: ToolConfig< @@ -17,6 +18,12 @@ export const gitlabCancelPipelineTool: ToolConfig< visibility: 'user-only', description: 'GitLab Personal Access Token', }, + host: { + type: 'string', + required: false, + visibility: 'user-only', + description: 'Self-managed GitLab host (e.g. gitlab.example.com). Defaults to gitlab.com.', + }, projectId: { type: 'string', required: true, @@ -34,7 +41,7 @@ export const gitlabCancelPipelineTool: ToolConfig< request: { url: (params) => { const encodedId = encodeURIComponent(String(params.projectId)) - return `https://gitlab.com/api/v4/projects/${encodedId}/pipelines/${params.pipelineId}/cancel` + return `${getGitLabApiBase(params.host)}/projects/${encodedId}/pipelines/${params.pipelineId}/cancel` }, method: 'POST', headers: (params) => ({ diff --git a/apps/sim/tools/gitlab/create_issue.ts b/apps/sim/tools/gitlab/create_issue.ts index 6a03e9ff970..cc4475831da 100644 --- a/apps/sim/tools/gitlab/create_issue.ts +++ b/apps/sim/tools/gitlab/create_issue.ts @@ -1,4 +1,5 @@ import type { GitLabCreateIssueParams, GitLabCreateIssueResponse } from '@/tools/gitlab/types' +import { getGitLabApiBase } from '@/tools/gitlab/utils' import type { ToolConfig } from '@/tools/types' export const gitlabCreateIssueTool: ToolConfig = @@ -15,6 +16,12 @@ export const gitlabCreateIssueTool: ToolConfig { const encodedId = encodeURIComponent(String(params.projectId)) - return `https://gitlab.com/api/v4/projects/${encodedId}/issues` + return `${getGitLabApiBase(params.host)}/projects/${encodedId}/issues` }, method: 'POST', headers: (params) => ({ diff --git a/apps/sim/tools/gitlab/create_issue_note.ts b/apps/sim/tools/gitlab/create_issue_note.ts index 150bf729d7e..0ad5c218bee 100644 --- a/apps/sim/tools/gitlab/create_issue_note.ts +++ b/apps/sim/tools/gitlab/create_issue_note.ts @@ -1,4 +1,5 @@ import type { GitLabCreateIssueNoteParams, GitLabCreateNoteResponse } from '@/tools/gitlab/types' +import { getGitLabApiBase } from '@/tools/gitlab/utils' import type { ToolConfig } from '@/tools/types' export const gitlabCreateIssueNoteTool: ToolConfig< @@ -17,6 +18,12 @@ export const gitlabCreateIssueNoteTool: ToolConfig< visibility: 'user-only', description: 'GitLab Personal Access Token', }, + host: { + type: 'string', + required: false, + visibility: 'user-only', + description: 'Self-managed GitLab host (e.g. gitlab.example.com). Defaults to gitlab.com.', + }, projectId: { type: 'string', required: true, @@ -40,7 +47,7 @@ export const gitlabCreateIssueNoteTool: ToolConfig< request: { url: (params) => { const encodedId = encodeURIComponent(String(params.projectId)) - return `https://gitlab.com/api/v4/projects/${encodedId}/issues/${params.issueIid}/notes` + return `${getGitLabApiBase(params.host)}/projects/${encodedId}/issues/${params.issueIid}/notes` }, method: 'POST', headers: (params) => ({ diff --git a/apps/sim/tools/gitlab/create_merge_request.ts b/apps/sim/tools/gitlab/create_merge_request.ts index fdaebbf842a..2c02c2dd0f4 100644 --- a/apps/sim/tools/gitlab/create_merge_request.ts +++ b/apps/sim/tools/gitlab/create_merge_request.ts @@ -2,6 +2,7 @@ import type { GitLabCreateMergeRequestParams, GitLabCreateMergeRequestResponse, } from '@/tools/gitlab/types' +import { getGitLabApiBase } from '@/tools/gitlab/utils' import type { ToolConfig } from '@/tools/types' export const gitlabCreateMergeRequestTool: ToolConfig< @@ -20,6 +21,12 @@ export const gitlabCreateMergeRequestTool: ToolConfig< visibility: 'user-only', description: 'GitLab Personal Access Token', }, + host: { + type: 'string', + required: false, + visibility: 'user-only', + description: 'Self-managed GitLab host (e.g. gitlab.example.com). Defaults to gitlab.com.', + }, projectId: { type: 'string', required: true, @@ -91,7 +98,7 @@ export const gitlabCreateMergeRequestTool: ToolConfig< request: { url: (params) => { const encodedId = encodeURIComponent(String(params.projectId)) - return `https://gitlab.com/api/v4/projects/${encodedId}/merge_requests` + return `${getGitLabApiBase(params.host)}/projects/${encodedId}/merge_requests` }, method: 'POST', headers: (params) => ({ diff --git a/apps/sim/tools/gitlab/create_merge_request_note.ts b/apps/sim/tools/gitlab/create_merge_request_note.ts index 599ec569719..f02f5fa35fb 100644 --- a/apps/sim/tools/gitlab/create_merge_request_note.ts +++ b/apps/sim/tools/gitlab/create_merge_request_note.ts @@ -2,6 +2,7 @@ import type { GitLabCreateMergeRequestNoteParams, GitLabCreateNoteResponse, } from '@/tools/gitlab/types' +import { getGitLabApiBase } from '@/tools/gitlab/utils' import type { ToolConfig } from '@/tools/types' export const gitlabCreateMergeRequestNoteTool: ToolConfig< @@ -20,6 +21,12 @@ export const gitlabCreateMergeRequestNoteTool: ToolConfig< visibility: 'user-only', description: 'GitLab Personal Access Token', }, + host: { + type: 'string', + required: false, + visibility: 'user-only', + description: 'Self-managed GitLab host (e.g. gitlab.example.com). Defaults to gitlab.com.', + }, projectId: { type: 'string', required: true, @@ -43,7 +50,7 @@ export const gitlabCreateMergeRequestNoteTool: ToolConfig< request: { url: (params) => { const encodedId = encodeURIComponent(String(params.projectId)) - return `https://gitlab.com/api/v4/projects/${encodedId}/merge_requests/${params.mergeRequestIid}/notes` + return `${getGitLabApiBase(params.host)}/projects/${encodedId}/merge_requests/${params.mergeRequestIid}/notes` }, method: 'POST', headers: (params) => ({ diff --git a/apps/sim/tools/gitlab/create_pipeline.ts b/apps/sim/tools/gitlab/create_pipeline.ts index 38a15df006b..a27ed7ba372 100644 --- a/apps/sim/tools/gitlab/create_pipeline.ts +++ b/apps/sim/tools/gitlab/create_pipeline.ts @@ -1,4 +1,5 @@ import type { GitLabCreatePipelineParams, GitLabCreatePipelineResponse } from '@/tools/gitlab/types' +import { getGitLabApiBase } from '@/tools/gitlab/utils' import type { ToolConfig } from '@/tools/types' export const gitlabCreatePipelineTool: ToolConfig< @@ -17,6 +18,12 @@ export const gitlabCreatePipelineTool: ToolConfig< visibility: 'user-only', description: 'GitLab Personal Access Token', }, + host: { + type: 'string', + required: false, + visibility: 'user-only', + description: 'Self-managed GitLab host (e.g. gitlab.example.com). Defaults to gitlab.com.', + }, projectId: { type: 'string', required: true, @@ -41,7 +48,7 @@ export const gitlabCreatePipelineTool: ToolConfig< request: { url: (params) => { const encodedId = encodeURIComponent(String(params.projectId)) - return `https://gitlab.com/api/v4/projects/${encodedId}/pipeline` + return `${getGitLabApiBase(params.host)}/projects/${encodedId}/pipeline` }, method: 'POST', headers: (params) => ({ diff --git a/apps/sim/tools/gitlab/delete_issue.ts b/apps/sim/tools/gitlab/delete_issue.ts index 64fbe881e53..475e52d77a8 100644 --- a/apps/sim/tools/gitlab/delete_issue.ts +++ b/apps/sim/tools/gitlab/delete_issue.ts @@ -1,4 +1,5 @@ import type { GitLabDeleteIssueParams, GitLabDeleteIssueResponse } from '@/tools/gitlab/types' +import { getGitLabApiBase } from '@/tools/gitlab/utils' import type { ToolConfig } from '@/tools/types' export const gitlabDeleteIssueTool: ToolConfig = @@ -15,6 +16,12 @@ export const gitlabDeleteIssueTool: ToolConfig { const encodedId = encodeURIComponent(String(params.projectId)) - return `https://gitlab.com/api/v4/projects/${encodedId}/issues/${params.issueIid}` + return `${getGitLabApiBase(params.host)}/projects/${encodedId}/issues/${params.issueIid}` }, method: 'DELETE', headers: (params) => ({ diff --git a/apps/sim/tools/gitlab/get_issue.ts b/apps/sim/tools/gitlab/get_issue.ts index 9a3c5821edb..aa87136e552 100644 --- a/apps/sim/tools/gitlab/get_issue.ts +++ b/apps/sim/tools/gitlab/get_issue.ts @@ -1,4 +1,5 @@ import type { GitLabGetIssueParams, GitLabGetIssueResponse } from '@/tools/gitlab/types' +import { getGitLabApiBase } from '@/tools/gitlab/utils' import type { ToolConfig } from '@/tools/types' export const gitlabGetIssueTool: ToolConfig = { @@ -14,6 +15,12 @@ export const gitlabGetIssueTool: ToolConfig { const encodedId = encodeURIComponent(String(params.projectId)) - return `https://gitlab.com/api/v4/projects/${encodedId}/issues/${params.issueIid}` + return `${getGitLabApiBase(params.host)}/projects/${encodedId}/issues/${params.issueIid}` }, method: 'GET', headers: (params) => ({ diff --git a/apps/sim/tools/gitlab/get_merge_request.ts b/apps/sim/tools/gitlab/get_merge_request.ts index 6e99d3cfbd3..f228cfba2eb 100644 --- a/apps/sim/tools/gitlab/get_merge_request.ts +++ b/apps/sim/tools/gitlab/get_merge_request.ts @@ -2,6 +2,7 @@ import type { GitLabGetMergeRequestParams, GitLabGetMergeRequestResponse, } from '@/tools/gitlab/types' +import { getGitLabApiBase } from '@/tools/gitlab/utils' import type { ToolConfig } from '@/tools/types' export const gitlabGetMergeRequestTool: ToolConfig< @@ -20,6 +21,12 @@ export const gitlabGetMergeRequestTool: ToolConfig< visibility: 'user-only', description: 'GitLab Personal Access Token', }, + host: { + type: 'string', + required: false, + visibility: 'user-only', + description: 'Self-managed GitLab host (e.g. gitlab.example.com). Defaults to gitlab.com.', + }, projectId: { type: 'string', required: true, @@ -37,7 +44,7 @@ export const gitlabGetMergeRequestTool: ToolConfig< request: { url: (params) => { const encodedId = encodeURIComponent(String(params.projectId)) - return `https://gitlab.com/api/v4/projects/${encodedId}/merge_requests/${params.mergeRequestIid}` + return `${getGitLabApiBase(params.host)}/projects/${encodedId}/merge_requests/${params.mergeRequestIid}` }, method: 'GET', headers: (params) => ({ diff --git a/apps/sim/tools/gitlab/get_pipeline.ts b/apps/sim/tools/gitlab/get_pipeline.ts index 5f4f25a0eaa..1494e65e4bb 100644 --- a/apps/sim/tools/gitlab/get_pipeline.ts +++ b/apps/sim/tools/gitlab/get_pipeline.ts @@ -1,4 +1,5 @@ import type { GitLabGetPipelineParams, GitLabGetPipelineResponse } from '@/tools/gitlab/types' +import { getGitLabApiBase } from '@/tools/gitlab/utils' import type { ToolConfig } from '@/tools/types' export const gitlabGetPipelineTool: ToolConfig = @@ -15,6 +16,12 @@ export const gitlabGetPipelineTool: ToolConfig { const encodedId = encodeURIComponent(String(params.projectId)) - return `https://gitlab.com/api/v4/projects/${encodedId}/pipelines/${params.pipelineId}` + return `${getGitLabApiBase(params.host)}/projects/${encodedId}/pipelines/${params.pipelineId}` }, method: 'GET', headers: (params) => ({ diff --git a/apps/sim/tools/gitlab/get_project.ts b/apps/sim/tools/gitlab/get_project.ts index c49369084f5..5ea42920584 100644 --- a/apps/sim/tools/gitlab/get_project.ts +++ b/apps/sim/tools/gitlab/get_project.ts @@ -1,4 +1,5 @@ import type { GitLabGetProjectParams, GitLabGetProjectResponse } from '@/tools/gitlab/types' +import { getGitLabApiBase } from '@/tools/gitlab/utils' import type { ToolConfig } from '@/tools/types' export const gitlabGetProjectTool: ToolConfig = { @@ -14,6 +15,12 @@ export const gitlabGetProjectTool: ToolConfig { const encodedId = encodeURIComponent(String(params.projectId)) - return `https://gitlab.com/api/v4/projects/${encodedId}` + return `${getGitLabApiBase(params.host)}/projects/${encodedId}` }, method: 'GET', headers: (params) => ({ diff --git a/apps/sim/tools/gitlab/list_issues.ts b/apps/sim/tools/gitlab/list_issues.ts index 1607920571e..40a016f3b34 100644 --- a/apps/sim/tools/gitlab/list_issues.ts +++ b/apps/sim/tools/gitlab/list_issues.ts @@ -1,4 +1,5 @@ import type { GitLabListIssuesParams, GitLabListIssuesResponse } from '@/tools/gitlab/types' +import { getGitLabApiBase } from '@/tools/gitlab/utils' import type { ToolConfig } from '@/tools/types' export const gitlabListIssuesTool: ToolConfig = { @@ -14,6 +15,12 @@ export const gitlabListIssuesTool: ToolConfig ({ diff --git a/apps/sim/tools/gitlab/list_merge_requests.ts b/apps/sim/tools/gitlab/list_merge_requests.ts index 0296bc3a24f..2cdae3301c4 100644 --- a/apps/sim/tools/gitlab/list_merge_requests.ts +++ b/apps/sim/tools/gitlab/list_merge_requests.ts @@ -2,6 +2,7 @@ import type { GitLabListMergeRequestsParams, GitLabListMergeRequestsResponse, } from '@/tools/gitlab/types' +import { getGitLabApiBase } from '@/tools/gitlab/utils' import type { ToolConfig } from '@/tools/types' export const gitlabListMergeRequestsTool: ToolConfig< @@ -20,6 +21,12 @@ export const gitlabListMergeRequestsTool: ToolConfig< visibility: 'user-only', description: 'GitLab Personal Access Token', }, + host: { + type: 'string', + required: false, + visibility: 'user-only', + description: 'Self-managed GitLab host (e.g. gitlab.example.com). Defaults to gitlab.com.', + }, projectId: { type: 'string', required: true, @@ -91,7 +98,7 @@ export const gitlabListMergeRequestsTool: ToolConfig< if (params.page) queryParams.append('page', String(params.page)) const query = queryParams.toString() - return `https://gitlab.com/api/v4/projects/${encodedId}/merge_requests${query ? `?${query}` : ''}` + return `${getGitLabApiBase(params.host)}/projects/${encodedId}/merge_requests${query ? `?${query}` : ''}` }, method: 'GET', headers: (params) => ({ diff --git a/apps/sim/tools/gitlab/list_pipelines.ts b/apps/sim/tools/gitlab/list_pipelines.ts index d4aed464736..80294e85f73 100644 --- a/apps/sim/tools/gitlab/list_pipelines.ts +++ b/apps/sim/tools/gitlab/list_pipelines.ts @@ -1,4 +1,5 @@ import type { GitLabListPipelinesParams, GitLabListPipelinesResponse } from '@/tools/gitlab/types' +import { getGitLabApiBase } from '@/tools/gitlab/utils' import type { ToolConfig } from '@/tools/types' export const gitlabListPipelinesTool: ToolConfig< @@ -17,6 +18,12 @@ export const gitlabListPipelinesTool: ToolConfig< visibility: 'user-only', description: 'GitLab Personal Access Token', }, + host: { + type: 'string', + required: false, + visibility: 'user-only', + description: 'Self-managed GitLab host (e.g. gitlab.example.com). Defaults to gitlab.com.', + }, projectId: { type: 'string', required: true, @@ -75,7 +82,7 @@ export const gitlabListPipelinesTool: ToolConfig< if (params.page) queryParams.append('page', String(params.page)) const query = queryParams.toString() - return `https://gitlab.com/api/v4/projects/${encodedId}/pipelines${query ? `?${query}` : ''}` + return `${getGitLabApiBase(params.host)}/projects/${encodedId}/pipelines${query ? `?${query}` : ''}` }, method: 'GET', headers: (params) => ({ diff --git a/apps/sim/tools/gitlab/list_projects.ts b/apps/sim/tools/gitlab/list_projects.ts index ec8018215a4..b6d6dd4c4a7 100644 --- a/apps/sim/tools/gitlab/list_projects.ts +++ b/apps/sim/tools/gitlab/list_projects.ts @@ -1,4 +1,5 @@ import type { GitLabListProjectsParams, GitLabListProjectsResponse } from '@/tools/gitlab/types' +import { getGitLabApiBase } from '@/tools/gitlab/utils' import type { ToolConfig } from '@/tools/types' export const gitlabListProjectsTool: ToolConfig< @@ -17,6 +18,12 @@ export const gitlabListProjectsTool: ToolConfig< visibility: 'user-only', description: 'GitLab Personal Access Token', }, + host: { + type: 'string', + required: false, + visibility: 'user-only', + description: 'Self-managed GitLab host (e.g. gitlab.example.com). Defaults to gitlab.com.', + }, owned: { type: 'boolean', required: false, @@ -80,7 +87,7 @@ export const gitlabListProjectsTool: ToolConfig< if (params.page) queryParams.append('page', String(params.page)) const query = queryParams.toString() - return `https://gitlab.com/api/v4/projects${query ? `?${query}` : ''}` + return `${getGitLabApiBase(params.host)}/projects${query ? `?${query}` : ''}` }, method: 'GET', headers: (params) => ({ diff --git a/apps/sim/tools/gitlab/merge_merge_request.ts b/apps/sim/tools/gitlab/merge_merge_request.ts index d63686a16f4..500e6ebfd07 100644 --- a/apps/sim/tools/gitlab/merge_merge_request.ts +++ b/apps/sim/tools/gitlab/merge_merge_request.ts @@ -2,6 +2,7 @@ import type { GitLabMergeMergeRequestParams, GitLabMergeMergeRequestResponse, } from '@/tools/gitlab/types' +import { getGitLabApiBase } from '@/tools/gitlab/utils' import type { ToolConfig } from '@/tools/types' export const gitlabMergeMergeRequestTool: ToolConfig< @@ -20,6 +21,12 @@ export const gitlabMergeMergeRequestTool: ToolConfig< visibility: 'user-only', description: 'GitLab Personal Access Token', }, + host: { + type: 'string', + required: false, + visibility: 'user-only', + description: 'Self-managed GitLab host (e.g. gitlab.example.com). Defaults to gitlab.com.', + }, projectId: { type: 'string', required: true, @@ -67,7 +74,7 @@ export const gitlabMergeMergeRequestTool: ToolConfig< request: { url: (params) => { const encodedId = encodeURIComponent(String(params.projectId)) - return `https://gitlab.com/api/v4/projects/${encodedId}/merge_requests/${params.mergeRequestIid}/merge` + return `${getGitLabApiBase(params.host)}/projects/${encodedId}/merge_requests/${params.mergeRequestIid}/merge` }, method: 'PUT', headers: (params) => ({ diff --git a/apps/sim/tools/gitlab/retry_pipeline.ts b/apps/sim/tools/gitlab/retry_pipeline.ts index 3c0fe6f2b4d..48143109c97 100644 --- a/apps/sim/tools/gitlab/retry_pipeline.ts +++ b/apps/sim/tools/gitlab/retry_pipeline.ts @@ -1,4 +1,5 @@ import type { GitLabRetryPipelineParams, GitLabRetryPipelineResponse } from '@/tools/gitlab/types' +import { getGitLabApiBase } from '@/tools/gitlab/utils' import type { ToolConfig } from '@/tools/types' export const gitlabRetryPipelineTool: ToolConfig< @@ -17,6 +18,12 @@ export const gitlabRetryPipelineTool: ToolConfig< visibility: 'user-only', description: 'GitLab Personal Access Token', }, + host: { + type: 'string', + required: false, + visibility: 'user-only', + description: 'Self-managed GitLab host (e.g. gitlab.example.com). Defaults to gitlab.com.', + }, projectId: { type: 'string', required: true, @@ -34,7 +41,7 @@ export const gitlabRetryPipelineTool: ToolConfig< request: { url: (params) => { const encodedId = encodeURIComponent(String(params.projectId)) - return `https://gitlab.com/api/v4/projects/${encodedId}/pipelines/${params.pipelineId}/retry` + return `${getGitLabApiBase(params.host)}/projects/${encodedId}/pipelines/${params.pipelineId}/retry` }, method: 'POST', headers: (params) => ({ diff --git a/apps/sim/tools/gitlab/types.ts b/apps/sim/tools/gitlab/types.ts index 9722c16422b..af865ed3ef8 100644 --- a/apps/sim/tools/gitlab/types.ts +++ b/apps/sim/tools/gitlab/types.ts @@ -194,6 +194,11 @@ interface GitLabMilestone { interface GitLabBaseParams { accessToken: string + /** + * Self-managed GitLab host (e.g. `gitlab.example.com`). Optional — defaults to + * `gitlab.com` so existing workflows keep working. + */ + host?: string } // ===== Project Parameters ===== diff --git a/apps/sim/tools/gitlab/update_issue.ts b/apps/sim/tools/gitlab/update_issue.ts index 27c1fb70164..acf7ca25402 100644 --- a/apps/sim/tools/gitlab/update_issue.ts +++ b/apps/sim/tools/gitlab/update_issue.ts @@ -1,4 +1,5 @@ import type { GitLabUpdateIssueParams, GitLabUpdateIssueResponse } from '@/tools/gitlab/types' +import { getGitLabApiBase } from '@/tools/gitlab/utils' import type { ToolConfig } from '@/tools/types' export const gitlabUpdateIssueTool: ToolConfig = @@ -15,6 +16,12 @@ export const gitlabUpdateIssueTool: ToolConfig { const encodedId = encodeURIComponent(String(params.projectId)) - return `https://gitlab.com/api/v4/projects/${encodedId}/issues/${params.issueIid}` + return `${getGitLabApiBase(params.host)}/projects/${encodedId}/issues/${params.issueIid}` }, method: 'PUT', headers: (params) => ({ diff --git a/apps/sim/tools/gitlab/update_merge_request.ts b/apps/sim/tools/gitlab/update_merge_request.ts index c02d4f13b08..69632a637d5 100644 --- a/apps/sim/tools/gitlab/update_merge_request.ts +++ b/apps/sim/tools/gitlab/update_merge_request.ts @@ -2,6 +2,7 @@ import type { GitLabUpdateMergeRequestParams, GitLabUpdateMergeRequestResponse, } from '@/tools/gitlab/types' +import { getGitLabApiBase } from '@/tools/gitlab/utils' import type { ToolConfig } from '@/tools/types' export const gitlabUpdateMergeRequestTool: ToolConfig< @@ -20,6 +21,12 @@ export const gitlabUpdateMergeRequestTool: ToolConfig< visibility: 'user-only', description: 'GitLab Personal Access Token', }, + host: { + type: 'string', + required: false, + visibility: 'user-only', + description: 'Self-managed GitLab host (e.g. gitlab.example.com). Defaults to gitlab.com.', + }, projectId: { type: 'string', required: true, @@ -97,7 +104,7 @@ export const gitlabUpdateMergeRequestTool: ToolConfig< request: { url: (params) => { const encodedId = encodeURIComponent(String(params.projectId)) - return `https://gitlab.com/api/v4/projects/${encodedId}/merge_requests/${params.mergeRequestIid}` + return `${getGitLabApiBase(params.host)}/projects/${encodedId}/merge_requests/${params.mergeRequestIid}` }, method: 'PUT', headers: (params) => ({ diff --git a/apps/sim/tools/gitlab/utils.test.ts b/apps/sim/tools/gitlab/utils.test.ts new file mode 100644 index 00000000000..f7eca36aef8 --- /dev/null +++ b/apps/sim/tools/gitlab/utils.test.ts @@ -0,0 +1,71 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import { getGitLabApiBase, normalizeGitLabHost, UnsafeGitLabHostError } from '@/tools/gitlab/utils' + +describe('normalizeGitLabHost', () => { + it('defaults to gitlab.com when the host is empty, blank, or not a string', () => { + expect(normalizeGitLabHost(undefined)).toBe('gitlab.com') + expect(normalizeGitLabHost(null)).toBe('gitlab.com') + expect(normalizeGitLabHost('')).toBe('gitlab.com') + expect(normalizeGitLabHost(' ')).toBe('gitlab.com') + expect(normalizeGitLabHost(42)).toBe('gitlab.com') + }) + + it('strips protocol and trailing slashes from a self-managed host', () => { + expect(normalizeGitLabHost('gitlab.example.com')).toBe('gitlab.example.com') + expect(normalizeGitLabHost('https://gitlab.example.com')).toBe('gitlab.example.com') + expect(normalizeGitLabHost('http://gitlab.example.com/')).toBe('gitlab.example.com') + expect(normalizeGitLabHost(' https://gitlab.example.com// ')).toBe('gitlab.example.com') + }) + + it('preserves an explicit port and IDN punycode labels', () => { + expect(normalizeGitLabHost('gitlab.example.com:8443')).toBe('gitlab.example.com:8443') + expect(normalizeGitLabHost('xn--80ak6aa92e.com')).toBe('xn--80ak6aa92e.com') + }) + + it('rejects hosts that could redirect the request authority (SSRF / token exfiltration)', () => { + const unsafe = [ + 'legit.com@evil.com', + 'user:pass@evil.com', + 'gitlab.com#@evil.com', + 'gitlab.com /api', + 'line\nbreak.com', + 'evil.com/path', + 'evil.com?x=1', + '[::1]', + 'a..b.com', + '.gitlab.com', + 'gitlab.com.', + ] + for (const host of unsafe) { + expect(() => normalizeGitLabHost(host), host).toThrow(UnsafeGitLabHostError) + } + }) + + it('accepts bare IP literals at the STRUCTURAL layer by design (private/metadata IPs are rejected later by the fetch-layer DNS guard)', () => { + // This guard is structural only — it prevents authority confusion (userinfo, + // path, whitespace). SSRF to private/loopback/metadata addresses is the + // responsibility of validateUrlWithDNS / secureFetchWithValidation at fetch + // time, the single SSRF chokepoint shared by tools, webhooks, and connectors. + // These hosts are therefore structurally valid here, then blocked at fetch. + expect(normalizeGitLabHost('127.0.0.1')).toBe('127.0.0.1') + expect(normalizeGitLabHost('169.254.169.254')).toBe('169.254.169.254') + expect(normalizeGitLabHost('localhost')).toBe('localhost') + }) +}) + +describe('getGitLabApiBase', () => { + it('builds the v4 REST base for the default and self-managed hosts', () => { + expect(getGitLabApiBase(undefined)).toBe('https://gitlab.com/api/v4') + expect(getGitLabApiBase('gitlab.example.com')).toBe('https://gitlab.example.com/api/v4') + expect(getGitLabApiBase('https://gitlab.example.com:8443/')).toBe( + 'https://gitlab.example.com:8443/api/v4' + ) + }) + + it('propagates rejection of unsafe hosts', () => { + expect(() => getGitLabApiBase('legit.com@evil.com')).toThrow(UnsafeGitLabHostError) + }) +}) diff --git a/apps/sim/tools/gitlab/utils.ts b/apps/sim/tools/gitlab/utils.ts new file mode 100644 index 00000000000..6334a7030ee --- /dev/null +++ b/apps/sim/tools/gitlab/utils.ts @@ -0,0 +1,68 @@ +const DEFAULT_GITLAB_HOST = 'gitlab.com' + +/** + * Error thrown when a user-supplied GitLab host is structurally unsafe to use + * as the target of a server-side request that carries the user's access token. + */ +export class UnsafeGitLabHostError extends Error { + constructor(rawHost: string) { + super(`Invalid GitLab host: ${rawHost}`) + this.name = 'UnsafeGitLabHostError' + } +} + +/** + * Rejects a host that is structurally unsafe to fetch with the caller's token. + * + * The host is later interpolated into `https:///api/v4`, so anything that + * could change the request's authority (userinfo `@`, an embedded path/query/ + * fragment, whitespace, or control characters) must be rejected to prevent the + * `PRIVATE-TOKEN` header from being sent to an attacker-controlled origin. The + * allowed alphabet is hostname labels plus an optional `:port`, so self-managed + * hosts such as `gitlab.example.com` or `gitlab.example.com:8443` keep working. + * This is a structural guard only; DNS-based private-IP/SSRF checks remain the + * responsibility of the fetch layer. + */ +function assertSafeGitLabHostString(host: string, rawHost: string): void { + const hostnameWithoutPort = host.replace(/:\d+$/, '') + const allowedHostChars = /^[A-Za-z0-9.-]+$/ + if (!allowedHostChars.test(hostnameWithoutPort)) { + throw new UnsafeGitLabHostError(rawHost) + } + if (hostnameWithoutPort.startsWith('.') || hostnameWithoutPort.endsWith('.')) { + throw new UnsafeGitLabHostError(rawHost) + } + if (hostnameWithoutPort.split('.').some((label) => label.length === 0)) { + throw new UnsafeGitLabHostError(rawHost) + } +} + +/** + * Normalizes a GitLab host value: trims whitespace, strips any protocol prefix + * and trailing slashes, validates that the result is a bare host (optionally + * with a port), and falls back to gitlab.com when empty. Mirrors the GitLab + * connector so tools, triggers, and connectors resolve hosts identically. + * + * @throws {UnsafeGitLabHostError} when a non-empty host is structurally unsafe. + */ +export function normalizeGitLabHost(rawHost: unknown): string { + const raw = typeof rawHost === 'string' ? rawHost.trim() : '' + if (!raw) return DEFAULT_GITLAB_HOST + const host = raw + .replace(/^https?:\/\//i, '') + .replace(/\/+$/, '') + .trim() + if (!host) return DEFAULT_GITLAB_HOST + assertSafeGitLabHostString(host, String(rawHost)) + return host +} + +/** + * Builds the REST API v4 base URL for the configured host. Defaults to + * gitlab.com so existing workflows that never set a host keep working. + * + * @throws {UnsafeGitLabHostError} when a non-empty host is structurally unsafe. + */ +export function getGitLabApiBase(rawHost: unknown): string { + return `https://${normalizeGitLabHost(rawHost)}/api/v4` +} diff --git a/apps/sim/triggers/gitlab/utils.ts b/apps/sim/triggers/gitlab/utils.ts index 6f7848ad2fe..a25d09e3447 100644 --- a/apps/sim/triggers/gitlab/utils.ts +++ b/apps/sim/triggers/gitlab/utils.ts @@ -103,6 +103,15 @@ export function buildGitLabExtraFields(triggerId: string): SubBlockConfig[] { mode: 'trigger', condition: { field: 'selectedTriggerId', value: triggerId }, }, + { + id: 'host', + title: 'GitLab Host', + type: 'short-input', + placeholder: 'gitlab.com', + description: 'Self-managed GitLab host. Leave blank for gitlab.com.', + mode: 'trigger', + condition: { field: 'selectedTriggerId', value: triggerId }, + }, ] } diff --git a/apps/sim/triggers/slack/capabilities.ts b/apps/sim/triggers/slack/capabilities.ts index 1a532e4ebf2..317c240621f 100644 --- a/apps/sim/triggers/slack/capabilities.ts +++ b/apps/sim/triggers/slack/capabilities.ts @@ -105,16 +105,7 @@ export const SLACK_CAPABILITIES: readonly SlackCapability[] = [ scopes: ['channels:history', 'groups:history', 'im:history', 'mpim:history'], events: [], }, - { - id: 'action_assistant', - label: 'Manage assistant threads', - description: - "Let the bot set the status indicator (the 'is thinking…' shimmer), title, and suggested prompts on AI app threads.", - defaultChecked: true, - group: 'action', - scopes: ['assistant:write'], - events: [], - }, + // TODO: Restore the 'action_assistant' capability (scope 'assistant:write') once Slack app review is approved { id: 'action_read_files', label: 'Read file attachments', From e1c3c7f6c97744bc1b193601a056963b32ef6413 Mon Sep 17 00:00:00 2001 From: Theodore Li Date: Wed, 24 Jun 2026 13:23:37 -0700 Subject: [PATCH 07/15] feat(secrets): ingest env secrets at container runtime instead of fanning into ECS taskdef (#5189) * feat(secrets): ingest env secrets at container runtime instead of fanning into ECS taskdef The app/socket ECS taskdefs were ~42KB, ~93% of which was the secrets[] array: 268 pointer entries each restating the full ~78-char secret ARN, marching toward the 64KB taskdef limit and growing ~150 bytes per hosted key added. The secret blob itself is only ~18KB/268 keys. Move secret delivery to container boot: new @sim/runtime-secrets loadRuntimeSecrets() reads SIM_ENV_SECRET_ID, fetches the combined secret once, and hydrates process.env (no-clobber, no-op when unset, fail-fast). Bootstrap entrypoints for app + realtime await it before importing the real server (env-flags reads env at module load). The app bootstrap is bun-bundled in the Dockerfile builder stage since it runs outside the Next standalone bundle; realtime keeps full node_modules and runs the TS entry. Backward-compatible: with the current fan-out taskdef the loader no-ops and the app reads the injected env vars unchanged. The matching infra change (empty secrets[] + SIM_ENV_SECRET_ID) ships separately, after this image is live. * fix(runtime-secrets): address review feedback - Move the binary-secret guard outside the retry loop (sendWithRetry) so a missing SecretString throws immediately instead of burning 3 attempts + backoff. - Bound each Secrets Manager request with AbortSignal.timeout(5s) so a stalled response can't hang boot indefinitely. - Drop the redundant @aws-sdk/client-secrets-manager pin from apps/realtime; it resolves transitively via @sim/runtime-secrets. - Add a test for the non-retriable binary-secret path. --- apps/realtime/package.json | 1 + apps/realtime/src/bootstrap.ts | 9 ++ apps/sim/bootstrap.ts | 13 +++ apps/sim/package.json | 1 + bun.lock | 24 +++++ docker/app.Dockerfile | 13 ++- docker/realtime.Dockerfile | 2 +- packages/runtime-secrets/package.json | 38 ++++++++ packages/runtime-secrets/src/index.test.ts | 91 ++++++++++++++++++ packages/runtime-secrets/src/index.ts | 102 +++++++++++++++++++++ packages/runtime-secrets/tsconfig.json | 5 + packages/runtime-secrets/vitest.config.ts | 7 ++ 12 files changed, 304 insertions(+), 2 deletions(-) create mode 100644 apps/realtime/src/bootstrap.ts create mode 100644 apps/sim/bootstrap.ts create mode 100644 packages/runtime-secrets/package.json create mode 100644 packages/runtime-secrets/src/index.test.ts create mode 100644 packages/runtime-secrets/src/index.ts create mode 100644 packages/runtime-secrets/tsconfig.json create mode 100644 packages/runtime-secrets/vitest.config.ts diff --git a/apps/realtime/package.json b/apps/realtime/package.json index 99867ef852d..e8b1e1607be 100644 --- a/apps/realtime/package.json +++ b/apps/realtime/package.json @@ -26,6 +26,7 @@ "@sim/logger": "workspace:*", "@sim/platform-authz": "workspace:*", "@sim/realtime-protocol": "workspace:*", + "@sim/runtime-secrets": "workspace:*", "@sim/security": "workspace:*", "@sim/utils": "workspace:*", "@sim/workflow-persistence": "workspace:*", diff --git a/apps/realtime/src/bootstrap.ts b/apps/realtime/src/bootstrap.ts new file mode 100644 index 00000000000..fe786372052 --- /dev/null +++ b/apps/realtime/src/bootstrap.ts @@ -0,0 +1,9 @@ +/** + * Container entrypoint. Hydrates `process.env` from the runtime secret before + * loading the Socket.IO server, whose modules (`@/env`, DB preflight) read env + * at import time. See `@sim/runtime-secrets`. + */ +import { loadRuntimeSecrets } from '@sim/runtime-secrets' + +await loadRuntimeSecrets() +await import('@/index') diff --git a/apps/sim/bootstrap.ts b/apps/sim/bootstrap.ts new file mode 100644 index 00000000000..bc2e92b882c --- /dev/null +++ b/apps/sim/bootstrap.ts @@ -0,0 +1,13 @@ +/** + * Container entrypoint. Hydrates `process.env` from the runtime secret before + * loading the Next.js standalone server, so application modules that read env at + * import time see the full configuration. See `@sim/runtime-secrets`. + */ +import { loadRuntimeSecrets } from '@sim/runtime-secrets' + +await loadRuntimeSecrets() +// `server.js` is the Next standalone build artifact, a sibling of this file in +// the image; it does not exist at type-check time, so the specifier is held in a +// variable to keep it out of static module resolution. +const standaloneServer = './server.js' +await import(standaloneServer) diff --git a/apps/sim/package.json b/apps/sim/package.json index 88e9575836d..dcb9c2bc649 100644 --- a/apps/sim/package.json +++ b/apps/sim/package.json @@ -99,6 +99,7 @@ "@sim/logger": "workspace:*", "@sim/platform-authz": "workspace:*", "@sim/realtime-protocol": "workspace:*", + "@sim/runtime-secrets": "workspace:*", "@sim/security": "workspace:*", "@sim/utils": "workspace:*", "@sim/workflow-persistence": "workspace:*", diff --git a/bun.lock b/bun.lock index e9bb4f978b3..c20f6bfa669 100644 --- a/bun.lock +++ b/bun.lock @@ -1,5 +1,6 @@ { "lockfileVersion": 1, + "configVersion": 0, "workspaces": { "": { "name": "simstudio", @@ -67,6 +68,7 @@ "@sim/logger": "workspace:*", "@sim/platform-authz": "workspace:*", "@sim/realtime-protocol": "workspace:*", + "@sim/runtime-secrets": "workspace:*", "@sim/security": "workspace:*", "@sim/utils": "workspace:*", "@sim/workflow-persistence": "workspace:*", @@ -158,6 +160,7 @@ "@sim/logger": "workspace:*", "@sim/platform-authz": "workspace:*", "@sim/realtime-protocol": "workspace:*", + "@sim/runtime-secrets": "workspace:*", "@sim/security": "workspace:*", "@sim/utils": "workspace:*", "@sim/workflow-persistence": "workspace:*", @@ -400,6 +403,21 @@ "typescript": "^5.7.3", }, }, + "packages/runtime-secrets": { + "name": "@sim/runtime-secrets", + "version": "0.1.0", + "dependencies": { + "@aws-sdk/client-secrets-manager": "3.1032.0", + "@sim/logger": "workspace:*", + "@sim/utils": "workspace:*", + }, + "devDependencies": { + "@sim/tsconfig": "workspace:*", + "@types/node": "24.2.1", + "typescript": "^5.7.3", + "vitest": "^4.1.0", + }, + }, "packages/security": { "name": "@sim/security", "version": "0.1.0", @@ -1469,6 +1487,8 @@ "@sim/realtime-protocol": ["@sim/realtime-protocol@workspace:packages/realtime-protocol"], + "@sim/runtime-secrets": ["@sim/runtime-secrets@workspace:packages/runtime-secrets"], + "@sim/security": ["@sim/security@workspace:packages/security"], "@sim/testing": ["@sim/testing@workspace:packages/testing"], @@ -4237,6 +4257,8 @@ "@sim/realtime/@types/node": ["@types/node@24.2.1", "", { "dependencies": { "undici-types": "~7.10.0" } }, "sha512-DRh5K+ka5eJic8CjH7td8QpYEV6Zo10gfRkjHCO3weqZHWDtAaSTFtl4+VMqOJ4N5jcuhZ9/l+yy8rVgw7BQeQ=="], + "@sim/runtime-secrets/@types/node": ["@types/node@24.2.1", "", { "dependencies": { "undici-types": "~7.10.0" } }, "sha512-DRh5K+ka5eJic8CjH7td8QpYEV6Zo10gfRkjHCO3weqZHWDtAaSTFtl4+VMqOJ4N5jcuhZ9/l+yy8rVgw7BQeQ=="], + "@sim/security/@types/node": ["@types/node@24.2.1", "", { "dependencies": { "undici-types": "~7.10.0" } }, "sha512-DRh5K+ka5eJic8CjH7td8QpYEV6Zo10gfRkjHCO3weqZHWDtAaSTFtl4+VMqOJ4N5jcuhZ9/l+yy8rVgw7BQeQ=="], "@smithy/middleware-compression/fflate": ["fflate@0.8.1", "", {}, "sha512-/exOvEuc+/iaUm105QIiOt4LpBdMTWsXxqR0HDF35vx3fmaKzw7354gTilCh5rkzEt8WYyG//ku3h3nRmd7CHQ=="], @@ -4749,6 +4771,8 @@ "@sim/realtime/@types/node/undici-types": ["undici-types@7.10.0", "", {}, "sha512-t5Fy/nfn+14LuOc2KNYg75vZqClpAiqscVvMygNnlsHBFpSXdJaYtXMcdNLpl/Qvc3P2cB3s6lOV51nqsFq4ag=="], + "@sim/runtime-secrets/@types/node/undici-types": ["undici-types@7.10.0", "", {}, "sha512-t5Fy/nfn+14LuOc2KNYg75vZqClpAiqscVvMygNnlsHBFpSXdJaYtXMcdNLpl/Qvc3P2cB3s6lOV51nqsFq4ag=="], + "@sim/security/@types/node/undici-types": ["undici-types@7.10.0", "", {}, "sha512-t5Fy/nfn+14LuOc2KNYg75vZqClpAiqscVvMygNnlsHBFpSXdJaYtXMcdNLpl/Qvc3P2cB3s6lOV51nqsFq4ag=="], "@trigger.dev/core/@opentelemetry/api-logs/@opentelemetry/api": ["@opentelemetry/api@1.9.1", "", {}, "sha512-gLyJlPHPZYdAk1JENA9LeHejZe1Ti77/pTeFm/nMXmQH/HFZlcS/O2XJB+L8fkbrNSqhdtlvjBVjxwUYanNH5Q=="], diff --git a/docker/app.Dockerfile b/docker/app.Dockerfile index ff0ea1ccc28..e6e7f22bb53 100644 --- a/docker/app.Dockerfile +++ b/docker/app.Dockerfile @@ -81,6 +81,13 @@ RUN --mount=type=cache,id=next-cache-${TARGETPLATFORM},target=/app/apps/sim/.nex --mount=type=cache,id=turbo-cache-${TARGETPLATFORM},target=/app/.turbo \ bun run build +# Bundle the secrets-loading bootstrap into a self-contained entrypoint. It runs +# before (and outside) the Next standalone server, so its dependencies +# (@sim/runtime-secrets, AWS SDK) are inlined here rather than resolved from the +# pruned standalone node_modules. The dynamic import of ./server.js stays a +# runtime import. +RUN bun build apps/sim/bootstrap.ts --target=bun --outfile=apps/sim/bootstrap.js + # ======================================== # Runner Stage: Run the actual app # ======================================== @@ -100,6 +107,10 @@ COPY --from=builder --chown=nextjs:nodejs /app/apps/sim/public ./apps/sim/public COPY --from=builder --chown=nextjs:nodejs /app/apps/sim/.next/standalone ./ COPY --from=builder --chown=nextjs:nodejs /app/apps/sim/.next/static ./apps/sim/.next/static +# Self-contained secrets-loading bootstrap (bundled in the builder stage). Runs +# before the standalone server.js to hydrate process.env from the runtime secret. +COPY --from=builder --chown=nextjs:nodejs /app/apps/sim/bootstrap.js ./apps/sim/bootstrap.js + # Copy blog/author content for runtime filesystem reads (not part of the JS bundle) COPY --from=builder --chown=nextjs:nodejs /app/apps/sim/content ./apps/sim/content @@ -128,4 +139,4 @@ EXPOSE 3000 ENV PORT=3000 \ HOSTNAME="0.0.0.0" -CMD ["bun", "apps/sim/server.js"] +CMD ["bun", "apps/sim/bootstrap.js"] diff --git a/docker/realtime.Dockerfile b/docker/realtime.Dockerfile index 16f3cd1c32f..d403c906462 100644 --- a/docker/realtime.Dockerfile +++ b/docker/realtime.Dockerfile @@ -49,4 +49,4 @@ USER nextjs EXPOSE 3002 -CMD ["bun", "apps/realtime/src/index.ts"] +CMD ["bun", "apps/realtime/src/bootstrap.ts"] diff --git a/packages/runtime-secrets/package.json b/packages/runtime-secrets/package.json new file mode 100644 index 00000000000..ee57201f3d1 --- /dev/null +++ b/packages/runtime-secrets/package.json @@ -0,0 +1,38 @@ +{ + "name": "@sim/runtime-secrets", + "version": "0.1.0", + "private": true, + "sideEffects": false, + "type": "module", + "license": "Apache-2.0", + "engines": { + "bun": ">=1.2.13", + "node": ">=20.0.0" + }, + "exports": { + ".": { + "types": "./src/index.ts", + "default": "./src/index.ts" + } + }, + "scripts": { + "type-check": "tsc --noEmit", + "lint": "biome check --write --unsafe .", + "lint:check": "biome check .", + "format": "biome format --write .", + "format:check": "biome format .", + "test": "vitest run", + "test:watch": "vitest" + }, + "dependencies": { + "@aws-sdk/client-secrets-manager": "3.1032.0", + "@sim/logger": "workspace:*", + "@sim/utils": "workspace:*" + }, + "devDependencies": { + "@sim/tsconfig": "workspace:*", + "@types/node": "24.2.1", + "typescript": "^5.7.3", + "vitest": "^4.1.0" + } +} diff --git a/packages/runtime-secrets/src/index.test.ts b/packages/runtime-secrets/src/index.test.ts new file mode 100644 index 00000000000..dfd5cec1f4b --- /dev/null +++ b/packages/runtime-secrets/src/index.test.ts @@ -0,0 +1,91 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockSend } = vi.hoisted(() => ({ mockSend: vi.fn() })) + +vi.mock('@aws-sdk/client-secrets-manager', () => ({ + SecretsManagerClient: class SecretsManagerClient { + send = mockSend + }, + GetSecretValueCommand: class GetSecretValueCommand { + constructor(public input: unknown) {} + }, +})) + +vi.mock('@sim/logger', () => ({ + createLogger: () => ({ info: vi.fn(), warn: vi.fn(), error: vi.fn() }), +})) + +vi.mock('@sim/utils/helpers', () => ({ + sleep: vi.fn().mockResolvedValue(undefined), +})) + +import { loadRuntimeSecrets } from './index' + +const TOUCHED = ['SIM_ENV_SECRET_ID', 'FOO', 'BAZ'] as const + +describe('loadRuntimeSecrets', () => { + beforeEach(() => { + vi.clearAllMocks() + for (const key of TOUCHED) delete process.env[key] + }) + + afterEach(() => { + for (const key of TOUCHED) delete process.env[key] + }) + + it('no-ops when SIM_ENV_SECRET_ID is unset', async () => { + await loadRuntimeSecrets() + expect(mockSend).not.toHaveBeenCalled() + }) + + it('hydrates process.env from the parsed secret JSON', async () => { + process.env.SIM_ENV_SECRET_ID = '/test/sim/env-vars' + mockSend.mockResolvedValue({ SecretString: JSON.stringify({ FOO: 'bar', BAZ: 'qux' }) }) + + await loadRuntimeSecrets() + + expect(process.env.FOO).toBe('bar') + expect(process.env.BAZ).toBe('qux') + }) + + it('never overwrites an already-set env var', async () => { + process.env.SIM_ENV_SECRET_ID = '/test/sim/env-vars' + process.env.FOO = 'existing' + mockSend.mockResolvedValue({ SecretString: JSON.stringify({ FOO: 'new', BAZ: 'qux' }) }) + + await loadRuntimeSecrets() + + expect(process.env.FOO).toBe('existing') + expect(process.env.BAZ).toBe('qux') + }) + + it('throws when the secret is not valid JSON', async () => { + process.env.SIM_ENV_SECRET_ID = '/test/sim/env-vars' + mockSend.mockResolvedValue({ SecretString: 'not json' }) + + await expect(loadRuntimeSecrets()).rejects.toThrow(/not valid JSON/) + }) + + it('throws when the secret JSON is not an object', async () => { + process.env.SIM_ENV_SECRET_ID = '/test/sim/env-vars' + mockSend.mockResolvedValue({ SecretString: JSON.stringify(['a', 'b']) }) + + await expect(loadRuntimeSecrets()).rejects.toThrow(/must be a JSON object/) + }) + + it('throws immediately on a binary secret (no SecretString), without retrying', async () => { + process.env.SIM_ENV_SECRET_ID = '/test/sim/env-vars' + mockSend.mockResolvedValue({}) + + await expect(loadRuntimeSecrets()).rejects.toThrow(/binary secrets/) + expect(mockSend).toHaveBeenCalledTimes(1) + }) + + it('retries then throws when the fetch keeps failing', async () => { + process.env.SIM_ENV_SECRET_ID = '/test/sim/env-vars' + mockSend.mockRejectedValue(new Error('boom')) + + await expect(loadRuntimeSecrets()).rejects.toThrow(/Failed to fetch runtime secrets/) + expect(mockSend).toHaveBeenCalledTimes(3) + }) +}) diff --git a/packages/runtime-secrets/src/index.ts b/packages/runtime-secrets/src/index.ts new file mode 100644 index 00000000000..86c79e7952d --- /dev/null +++ b/packages/runtime-secrets/src/index.ts @@ -0,0 +1,102 @@ +import type { GetSecretValueCommandOutput } from '@aws-sdk/client-secrets-manager' +import { GetSecretValueCommand, SecretsManagerClient } from '@aws-sdk/client-secrets-manager' +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import { sleep } from '@sim/utils/helpers' +import { backoffWithJitter } from '@sim/utils/retry' + +const logger = createLogger('RuntimeSecrets') + +/** Plaintext env var (set in the ECS task definition) naming the secret to ingest. */ +const SECRET_ID_ENV = 'SIM_ENV_SECRET_ID' + +const MAX_ATTEMPTS = 3 + +/** Bounds each Secrets Manager request so a stalled response can't hang boot. */ +const REQUEST_TIMEOUT_MS = 5000 + +/** + * Fetches the combined `/{env}/sim/env-vars` secret once at container boot and + * hydrates `process.env`, so secrets no longer have to be fanned out into the + * ECS task definition (which is approaching the 64 KB rendered-document limit). + * + * Must run before any application module that reads env at import time. No-ops + * when {@link SECRET_ID_ENV} is unset (local dev / self-hosted keep using their + * own env). Existing `process.env` keys are never overwritten, so explicit + * task-definition `environment` entries win. Throws on any fetch/parse failure + * so a misconfigured container crashes instead of booting without its config. + */ +export async function loadRuntimeSecrets(): Promise { + const secretId = process.env[SECRET_ID_ENV] + if (!secretId) { + logger.info(`${SECRET_ID_ENV} not set; skipping runtime secret ingestion`) + return + } + + const client = new SecretsManagerClient( + process.env.AWS_REGION ? { region: process.env.AWS_REGION } : {} + ) + + const secretString = await fetchSecretString(client, secretId) + const entries = parseSecretJson(secretString) + + let loaded = 0 + let skipped = 0 + for (const [key, value] of Object.entries(entries)) { + if (key in process.env) { + skipped++ + continue + } + process.env[key] = typeof value === 'string' ? value : JSON.stringify(value) + loaded++ + } + + logger.info('Runtime secrets ingested', { secretId, loaded, skipped }) +} + +async function fetchSecretString(client: SecretsManagerClient, secretId: string): Promise { + const response = await sendWithRetry(client, secretId) + if (!response.SecretString) { + // Non-retriable: a binary secret will never become a string between attempts. + throw new Error('Secret has no SecretString (binary secrets are not supported)') + } + return response.SecretString +} + +async function sendWithRetry( + client: SecretsManagerClient, + secretId: string +): Promise { + let lastError: unknown + for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) { + try { + return await client.send(new GetSecretValueCommand({ SecretId: secretId }), { + abortSignal: AbortSignal.timeout(REQUEST_TIMEOUT_MS), + }) + } catch (error) { + lastError = error + if (attempt < MAX_ATTEMPTS) { + const delay = backoffWithJitter(attempt, null, { baseMs: 200, maxMs: 2000 }) + logger.warn( + `Failed to fetch runtime secrets (attempt ${attempt}/${MAX_ATTEMPTS}), retrying`, + { error: getErrorMessage(error) } + ) + await sleep(delay) + } + } + } + throw new Error(`Failed to fetch runtime secrets from ${secretId}: ${getErrorMessage(lastError)}`) +} + +function parseSecretJson(secretString: string): Record { + let parsed: unknown + try { + parsed = JSON.parse(secretString) + } catch (error) { + throw new Error(`Runtime secret is not valid JSON: ${getErrorMessage(error)}`) + } + if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) { + throw new Error('Runtime secret must be a JSON object of key/value pairs') + } + return parsed as Record +} diff --git a/packages/runtime-secrets/tsconfig.json b/packages/runtime-secrets/tsconfig.json new file mode 100644 index 00000000000..1ffa3d2e844 --- /dev/null +++ b/packages/runtime-secrets/tsconfig.json @@ -0,0 +1,5 @@ +{ + "extends": "@sim/tsconfig/library.json", + "include": ["src/**/*"], + "exclude": ["node_modules", "dist"] +} diff --git a/packages/runtime-secrets/vitest.config.ts b/packages/runtime-secrets/vitest.config.ts new file mode 100644 index 00000000000..2b1c323fe22 --- /dev/null +++ b/packages/runtime-secrets/vitest.config.ts @@ -0,0 +1,7 @@ +import { defineConfig } from 'vitest/config' + +export default defineConfig({ + test: { + environment: 'node', + }, +}) From 5a938e5d564eaaf0a519f137a45aa6a8f3c5828a Mon Sep 17 00:00:00 2001 From: Theodore Li Date: Wed, 24 Jun 2026 16:29:00 -0700 Subject: [PATCH 08/15] improvement(sandbox): mount workspace files by presigned URL instead of buffering bytes (#5202) * improvement(sandbox): mount workspace files by presigned URL instead of buffering bytes Files and directories mounted into the function_execute sandbox were downloaded into the web process, re-encoded, and shipped inline. Mirror the table-snapshot path: under cloud storage, presign each file and let the sandbox curl it directly (no web-heap transit). Local storage keeps the buffered fallback. Add a count cap on the inputFiles list and a generous aggregate URL-mount byte ceiling so oversized requests fail fast instead of filling sandbox disk. * improvement(sandbox): use mount path in size-limit errors, display GB, add directory local-fallback test --- .../tools/handlers/function-execute.test.ts | 149 +++++++++++++++++- .../tools/handlers/function-execute.ts | 147 +++++++++++------ 2 files changed, 244 insertions(+), 52 deletions(-) diff --git a/apps/sim/lib/copilot/tools/handlers/function-execute.test.ts b/apps/sim/lib/copilot/tools/handlers/function-execute.test.ts index b47286a03b2..f6494b14aa0 100644 --- a/apps/sim/lib/copilot/tools/handlers/function-execute.test.ts +++ b/apps/sim/lib/copilot/tools/handlers/function-execute.test.ts @@ -13,6 +13,11 @@ const { mockGeneratePresignedDownloadUrl, mockHasCloudStorage, mockExecuteTool, + mockListWorkspaceFiles, + mockFindWorkspaceFileRecord, + mockFetchWorkspaceFileBuffer, + mockGetSandboxWorkspaceFilePath, + mockListWorkspaceFileFolders, } = vi.hoisted(() => ({ mockIsFeatureEnabled: vi.fn(), mockGetTableById: vi.fn(), @@ -23,6 +28,11 @@ const { mockGeneratePresignedDownloadUrl: vi.fn(), mockHasCloudStorage: vi.fn(), mockExecuteTool: vi.fn(), + mockListWorkspaceFiles: vi.fn(), + mockFindWorkspaceFileRecord: vi.fn(), + mockFetchWorkspaceFileBuffer: vi.fn(), + mockGetSandboxWorkspaceFilePath: vi.fn(), + mockListWorkspaceFileFolders: vi.fn(), })) vi.mock('@/lib/core/config/feature-flags', () => ({ isFeatureEnabled: mockIsFeatureEnabled })) @@ -41,15 +51,14 @@ vi.mock('@/lib/uploads/core/storage-service', () => ({ hasCloudStorage: mockHasCloudStorage, })) vi.mock('@/tools', () => ({ executeTool: mockExecuteTool })) -// Workspace-file + VFS surfaces are unused on the tables-only path; stub to avoid heavy loads. vi.mock('@/lib/uploads/contexts/workspace/workspace-file-manager', () => ({ - fetchWorkspaceFileBuffer: vi.fn(), - findWorkspaceFileRecord: vi.fn(), - getSandboxWorkspaceFilePath: vi.fn(), - listWorkspaceFiles: vi.fn(), + fetchWorkspaceFileBuffer: mockFetchWorkspaceFileBuffer, + findWorkspaceFileRecord: mockFindWorkspaceFileRecord, + getSandboxWorkspaceFilePath: mockGetSandboxWorkspaceFilePath, + listWorkspaceFiles: mockListWorkspaceFiles, })) vi.mock('@/lib/uploads/contexts/workspace/workspace-file-folder-manager', () => ({ - listWorkspaceFileFolders: vi.fn(), + listWorkspaceFileFolders: mockListWorkspaceFileFolders, })) vi.mock('@/lib/copilot/vfs/path-utils', () => ({ decodeVfsPathSegments: (p: string) => p.split('/'), @@ -247,3 +256,131 @@ describe('executeFunctionExecute table mounts', () => { expect(mockGetOrCreateTableSnapshot).not.toHaveBeenCalled() }) }) + +const fileRecord = { + id: 'file_1', + workspaceId: 'ws_1', + name: 'data.csv', + key: 'workspace/ws_1/data.csv', + path: '/api/files/serve/workspace%2Fws_1%2Fdata.csv', + size: 100, + type: 'text/csv', + storageContext: 'workspace' as const, +} + +describe('executeFunctionExecute file mounts', () => { + beforeEach(() => { + vi.clearAllMocks() + mockExecuteTool.mockResolvedValue({ success: true }) + mockIsFeatureEnabled.mockResolvedValue(false) + mockHasCloudStorage.mockReturnValue(true) + mockGeneratePresignedDownloadUrl.mockResolvedValue('https://s3.example/file?sig=abc') + mockListWorkspaceFiles.mockResolvedValue([fileRecord]) + mockFindWorkspaceFileRecord.mockReturnValue(fileRecord) + mockGetSandboxWorkspaceFilePath.mockReturnValue('/home/user/files/data.csv') + }) + + it('cloud storage: mounts by presigned URL with the record context, no bytes through web', async () => { + await executeFunctionExecute({ inputFiles: ['files/data.csv'] }, context as never) + + expect(mockFetchWorkspaceFileBuffer).not.toHaveBeenCalled() + expect(mockGeneratePresignedDownloadUrl).toHaveBeenCalledWith( + 'workspace/ws_1/data.csv', + 'workspace', + expect.any(Number) + ) + expect(mountedFiles()[0]).toEqual({ + type: 'url', + path: '/home/user/files/data.csv', + url: 'https://s3.example/file?sig=abc', + }) + }) + + it('local storage: falls back to a buffered inline content mount', async () => { + mockHasCloudStorage.mockReturnValue(false) + mockFetchWorkspaceFileBuffer.mockResolvedValue(Buffer.from('name\nAda\n')) + + await executeFunctionExecute({ inputFiles: ['files/data.csv'] }, context as never) + + expect(mockGeneratePresignedDownloadUrl).not.toHaveBeenCalled() + const file = mountedFiles()[0] + expect(file.path).toBe('/home/user/files/data.csv') + expect(file.content).toBe('name\nAda\n') + expect(file.type).toBeUndefined() + }) + + it('cloud storage: throws when a file exceeds the per-file URL mount limit', async () => { + mockFindWorkspaceFileRecord.mockReturnValue({ ...fileRecord, size: 600 * 1024 * 1024 }) + + await expect( + executeFunctionExecute({ inputFiles: ['files/data.csv'] }, context as never) + ).rejects.toThrow(/per-file mount limit/) + expect(mockGeneratePresignedDownloadUrl).not.toHaveBeenCalled() + }) + + it('cloud storage: throws when mounts exceed the aggregate URL mount limit', async () => { + // Each file is at the 500MB per-file cap; the 5th pushes the running total past 2GB. + mockFindWorkspaceFileRecord.mockReturnValue({ ...fileRecord, size: 500 * 1024 * 1024 }) + const paths = Array.from({ length: 5 }, (_, i) => `files/big-${i}.csv`) + + await expect(executeFunctionExecute({ inputFiles: paths }, context as never)).rejects.toThrow( + /total mount limit/ + ) + expect(mockGeneratePresignedDownloadUrl).toHaveBeenCalledTimes(4) + }) + + it('throws when the inputFiles list exceeds the mounted-file count cap', async () => { + const paths = Array.from({ length: 501 }, (_, i) => `files/f-${i}.csv`) + + await expect(executeFunctionExecute({ inputFiles: paths }, context as never)).rejects.toThrow( + /Too many input files/ + ) + expect(mockListWorkspaceFiles).not.toHaveBeenCalled() + }) + + it('cloud storage: mounts each directory descendant by presigned URL', async () => { + mockListWorkspaceFileFolders.mockResolvedValue([{ path: 'Reports' }]) + const descendant = { + ...fileRecord, + name: 'q1.csv', + key: 'workspace/ws_1/q1.csv', + folderPath: 'Reports', + } + mockListWorkspaceFiles.mockResolvedValue([descendant]) + + await executeFunctionExecute({ inputs: { directories: ['files/Reports'] } }, context as never) + + expect(mockFetchWorkspaceFileBuffer).not.toHaveBeenCalled() + expect(mockGeneratePresignedDownloadUrl).toHaveBeenCalledWith( + 'workspace/ws_1/q1.csv', + 'workspace', + expect.any(Number) + ) + expect(mountedFiles()[0]).toEqual({ + type: 'url', + path: '/home/user/files/Reports/q1.csv', + url: 'https://s3.example/file?sig=abc', + }) + }) + + it('local storage: buffers directory descendants via inline content', async () => { + mockHasCloudStorage.mockReturnValue(false) + mockListWorkspaceFileFolders.mockResolvedValue([{ path: 'Reports' }]) + const descendant = { + ...fileRecord, + name: 'q1.csv', + key: 'workspace/ws_1/q1.csv', + folderPath: 'Reports', + } + mockListWorkspaceFiles.mockResolvedValue([descendant]) + mockFetchWorkspaceFileBuffer.mockResolvedValue(Buffer.from('a,b\n1,2\n')) + + await executeFunctionExecute({ inputs: { directories: ['files/Reports'] } }, context as never) + + expect(mockGeneratePresignedDownloadUrl).not.toHaveBeenCalled() + const file = mountedFiles()[0] + expect(file.path).toBe('/home/user/files/Reports/q1.csv') + expect(file.content).toBe('a,b\n1,2\n') + expect(file.type).toBeUndefined() + }) +}) diff --git a/apps/sim/lib/copilot/tools/handlers/function-execute.ts b/apps/sim/lib/copilot/tools/handlers/function-execute.ts index 2550b63dde1..bc32699d66a 100644 --- a/apps/sim/lib/copilot/tools/handlers/function-execute.ts +++ b/apps/sim/lib/copilot/tools/handlers/function-execute.ts @@ -14,6 +14,7 @@ import { findWorkspaceFileRecord, getSandboxWorkspaceFilePath, listWorkspaceFiles, + type WorkspaceFileRecord, } from '@/lib/uploads/contexts/workspace/workspace-file-manager' import { downloadFile, @@ -37,15 +38,98 @@ const MAX_MOUNTED_FILES = 500 const SNAPSHOT_MIN_ROWS = 500 /** - * Lifetime of the presigned URL handed to the sandbox to fetch a snapshot. Long enough to download - * a large file at sandbox startup; the URL grants read to only that one version-pinned object. + * Lifetime of a presigned URL handed to the sandbox to fetch a mounted object (table snapshot or + * workspace file). Long enough to download a large file at sandbox startup; the URL grants read to + * only that one object. */ -const SNAPSHOT_URL_TTL_SECONDS = 600 +const MOUNT_URL_TTL_SECONDS = 600 + +/** + * Per-file ceiling for URL-mounted workspace files. The bytes never transit the web process — the + * sandbox curls them straight from storage — so the bound is sandbox disk, not web heap (unlike the + * inline MAX_FILE_SIZE path). + */ +const MOUNT_URL_MAX_BYTES = 500 * 1024 * 1024 + +/** + * Aggregate ceiling across all URL-mounted files in one request. URL mounts bypass the web heap (so + * they don't count against MAX_TOTAL_SIZE), but the sandbox still curls every byte onto its disk — + * this rejects an oversized request up front instead of filling the sandbox disk one slow curl at a + * time. Generous vs MAX_TOTAL_SIZE since the bytes never transit web memory. + */ +const MAX_TOTAL_URL_BYTES = 2 * 1024 * 1024 * 1024 type SandboxFile = | { type?: 'content'; path: string; content: string; encoding?: 'base64' } | { type: 'url'; path: string; url: string } +/** + * Running byte totals for one resolveInputFiles call. `buffered` bytes pass through the web process + * (capped by MAX_TOTAL_SIZE); `url` bytes are curled straight into the sandbox (capped by + * MAX_TOTAL_URL_BYTES). Tracked separately because the two ceilings protect different resources — + * web heap vs sandbox disk. + */ +interface MountedBytes { + buffered: number + url: number +} + +/** + * Mounts a stored workspace file into the sandbox and records its bytes against the running totals. + * With cloud storage the sandbox fetches the bytes itself from a presigned URL (no web-heap transit, + * per-file ceiling MOUNT_URL_MAX_BYTES, aggregate ceiling MAX_TOTAL_URL_BYTES); with local storage a + * presigned URL is an app-internal serve path a remote sandbox can't reach, so we buffer the bytes + * through the web process under the inline MAX_FILE_SIZE / MAX_TOTAL_SIZE guards. + */ +async function pushWorkspaceFileMount( + sandboxFiles: SandboxFile[], + record: WorkspaceFileRecord, + mountPath: string, + mounted: MountedBytes +): Promise { + if (hasCloudStorage()) { + if (record.size > MOUNT_URL_MAX_BYTES) { + throw new Error( + `Input file "${mountPath}" is ${Math.round(record.size / 1024 / 1024)}MB, over the ${MOUNT_URL_MAX_BYTES / 1024 / 1024}MB per-file mount limit.` + ) + } + if (mounted.url + record.size > MAX_TOTAL_URL_BYTES) { + throw new Error( + `Mounting "${mountPath}" would exceed the ${MAX_TOTAL_URL_BYTES / 1024 / 1024 / 1024}GB total mount limit. Mount fewer or smaller files.` + ) + } + const url = await generatePresignedDownloadUrl( + record.key, + record.storageContext ?? 'workspace', + MOUNT_URL_TTL_SECONDS + ) + sandboxFiles.push({ type: 'url', path: mountPath, url }) + mounted.url += record.size + return + } + + if (record.size > MAX_FILE_SIZE) { + throw new Error( + `Input file "${mountPath}" is ${Math.round(record.size / 1024 / 1024)}MB, over the ${MAX_FILE_SIZE / 1024 / 1024}MB per-file mount limit.` + ) + } + if (mounted.buffered + record.size > MAX_TOTAL_SIZE) { + throw new Error( + `Mounting "${mountPath}" would exceed the ${MAX_TOTAL_SIZE / 1024 / 1024}MB total mount limit. Mount fewer or smaller files.` + ) + } + const buffer = await fetchWorkspaceFileBuffer(record) + const isText = /^text\/|application\/json|application\/xml|application\/csv/.test( + record.type || '' + ) + sandboxFiles.push({ + path: mountPath, + content: isText ? buffer.toString('utf-8') : buffer.toString('base64'), + encoding: isText ? undefined : 'base64', + }) + mounted.buffered += buffer.length +} + interface CanonicalFileInput { path: string sandboxPath?: string @@ -89,10 +173,15 @@ export async function resolveInputFiles( inputDirectories?: unknown[] ): Promise { const sandboxFiles: SandboxFile[] = [] - let totalSize = 0 + const mounted: MountedBytes = { buffered: 0, url: 0 } const betaEnabled = await isFeatureEnabled('mothership-beta') if (inputFiles?.length && workspaceId) { + if (inputFiles.length > MAX_MOUNTED_FILES) { + throw new Error( + `Too many input files (${inputFiles.length}). Maximum is ${MAX_MOUNTED_FILES}. Mount fewer files.` + ) + } const allFiles = await listWorkspaceFiles(workspaceId, { includeReservedSystemFiles: betaEnabled, }) @@ -124,33 +213,14 @@ export async function resolveInputFiles( `Input file not found: "${filePath}". Pass the exact canonical VFS path copied from glob/read (e.g. "files/Reports/data.csv").` ) } - if (record.size > MAX_FILE_SIZE) { - throw new Error( - `Input file "${filePath}" is ${Math.round(record.size / 1024 / 1024)}MB, over the ${MAX_FILE_SIZE / 1024 / 1024}MB per-file mount limit.` - ) - } - if (totalSize + record.size > MAX_TOTAL_SIZE) { - throw new Error( - `Mounting "${filePath}" would exceed the ${MAX_TOTAL_SIZE / 1024 / 1024}MB total mount limit. Mount fewer or smaller files.` - ) - } - const buffer = await fetchWorkspaceFileBuffer(record) - totalSize += buffer.length - const isText = /^text\/|application\/json|application\/xml|application\/csv/.test( - record.type || '' - ) - const content = isText ? buffer.toString('utf-8') : buffer.toString('base64') const explicitSandboxPath = typeof fileRef === 'object' && fileRef !== null ? (fileRef as CanonicalFileInput).sandboxPath : undefined - sandboxFiles.push({ - path: - explicitSandboxPath || - (alias ? workflowAliasSandboxPath(alias.aliasPath) : getSandboxWorkspaceFilePath(record)), - content, - encoding: isText ? undefined : 'base64', - }) + const mountPath = + explicitSandboxPath || + (alias ? workflowAliasSandboxPath(alias.aliasPath) : getSandboxWorkspaceFilePath(record)) + await pushWorkspaceFileMount(sandboxFiles, record, mountPath, mounted) } } @@ -228,17 +298,6 @@ export async function resolveInputFiles( } } for (const record of descendants) { - if (record.size > MAX_FILE_SIZE) { - throw new Error(`Input file exceeds size limit: ${record.name}`) - } - if (totalSize + record.size > MAX_TOTAL_SIZE) { - throw new Error('Total input size limit exceeded while mounting directory') - } - const buffer = await fetchWorkspaceFileBuffer(record) - totalSize += buffer.length - const isText = /^text\/|application\/json|application\/xml|application\/csv/.test( - record.type || '' - ) const relativeFolder = record.folderPath?.slice(folder.path.length).replace(/^\/+/, '') ?? '' const relativePath = alias @@ -246,11 +305,7 @@ export async function resolveInputFiles( [relativeFolder, record.name].filter(Boolean).join('/').split('/') ) : [relativeFolder, record.name].filter(Boolean).join('/') - sandboxFiles.push({ - path: `${mountRoot}/${relativePath}`, - content: isText ? buffer.toString('utf-8') : buffer.toString('base64'), - encoding: isText ? undefined : 'base64', - }) + await pushWorkspaceFileMount(sandboxFiles, record, `${mountRoot}/${relativePath}`, mounted) } } } @@ -305,7 +360,7 @@ export async function resolveInputFiles( const url = await generatePresignedDownloadUrl( snapshot.key, 'execution', - SNAPSHOT_URL_TTL_SECONDS + MOUNT_URL_TTL_SECONDS ) sandboxFiles.push({ type: 'url', path: mountPath, url }) continue @@ -318,7 +373,7 @@ export async function resolveInputFiles( `Input table "${tableId}" is ${Math.round(snapshot.size / 1024 / 1024)}MB, over the ${MAX_FILE_SIZE / 1024 / 1024}MB per-file mount limit.` ) } - if (totalSize + snapshot.size > MAX_TOTAL_SIZE) { + if (mounted.buffered + snapshot.size > MAX_TOTAL_SIZE) { throw new Error( `Mounting "${tableId}" would exceed the ${MAX_TOTAL_SIZE / 1024 / 1024}MB total mount limit. Mount fewer or smaller tables.` ) @@ -328,7 +383,7 @@ export async function resolveInputFiles( context: 'execution', maxBytes: MAX_FILE_SIZE, }) - totalSize += buffer.length + mounted.buffered += buffer.length sandboxFiles.push({ path: mountPath, content: buffer.toString('utf-8') }) continue } From c3a09694aba809f91021b5bf3246c95b4a2d7eaa Mon Sep 17 00:00:00 2001 From: Waleed Date: Wed, 24 Jun 2026 17:37:37 -0700 Subject: [PATCH 09/15] fix(tables): SSR crash from tableKeys in a 'use client' module + drop redundant flushChunks (#5204) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(tables): move tableKeys to a non-client module so the SSR prefetch works The tables list page crashed at SSR ('tableKeys.list is not a function') because tables/prefetch.ts (a server component) imported tableKeys from hooks/queries/tables.ts — a 'use client' module whose exports resolve to client-reference stubs on the server. Extract the key factory into hooks/queries/utils/table-keys.ts (no 'use client'), mirroring folder-keys.ts, and import it from there in the prefetch, hook, trigger, and consumers. * refactor(chat): drop redundant flushChunks on the SSE error path On an error 'final' event the reader stops via return true, so the post-loop flush is the single flush point. Defer the error append to after that flush (single flush, correct ordering) instead of flushing inside onEvent and again post-loop. No behavior change. * fix(sse): process the final unterminated line on stream end readSSELines broke out of the read loop on 'done' without flushing the TextDecoder or processing the trailing buffer, so a final 'data:' line not terminated by a newline (and any buffered multi-byte character) was dropped. Flush the decoder on end-of-stream and process the remaining buffer. Addresses a Cursor Medium finding on the consolidated SSE reader. --- .../resource-registry/resource-registry.tsx | 2 +- .../[workspaceId]/lib/prefetch.test.ts | 2 +- .../[tableId]/hooks/use-table-event-stream.ts | 2 +- .../[workspaceId]/tables/prefetch.ts | 2 +- .../w/[workflowId]/components/chat/chat.tsx | 14 ++++---- apps/sim/hooks/queries/tables.test.ts | 2 +- apps/sim/hooks/queries/tables.ts | 26 +------------- apps/sim/hooks/queries/utils/table-keys.ts | 34 +++++++++++++++++++ apps/sim/lib/core/utils/sse.test.ts | 26 ++++++++++++++ apps/sim/lib/core/utils/sse.ts | 7 ++-- apps/sim/triggers/table/poller.ts | 2 +- 11 files changed, 79 insertions(+), 40 deletions(-) create mode 100644 apps/sim/hooks/queries/utils/table-keys.ts diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-registry/resource-registry.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-registry/resource-registry.tsx index 460950dfa18..7c3f6a4db5e 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-registry/resource-registry.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-registry/resource-registry.tsx @@ -25,9 +25,9 @@ import { knowledgeKeys } from '@/hooks/queries/kb/knowledge' import { logKeys } from '@/hooks/queries/logs' import { mothershipChatKeys } from '@/hooks/queries/mothership-chats' import { scheduleKeys } from '@/hooks/queries/schedules' -import { tableKeys } from '@/hooks/queries/tables' import { folderKeys } from '@/hooks/queries/utils/folder-keys' import { invalidateWorkflowLists } from '@/hooks/queries/utils/invalidate-workflow-lists' +import { tableKeys } from '@/hooks/queries/utils/table-keys' import { workspaceFileFolderKeys } from '@/hooks/queries/workspace-file-folders' import { workspaceFilesKeys } from '@/hooks/queries/workspace-files' diff --git a/apps/sim/app/workspace/[workspaceId]/lib/prefetch.test.ts b/apps/sim/app/workspace/[workspaceId]/lib/prefetch.test.ts index d031c0648ef..2f8375c836f 100644 --- a/apps/sim/app/workspace/[workspaceId]/lib/prefetch.test.ts +++ b/apps/sim/app/workspace/[workspaceId]/lib/prefetch.test.ts @@ -21,8 +21,8 @@ import { prefetchHomeLists } from '@/app/workspace/[workspaceId]/home/prefetch' import { prefetchKnowledgeBases } from '@/app/workspace/[workspaceId]/knowledge/prefetch' import { prefetchTables } from '@/app/workspace/[workspaceId]/tables/prefetch' import { knowledgeKeys } from '@/hooks/queries/kb/knowledge' -import { tableKeys } from '@/hooks/queries/tables' import { folderKeys } from '@/hooks/queries/utils/folder-keys' +import { tableKeys } from '@/hooks/queries/utils/table-keys' import { workspaceFileFolderKeys } from '@/hooks/queries/workspace-file-folders' import { workspaceFilesKeys } from '@/hooks/queries/workspace-files' diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/hooks/use-table-event-stream.ts b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/hooks/use-table-event-stream.ts index 34789dff546..6a0efe9695d 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/hooks/use-table-event-stream.ts +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/hooks/use-table-event-stream.ts @@ -13,8 +13,8 @@ import { downloadExportResult, snapshotAndMutateRows, type TableRunState, - tableKeys, } from '@/hooks/queries/tables' +import { tableKeys } from '@/hooks/queries/utils/table-keys' const logger = createLogger('useTableEventStream') diff --git a/apps/sim/app/workspace/[workspaceId]/tables/prefetch.ts b/apps/sim/app/workspace/[workspaceId]/tables/prefetch.ts index 60d6a79a735..8d41a1d6680 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/prefetch.ts +++ b/apps/sim/app/workspace/[workspaceId]/tables/prefetch.ts @@ -1,7 +1,7 @@ import type { QueryClient } from '@tanstack/react-query' import type { TableDefinition } from '@/lib/table' import { prefetchInternalJson } from '@/app/workspace/[workspaceId]/lib/prefetch-internal-fetch' -import { tableKeys } from '@/hooks/queries/tables' +import { tableKeys } from '@/hooks/queries/utils/table-keys' /** * Prefetches the workspace's tables list under the same query key the client diff --git a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/chat/chat.tsx b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/chat/chat.tsx index d4caa19424f..fe3892ec05c 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/chat/chat.tsx +++ b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/chat/chat.tsx @@ -561,6 +561,7 @@ export function Chat() { } } + let finalError: string | null = null try { await readSSEEvents<{ event?: string; data?: ExecutionResult; chunk?: string }>(reader, { onParseError: (_data, e) => { @@ -571,12 +572,7 @@ export function Chat() { if (event === 'final' && eventData) { if ('success' in eventData && !eventData.success) { - const errorMessage = eventData.error || 'Workflow execution failed' - flushChunks() - appendMessageContent( - responseMessageId, - `${accumulatedContent ? '\n\n' : ''}Error: ${errorMessage}` - ) + finalError = eventData.error || 'Workflow execution failed' } return true } @@ -589,6 +585,12 @@ export function Chat() { }, }) flushChunks() + if (finalError) { + appendMessageContent( + responseMessageId, + `${accumulatedContent ? '\n\n' : ''}Error: ${finalError}` + ) + } finalizeMessageStream(responseMessageId) } catch (error) { if ((error as Error)?.name !== 'AbortError') { diff --git a/apps/sim/hooks/queries/tables.test.ts b/apps/sim/hooks/queries/tables.test.ts index da4c8c1cc04..d8245e50c31 100644 --- a/apps/sim/hooks/queries/tables.test.ts +++ b/apps/sim/hooks/queries/tables.test.ts @@ -83,13 +83,13 @@ vi.mock('@/components/emcn', () => ({ })) import { - tableKeys, tableRowsInfiniteOptions, tableRowsParamsKey, useDeleteColumn, useRestoreTable, useUpdateColumn, } from '@/hooks/queries/tables' +import { tableKeys } from '@/hooks/queries/utils/table-keys' const TABLE_ID = 'tbl-1' const WORKSPACE_ID = 'ws-1' diff --git a/apps/sim/hooks/queries/tables.ts b/apps/sim/hooks/queries/tables.ts index df49de32b17..cbf69d8c57a 100644 --- a/apps/sim/hooks/queries/tables.ts +++ b/apps/sim/hooks/queries/tables.ts @@ -96,34 +96,10 @@ import { optimisticallyScheduleNewlyEligibleGroups, } from '@/lib/table/deps' import { runUploadStrategy } from '@/lib/uploads/client/direct-upload' +import { type TableQueryScope, tableKeys } from '@/hooks/queries/utils/table-keys' const logger = createLogger('TableQueries') -type TableQueryScope = 'active' | 'archived' | 'all' - -export const tableKeys = { - all: ['tables'] as const, - lists: () => [...tableKeys.all, 'list'] as const, - list: (workspaceId?: string, scope: TableQueryScope = 'active') => - [...tableKeys.lists(), workspaceId ?? '', scope] as const, - details: () => [...tableKeys.all, 'detail'] as const, - detail: (tableId: string) => [...tableKeys.details(), tableId] as const, - exportJobs: (workspaceId?: string) => - [...tableKeys.all, 'export-jobs', workspaceId ?? ''] as const, - rowsRoot: (tableId: string) => [...tableKeys.detail(tableId), 'rows'] as const, - infiniteRows: (tableId: string, paramsKey: string) => - [...tableKeys.rowsRoot(tableId), 'infinite', paramsKey] as const, - rowWrites: (tableId: string) => [...tableKeys.rowsRoot(tableId), 'write'] as const, - find: (tableId: string, paramsKey: string) => - [...tableKeys.rowsRoot(tableId), 'find', paramsKey] as const, - activeDispatches: (tableId: string) => - [...tableKeys.detail(tableId), 'active-dispatches'] as const, - enrichmentDetails: (tableId: string) => - [...tableKeys.detail(tableId), 'enrichment-detail'] as const, - enrichmentDetail: (tableId: string, rowId: string, groupId: string) => - [...tableKeys.enrichmentDetails(tableId), rowId, groupId] as const, -} - type TableRowsParams = Omit & TableIdParamsInput & { filter?: Filter | null diff --git a/apps/sim/hooks/queries/utils/table-keys.ts b/apps/sim/hooks/queries/utils/table-keys.ts new file mode 100644 index 00000000000..cf27bddd013 --- /dev/null +++ b/apps/sim/hooks/queries/utils/table-keys.ts @@ -0,0 +1,34 @@ +/** + * React Query key factory for user-defined tables. + * + * Lives in this standalone (non-`'use client'`) module — like + * {@link file://./folder-keys.ts} — so it can be imported from server + * components (e.g. the tables page prefetch) without pulling in the + * `'use client'` `@/hooks/queries/tables` module, whose exports would + * otherwise resolve to client-reference stubs on the server. + */ + +export type TableQueryScope = 'active' | 'archived' | 'all' + +export const tableKeys = { + all: ['tables'] as const, + lists: () => [...tableKeys.all, 'list'] as const, + list: (workspaceId?: string, scope: TableQueryScope = 'active') => + [...tableKeys.lists(), workspaceId ?? '', scope] as const, + details: () => [...tableKeys.all, 'detail'] as const, + detail: (tableId: string) => [...tableKeys.details(), tableId] as const, + exportJobs: (workspaceId?: string) => + [...tableKeys.all, 'export-jobs', workspaceId ?? ''] as const, + rowsRoot: (tableId: string) => [...tableKeys.detail(tableId), 'rows'] as const, + infiniteRows: (tableId: string, paramsKey: string) => + [...tableKeys.rowsRoot(tableId), 'infinite', paramsKey] as const, + rowWrites: (tableId: string) => [...tableKeys.rowsRoot(tableId), 'write'] as const, + find: (tableId: string, paramsKey: string) => + [...tableKeys.rowsRoot(tableId), 'find', paramsKey] as const, + activeDispatches: (tableId: string) => + [...tableKeys.detail(tableId), 'active-dispatches'] as const, + enrichmentDetails: (tableId: string) => + [...tableKeys.detail(tableId), 'enrichment-detail'] as const, + enrichmentDetail: (tableId: string, rowId: string, groupId: string) => + [...tableKeys.enrichmentDetails(tableId), rowId, groupId] as const, +} diff --git a/apps/sim/lib/core/utils/sse.test.ts b/apps/sim/lib/core/utils/sse.test.ts index 579e23fd607..7ddf99cbd3b 100644 --- a/apps/sim/lib/core/utils/sse.test.ts +++ b/apps/sim/lib/core/utils/sse.test.ts @@ -361,6 +361,32 @@ describe('readSSEEvents', () => { expect(events).toEqual([{ msg: 'hello' }]) }) + it('emits a final data: line that has no trailing newline (stream tail)', async () => { + const stream = streamFromStringChunks(['data: {"n":1}\n', 'data: {"n":2}']) + const events: number[] = [] + await readSSEEvents<{ n: number }>(stream, { + onEvent: (e) => { + events.push(e.n) + }, + }) + expect(events).toEqual([1, 2]) + }) + + it('flushes a multi-byte character in the final unterminated line', async () => { + const encoder = new TextEncoder() + const euro = encoder.encode('€') + const chunk1 = new Uint8Array([...encoder.encode('data: {"s":"'), euro[0], euro[1]]) + const chunk2 = new Uint8Array([euro[2], ...encoder.encode('"}')]) + const stream = createStreamFromChunks([chunk1, chunk2]) + const events: Array<{ s: string }> = [] + await readSSEEvents<{ s: string }>(stream, { + onEvent: (e) => { + events.push(e) + }, + }) + expect(events).toEqual([{ s: '€' }]) + }) + it('skips the [DONE] sentinel', async () => { const stream = streamFromStringChunks(['data: {"n":1}\n\n', 'data: [DONE]\n\n']) const events: number[] = [] diff --git a/apps/sim/lib/core/utils/sse.ts b/apps/sim/lib/core/utils/sse.ts index 50c758f0013..2651147136e 100644 --- a/apps/sim/lib/core/utils/sse.ts +++ b/apps/sim/lib/core/utils/sse.ts @@ -138,11 +138,10 @@ export async function readSSELines(source: SSESource, options: ReadSSELinesOptio if (signal?.aborted) break const { done, value } = await reader.read() - if (done) break - buffer += decoder.decode(value, { stream: true }) + buffer += done ? decoder.decode() : decoder.decode(value, { stream: true }) const lines = buffer.split('\n') - buffer = lines.pop() ?? '' + buffer = done ? '' : (lines.pop() ?? '') for (const rawLine of lines) { if (signal?.aborted) return @@ -156,6 +155,8 @@ export async function readSSELines(source: SSESource, options: ReadSSELinesOptio if ((await onData(data)) === true) return } + + if (done) break } } finally { if (ownsLock) reader.releaseLock() diff --git a/apps/sim/triggers/table/poller.ts b/apps/sim/triggers/table/poller.ts index 6fe6ad17f81..e922c0502ec 100644 --- a/apps/sim/triggers/table/poller.ts +++ b/apps/sim/triggers/table/poller.ts @@ -3,7 +3,7 @@ import { requestJson } from '@/lib/api/client/request' import { listTablesContract } from '@/lib/api/contracts/tables' import type { TableDefinition } from '@/lib/table' import { getQueryClient } from '@/app/_shell/providers/get-query-client' -import { tableKeys } from '@/hooks/queries/tables' +import { tableKeys } from '@/hooks/queries/utils/table-keys' import { useWorkflowRegistry } from '@/stores/workflows/registry/store' import { useSubBlockStore } from '@/stores/workflows/subblock/store' import type { TriggerConfig } from '@/triggers/types' From ae4bc05e607d3fe69e7b1ffc2723647df810743a Mon Sep 17 00:00:00 2001 From: Waleed Date: Wed, 24 Jun 2026 18:05:40 -0700 Subject: [PATCH 10/15] feat(gitlab): add repository, code-review, and CI job tools + validation fixes (#5205) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(gitlab): add repository, code-review, and CI job tools + validation fixes Expand the GitLab integration with 12 new tools (all host-aware via getGitLabApiBase, wired through types/index/registry/block): - Repository: list_repository_tree, get_file, create_file, update_file, create_branch, list_branches, list_commits - Code review: get_merge_request_changes, approve_merge_request - CI jobs: list_pipeline_jobs, get_job_log, play_job Validation fixes from /validate-integration: - Correct the block inputs key (credential -> accessToken) so it matches the subBlock id and the params the block reads - Trim projectId before encoding in all tool request URLs (input hygiene) /validate-connector and /validate-trigger passed clean against the GitLab REST API v4 docs — no changes required. * fix(gitlab): address review feedback + regen docs - get_merge_request_changes: use the /diffs endpoint (/changes was removed in GitLab 18.0); return the diff array + count (drops the MR envelope that /diffs no longer provides), fetch max page size in a single call - create_file/update_file: send explicit `encoding: 'text'` for clarity - Remove `// =====` separator comments from types.ts (repo convention) - Regenerate GitLab integration docs + catalog for the 12 new tools --- .../content/docs/en/integrations/gitlab.mdx | 464 ++++++++++++++++++ apps/sim/blocks/blocks/gitlab.ts | 347 ++++++++++++- apps/sim/lib/integrations/integrations.json | 152 +++++- .../sim/tools/gitlab/approve_merge_request.ts | 105 ++++ apps/sim/tools/gitlab/cancel_pipeline.ts | 2 +- apps/sim/tools/gitlab/create_branch.ts | 102 ++++ apps/sim/tools/gitlab/create_file.ts | 106 ++++ apps/sim/tools/gitlab/create_issue.ts | 2 +- apps/sim/tools/gitlab/create_issue_note.ts | 2 +- apps/sim/tools/gitlab/create_merge_request.ts | 2 +- .../tools/gitlab/create_merge_request_note.ts | 2 +- apps/sim/tools/gitlab/create_pipeline.ts | 2 +- apps/sim/tools/gitlab/delete_issue.ts | 2 +- apps/sim/tools/gitlab/get_file.ts | 114 +++++ apps/sim/tools/gitlab/get_issue.ts | 2 +- apps/sim/tools/gitlab/get_job_log.ts | 75 +++ apps/sim/tools/gitlab/get_merge_request.ts | 2 +- .../tools/gitlab/get_merge_request_changes.ts | 98 ++++ apps/sim/tools/gitlab/get_pipeline.ts | 2 +- apps/sim/tools/gitlab/get_project.ts | 2 +- apps/sim/tools/gitlab/index.ts | 28 ++ apps/sim/tools/gitlab/list_branches.ts | 103 ++++ apps/sim/tools/gitlab/list_commits.ts | 129 +++++ apps/sim/tools/gitlab/list_issues.ts | 2 +- apps/sim/tools/gitlab/list_merge_requests.ts | 2 +- apps/sim/tools/gitlab/list_pipeline_jobs.ts | 119 +++++ apps/sim/tools/gitlab/list_pipelines.ts | 2 +- apps/sim/tools/gitlab/list_repository_tree.ts | 120 +++++ apps/sim/tools/gitlab/merge_merge_request.ts | 2 +- apps/sim/tools/gitlab/play_job.ts | 90 ++++ apps/sim/tools/gitlab/retry_pipeline.ts | 2 +- apps/sim/tools/gitlab/types.ts | 234 ++++++++- apps/sim/tools/gitlab/update_file.ts | 118 +++++ apps/sim/tools/gitlab/update_issue.ts | 2 +- apps/sim/tools/gitlab/update_merge_request.ts | 2 +- apps/sim/tools/registry.ts | 50 +- 36 files changed, 2521 insertions(+), 69 deletions(-) create mode 100644 apps/sim/tools/gitlab/approve_merge_request.ts create mode 100644 apps/sim/tools/gitlab/create_branch.ts create mode 100644 apps/sim/tools/gitlab/create_file.ts create mode 100644 apps/sim/tools/gitlab/get_file.ts create mode 100644 apps/sim/tools/gitlab/get_job_log.ts create mode 100644 apps/sim/tools/gitlab/get_merge_request_changes.ts create mode 100644 apps/sim/tools/gitlab/list_branches.ts create mode 100644 apps/sim/tools/gitlab/list_commits.ts create mode 100644 apps/sim/tools/gitlab/list_pipeline_jobs.ts create mode 100644 apps/sim/tools/gitlab/list_repository_tree.ts create mode 100644 apps/sim/tools/gitlab/play_job.ts create mode 100644 apps/sim/tools/gitlab/update_file.ts diff --git a/apps/docs/content/docs/en/integrations/gitlab.mdx b/apps/docs/content/docs/en/integrations/gitlab.mdx index 6d7340bac8e..eb37db7210b 100644 --- a/apps/docs/content/docs/en/integrations/gitlab.mdx +++ b/apps/docs/content/docs/en/integrations/gitlab.mdx @@ -41,6 +41,7 @@ List GitLab projects accessible to the authenticated user | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | +| `host` | string | No | Self-managed GitLab host \(e.g. gitlab.example.com\). Defaults to gitlab.com. | | `owned` | boolean | No | Limit to projects owned by the current user | | `membership` | boolean | No | Limit to projects the current user is a member of | | `search` | string | No | Search projects by name | @@ -65,6 +66,7 @@ Get details of a specific GitLab project | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | +| `host` | string | No | Self-managed GitLab host \(e.g. gitlab.example.com\). Defaults to gitlab.com. | | `projectId` | string | Yes | Project ID or URL-encoded path \(e.g., "namespace/project"\) | #### Output @@ -81,6 +83,7 @@ List issues in a GitLab project | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | +| `host` | string | No | Self-managed GitLab host \(e.g. gitlab.example.com\). Defaults to gitlab.com. | | `projectId` | string | Yes | Project ID or URL-encoded path | | `state` | string | No | Filter by state \(opened, closed, all\) | | `labels` | string | No | Comma-separated list of label names | @@ -107,6 +110,7 @@ Get details of a specific GitLab issue | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | +| `host` | string | No | Self-managed GitLab host \(e.g. gitlab.example.com\). Defaults to gitlab.com. | | `projectId` | string | Yes | Project ID or URL-encoded path | | `issueIid` | number | Yes | Issue number within the project \(the # shown in GitLab UI\) | @@ -124,6 +128,7 @@ Create a new issue in a GitLab project | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | +| `host` | string | No | Self-managed GitLab host \(e.g. gitlab.example.com\). Defaults to gitlab.com. | | `projectId` | string | Yes | Project ID or URL-encoded path | | `title` | string | Yes | Issue title | | `description` | string | No | Issue description \(Markdown supported\) | @@ -147,6 +152,7 @@ Update an existing issue in a GitLab project | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | +| `host` | string | No | Self-managed GitLab host \(e.g. gitlab.example.com\). Defaults to gitlab.com. | | `projectId` | string | Yes | Project ID or URL-encoded path | | `issueIid` | number | Yes | Issue internal ID \(IID\) | | `title` | string | No | New issue title | @@ -172,6 +178,7 @@ Delete an issue from a GitLab project | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | +| `host` | string | No | Self-managed GitLab host \(e.g. gitlab.example.com\). Defaults to gitlab.com. | | `projectId` | string | Yes | Project ID or URL-encoded path | | `issueIid` | number | Yes | Issue internal ID \(IID\) | @@ -189,6 +196,7 @@ Add a comment to a GitLab issue | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | +| `host` | string | No | Self-managed GitLab host \(e.g. gitlab.example.com\). Defaults to gitlab.com. | | `projectId` | string | Yes | Project ID or URL-encoded path | | `issueIid` | number | Yes | Issue internal ID \(IID\) | | `body` | string | Yes | Comment body \(Markdown supported\) | @@ -207,6 +215,7 @@ List merge requests in a GitLab project | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | +| `host` | string | No | Self-managed GitLab host \(e.g. gitlab.example.com\). Defaults to gitlab.com. | | `projectId` | string | Yes | Project ID or URL-encoded path | | `state` | string | No | Filter by state \(opened, closed, merged, all\) | | `labels` | string | No | Comma-separated list of label names | @@ -232,6 +241,7 @@ Get details of a specific GitLab merge request | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | +| `host` | string | No | Self-managed GitLab host \(e.g. gitlab.example.com\). Defaults to gitlab.com. | | `projectId` | string | Yes | Project ID or URL-encoded path | | `mergeRequestIid` | number | Yes | Merge request internal ID \(IID\) | @@ -249,6 +259,7 @@ Create a new merge request in a GitLab project | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | +| `host` | string | No | Self-managed GitLab host \(e.g. gitlab.example.com\). Defaults to gitlab.com. | | `projectId` | string | Yes | Project ID or URL-encoded path | | `sourceBranch` | string | Yes | Source branch name | | `targetBranch` | string | Yes | Target branch name | @@ -275,6 +286,7 @@ Update an existing merge request in a GitLab project | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | +| `host` | string | No | Self-managed GitLab host \(e.g. gitlab.example.com\). Defaults to gitlab.com. | | `projectId` | string | Yes | Project ID or URL-encoded path | | `mergeRequestIid` | number | Yes | Merge request internal ID \(IID\) | | `title` | string | No | New merge request title | @@ -302,6 +314,7 @@ Merge a merge request in a GitLab project | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | +| `host` | string | No | Self-managed GitLab host \(e.g. gitlab.example.com\). Defaults to gitlab.com. | | `projectId` | string | Yes | Project ID or URL-encoded path | | `mergeRequestIid` | number | Yes | Merge request internal ID \(IID\) | | `mergeCommitMessage` | string | No | Custom merge commit message | @@ -324,6 +337,7 @@ Add a comment to a GitLab merge request | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | +| `host` | string | No | Self-managed GitLab host \(e.g. gitlab.example.com\). Defaults to gitlab.com. | | `projectId` | string | Yes | Project ID or URL-encoded path | | `mergeRequestIid` | number | Yes | Merge request internal ID \(IID\) | | `body` | string | Yes | Comment body \(Markdown supported\) | @@ -342,6 +356,7 @@ List pipelines in a GitLab project | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | +| `host` | string | No | Self-managed GitLab host \(e.g. gitlab.example.com\). Defaults to gitlab.com. | | `projectId` | string | Yes | Project ID or URL-encoded path | | `ref` | string | No | Filter by ref \(branch or tag\) | | `status` | string | No | Filter by status \(created, waiting_for_resource, preparing, pending, running, success, failed, canceled, skipped, manual, scheduled\) | @@ -365,6 +380,7 @@ Get details of a specific GitLab pipeline | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | +| `host` | string | No | Self-managed GitLab host \(e.g. gitlab.example.com\). Defaults to gitlab.com. | | `projectId` | string | Yes | Project ID or URL-encoded path | | `pipelineId` | number | Yes | Pipeline ID | @@ -382,6 +398,7 @@ Trigger a new pipeline in a GitLab project | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | +| `host` | string | No | Self-managed GitLab host \(e.g. gitlab.example.com\). Defaults to gitlab.com. | | `projectId` | string | Yes | Project ID or URL-encoded path | | `ref` | string | Yes | Branch or tag to run the pipeline on | | `variables` | array | No | Array of variables for the pipeline \(each with key, value, and optional variable_type\) | @@ -400,6 +417,7 @@ Retry a failed GitLab pipeline | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | +| `host` | string | No | Self-managed GitLab host \(e.g. gitlab.example.com\). Defaults to gitlab.com. | | `projectId` | string | Yes | Project ID or URL-encoded path | | `pipelineId` | number | Yes | Pipeline ID | @@ -417,6 +435,7 @@ Cancel a running GitLab pipeline | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | +| `host` | string | No | Self-managed GitLab host \(e.g. gitlab.example.com\). Defaults to gitlab.com. | | `projectId` | string | Yes | Project ID or URL-encoded path | | `pipelineId` | number | Yes | Pipeline ID | @@ -426,4 +445,449 @@ Cancel a running GitLab pipeline | --------- | ---- | ----------- | | `pipeline` | object | The cancelled GitLab pipeline | +### `gitlab_list_repository_tree` + +List files and directories in a GitLab project repository + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `host` | string | No | Self-managed GitLab host \(e.g. gitlab.example.com\). Defaults to gitlab.com. | +| `projectId` | string | Yes | Project ID or URL-encoded path | +| `path` | string | No | Path inside the repository to list | +| `ref` | string | No | Branch, tag, or commit SHA to list from | +| `recursive` | boolean | No | Whether to list files recursively | +| `perPage` | number | No | Number of results per page \(default 20, max 100\) | +| `page` | number | No | Page number for pagination | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `tree` | array | List of repository tree entries | +| `total` | number | Total number of tree entries | + +### `gitlab_get_file` + +Get the contents of a file from a GitLab project repository + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `host` | string | No | Self-managed GitLab host \(e.g. gitlab.example.com\). Defaults to gitlab.com. | +| `projectId` | string | Yes | Project ID or URL-encoded path | +| `filePath` | string | Yes | Path to the file in the repository | +| `ref` | string | Yes | Branch, tag, or commit SHA | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `filePath` | string | The file path | +| `fileName` | string | The file name | +| `size` | number | The file size in bytes | +| `ref` | string | The branch, tag, or commit SHA | +| `blobId` | string | The blob ID | +| `lastCommitId` | string | The last commit ID that modified the file | +| `content` | string | The decoded file content | + +### `gitlab_create_file` + +Create a new file in a GitLab project repository + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `host` | string | No | Self-managed GitLab host \(e.g. gitlab.example.com\). Defaults to gitlab.com. | +| `projectId` | string | Yes | Project ID or URL-encoded path | +| `filePath` | string | Yes | Path to the file in the repository | +| `branch` | string | Yes | Branch to commit the new file to | +| `content` | string | Yes | File content | +| `commitMessage` | string | Yes | Commit message | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `filePath` | string | The created file path | +| `branch` | string | The branch the file was committed to | + +### `gitlab_update_file` + +Update an existing file in a GitLab project repository + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `host` | string | No | Self-managed GitLab host \(e.g. gitlab.example.com\). Defaults to gitlab.com. | +| `projectId` | string | Yes | Project ID or URL-encoded path | +| `filePath` | string | Yes | Path to the file in the repository | +| `branch` | string | Yes | Branch to commit the update to | +| `content` | string | Yes | New file content | +| `commitMessage` | string | Yes | Commit message | +| `lastCommitId` | string | No | Last known commit ID for the file \(optimistic locking\) | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `filePath` | string | The updated file path | +| `branch` | string | The branch the update was committed to | + +### `gitlab_create_branch` + +Create a new branch in a GitLab project repository + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `host` | string | No | Self-managed GitLab host \(e.g. gitlab.example.com\). Defaults to gitlab.com. | +| `projectId` | string | Yes | Project ID or URL-encoded path | +| `branch` | string | Yes | Name of the new branch | +| `ref` | string | Yes | Source branch/tag/SHA | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `name` | string | The created branch name | +| `webUrl` | string | The web URL of the branch | +| `protected` | boolean | Whether the branch is protected | +| `commit` | object | The commit the branch points to | + +### `gitlab_list_branches` + +List branches in a GitLab project repository + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `host` | string | No | Self-managed GitLab host \(e.g. gitlab.example.com\). Defaults to gitlab.com. | +| `projectId` | string | Yes | Project ID or URL-encoded path | +| `search` | string | No | Filter branches by name | +| `perPage` | number | No | Number of results per page \(default 20, max 100\) | +| `page` | number | No | Page number for pagination | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `branches` | array | List of branches | +| `total` | number | Total number of branches | + +### `gitlab_list_commits` + +List commits in a GitLab project repository + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `host` | string | No | Self-managed GitLab host \(e.g. gitlab.example.com\). Defaults to gitlab.com. | +| `projectId` | string | Yes | Project ID or URL-encoded path | +| `refName` | string | No | Branch, tag, or revision range to list commits from | +| `since` | string | No | Only commits after this ISO 8601 date | +| `until` | string | No | Only commits before this ISO 8601 date | +| `path` | string | No | Only commits affecting this file path | +| `author` | string | No | Filter commits by author | +| `perPage` | number | No | Number of results per page \(default 20, max 100\) | +| `page` | number | No | Page number for pagination | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `commits` | array | List of commits | +| `total` | number | Total number of commits | + +### `gitlab_get_merge_request_changes` + +Get the file changes (diffs) of a GitLab merge request + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `host` | string | No | Self-managed GitLab host \(e.g. gitlab.example.com\). Defaults to gitlab.com. | +| `projectId` | string | Yes | Project ID or URL-encoded path | +| `mergeRequestIid` | number | Yes | Merge request internal ID \(IID\) | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `mergeRequestIid` | number | The merge request internal ID \(IID\) | +| `changes` | array | List of file changes \(diffs\) | +| `changesCount` | number | Number of changed files returned | + +### `gitlab_approve_merge_request` + +Approve a GitLab merge request + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `host` | string | No | Self-managed GitLab host \(e.g. gitlab.example.com\). Defaults to gitlab.com. | +| `projectId` | string | Yes | Project ID or URL-encoded path | +| `mergeRequestIid` | number | Yes | Merge request internal ID \(IID\) | +| `sha` | string | No | HEAD SHA of the merge request to approve | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `approvalsRequired` | number | Number of approvals required | +| `approvalsLeft` | number | Number of approvals still needed | +| `approvedBy` | array | List of approvers | + +### `gitlab_list_pipeline_jobs` + +List jobs for a GitLab pipeline + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `host` | string | No | Self-managed GitLab host \(e.g. gitlab.example.com\). Defaults to gitlab.com. | +| `projectId` | string | Yes | Project ID or URL-encoded path | +| `pipelineId` | number | Yes | Pipeline ID | +| `scope` | string | No | Filter jobs by scope \(e.g. created, running, success, failed\) | +| `includeRetried` | boolean | No | Whether to include retried jobs | +| `perPage` | number | No | Number of results per page \(default 20, max 100\) | +| `page` | number | No | Page number for pagination | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `jobs` | array | List of pipeline jobs | +| `total` | number | Total number of jobs | + +### `gitlab_get_job_log` + +Get the log (trace) of a GitLab job + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `host` | string | No | Self-managed GitLab host \(e.g. gitlab.example.com\). Defaults to gitlab.com. | +| `projectId` | string | Yes | Project ID or URL-encoded path | +| `jobId` | number | Yes | Job ID | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `log` | string | The job log \(trace\) output | + +### `gitlab_play_job` + +Trigger (play) a manual GitLab job + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `host` | string | No | Self-managed GitLab host \(e.g. gitlab.example.com\). Defaults to gitlab.com. | +| `projectId` | string | Yes | Project ID or URL-encoded path | +| `jobId` | number | Yes | Job ID | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `id` | number | The job ID | +| `name` | string | The job name | +| `status` | string | The job status | +| `webUrl` | string | The web URL of the job | + + + +## Triggers + +A **Trigger** is a block that starts a workflow when an event happens in this service. + +### GitLab Comment + +Trigger workflow when a comment is added on a commit, merge request, or issue + +#### Configuration + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `accessToken` | string | Yes | Used to create the webhook in your project. Requires the Maintainer or Owner role. | +| `projectId` | string | Yes | The GitLab project to register the webhook on. | +| `host` | string | No | Self-managed GitLab host. Leave blank for gitlab.com. | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `object_kind` | string | Event kind \(note\) | +| `event_type` | string | GitLab event type from the X-Gitlab-Event header | +| `object_attributes` | object | object_attributes output from the tool | +| ↳ `id` | number | Comment ID | +| ↳ `note` | string | Comment body | +| ↳ `noteable_type` | string | What the comment is on \(Commit, MergeRequest, Issue, Snippet\) | +| ↳ `action` | string | Action \(create, update\) | +| ↳ `url` | string | Comment URL | + + +--- + +### GitLab Event + +Trigger workflow from any GitLab webhook event + +#### Configuration + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `accessToken` | string | Yes | Used to create the webhook in your project. Requires the Maintainer or Owner role. | +| `projectId` | string | Yes | The GitLab project to register the webhook on. | +| `host` | string | No | Self-managed GitLab host. Leave blank for gitlab.com. | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `object_kind` | string | Event kind \(push, merge_request, issue, etc.\) | +| `event_type` | string | GitLab event type from the X-Gitlab-Event header | +| `user` | json | Actor that triggered the event \(when present\) | +| `object_attributes` | json | Event-specific attributes \(varies by object_kind\) | + + +--- + +### GitLab Issue + +Trigger workflow when an issue is opened, updated, or closed in GitLab + +#### Configuration + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `accessToken` | string | Yes | Used to create the webhook in your project. Requires the Maintainer or Owner role. | +| `projectId` | string | Yes | The GitLab project to register the webhook on. | +| `host` | string | No | Self-managed GitLab host. Leave blank for gitlab.com. | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `object_kind` | string | Event kind \(issue\) | +| `event_type` | string | GitLab event type from the X-Gitlab-Event header | +| `object_attributes` | object | object_attributes output from the tool | +| ↳ `id` | number | Global issue ID | +| ↳ `iid` | number | Project-scoped issue number | +| ↳ `title` | string | Issue title | +| ↳ `state` | string | State \(opened, closed\) | +| ↳ `action` | string | Action \(open, close, reopen, update\) | +| ↳ `description` | string | Issue description | +| ↳ `confidential` | boolean | Whether the issue is confidential | +| ↳ `url` | string | Issue URL | + + +--- + +### GitLab Merge Request + +Trigger workflow when a merge request is opened, updated, or merged in GitLab + +#### Configuration + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `accessToken` | string | Yes | Used to create the webhook in your project. Requires the Maintainer or Owner role. | +| `projectId` | string | Yes | The GitLab project to register the webhook on. | +| `host` | string | No | Self-managed GitLab host. Leave blank for gitlab.com. | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `object_kind` | string | Event kind \(merge_request\) | +| `event_type` | string | GitLab event type from the X-Gitlab-Event header | +| `object_attributes` | object | object_attributes output from the tool | +| ↳ `id` | number | Global merge request ID | +| ↳ `iid` | number | Project-scoped merge request number | +| ↳ `title` | string | Merge request title | +| ↳ `state` | string | State \(opened, closed, merged, locked\) | +| ↳ `action` | string | Action \(open, close, reopen, update, merge, etc.\) | +| ↳ `source_branch` | string | Source branch | +| ↳ `target_branch` | string | Target branch | +| ↳ `merge_status` | string | Merge status | +| ↳ `draft` | boolean | Whether the merge request is a draft | +| ↳ `url` | string | Merge request URL | + + +--- + +### GitLab Pipeline + +Trigger workflow when a pipeline status changes in GitLab + +#### Configuration + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `accessToken` | string | Yes | Used to create the webhook in your project. Requires the Maintainer or Owner role. | +| `projectId` | string | Yes | The GitLab project to register the webhook on. | +| `host` | string | No | Self-managed GitLab host. Leave blank for gitlab.com. | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `object_kind` | string | Event kind \(pipeline\) | +| `event_type` | string | GitLab event type from the X-Gitlab-Event header | +| `object_attributes` | object | object_attributes output from the tool | +| ↳ `id` | number | Pipeline ID | +| ↳ `status` | string | Pipeline status \(success, failed, running, etc.\) | +| ↳ `detailed_status` | string | Detailed pipeline status | +| ↳ `ref` | string | Ref the pipeline ran on | +| ↳ `sha` | string | Commit SHA | +| ↳ `source` | string | Pipeline source \(push, web, schedule, etc.\) | +| ↳ `duration` | number | Pipeline duration in seconds | +| ↳ `url` | string | Pipeline URL | + + +--- + +### GitLab Push + +Trigger workflow when commits are pushed to a GitLab project + +#### Configuration + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `accessToken` | string | Yes | Used to create the webhook in your project. Requires the Maintainer or Owner role. | +| `projectId` | string | Yes | The GitLab project to register the webhook on. | +| `host` | string | No | Self-managed GitLab host. Leave blank for gitlab.com. | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `object_kind` | string | Event kind \(push\) | +| `event_type` | string | GitLab event type from the X-Gitlab-Event header | +| `ref` | string | Git ref that was pushed \(e.g. refs/heads/main\) | +| `branch` | string | Branch name derived from ref | +| `before` | string | SHA before the push | +| `after` | string | SHA after the push | +| `checkout_sha` | string | SHA of the most recent commit | +| `user_username` | string | Username of the pusher | +| `user_name` | string | Display name of the pusher | +| `user_email` | string | Email of the pusher | +| `total_commits_count` | number | Number of commits in the push | +| `commits` | json | Array of commit objects included in this push | diff --git a/apps/sim/blocks/blocks/gitlab.ts b/apps/sim/blocks/blocks/gitlab.ts index 1cfe24a9e25..078422bedac 100644 --- a/apps/sim/blocks/blocks/gitlab.ts +++ b/apps/sim/blocks/blocks/gitlab.ts @@ -46,6 +46,21 @@ export const GitLabBlock: BlockConfig = { { label: 'Create Pipeline', id: 'gitlab_create_pipeline' }, { label: 'Retry Pipeline', id: 'gitlab_retry_pipeline' }, { label: 'Cancel Pipeline', id: 'gitlab_cancel_pipeline' }, + // Repository Operations + { label: 'List Repository Tree', id: 'gitlab_list_repository_tree' }, + { label: 'Get File', id: 'gitlab_get_file' }, + { label: 'Create File', id: 'gitlab_create_file' }, + { label: 'Update File', id: 'gitlab_update_file' }, + { label: 'List Commits', id: 'gitlab_list_commits' }, + { label: 'List Branches', id: 'gitlab_list_branches' }, + { label: 'Create Branch', id: 'gitlab_create_branch' }, + // Additional Merge Request Operations + { label: 'Get MR Changes', id: 'gitlab_get_merge_request_changes' }, + { label: 'Approve Merge Request', id: 'gitlab_approve_merge_request' }, + // Job Operations + { label: 'List Pipeline Jobs', id: 'gitlab_list_pipeline_jobs' }, + { label: 'Get Job Log', id: 'gitlab_get_job_log' }, + { label: 'Play Job', id: 'gitlab_play_job' }, ], value: () => 'gitlab_list_projects', }, @@ -94,6 +109,18 @@ export const GitLabBlock: BlockConfig = { 'gitlab_create_pipeline', 'gitlab_retry_pipeline', 'gitlab_cancel_pipeline', + 'gitlab_list_repository_tree', + 'gitlab_get_file', + 'gitlab_create_file', + 'gitlab_update_file', + 'gitlab_list_commits', + 'gitlab_list_branches', + 'gitlab_create_branch', + 'gitlab_get_merge_request_changes', + 'gitlab_approve_merge_request', + 'gitlab_list_pipeline_jobs', + 'gitlab_get_job_log', + 'gitlab_play_job', ], }, }, @@ -128,6 +155,8 @@ export const GitLabBlock: BlockConfig = { 'gitlab_update_merge_request', 'gitlab_merge_merge_request', 'gitlab_create_merge_request_note', + 'gitlab_get_merge_request_changes', + 'gitlab_approve_merge_request', ], }, }, @@ -140,7 +169,12 @@ export const GitLabBlock: BlockConfig = { required: true, condition: { field: 'operation', - value: ['gitlab_get_pipeline', 'gitlab_retry_pipeline', 'gitlab_cancel_pipeline'], + value: [ + 'gitlab_get_pipeline', + 'gitlab_retry_pipeline', + 'gitlab_cancel_pipeline', + 'gitlab_list_pipeline_jobs', + ], }, }, // Title (for issue/MR creation) @@ -247,7 +281,135 @@ Return ONLY the comment text - no explanations, no extra formatting.`, required: true, condition: { field: 'operation', - value: ['gitlab_create_pipeline'], + value: ['gitlab_create_pipeline', 'gitlab_get_file', 'gitlab_create_branch'], + }, + }, + // File Path + { + id: 'filePath', + title: 'File Path', + type: 'short-input', + placeholder: 'Path to file (e.g., src/index.ts)', + required: true, + condition: { + field: 'operation', + value: ['gitlab_get_file', 'gitlab_create_file', 'gitlab_update_file'], + }, + }, + // Branch + { + id: 'branch', + title: 'Branch', + type: 'short-input', + placeholder: 'Branch name', + required: true, + condition: { + field: 'operation', + value: ['gitlab_create_file', 'gitlab_update_file', 'gitlab_create_branch'], + }, + }, + // File Content + { + id: 'content', + title: 'File Content', + type: 'long-input', + placeholder: 'File content', + required: true, + condition: { + field: 'operation', + value: ['gitlab_create_file', 'gitlab_update_file'], + }, + }, + // Commit Message + { + id: 'commitMessage', + title: 'Commit Message', + type: 'short-input', + placeholder: 'Commit message', + required: true, + condition: { + field: 'operation', + value: ['gitlab_create_file', 'gitlab_update_file'], + }, + }, + // Job ID + { + id: 'jobId', + title: 'Job ID', + type: 'short-input', + placeholder: 'Enter job ID', + required: true, + condition: { + field: 'operation', + value: ['gitlab_get_job_log', 'gitlab_play_job'], + }, + }, + // Subdirectory path (for repository tree) + { + id: 'path', + title: 'Path', + type: 'short-input', + placeholder: 'Subdirectory path (optional)', + mode: 'advanced', + condition: { + field: 'operation', + value: ['gitlab_list_repository_tree'], + }, + }, + // Recursive tree listing + { + id: 'recursive', + title: 'Recursive', + type: 'switch', + mode: 'advanced', + condition: { + field: 'operation', + value: ['gitlab_list_repository_tree'], + }, + }, + // Ref name filter (for list commits) + { + id: 'refName', + title: 'Ref (branch/tag)', + type: 'short-input', + placeholder: 'Branch or tag (optional)', + mode: 'advanced', + condition: { + field: 'operation', + value: ['gitlab_list_commits'], + }, + }, + // Job scope filter (for list pipeline jobs) + { + id: 'scope', + title: 'Job Scope', + type: 'dropdown', + options: [ + { label: 'All', id: '' }, + { label: 'Failed', id: 'failed' }, + { label: 'Success', id: 'success' }, + { label: 'Running', id: 'running' }, + { label: 'Pending', id: 'pending' }, + { label: 'Canceled', id: 'canceled' }, + { label: 'Manual', id: 'manual' }, + ], + value: () => '', + mode: 'advanced', + condition: { + field: 'operation', + value: ['gitlab_list_pipeline_jobs'], + }, + }, + // Commit SHA (for approve merge request) + { + id: 'sha', + title: 'Commit SHA', + type: 'short-input', + placeholder: 'Optional HEAD SHA to approve', + mode: 'advanced', + condition: { + field: 'operation', + value: ['gitlab_approve_merge_request'], }, }, // Labels @@ -427,6 +589,10 @@ Return ONLY the commit message - no explanations, no extra text.`, 'gitlab_list_issues', 'gitlab_list_merge_requests', 'gitlab_list_pipelines', + 'gitlab_list_repository_tree', + 'gitlab_list_branches', + 'gitlab_list_commits', + 'gitlab_list_pipeline_jobs', ], }, }, @@ -444,6 +610,10 @@ Return ONLY the commit message - no explanations, no extra text.`, 'gitlab_list_issues', 'gitlab_list_merge_requests', 'gitlab_list_pipelines', + 'gitlab_list_repository_tree', + 'gitlab_list_branches', + 'gitlab_list_commits', + 'gitlab_list_pipeline_jobs', ], }, }, @@ -475,6 +645,18 @@ Return ONLY the commit message - no explanations, no extra text.`, 'gitlab_create_pipeline', 'gitlab_retry_pipeline', 'gitlab_cancel_pipeline', + 'gitlab_list_repository_tree', + 'gitlab_get_file', + 'gitlab_create_file', + 'gitlab_update_file', + 'gitlab_create_branch', + 'gitlab_list_branches', + 'gitlab_list_commits', + 'gitlab_get_merge_request_changes', + 'gitlab_approve_merge_request', + 'gitlab_list_pipeline_jobs', + 'gitlab_get_job_log', + 'gitlab_play_job', ], config: { tool: (params) => { @@ -710,6 +892,140 @@ Return ONLY the commit message - no explanations, no extra text.`, pipelineId: Number(params.pipelineId), } + case 'gitlab_list_repository_tree': + if (!params.projectId?.trim()) { + throw new Error('Project ID is required.') + } + return { + ...baseParams, + projectId: params.projectId.trim(), + path: params.path?.trim() || undefined, + recursive: params.recursive || undefined, + perPage: params.perPage ? Number(params.perPage) : undefined, + page: params.page ? Number(params.page) : undefined, + } + + case 'gitlab_get_file': + if (!params.projectId?.trim() || !params.filePath?.trim() || !params.ref?.trim()) { + throw new Error('Project ID, file path, and ref are required.') + } + return { + ...baseParams, + projectId: params.projectId.trim(), + filePath: params.filePath.trim(), + ref: params.ref.trim(), + } + + case 'gitlab_create_file': + case 'gitlab_update_file': + if ( + !params.projectId?.trim() || + !params.filePath?.trim() || + !params.branch?.trim() || + !params.content || + !params.commitMessage?.trim() + ) { + throw new Error( + 'Project ID, file path, branch, content, and commit message are required.' + ) + } + return { + ...baseParams, + projectId: params.projectId.trim(), + filePath: params.filePath.trim(), + branch: params.branch.trim(), + content: params.content, + commitMessage: params.commitMessage.trim(), + } + + case 'gitlab_create_branch': + if (!params.projectId?.trim() || !params.branch?.trim() || !params.ref?.trim()) { + throw new Error('Project ID, branch name, and source ref are required.') + } + return { + ...baseParams, + projectId: params.projectId.trim(), + branch: params.branch.trim(), + ref: params.ref.trim(), + } + + case 'gitlab_list_branches': + if (!params.projectId?.trim()) { + throw new Error('Project ID is required.') + } + return { + ...baseParams, + projectId: params.projectId.trim(), + perPage: params.perPage ? Number(params.perPage) : undefined, + page: params.page ? Number(params.page) : undefined, + } + + case 'gitlab_list_commits': + if (!params.projectId?.trim()) { + throw new Error('Project ID is required.') + } + return { + ...baseParams, + projectId: params.projectId.trim(), + refName: params.refName?.trim() || undefined, + perPage: params.perPage ? Number(params.perPage) : undefined, + page: params.page ? Number(params.page) : undefined, + } + + case 'gitlab_get_merge_request_changes': + if (!params.projectId?.trim() || !params.mergeRequestIid) { + throw new Error('Project ID and Merge Request IID are required.') + } + return { + ...baseParams, + projectId: params.projectId.trim(), + mergeRequestIid: Number(params.mergeRequestIid), + } + + case 'gitlab_approve_merge_request': + if (!params.projectId?.trim() || !params.mergeRequestIid) { + throw new Error('Project ID and Merge Request IID are required.') + } + return { + ...baseParams, + projectId: params.projectId.trim(), + mergeRequestIid: Number(params.mergeRequestIid), + sha: params.sha?.trim() || undefined, + } + + case 'gitlab_list_pipeline_jobs': + if (!params.projectId?.trim() || !params.pipelineId) { + throw new Error('Project ID and Pipeline ID are required.') + } + return { + ...baseParams, + projectId: params.projectId.trim(), + pipelineId: Number(params.pipelineId), + scope: params.scope || undefined, + perPage: params.perPage ? Number(params.perPage) : undefined, + page: params.page ? Number(params.page) : undefined, + } + + case 'gitlab_get_job_log': + if (!params.projectId?.trim() || !params.jobId) { + throw new Error('Project ID and Job ID are required.') + } + return { + ...baseParams, + projectId: params.projectId.trim(), + jobId: Number(params.jobId), + } + + case 'gitlab_play_job': + if (!params.projectId?.trim() || !params.jobId) { + throw new Error('Project ID and Job ID are required.') + } + return { + ...baseParams, + projectId: params.projectId.trim(), + jobId: Number(params.jobId), + } + default: return baseParams } @@ -718,7 +1034,7 @@ Return ONLY the commit message - no explanations, no extra text.`, }, inputs: { operation: { type: 'string', description: 'Operation to perform' }, - credential: { type: 'string', description: 'GitLab access token' }, + accessToken: { type: 'string', description: 'GitLab Personal Access Token' }, host: { type: 'string', description: 'Self-managed GitLab host (defaults to gitlab.com)' }, projectId: { type: 'string', description: 'Project ID or URL-encoded path' }, issueIid: { type: 'number', description: 'Issue internal ID' }, @@ -745,6 +1061,16 @@ Return ONLY the commit message - no explanations, no extra text.`, mergeCommitMessage: { type: 'string', description: 'Custom merge commit message' }, perPage: { type: 'number', description: 'Results per page' }, page: { type: 'number', description: 'Page number' }, + filePath: { type: 'string', description: 'Path to file in the repository' }, + branch: { type: 'string', description: 'Branch name' }, + content: { type: 'string', description: 'File content' }, + commitMessage: { type: 'string', description: 'Commit message' }, + jobId: { type: 'number', description: 'Job ID' }, + path: { type: 'string', description: 'Subdirectory path for repository tree' }, + recursive: { type: 'boolean', description: 'Recursively list repository tree' }, + refName: { type: 'string', description: 'Branch or tag name filter' }, + scope: { type: 'string', description: 'Job scope filter' }, + sha: { type: 'string', description: 'Commit SHA' }, }, outputs: { // Project outputs @@ -761,6 +1087,21 @@ Return ONLY the commit message - no explanations, no extra text.`, pipeline: { type: 'json', description: 'Pipeline details' }, // Note outputs note: { type: 'json', description: 'Comment/note details' }, + // Repository outputs + tree: { type: 'json', description: 'Repository tree entries' }, + content: { type: 'string', description: 'File contents (decoded)' }, + fileName: { type: 'string', description: 'File name' }, + branches: { type: 'json', description: 'List of branches' }, + commits: { type: 'json', description: 'List of commits' }, + name: { type: 'string', description: 'Created branch name' }, + webUrl: { type: 'string', description: 'Web URL' }, + // Merge request change outputs + changes: { type: 'json', description: 'Merge request file changes/diffs' }, + approvalsRequired: { type: 'number', description: 'Approvals required' }, + approvalsLeft: { type: 'number', description: 'Approvals remaining' }, + // Job outputs + jobs: { type: 'json', description: 'Pipeline jobs' }, + log: { type: 'string', description: 'Job log output' }, // Success indicator success: { type: 'boolean', description: 'Operation success status' }, }, diff --git a/apps/sim/lib/integrations/integrations.json b/apps/sim/lib/integrations/integrations.json index cf2683bc099..12543d0f629 100644 --- a/apps/sim/lib/integrations/integrations.json +++ b/apps/sim/lib/integrations/integrations.json @@ -1,5 +1,5 @@ { - "updatedAt": "2026-06-18", + "updatedAt": "2026-06-25", "integrations": [ { "type": "onepassword", @@ -5628,11 +5628,90 @@ { "name": "Cancel Pipeline", "description": "Cancel a running GitLab pipeline" + }, + { + "name": "List Repository Tree", + "description": "List files and directories in a GitLab project repository" + }, + { + "name": "Get File", + "description": "Get the contents of a file from a GitLab project repository" + }, + { + "name": "Create File", + "description": "Create a new file in a GitLab project repository" + }, + { + "name": "Update File", + "description": "Update an existing file in a GitLab project repository" + }, + { + "name": "List Commits", + "description": "List commits in a GitLab project repository" + }, + { + "name": "List Branches", + "description": "List branches in a GitLab project repository" + }, + { + "name": "Create Branch", + "description": "Create a new branch in a GitLab project repository" + }, + { + "name": "Get MR Changes", + "description": "Get the file changes (diffs) of a GitLab merge request" + }, + { + "name": "Approve Merge Request", + "description": "Approve a GitLab merge request" + }, + { + "name": "List Pipeline Jobs", + "description": "List jobs for a GitLab pipeline" + }, + { + "name": "Get Job Log", + "description": "Get the log (trace) of a GitLab job" + }, + { + "name": "Play Job", + "description": "Trigger (play) a manual GitLab job" } ], - "operationCount": 19, - "triggers": [], - "triggerCount": 0, + "operationCount": 31, + "triggers": [ + { + "id": "gitlab_push", + "name": "GitLab Push", + "description": "Trigger workflow when commits are pushed to a GitLab project" + }, + { + "id": "gitlab_merge_request", + "name": "GitLab Merge Request", + "description": "Trigger workflow when a merge request is opened, updated, or merged in GitLab" + }, + { + "id": "gitlab_issue", + "name": "GitLab Issue", + "description": "Trigger workflow when an issue is opened, updated, or closed in GitLab" + }, + { + "id": "gitlab_pipeline", + "name": "GitLab Pipeline", + "description": "Trigger workflow when a pipeline status changes in GitLab" + }, + { + "id": "gitlab_comment", + "name": "GitLab Comment", + "description": "Trigger workflow when a comment is added on a commit, merge request, or issue" + }, + { + "id": "gitlab_webhook", + "name": "GitLab Event", + "description": "Trigger workflow from any GitLab webhook event" + } + ], + "triggerCount": 6, "authType": "api-key", "category": "tools", "integrationType": "devops", @@ -11233,8 +11312,39 @@ } ], "operationCount": 6, - "triggers": [], - "triggerCount": 0, + "triggers": [ + { + "id": "pagerduty_incident_triggered", + "name": "PagerDuty Incident Triggered", + "description": "Trigger workflow when a new incident is triggered in PagerDuty" + }, + { + "id": "pagerduty_incident_acknowledged", + "name": "PagerDuty Incident Acknowledged", + "description": "Trigger workflow when an incident is acknowledged in PagerDuty" + }, + { + "id": "pagerduty_incident_resolved", + "name": "PagerDuty Incident Resolved", + "description": "Trigger workflow when an incident is resolved in PagerDuty" + }, + { + "id": "pagerduty_incident_escalated", + "name": "PagerDuty Incident Escalated", + "description": "Trigger workflow when an incident is escalated in PagerDuty" + }, + { + "id": "pagerduty_incident_reassigned", + "name": "PagerDuty Incident Reassigned", + "description": "Trigger workflow when an incident is reassigned in PagerDuty" + }, + { + "id": "pagerduty_webhook", + "name": "PagerDuty Incident Event", + "description": "Trigger workflow from any PagerDuty incident event" + } + ], + "triggerCount": 6, "authType": "api-key", "category": "tools", "integrationType": "observability", @@ -18289,8 +18399,34 @@ } ], "operationCount": 26, - "triggers": [], - "triggerCount": 0, + "triggers": [ + { + "id": "zendesk_ticket_created", + "name": "Zendesk Ticket Created", + "description": "Trigger workflow when a new ticket is created in Zendesk" + }, + { + "id": "zendesk_ticket_status_changed", + "name": "Zendesk Ticket Status Changed", + "description": "Trigger workflow when a ticket status changes in Zendesk" + }, + { + "id": "zendesk_ticket_comment_added", + "name": "Zendesk Ticket Comment Added", + "description": "Trigger workflow when a comment is added to a Zendesk ticket" + }, + { + "id": "zendesk_ticket_priority_changed", + "name": "Zendesk Ticket Priority Changed", + "description": "Trigger workflow when a ticket priority changes in Zendesk" + }, + { + "id": "zendesk_webhook", + "name": "Zendesk Ticket Event", + "description": "Trigger workflow from any Zendesk ticket event" + } + ], + "triggerCount": 5, "authType": "api-key", "category": "tools", "integrationType": "support", diff --git a/apps/sim/tools/gitlab/approve_merge_request.ts b/apps/sim/tools/gitlab/approve_merge_request.ts new file mode 100644 index 00000000000..0db63c53539 --- /dev/null +++ b/apps/sim/tools/gitlab/approve_merge_request.ts @@ -0,0 +1,105 @@ +import type { + GitLabApproveMergeRequestParams, + GitLabApproveMergeRequestResponse, +} from '@/tools/gitlab/types' +import { getGitLabApiBase } from '@/tools/gitlab/utils' +import type { ToolConfig } from '@/tools/types' + +export const gitlabApproveMergeRequestTool: ToolConfig< + GitLabApproveMergeRequestParams, + GitLabApproveMergeRequestResponse +> = { + id: 'gitlab_approve_merge_request', + name: 'GitLab Approve Merge Request', + description: 'Approve a GitLab merge request', + version: '1.0.0', + + params: { + accessToken: { + type: 'string', + required: true, + visibility: 'user-only', + description: 'GitLab Personal Access Token', + }, + host: { + type: 'string', + required: false, + visibility: 'user-only', + description: 'Self-managed GitLab host (e.g. gitlab.example.com). Defaults to gitlab.com.', + }, + projectId: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'Project ID or URL-encoded path', + }, + mergeRequestIid: { + type: 'number', + required: true, + visibility: 'user-or-llm', + description: 'Merge request internal ID (IID)', + }, + sha: { + type: 'string', + required: false, + visibility: 'user-or-llm', + description: 'HEAD SHA of the merge request to approve', + }, + }, + + request: { + url: (params) => { + const encodedId = encodeURIComponent(String(params.projectId).trim()) + return `${getGitLabApiBase(params.host)}/projects/${encodedId}/merge_requests/${params.mergeRequestIid}/approve` + }, + method: 'POST', + headers: (params) => ({ + 'Content-Type': 'application/json', + 'PRIVATE-TOKEN': params.accessToken, + }), + body: (params) => { + const body: Record = {} + + if (params.sha) body.sha = params.sha + + return body + }, + }, + + transformResponse: async (response) => { + if (!response.ok) { + const errorText = await response.text() + return { + success: false, + error: `GitLab API error: ${response.status} ${errorText}`, + output: {}, + } + } + + const data = await response.json() + + return { + success: true, + output: { + approvalsRequired: data.approvals_required ?? null, + approvalsLeft: data.approvals_left ?? null, + approvedBy: data.approved_by ?? [], + }, + } + }, + + outputs: { + approvalsRequired: { + type: 'number', + description: 'Number of approvals required', + }, + approvalsLeft: { + type: 'number', + description: 'Number of approvals still needed', + }, + approvedBy: { + type: 'array', + description: 'List of approvers', + }, + }, +} diff --git a/apps/sim/tools/gitlab/cancel_pipeline.ts b/apps/sim/tools/gitlab/cancel_pipeline.ts index 9707f758a0e..b16ef0534e9 100644 --- a/apps/sim/tools/gitlab/cancel_pipeline.ts +++ b/apps/sim/tools/gitlab/cancel_pipeline.ts @@ -40,7 +40,7 @@ export const gitlabCancelPipelineTool: ToolConfig< request: { url: (params) => { - const encodedId = encodeURIComponent(String(params.projectId)) + const encodedId = encodeURIComponent(String(params.projectId).trim()) return `${getGitLabApiBase(params.host)}/projects/${encodedId}/pipelines/${params.pipelineId}/cancel` }, method: 'POST', diff --git a/apps/sim/tools/gitlab/create_branch.ts b/apps/sim/tools/gitlab/create_branch.ts new file mode 100644 index 00000000000..8fe6a8697c2 --- /dev/null +++ b/apps/sim/tools/gitlab/create_branch.ts @@ -0,0 +1,102 @@ +import type { GitLabCreateBranchParams, GitLabCreateBranchResponse } from '@/tools/gitlab/types' +import { getGitLabApiBase } from '@/tools/gitlab/utils' +import type { ToolConfig } from '@/tools/types' + +export const gitlabCreateBranchTool: ToolConfig< + GitLabCreateBranchParams, + GitLabCreateBranchResponse +> = { + id: 'gitlab_create_branch', + name: 'GitLab Create Branch', + description: 'Create a new branch in a GitLab project repository', + version: '1.0.0', + + params: { + accessToken: { + type: 'string', + required: true, + visibility: 'user-only', + description: 'GitLab Personal Access Token', + }, + host: { + type: 'string', + required: false, + visibility: 'user-only', + description: 'Self-managed GitLab host (e.g. gitlab.example.com). Defaults to gitlab.com.', + }, + projectId: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'Project ID or URL-encoded path', + }, + branch: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'Name of the new branch', + }, + ref: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'Source branch/tag/SHA', + }, + }, + + request: { + url: (params) => { + const encodedId = encodeURIComponent(String(params.projectId).trim()) + const queryParams = new URLSearchParams() + queryParams.append('branch', String(params.branch)) + queryParams.append('ref', String(params.ref)) + return `${getGitLabApiBase(params.host)}/projects/${encodedId}/repository/branches?${queryParams.toString()}` + }, + method: 'POST', + headers: (params) => ({ + 'PRIVATE-TOKEN': params.accessToken, + }), + }, + + transformResponse: async (response) => { + if (!response.ok) { + const errorText = await response.text() + return { + success: false, + error: `GitLab API error: ${response.status} ${errorText}`, + output: {}, + } + } + + const data = await response.json() + + return { + success: true, + output: { + name: data.name ?? null, + webUrl: data.web_url ?? null, + protected: data.protected ?? null, + commit: data.commit ?? null, + }, + } + }, + + outputs: { + name: { + type: 'string', + description: 'The created branch name', + }, + webUrl: { + type: 'string', + description: 'The web URL of the branch', + }, + protected: { + type: 'boolean', + description: 'Whether the branch is protected', + }, + commit: { + type: 'object', + description: 'The commit the branch points to', + }, + }, +} diff --git a/apps/sim/tools/gitlab/create_file.ts b/apps/sim/tools/gitlab/create_file.ts new file mode 100644 index 00000000000..f0468959b4a --- /dev/null +++ b/apps/sim/tools/gitlab/create_file.ts @@ -0,0 +1,106 @@ +import type { GitLabCreateFileParams, GitLabCreateFileResponse } from '@/tools/gitlab/types' +import { getGitLabApiBase } from '@/tools/gitlab/utils' +import type { ToolConfig } from '@/tools/types' + +export const gitlabCreateFileTool: ToolConfig = { + id: 'gitlab_create_file', + name: 'GitLab Create File', + description: 'Create a new file in a GitLab project repository', + version: '1.0.0', + + params: { + accessToken: { + type: 'string', + required: true, + visibility: 'user-only', + description: 'GitLab Personal Access Token', + }, + host: { + type: 'string', + required: false, + visibility: 'user-only', + description: 'Self-managed GitLab host (e.g. gitlab.example.com). Defaults to gitlab.com.', + }, + projectId: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'Project ID or URL-encoded path', + }, + filePath: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'Path to the file in the repository', + }, + branch: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'Branch to commit the new file to', + }, + content: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'File content', + }, + commitMessage: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'Commit message', + }, + }, + + request: { + url: (params) => { + const encodedId = encodeURIComponent(String(params.projectId).trim()) + const encodedPath = encodeURIComponent(String(params.filePath)) + return `${getGitLabApiBase(params.host)}/projects/${encodedId}/repository/files/${encodedPath}` + }, + method: 'POST', + headers: (params) => ({ + 'Content-Type': 'application/json', + 'PRIVATE-TOKEN': params.accessToken, + }), + body: (params) => ({ + branch: params.branch, + content: params.content, + commit_message: params.commitMessage, + encoding: 'text', + }), + }, + + transformResponse: async (response) => { + if (!response.ok) { + const errorText = await response.text() + return { + success: false, + error: `GitLab API error: ${response.status} ${errorText}`, + output: {}, + } + } + + const data = await response.json() + + return { + success: true, + output: { + filePath: data.file_path ?? null, + branch: data.branch ?? null, + }, + } + }, + + outputs: { + filePath: { + type: 'string', + description: 'The created file path', + }, + branch: { + type: 'string', + description: 'The branch the file was committed to', + }, + }, +} diff --git a/apps/sim/tools/gitlab/create_issue.ts b/apps/sim/tools/gitlab/create_issue.ts index cc4475831da..f3830a81f0a 100644 --- a/apps/sim/tools/gitlab/create_issue.ts +++ b/apps/sim/tools/gitlab/create_issue.ts @@ -74,7 +74,7 @@ export const gitlabCreateIssueTool: ToolConfig { - const encodedId = encodeURIComponent(String(params.projectId)) + const encodedId = encodeURIComponent(String(params.projectId).trim()) return `${getGitLabApiBase(params.host)}/projects/${encodedId}/issues` }, method: 'POST', diff --git a/apps/sim/tools/gitlab/create_issue_note.ts b/apps/sim/tools/gitlab/create_issue_note.ts index 0ad5c218bee..c1ef8447405 100644 --- a/apps/sim/tools/gitlab/create_issue_note.ts +++ b/apps/sim/tools/gitlab/create_issue_note.ts @@ -46,7 +46,7 @@ export const gitlabCreateIssueNoteTool: ToolConfig< request: { url: (params) => { - const encodedId = encodeURIComponent(String(params.projectId)) + const encodedId = encodeURIComponent(String(params.projectId).trim()) return `${getGitLabApiBase(params.host)}/projects/${encodedId}/issues/${params.issueIid}/notes` }, method: 'POST', diff --git a/apps/sim/tools/gitlab/create_merge_request.ts b/apps/sim/tools/gitlab/create_merge_request.ts index 2c02c2dd0f4..9c47556553e 100644 --- a/apps/sim/tools/gitlab/create_merge_request.ts +++ b/apps/sim/tools/gitlab/create_merge_request.ts @@ -97,7 +97,7 @@ export const gitlabCreateMergeRequestTool: ToolConfig< request: { url: (params) => { - const encodedId = encodeURIComponent(String(params.projectId)) + const encodedId = encodeURIComponent(String(params.projectId).trim()) return `${getGitLabApiBase(params.host)}/projects/${encodedId}/merge_requests` }, method: 'POST', diff --git a/apps/sim/tools/gitlab/create_merge_request_note.ts b/apps/sim/tools/gitlab/create_merge_request_note.ts index f02f5fa35fb..7f364efba51 100644 --- a/apps/sim/tools/gitlab/create_merge_request_note.ts +++ b/apps/sim/tools/gitlab/create_merge_request_note.ts @@ -49,7 +49,7 @@ export const gitlabCreateMergeRequestNoteTool: ToolConfig< request: { url: (params) => { - const encodedId = encodeURIComponent(String(params.projectId)) + const encodedId = encodeURIComponent(String(params.projectId).trim()) return `${getGitLabApiBase(params.host)}/projects/${encodedId}/merge_requests/${params.mergeRequestIid}/notes` }, method: 'POST', diff --git a/apps/sim/tools/gitlab/create_pipeline.ts b/apps/sim/tools/gitlab/create_pipeline.ts index a27ed7ba372..6a9777ea53b 100644 --- a/apps/sim/tools/gitlab/create_pipeline.ts +++ b/apps/sim/tools/gitlab/create_pipeline.ts @@ -47,7 +47,7 @@ export const gitlabCreatePipelineTool: ToolConfig< request: { url: (params) => { - const encodedId = encodeURIComponent(String(params.projectId)) + const encodedId = encodeURIComponent(String(params.projectId).trim()) return `${getGitLabApiBase(params.host)}/projects/${encodedId}/pipeline` }, method: 'POST', diff --git a/apps/sim/tools/gitlab/delete_issue.ts b/apps/sim/tools/gitlab/delete_issue.ts index 475e52d77a8..7521b826629 100644 --- a/apps/sim/tools/gitlab/delete_issue.ts +++ b/apps/sim/tools/gitlab/delete_issue.ts @@ -38,7 +38,7 @@ export const gitlabDeleteIssueTool: ToolConfig { - const encodedId = encodeURIComponent(String(params.projectId)) + const encodedId = encodeURIComponent(String(params.projectId).trim()) return `${getGitLabApiBase(params.host)}/projects/${encodedId}/issues/${params.issueIid}` }, method: 'DELETE', diff --git a/apps/sim/tools/gitlab/get_file.ts b/apps/sim/tools/gitlab/get_file.ts new file mode 100644 index 00000000000..7c394b2591f --- /dev/null +++ b/apps/sim/tools/gitlab/get_file.ts @@ -0,0 +1,114 @@ +import type { GitLabGetFileParams, GitLabGetFileResponse } from '@/tools/gitlab/types' +import { getGitLabApiBase } from '@/tools/gitlab/utils' +import type { ToolConfig } from '@/tools/types' + +export const gitlabGetFileTool: ToolConfig = { + id: 'gitlab_get_file', + name: 'GitLab Get File', + description: 'Get the contents of a file from a GitLab project repository', + version: '1.0.0', + + params: { + accessToken: { + type: 'string', + required: true, + visibility: 'user-only', + description: 'GitLab Personal Access Token', + }, + host: { + type: 'string', + required: false, + visibility: 'user-only', + description: 'Self-managed GitLab host (e.g. gitlab.example.com). Defaults to gitlab.com.', + }, + projectId: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'Project ID or URL-encoded path', + }, + filePath: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'Path to the file in the repository', + }, + ref: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'Branch, tag, or commit SHA', + }, + }, + + request: { + url: (params) => { + const encodedId = encodeURIComponent(String(params.projectId).trim()) + const encodedPath = encodeURIComponent(String(params.filePath)) + const ref = encodeURIComponent(String(params.ref)) + return `${getGitLabApiBase(params.host)}/projects/${encodedId}/repository/files/${encodedPath}?ref=${ref}` + }, + method: 'GET', + headers: (params) => ({ + 'PRIVATE-TOKEN': params.accessToken, + }), + }, + + transformResponse: async (response) => { + if (!response.ok) { + const errorText = await response.text() + return { + success: false, + error: `GitLab API error: ${response.status} ${errorText}`, + output: {}, + } + } + + const data = await response.json() + const decoded = Buffer.from(data.content ?? '', 'base64').toString('utf-8') + + return { + success: true, + output: { + filePath: data.file_path ?? null, + fileName: data.file_name ?? null, + size: data.size ?? null, + ref: data.ref ?? null, + blobId: data.blob_id ?? null, + lastCommitId: data.last_commit_id ?? null, + content: decoded, + }, + } + }, + + outputs: { + filePath: { + type: 'string', + description: 'The file path', + }, + fileName: { + type: 'string', + description: 'The file name', + }, + size: { + type: 'number', + description: 'The file size in bytes', + }, + ref: { + type: 'string', + description: 'The branch, tag, or commit SHA', + }, + blobId: { + type: 'string', + description: 'The blob ID', + }, + lastCommitId: { + type: 'string', + description: 'The last commit ID that modified the file', + }, + content: { + type: 'string', + description: 'The decoded file content', + }, + }, +} diff --git a/apps/sim/tools/gitlab/get_issue.ts b/apps/sim/tools/gitlab/get_issue.ts index aa87136e552..edd41e86f6c 100644 --- a/apps/sim/tools/gitlab/get_issue.ts +++ b/apps/sim/tools/gitlab/get_issue.ts @@ -37,7 +37,7 @@ export const gitlabGetIssueTool: ToolConfig { - const encodedId = encodeURIComponent(String(params.projectId)) + const encodedId = encodeURIComponent(String(params.projectId).trim()) return `${getGitLabApiBase(params.host)}/projects/${encodedId}/issues/${params.issueIid}` }, method: 'GET', diff --git a/apps/sim/tools/gitlab/get_job_log.ts b/apps/sim/tools/gitlab/get_job_log.ts new file mode 100644 index 00000000000..f3cd4b58314 --- /dev/null +++ b/apps/sim/tools/gitlab/get_job_log.ts @@ -0,0 +1,75 @@ +import type { GitLabGetJobLogParams, GitLabGetJobLogResponse } from '@/tools/gitlab/types' +import { getGitLabApiBase } from '@/tools/gitlab/utils' +import type { ToolConfig } from '@/tools/types' + +export const gitlabGetJobLogTool: ToolConfig = { + id: 'gitlab_get_job_log', + name: 'GitLab Get Job Log', + description: 'Get the log (trace) of a GitLab job', + version: '1.0.0', + + params: { + accessToken: { + type: 'string', + required: true, + visibility: 'user-only', + description: 'GitLab Personal Access Token', + }, + host: { + type: 'string', + required: false, + visibility: 'user-only', + description: 'Self-managed GitLab host (e.g. gitlab.example.com). Defaults to gitlab.com.', + }, + projectId: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'Project ID or URL-encoded path', + }, + jobId: { + type: 'number', + required: true, + visibility: 'user-or-llm', + description: 'Job ID', + }, + }, + + request: { + url: (params) => { + const encodedId = encodeURIComponent(String(params.projectId).trim()) + return `${getGitLabApiBase(params.host)}/projects/${encodedId}/jobs/${params.jobId}/trace` + }, + method: 'GET', + headers: (params) => ({ + 'PRIVATE-TOKEN': params.accessToken, + }), + }, + + transformResponse: async (response) => { + if (!response.ok) { + const errorText = await response.text() + return { + success: false, + error: `GitLab API error: ${response.status} ${errorText}`, + output: {}, + } + } + + const log = await response.text() + + return { + success: true, + output: { + log, + }, + } + }, + + outputs: { + log: { + type: 'string', + description: 'The job log (trace) output', + }, + }, +} diff --git a/apps/sim/tools/gitlab/get_merge_request.ts b/apps/sim/tools/gitlab/get_merge_request.ts index f228cfba2eb..b9b45b0d98d 100644 --- a/apps/sim/tools/gitlab/get_merge_request.ts +++ b/apps/sim/tools/gitlab/get_merge_request.ts @@ -43,7 +43,7 @@ export const gitlabGetMergeRequestTool: ToolConfig< request: { url: (params) => { - const encodedId = encodeURIComponent(String(params.projectId)) + const encodedId = encodeURIComponent(String(params.projectId).trim()) return `${getGitLabApiBase(params.host)}/projects/${encodedId}/merge_requests/${params.mergeRequestIid}` }, method: 'GET', diff --git a/apps/sim/tools/gitlab/get_merge_request_changes.ts b/apps/sim/tools/gitlab/get_merge_request_changes.ts new file mode 100644 index 00000000000..4bd6ba78e5e --- /dev/null +++ b/apps/sim/tools/gitlab/get_merge_request_changes.ts @@ -0,0 +1,98 @@ +import type { + GitLabGetMergeRequestChangesParams, + GitLabGetMergeRequestChangesResponse, +} from '@/tools/gitlab/types' +import { getGitLabApiBase } from '@/tools/gitlab/utils' +import type { ToolConfig } from '@/tools/types' + +export const gitlabGetMergeRequestChangesTool: ToolConfig< + GitLabGetMergeRequestChangesParams, + GitLabGetMergeRequestChangesResponse +> = { + id: 'gitlab_get_merge_request_changes', + name: 'GitLab Get Merge Request Changes', + description: 'Get the file changes (diffs) of a GitLab merge request', + version: '1.0.0', + + params: { + accessToken: { + type: 'string', + required: true, + visibility: 'user-only', + description: 'GitLab Personal Access Token', + }, + host: { + type: 'string', + required: false, + visibility: 'user-only', + description: 'Self-managed GitLab host (e.g. gitlab.example.com). Defaults to gitlab.com.', + }, + projectId: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'Project ID or URL-encoded path', + }, + mergeRequestIid: { + type: 'number', + required: true, + visibility: 'user-or-llm', + description: 'Merge request internal ID (IID)', + }, + }, + + request: { + /** + * Uses the `/diffs` endpoint (the `/changes` endpoint was deprecated in + * GitLab 15.7 and removed in 18.0). `/diffs` returns the diff array directly + * and is paginated; we request the max page size (100) to return the changes + * in a single call, which covers the vast majority of merge requests. + */ + url: (params) => { + const encodedId = encodeURIComponent(String(params.projectId).trim()) + return `${getGitLabApiBase(params.host)}/projects/${encodedId}/merge_requests/${params.mergeRequestIid}/diffs?per_page=100` + }, + method: 'GET', + headers: (params) => ({ + 'PRIVATE-TOKEN': params.accessToken, + }), + }, + + transformResponse: async (response, params) => { + if (!response.ok) { + const errorText = await response.text() + return { + success: false, + error: `GitLab API error: ${response.status} ${errorText}`, + output: {}, + } + } + + const data = await response.json() + const changes = Array.isArray(data) ? data : [] + + return { + success: true, + output: { + mergeRequestIid: params?.mergeRequestIid ?? null, + changes, + changesCount: changes.length, + }, + } + }, + + outputs: { + mergeRequestIid: { + type: 'number', + description: 'The merge request internal ID (IID)', + }, + changes: { + type: 'array', + description: 'List of file changes (diffs)', + }, + changesCount: { + type: 'number', + description: 'Number of changed files returned', + }, + }, +} diff --git a/apps/sim/tools/gitlab/get_pipeline.ts b/apps/sim/tools/gitlab/get_pipeline.ts index 1494e65e4bb..c69cfd61185 100644 --- a/apps/sim/tools/gitlab/get_pipeline.ts +++ b/apps/sim/tools/gitlab/get_pipeline.ts @@ -38,7 +38,7 @@ export const gitlabGetPipelineTool: ToolConfig { - const encodedId = encodeURIComponent(String(params.projectId)) + const encodedId = encodeURIComponent(String(params.projectId).trim()) return `${getGitLabApiBase(params.host)}/projects/${encodedId}/pipelines/${params.pipelineId}` }, method: 'GET', diff --git a/apps/sim/tools/gitlab/get_project.ts b/apps/sim/tools/gitlab/get_project.ts index 5ea42920584..882da20092a 100644 --- a/apps/sim/tools/gitlab/get_project.ts +++ b/apps/sim/tools/gitlab/get_project.ts @@ -31,7 +31,7 @@ export const gitlabGetProjectTool: ToolConfig { - const encodedId = encodeURIComponent(String(params.projectId)) + const encodedId = encodeURIComponent(String(params.projectId).trim()) return `${getGitLabApiBase(params.host)}/projects/${encodedId}` }, method: 'GET', diff --git a/apps/sim/tools/gitlab/index.ts b/apps/sim/tools/gitlab/index.ts index 0133af843ab..25d7ba9e6cd 100644 --- a/apps/sim/tools/gitlab/index.ts +++ b/apps/sim/tools/gitlab/index.ts @@ -1,20 +1,32 @@ +import { gitlabApproveMergeRequestTool } from '@/tools/gitlab/approve_merge_request' import { gitlabCancelPipelineTool } from '@/tools/gitlab/cancel_pipeline' +import { gitlabCreateBranchTool } from '@/tools/gitlab/create_branch' +import { gitlabCreateFileTool } from '@/tools/gitlab/create_file' import { gitlabCreateIssueTool } from '@/tools/gitlab/create_issue' import { gitlabCreateIssueNoteTool } from '@/tools/gitlab/create_issue_note' import { gitlabCreateMergeRequestTool } from '@/tools/gitlab/create_merge_request' import { gitlabCreateMergeRequestNoteTool } from '@/tools/gitlab/create_merge_request_note' import { gitlabCreatePipelineTool } from '@/tools/gitlab/create_pipeline' import { gitlabDeleteIssueTool } from '@/tools/gitlab/delete_issue' +import { gitlabGetFileTool } from '@/tools/gitlab/get_file' import { gitlabGetIssueTool } from '@/tools/gitlab/get_issue' +import { gitlabGetJobLogTool } from '@/tools/gitlab/get_job_log' import { gitlabGetMergeRequestTool } from '@/tools/gitlab/get_merge_request' +import { gitlabGetMergeRequestChangesTool } from '@/tools/gitlab/get_merge_request_changes' import { gitlabGetPipelineTool } from '@/tools/gitlab/get_pipeline' import { gitlabGetProjectTool } from '@/tools/gitlab/get_project' +import { gitlabListBranchesTool } from '@/tools/gitlab/list_branches' +import { gitlabListCommitsTool } from '@/tools/gitlab/list_commits' import { gitlabListIssuesTool } from '@/tools/gitlab/list_issues' import { gitlabListMergeRequestsTool } from '@/tools/gitlab/list_merge_requests' +import { gitlabListPipelineJobsTool } from '@/tools/gitlab/list_pipeline_jobs' import { gitlabListPipelinesTool } from '@/tools/gitlab/list_pipelines' import { gitlabListProjectsTool } from '@/tools/gitlab/list_projects' +import { gitlabListRepositoryTreeTool } from '@/tools/gitlab/list_repository_tree' import { gitlabMergeMergeRequestTool } from '@/tools/gitlab/merge_merge_request' +import { gitlabPlayJobTool } from '@/tools/gitlab/play_job' import { gitlabRetryPipelineTool } from '@/tools/gitlab/retry_pipeline' +import { gitlabUpdateFileTool } from '@/tools/gitlab/update_file' import { gitlabUpdateIssueTool } from '@/tools/gitlab/update_issue' import { gitlabUpdateMergeRequestTool } from '@/tools/gitlab/update_merge_request' @@ -36,10 +48,26 @@ export { gitlabUpdateMergeRequestTool, gitlabMergeMergeRequestTool, gitlabCreateMergeRequestNoteTool, + gitlabGetMergeRequestChangesTool, + gitlabApproveMergeRequestTool, // Pipelines gitlabListPipelinesTool, gitlabGetPipelineTool, gitlabCreatePipelineTool, gitlabRetryPipelineTool, gitlabCancelPipelineTool, + // Jobs + gitlabListPipelineJobsTool, + gitlabGetJobLogTool, + gitlabPlayJobTool, + // Repository Files & Tree + gitlabListRepositoryTreeTool, + gitlabGetFileTool, + gitlabCreateFileTool, + gitlabUpdateFileTool, + // Branches + gitlabListBranchesTool, + gitlabCreateBranchTool, + // Commits + gitlabListCommitsTool, } diff --git a/apps/sim/tools/gitlab/list_branches.ts b/apps/sim/tools/gitlab/list_branches.ts new file mode 100644 index 00000000000..2943c250285 --- /dev/null +++ b/apps/sim/tools/gitlab/list_branches.ts @@ -0,0 +1,103 @@ +import type { GitLabListBranchesParams, GitLabListBranchesResponse } from '@/tools/gitlab/types' +import { getGitLabApiBase } from '@/tools/gitlab/utils' +import type { ToolConfig } from '@/tools/types' + +export const gitlabListBranchesTool: ToolConfig< + GitLabListBranchesParams, + GitLabListBranchesResponse +> = { + id: 'gitlab_list_branches', + name: 'GitLab List Branches', + description: 'List branches in a GitLab project repository', + version: '1.0.0', + + params: { + accessToken: { + type: 'string', + required: true, + visibility: 'user-only', + description: 'GitLab Personal Access Token', + }, + host: { + type: 'string', + required: false, + visibility: 'user-only', + description: 'Self-managed GitLab host (e.g. gitlab.example.com). Defaults to gitlab.com.', + }, + projectId: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'Project ID or URL-encoded path', + }, + search: { + type: 'string', + required: false, + visibility: 'user-or-llm', + description: 'Filter branches by name', + }, + perPage: { + type: 'number', + required: false, + visibility: 'user-or-llm', + description: 'Number of results per page (default 20, max 100)', + }, + page: { + type: 'number', + required: false, + visibility: 'user-or-llm', + description: 'Page number for pagination', + }, + }, + + request: { + url: (params) => { + const encodedId = encodeURIComponent(String(params.projectId).trim()) + const queryParams = new URLSearchParams() + + if (params.search) queryParams.append('search', params.search) + if (params.perPage) queryParams.append('per_page', String(params.perPage)) + if (params.page) queryParams.append('page', String(params.page)) + + const query = queryParams.toString() + return `${getGitLabApiBase(params.host)}/projects/${encodedId}/repository/branches${query ? `?${query}` : ''}` + }, + method: 'GET', + headers: (params) => ({ + 'PRIVATE-TOKEN': params.accessToken, + }), + }, + + transformResponse: async (response) => { + if (!response.ok) { + const errorText = await response.text() + return { + success: false, + error: `GitLab API error: ${response.status} ${errorText}`, + output: {}, + } + } + + const branches = await response.json() + const total = response.headers.get('x-total') + + return { + success: true, + output: { + branches: branches ?? [], + total: total ? Number.parseInt(total, 10) : (branches?.length ?? 0), + }, + } + }, + + outputs: { + branches: { + type: 'array', + description: 'List of branches', + }, + total: { + type: 'number', + description: 'Total number of branches', + }, + }, +} diff --git a/apps/sim/tools/gitlab/list_commits.ts b/apps/sim/tools/gitlab/list_commits.ts new file mode 100644 index 00000000000..b3be01049d2 --- /dev/null +++ b/apps/sim/tools/gitlab/list_commits.ts @@ -0,0 +1,129 @@ +import type { GitLabListCommitsParams, GitLabListCommitsResponse } from '@/tools/gitlab/types' +import { getGitLabApiBase } from '@/tools/gitlab/utils' +import type { ToolConfig } from '@/tools/types' + +export const gitlabListCommitsTool: ToolConfig = + { + id: 'gitlab_list_commits', + name: 'GitLab List Commits', + description: 'List commits in a GitLab project repository', + version: '1.0.0', + + params: { + accessToken: { + type: 'string', + required: true, + visibility: 'user-only', + description: 'GitLab Personal Access Token', + }, + host: { + type: 'string', + required: false, + visibility: 'user-only', + description: 'Self-managed GitLab host (e.g. gitlab.example.com). Defaults to gitlab.com.', + }, + projectId: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'Project ID or URL-encoded path', + }, + refName: { + type: 'string', + required: false, + visibility: 'user-or-llm', + description: 'Branch, tag, or revision range to list commits from', + }, + since: { + type: 'string', + required: false, + visibility: 'user-or-llm', + description: 'Only commits after this ISO 8601 date', + }, + until: { + type: 'string', + required: false, + visibility: 'user-or-llm', + description: 'Only commits before this ISO 8601 date', + }, + path: { + type: 'string', + required: false, + visibility: 'user-or-llm', + description: 'Only commits affecting this file path', + }, + author: { + type: 'string', + required: false, + visibility: 'user-or-llm', + description: 'Filter commits by author', + }, + perPage: { + type: 'number', + required: false, + visibility: 'user-or-llm', + description: 'Number of results per page (default 20, max 100)', + }, + page: { + type: 'number', + required: false, + visibility: 'user-or-llm', + description: 'Page number for pagination', + }, + }, + + request: { + url: (params) => { + const encodedId = encodeURIComponent(String(params.projectId).trim()) + const queryParams = new URLSearchParams() + + if (params.refName) queryParams.append('ref_name', params.refName) + if (params.since) queryParams.append('since', params.since) + if (params.until) queryParams.append('until', params.until) + if (params.path) queryParams.append('path', params.path) + if (params.author) queryParams.append('author', params.author) + if (params.perPage) queryParams.append('per_page', String(params.perPage)) + if (params.page) queryParams.append('page', String(params.page)) + + const query = queryParams.toString() + return `${getGitLabApiBase(params.host)}/projects/${encodedId}/repository/commits${query ? `?${query}` : ''}` + }, + method: 'GET', + headers: (params) => ({ + 'PRIVATE-TOKEN': params.accessToken, + }), + }, + + transformResponse: async (response) => { + if (!response.ok) { + const errorText = await response.text() + return { + success: false, + error: `GitLab API error: ${response.status} ${errorText}`, + output: {}, + } + } + + const commits = await response.json() + const total = response.headers.get('x-total') + + return { + success: true, + output: { + commits: commits ?? [], + total: total ? Number.parseInt(total, 10) : (commits?.length ?? 0), + }, + } + }, + + outputs: { + commits: { + type: 'array', + description: 'List of commits', + }, + total: { + type: 'number', + description: 'Total number of commits', + }, + }, + } diff --git a/apps/sim/tools/gitlab/list_issues.ts b/apps/sim/tools/gitlab/list_issues.ts index 40a016f3b34..8a46a915314 100644 --- a/apps/sim/tools/gitlab/list_issues.ts +++ b/apps/sim/tools/gitlab/list_issues.ts @@ -85,7 +85,7 @@ export const gitlabListIssuesTool: ToolConfig { - const encodedId = encodeURIComponent(String(params.projectId)) + const encodedId = encodeURIComponent(String(params.projectId).trim()) const queryParams = new URLSearchParams() if (params.state) queryParams.append('state', params.state) diff --git a/apps/sim/tools/gitlab/list_merge_requests.ts b/apps/sim/tools/gitlab/list_merge_requests.ts index 2cdae3301c4..65ad7efe797 100644 --- a/apps/sim/tools/gitlab/list_merge_requests.ts +++ b/apps/sim/tools/gitlab/list_merge_requests.ts @@ -85,7 +85,7 @@ export const gitlabListMergeRequestsTool: ToolConfig< request: { url: (params) => { - const encodedId = encodeURIComponent(String(params.projectId)) + const encodedId = encodeURIComponent(String(params.projectId).trim()) const queryParams = new URLSearchParams() if (params.state) queryParams.append('state', params.state) diff --git a/apps/sim/tools/gitlab/list_pipeline_jobs.ts b/apps/sim/tools/gitlab/list_pipeline_jobs.ts new file mode 100644 index 00000000000..db63ccb8df6 --- /dev/null +++ b/apps/sim/tools/gitlab/list_pipeline_jobs.ts @@ -0,0 +1,119 @@ +import type { + GitLabListPipelineJobsParams, + GitLabListPipelineJobsResponse, +} from '@/tools/gitlab/types' +import { getGitLabApiBase } from '@/tools/gitlab/utils' +import type { ToolConfig } from '@/tools/types' + +export const gitlabListPipelineJobsTool: ToolConfig< + GitLabListPipelineJobsParams, + GitLabListPipelineJobsResponse +> = { + id: 'gitlab_list_pipeline_jobs', + name: 'GitLab List Pipeline Jobs', + description: 'List jobs for a GitLab pipeline', + version: '1.0.0', + + params: { + accessToken: { + type: 'string', + required: true, + visibility: 'user-only', + description: 'GitLab Personal Access Token', + }, + host: { + type: 'string', + required: false, + visibility: 'user-only', + description: 'Self-managed GitLab host (e.g. gitlab.example.com). Defaults to gitlab.com.', + }, + projectId: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'Project ID or URL-encoded path', + }, + pipelineId: { + type: 'number', + required: true, + visibility: 'user-or-llm', + description: 'Pipeline ID', + }, + scope: { + type: 'string', + required: false, + visibility: 'user-or-llm', + description: 'Filter jobs by scope (e.g. created, running, success, failed)', + }, + includeRetried: { + type: 'boolean', + required: false, + visibility: 'user-or-llm', + description: 'Whether to include retried jobs', + }, + perPage: { + type: 'number', + required: false, + visibility: 'user-or-llm', + description: 'Number of results per page (default 20, max 100)', + }, + page: { + type: 'number', + required: false, + visibility: 'user-or-llm', + description: 'Page number for pagination', + }, + }, + + request: { + url: (params) => { + const encodedId = encodeURIComponent(String(params.projectId).trim()) + const queryParams = new URLSearchParams() + + if (params.scope) queryParams.append('scope', params.scope) + if (params.includeRetried) queryParams.append('include_retried', 'true') + if (params.perPage) queryParams.append('per_page', String(params.perPage)) + if (params.page) queryParams.append('page', String(params.page)) + + const query = queryParams.toString() + return `${getGitLabApiBase(params.host)}/projects/${encodedId}/pipelines/${params.pipelineId}/jobs${query ? `?${query}` : ''}` + }, + method: 'GET', + headers: (params) => ({ + 'PRIVATE-TOKEN': params.accessToken, + }), + }, + + transformResponse: async (response) => { + if (!response.ok) { + const errorText = await response.text() + return { + success: false, + error: `GitLab API error: ${response.status} ${errorText}`, + output: {}, + } + } + + const jobs = await response.json() + const total = response.headers.get('x-total') + + return { + success: true, + output: { + jobs: jobs ?? [], + total: total ? Number.parseInt(total, 10) : (jobs?.length ?? 0), + }, + } + }, + + outputs: { + jobs: { + type: 'array', + description: 'List of pipeline jobs', + }, + total: { + type: 'number', + description: 'Total number of jobs', + }, + }, +} diff --git a/apps/sim/tools/gitlab/list_pipelines.ts b/apps/sim/tools/gitlab/list_pipelines.ts index 80294e85f73..bb21081e2ea 100644 --- a/apps/sim/tools/gitlab/list_pipelines.ts +++ b/apps/sim/tools/gitlab/list_pipelines.ts @@ -71,7 +71,7 @@ export const gitlabListPipelinesTool: ToolConfig< request: { url: (params) => { - const encodedId = encodeURIComponent(String(params.projectId)) + const encodedId = encodeURIComponent(String(params.projectId).trim()) const queryParams = new URLSearchParams() if (params.ref) queryParams.append('ref', params.ref) diff --git a/apps/sim/tools/gitlab/list_repository_tree.ts b/apps/sim/tools/gitlab/list_repository_tree.ts new file mode 100644 index 00000000000..efbc439b852 --- /dev/null +++ b/apps/sim/tools/gitlab/list_repository_tree.ts @@ -0,0 +1,120 @@ +import type { + GitLabListRepositoryTreeParams, + GitLabListRepositoryTreeResponse, +} from '@/tools/gitlab/types' +import { getGitLabApiBase } from '@/tools/gitlab/utils' +import type { ToolConfig } from '@/tools/types' + +export const gitlabListRepositoryTreeTool: ToolConfig< + GitLabListRepositoryTreeParams, + GitLabListRepositoryTreeResponse +> = { + id: 'gitlab_list_repository_tree', + name: 'GitLab List Repository Tree', + description: 'List files and directories in a GitLab project repository', + version: '1.0.0', + + params: { + accessToken: { + type: 'string', + required: true, + visibility: 'user-only', + description: 'GitLab Personal Access Token', + }, + host: { + type: 'string', + required: false, + visibility: 'user-only', + description: 'Self-managed GitLab host (e.g. gitlab.example.com). Defaults to gitlab.com.', + }, + projectId: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'Project ID or URL-encoded path', + }, + path: { + type: 'string', + required: false, + visibility: 'user-or-llm', + description: 'Path inside the repository to list', + }, + ref: { + type: 'string', + required: false, + visibility: 'user-or-llm', + description: 'Branch, tag, or commit SHA to list from', + }, + recursive: { + type: 'boolean', + required: false, + visibility: 'user-or-llm', + description: 'Whether to list files recursively', + }, + perPage: { + type: 'number', + required: false, + visibility: 'user-or-llm', + description: 'Number of results per page (default 20, max 100)', + }, + page: { + type: 'number', + required: false, + visibility: 'user-or-llm', + description: 'Page number for pagination', + }, + }, + + request: { + url: (params) => { + const encodedId = encodeURIComponent(String(params.projectId).trim()) + const queryParams = new URLSearchParams() + + if (params.path) queryParams.append('path', params.path) + if (params.ref) queryParams.append('ref', params.ref) + if (params.recursive) queryParams.append('recursive', 'true') + if (params.perPage) queryParams.append('per_page', String(params.perPage)) + if (params.page) queryParams.append('page', String(params.page)) + + const query = queryParams.toString() + return `${getGitLabApiBase(params.host)}/projects/${encodedId}/repository/tree${query ? `?${query}` : ''}` + }, + method: 'GET', + headers: (params) => ({ + 'PRIVATE-TOKEN': params.accessToken, + }), + }, + + transformResponse: async (response) => { + if (!response.ok) { + const errorText = await response.text() + return { + success: false, + error: `GitLab API error: ${response.status} ${errorText}`, + output: {}, + } + } + + const tree = await response.json() + const total = response.headers.get('x-total') + + return { + success: true, + output: { + tree: tree ?? [], + total: total ? Number.parseInt(total, 10) : (tree?.length ?? 0), + }, + } + }, + + outputs: { + tree: { + type: 'array', + description: 'List of repository tree entries', + }, + total: { + type: 'number', + description: 'Total number of tree entries', + }, + }, +} diff --git a/apps/sim/tools/gitlab/merge_merge_request.ts b/apps/sim/tools/gitlab/merge_merge_request.ts index 500e6ebfd07..4991e67e209 100644 --- a/apps/sim/tools/gitlab/merge_merge_request.ts +++ b/apps/sim/tools/gitlab/merge_merge_request.ts @@ -73,7 +73,7 @@ export const gitlabMergeMergeRequestTool: ToolConfig< request: { url: (params) => { - const encodedId = encodeURIComponent(String(params.projectId)) + const encodedId = encodeURIComponent(String(params.projectId).trim()) return `${getGitLabApiBase(params.host)}/projects/${encodedId}/merge_requests/${params.mergeRequestIid}/merge` }, method: 'PUT', diff --git a/apps/sim/tools/gitlab/play_job.ts b/apps/sim/tools/gitlab/play_job.ts new file mode 100644 index 00000000000..bd8a37effca --- /dev/null +++ b/apps/sim/tools/gitlab/play_job.ts @@ -0,0 +1,90 @@ +import type { GitLabPlayJobParams, GitLabPlayJobResponse } from '@/tools/gitlab/types' +import { getGitLabApiBase } from '@/tools/gitlab/utils' +import type { ToolConfig } from '@/tools/types' + +export const gitlabPlayJobTool: ToolConfig = { + id: 'gitlab_play_job', + name: 'GitLab Play Job', + description: 'Trigger (play) a manual GitLab job', + version: '1.0.0', + + params: { + accessToken: { + type: 'string', + required: true, + visibility: 'user-only', + description: 'GitLab Personal Access Token', + }, + host: { + type: 'string', + required: false, + visibility: 'user-only', + description: 'Self-managed GitLab host (e.g. gitlab.example.com). Defaults to gitlab.com.', + }, + projectId: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'Project ID or URL-encoded path', + }, + jobId: { + type: 'number', + required: true, + visibility: 'user-or-llm', + description: 'Job ID', + }, + }, + + request: { + url: (params) => { + const encodedId = encodeURIComponent(String(params.projectId).trim()) + return `${getGitLabApiBase(params.host)}/projects/${encodedId}/jobs/${params.jobId}/play` + }, + method: 'POST', + headers: (params) => ({ + 'PRIVATE-TOKEN': params.accessToken, + }), + }, + + transformResponse: async (response) => { + if (!response.ok) { + const errorText = await response.text() + return { + success: false, + error: `GitLab API error: ${response.status} ${errorText}`, + output: {}, + } + } + + const data = await response.json() + + return { + success: true, + output: { + id: data.id ?? null, + name: data.name ?? null, + status: data.status ?? null, + webUrl: data.web_url ?? null, + }, + } + }, + + outputs: { + id: { + type: 'number', + description: 'The job ID', + }, + name: { + type: 'string', + description: 'The job name', + }, + status: { + type: 'string', + description: 'The job status', + }, + webUrl: { + type: 'string', + description: 'The web URL of the job', + }, + }, +} diff --git a/apps/sim/tools/gitlab/retry_pipeline.ts b/apps/sim/tools/gitlab/retry_pipeline.ts index 48143109c97..470ade6d9f0 100644 --- a/apps/sim/tools/gitlab/retry_pipeline.ts +++ b/apps/sim/tools/gitlab/retry_pipeline.ts @@ -40,7 +40,7 @@ export const gitlabRetryPipelineTool: ToolConfig< request: { url: (params) => { - const encodedId = encodeURIComponent(String(params.projectId)) + const encodedId = encodeURIComponent(String(params.projectId).trim()) return `${getGitLabApiBase(params.host)}/projects/${encodedId}/pipelines/${params.pipelineId}/retry` }, method: 'POST', diff --git a/apps/sim/tools/gitlab/types.ts b/apps/sim/tools/gitlab/types.ts index af865ed3ef8..d87a10d67b6 100644 --- a/apps/sim/tools/gitlab/types.ts +++ b/apps/sim/tools/gitlab/types.ts @@ -1,7 +1,5 @@ import type { ToolResponse } from '@/tools/types' -// ===== Core Types ===== - interface GitLabProject { id: number name: string @@ -124,6 +122,46 @@ interface GitLabPipeline { } } +interface GitLabTreeEntry { + id: string + name: string + type: string + path: string + mode: string +} + +interface GitLabCommit { + id: string + short_id: string + title: string + message: string + author_name: string + authored_date: string + created_at: string + web_url: string +} + +interface GitLabJob { + id: number + name: string + stage: string + status: string + started_at?: string | null + finished_at?: string | null + duration?: number | null + web_url: string + ref?: string +} + +interface GitLabMergeRequestChange { + old_path: string + new_path: string + diff: string + new_file: boolean + deleted_file: boolean + renamed_file: boolean +} + interface GitLabBranch { name: string merged: boolean @@ -190,8 +228,6 @@ interface GitLabMilestone { web_url: string } -// ===== Common Parameters ===== - interface GitLabBaseParams { accessToken: string /** @@ -201,8 +237,6 @@ interface GitLabBaseParams { host?: string } -// ===== Project Parameters ===== - export interface GitLabListProjectsParams extends GitLabBaseParams { owned?: boolean membership?: boolean @@ -218,8 +252,6 @@ export interface GitLabGetProjectParams extends GitLabBaseParams { projectId: string | number } -// ===== Issue Parameters ===== - export interface GitLabListIssuesParams extends GitLabBaseParams { projectId: string | number state?: 'opened' | 'closed' | 'all' @@ -267,8 +299,6 @@ export interface GitLabDeleteIssueParams extends GitLabBaseParams { issueIid: number } -// ===== Merge Request Parameters ===== - export interface GitLabListMergeRequestsParams extends GitLabBaseParams { projectId: string | number state?: 'opened' | 'closed' | 'merged' | 'all' @@ -325,8 +355,6 @@ export interface GitLabMergeMergeRequestParams extends GitLabBaseParams { mergeWhenPipelineSucceeds?: boolean } -// ===== Pipeline Parameters ===== - export interface GitLabListPipelinesParams extends GitLabBaseParams { projectId: string | number ref?: string @@ -369,9 +397,7 @@ export interface GitLabCancelPipelineParams extends GitLabBaseParams { pipelineId: number } -// ===== Branch Parameters ===== - -interface GitLabListBranchesParams extends GitLabBaseParams { +export interface GitLabListBranchesParams extends GitLabBaseParams { projectId: string | number search?: string perPage?: number @@ -383,7 +409,7 @@ interface GitLabGetBranchParams extends GitLabBaseParams { branch: string } -interface GitLabCreateBranchParams extends GitLabBaseParams { +export interface GitLabCreateBranchParams extends GitLabBaseParams { projectId: string | number branch: string ref: string @@ -394,7 +420,78 @@ interface GitLabDeleteBranchParams extends GitLabBaseParams { branch: string } -// ===== Note/Comment Parameters ===== +export interface GitLabListRepositoryTreeParams extends GitLabBaseParams { + projectId: string | number + path?: string + ref?: string + recursive?: boolean + perPage?: number + page?: number +} + +export interface GitLabGetFileParams extends GitLabBaseParams { + projectId: string | number + filePath: string + ref: string +} + +export interface GitLabCreateFileParams extends GitLabBaseParams { + projectId: string | number + filePath: string + branch: string + content: string + commitMessage: string +} + +export interface GitLabUpdateFileParams extends GitLabBaseParams { + projectId: string | number + filePath: string + branch: string + content: string + commitMessage: string + lastCommitId?: string +} + +export interface GitLabListCommitsParams extends GitLabBaseParams { + projectId: string | number + refName?: string + since?: string + until?: string + path?: string + author?: string + perPage?: number + page?: number +} + +export interface GitLabGetMergeRequestChangesParams extends GitLabBaseParams { + projectId: string | number + mergeRequestIid: number +} + +export interface GitLabApproveMergeRequestParams extends GitLabBaseParams { + projectId: string | number + mergeRequestIid: number + sha?: string +} + +export interface GitLabListPipelineJobsParams extends GitLabBaseParams { + projectId: string | number + pipelineId: number + scope?: string + includeRetried?: boolean + perPage?: number + page?: number +} + +export interface GitLabGetJobLogParams extends GitLabBaseParams { + projectId: string | number + jobId: number +} + +export interface GitLabPlayJobParams extends GitLabBaseParams { + projectId: string | number + jobId: number +} interface GitLabListIssueNotesParams extends GitLabBaseParams { projectId: string | number @@ -426,8 +523,6 @@ export interface GitLabCreateMergeRequestNoteParams extends GitLabBaseParams { body: string } -// ===== Label Parameters ===== - interface GitLabListLabelsParams extends GitLabBaseParams { projectId: string | number search?: string @@ -442,8 +537,6 @@ interface GitLabCreateLabelParams extends GitLabBaseParams { description?: string } -// ===== User Parameters ===== - interface GitLabGetCurrentUserParams extends GitLabBaseParams {} interface GitLabListUsersParams extends GitLabBaseParams { @@ -452,8 +545,6 @@ interface GitLabListUsersParams extends GitLabBaseParams { page?: number } -// ===== Response Types ===== - export interface GitLabListProjectsResponse extends ToolResponse { output: { projects?: GitLabProject[] @@ -560,7 +651,7 @@ export interface GitLabCancelPipelineResponse extends ToolResponse { } } -interface GitLabListBranchesResponse extends ToolResponse { +export interface GitLabListBranchesResponse extends ToolResponse { output: { branches?: GitLabBranch[] total?: number @@ -573,9 +664,12 @@ interface GitLabGetBranchResponse extends ToolResponse { } } -interface GitLabCreateBranchResponse extends ToolResponse { +export interface GitLabCreateBranchResponse extends ToolResponse { output: { - branch?: GitLabBranch + name?: string | null + webUrl?: string | null + protected?: boolean | null + commit?: GitLabBranch['commit'] | null } } @@ -624,7 +718,83 @@ interface GitLabListUsersResponse extends ToolResponse { } } -// ===== Union Response Type ===== +export interface GitLabListRepositoryTreeResponse extends ToolResponse { + output: { + tree?: GitLabTreeEntry[] + total?: number + } +} + +export interface GitLabGetFileResponse extends ToolResponse { + output: { + filePath?: string | null + fileName?: string | null + size?: number | null + ref?: string | null + blobId?: string | null + lastCommitId?: string | null + content?: string + } +} + +export interface GitLabCreateFileResponse extends ToolResponse { + output: { + filePath?: string | null + branch?: string | null + } +} + +export interface GitLabUpdateFileResponse extends ToolResponse { + output: { + filePath?: string | null + branch?: string | null + } +} + +export interface GitLabListCommitsResponse extends ToolResponse { + output: { + commits?: GitLabCommit[] + total?: number + } +} + +export interface GitLabGetMergeRequestChangesResponse extends ToolResponse { + output: { + mergeRequestIid?: number | null + changes?: GitLabMergeRequestChange[] + changesCount?: number + } +} + +export interface GitLabApproveMergeRequestResponse extends ToolResponse { + output: { + approvalsRequired?: number | null + approvalsLeft?: number | null + approvedBy?: unknown[] + } +} + +export interface GitLabListPipelineJobsResponse extends ToolResponse { + output: { + jobs?: GitLabJob[] + total?: number + } +} + +export interface GitLabGetJobLogResponse extends ToolResponse { + output: { + log?: string + } +} + +export interface GitLabPlayJobResponse extends ToolResponse { + output: { + id?: number | null + name?: string | null + status?: string | null + webUrl?: string | null + } +} export type GitLabResponse = | GitLabListProjectsResponse @@ -654,3 +824,13 @@ export type GitLabResponse = | GitLabCreateLabelResponse | GitLabGetCurrentUserResponse | GitLabListUsersResponse + | GitLabListRepositoryTreeResponse + | GitLabGetFileResponse + | GitLabCreateFileResponse + | GitLabUpdateFileResponse + | GitLabListCommitsResponse + | GitLabGetMergeRequestChangesResponse + | GitLabApproveMergeRequestResponse + | GitLabListPipelineJobsResponse + | GitLabGetJobLogResponse + | GitLabPlayJobResponse diff --git a/apps/sim/tools/gitlab/update_file.ts b/apps/sim/tools/gitlab/update_file.ts new file mode 100644 index 00000000000..38a7beece3d --- /dev/null +++ b/apps/sim/tools/gitlab/update_file.ts @@ -0,0 +1,118 @@ +import type { GitLabUpdateFileParams, GitLabUpdateFileResponse } from '@/tools/gitlab/types' +import { getGitLabApiBase } from '@/tools/gitlab/utils' +import type { ToolConfig } from '@/tools/types' + +export const gitlabUpdateFileTool: ToolConfig = { + id: 'gitlab_update_file', + name: 'GitLab Update File', + description: 'Update an existing file in a GitLab project repository', + version: '1.0.0', + + params: { + accessToken: { + type: 'string', + required: true, + visibility: 'user-only', + description: 'GitLab Personal Access Token', + }, + host: { + type: 'string', + required: false, + visibility: 'user-only', + description: 'Self-managed GitLab host (e.g. gitlab.example.com). Defaults to gitlab.com.', + }, + projectId: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'Project ID or URL-encoded path', + }, + filePath: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'Path to the file in the repository', + }, + branch: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'Branch to commit the update to', + }, + content: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'New file content', + }, + commitMessage: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'Commit message', + }, + lastCommitId: { + type: 'string', + required: false, + visibility: 'user-or-llm', + description: 'Last known commit ID for the file (optimistic locking)', + }, + }, + + request: { + url: (params) => { + const encodedId = encodeURIComponent(String(params.projectId).trim()) + const encodedPath = encodeURIComponent(String(params.filePath)) + return `${getGitLabApiBase(params.host)}/projects/${encodedId}/repository/files/${encodedPath}` + }, + method: 'PUT', + headers: (params) => ({ + 'Content-Type': 'application/json', + 'PRIVATE-TOKEN': params.accessToken, + }), + body: (params) => { + const body: Record = { + branch: params.branch, + content: params.content, + commit_message: params.commitMessage, + encoding: 'text', + } + + if (params.lastCommitId) body.last_commit_id = params.lastCommitId + + return body + }, + }, + + transformResponse: async (response) => { + if (!response.ok) { + const errorText = await response.text() + return { + success: false, + error: `GitLab API error: ${response.status} ${errorText}`, + output: {}, + } + } + + const data = await response.json() + + return { + success: true, + output: { + filePath: data.file_path ?? null, + branch: data.branch ?? null, + }, + } + }, + + outputs: { + filePath: { + type: 'string', + description: 'The updated file path', + }, + branch: { + type: 'string', + description: 'The branch the update was committed to', + }, + }, +} diff --git a/apps/sim/tools/gitlab/update_issue.ts b/apps/sim/tools/gitlab/update_issue.ts index acf7ca25402..b2e8b8503aa 100644 --- a/apps/sim/tools/gitlab/update_issue.ts +++ b/apps/sim/tools/gitlab/update_issue.ts @@ -86,7 +86,7 @@ export const gitlabUpdateIssueTool: ToolConfig { - const encodedId = encodeURIComponent(String(params.projectId)) + const encodedId = encodeURIComponent(String(params.projectId).trim()) return `${getGitLabApiBase(params.host)}/projects/${encodedId}/issues/${params.issueIid}` }, method: 'PUT', diff --git a/apps/sim/tools/gitlab/update_merge_request.ts b/apps/sim/tools/gitlab/update_merge_request.ts index 69632a637d5..a0930567db9 100644 --- a/apps/sim/tools/gitlab/update_merge_request.ts +++ b/apps/sim/tools/gitlab/update_merge_request.ts @@ -103,7 +103,7 @@ export const gitlabUpdateMergeRequestTool: ToolConfig< request: { url: (params) => { - const encodedId = encodeURIComponent(String(params.projectId)) + const encodedId = encodeURIComponent(String(params.projectId).trim()) return `${getGitLabApiBase(params.host)}/projects/${encodedId}/merge_requests/${params.mergeRequestIid}` }, method: 'PUT', diff --git a/apps/sim/tools/registry.ts b/apps/sim/tools/registry.ts index df8a070ec5c..7f8f537c98d 100644 --- a/apps/sim/tools/registry.ts +++ b/apps/sim/tools/registry.ts @@ -1082,23 +1082,35 @@ import { githubUpdateReleaseV2Tool, } from '@/tools/github' import { + gitlabApproveMergeRequestTool, gitlabCancelPipelineTool, + gitlabCreateBranchTool, + gitlabCreateFileTool, gitlabCreateIssueNoteTool, gitlabCreateIssueTool, gitlabCreateMergeRequestNoteTool, gitlabCreateMergeRequestTool, gitlabCreatePipelineTool, gitlabDeleteIssueTool, + gitlabGetFileTool, gitlabGetIssueTool, + gitlabGetJobLogTool, + gitlabGetMergeRequestChangesTool, gitlabGetMergeRequestTool, gitlabGetPipelineTool, gitlabGetProjectTool, + gitlabListBranchesTool, + gitlabListCommitsTool, gitlabListIssuesTool, gitlabListMergeRequestsTool, + gitlabListPipelineJobsTool, gitlabListPipelinesTool, gitlabListProjectsTool, + gitlabListRepositoryTreeTool, gitlabMergeMergeRequestTool, + gitlabPlayJobTool, gitlabRetryPipelineTool, + gitlabUpdateFileTool, gitlabUpdateIssueTool, gitlabUpdateMergeRequestTool, } from '@/tools/gitlab' @@ -5477,25 +5489,37 @@ export const tools: Record = { github_check_star_v2: githubCheckStarV2Tool, github_list_stargazers: githubListStargazersTool, github_list_stargazers_v2: githubListStargazersV2Tool, - gitlab_list_projects: gitlabListProjectsTool, - gitlab_get_project: gitlabGetProjectTool, - gitlab_list_issues: gitlabListIssuesTool, - gitlab_get_issue: gitlabGetIssueTool, + gitlab_approve_merge_request: gitlabApproveMergeRequestTool, + gitlab_cancel_pipeline: gitlabCancelPipelineTool, + gitlab_create_branch: gitlabCreateBranchTool, + gitlab_create_file: gitlabCreateFileTool, gitlab_create_issue: gitlabCreateIssueTool, - gitlab_update_issue: gitlabUpdateIssueTool, - gitlab_delete_issue: gitlabDeleteIssueTool, gitlab_create_issue_note: gitlabCreateIssueNoteTool, - gitlab_list_merge_requests: gitlabListMergeRequestsTool, - gitlab_get_merge_request: gitlabGetMergeRequestTool, gitlab_create_merge_request: gitlabCreateMergeRequestTool, - gitlab_update_merge_request: gitlabUpdateMergeRequestTool, - gitlab_merge_merge_request: gitlabMergeMergeRequestTool, gitlab_create_merge_request_note: gitlabCreateMergeRequestNoteTool, - gitlab_list_pipelines: gitlabListPipelinesTool, - gitlab_get_pipeline: gitlabGetPipelineTool, gitlab_create_pipeline: gitlabCreatePipelineTool, + gitlab_delete_issue: gitlabDeleteIssueTool, + gitlab_get_file: gitlabGetFileTool, + gitlab_get_issue: gitlabGetIssueTool, + gitlab_get_job_log: gitlabGetJobLogTool, + gitlab_get_merge_request: gitlabGetMergeRequestTool, + gitlab_get_merge_request_changes: gitlabGetMergeRequestChangesTool, + gitlab_get_pipeline: gitlabGetPipelineTool, + gitlab_get_project: gitlabGetProjectTool, + gitlab_list_branches: gitlabListBranchesTool, + gitlab_list_commits: gitlabListCommitsTool, + gitlab_list_issues: gitlabListIssuesTool, + gitlab_list_merge_requests: gitlabListMergeRequestsTool, + gitlab_list_pipeline_jobs: gitlabListPipelineJobsTool, + gitlab_list_pipelines: gitlabListPipelinesTool, + gitlab_list_projects: gitlabListProjectsTool, + gitlab_list_repository_tree: gitlabListRepositoryTreeTool, + gitlab_merge_merge_request: gitlabMergeMergeRequestTool, + gitlab_play_job: gitlabPlayJobTool, gitlab_retry_pipeline: gitlabRetryPipelineTool, - gitlab_cancel_pipeline: gitlabCancelPipelineTool, + gitlab_update_file: gitlabUpdateFileTool, + gitlab_update_issue: gitlabUpdateIssueTool, + gitlab_update_merge_request: gitlabUpdateMergeRequestTool, grain_list_recordings: grainListRecordingsTool, grain_get_recording: grainGetRecordingTool, grain_get_transcript: grainGetTranscriptTool, From cff7a4931008a893c270dcf7fa5a21f654361153 Mon Sep 17 00:00:00 2001 From: Theodore Li Date: Wed, 24 Jun 2026 18:07:39 -0700 Subject: [PATCH 11/15] feat(file): workspace-scoped inline images + public-share cascade (#5203) * feat(file): workspace-scoped inline images + public-share cascade Embedded markdown images now resolve only within the document's workspace, and public file shares cascade to the images the shared document embeds. - New /api/workspaces/[id]/files/inline (in-app, workspace-scoped) and /api/files/public/[token]/inline (public cascade) routes; the public one serves an embed only when it is referenced-by-doc, same-workspace, and passes a magic-byte image sniff - Embed srcs (serve-key and view-id forms) rewrite through one scoped inline route; one shared isomorphic parser owns the embed grammar for both the frontend renderer and the server doc scan - Accept wf_ file ids on the view/export routes (were 400ing on .uuid()) * feat(file): add Image command to the markdown editor slash menu - New /Image slash command uploads an image via a file picker and inserts it at the caret (same upload+insert path as paste/drop) - Inserted src is the workspace serve URL, so it renders in-app and cascades to public shares like any other embed - Per-editor handler wired through slash-command storage (the extension set is a shared singleton); only active when the editor is editable * fix(file): export rewrites all embed forms; cap embedded refs combined Addresses PR review: - Markdown export now rewrites the in-app `/workspace//files/` embed form too (not just `/api/files/view/`), so a bundled asset never leaves a broken link in an offline export (Bugbot) - extractEmbeddedFileRefs bounds total references (keys + ids) to 50 combined rather than 50 each, matching MAX_EMBEDDED_IMAGES intent --- apps/sim/app/api/files/export/[id]/route.ts | 18 ++- .../files/public/[token]/inline/route.test.ts | 116 ++++++++++++++++++ .../api/files/public/[token]/inline/route.ts | 99 +++++++++++++++ apps/sim/app/api/files/serve-inline-image.ts | 44 +++++++ .../[id]/files/inline/route.test.ts | 77 ++++++++++++ .../api/workspaces/[id]/files/inline/route.ts | 59 +++++++++ apps/sim/app/f/[token]/public-file-view.tsx | 19 ++- .../components/file-viewer/file-viewer.tsx | 26 +++- .../rich-markdown-editor/image.test.ts | 45 ++++--- .../rich-markdown-editor/image.tsx | 22 +--- .../rich-markdown-editor.tsx | 34 +++++ .../slash-command/commands.test.ts | 36 +++++- .../slash-command/commands.ts | 22 ++++ .../slash-command/slash-command.ts | 19 ++- apps/sim/hooks/use-file-content-source.tsx | 77 ++++++++++-- apps/sim/lib/api/contracts/primitives.ts | 27 ++++ apps/sim/lib/api/contracts/public-shares.ts | 18 ++- .../sim/lib/api/contracts/storage-transfer.ts | 5 +- apps/sim/lib/api/contracts/workspace-files.ts | 16 +++ .../server/files/embedded-image-refs.test.ts | 43 +++++++ .../tools/server/files/embedded-image-refs.ts | 20 ++- .../lib/uploads/server/inline-image.test.ts | 63 ++++++++++ apps/sim/lib/uploads/server/inline-image.ts | 41 +++++++ .../uploads/utils/embedded-image-ref.test.ts | 60 +++++++++ .../lib/uploads/utils/embedded-image-ref.ts | 73 +++++++++++ apps/sim/lib/uploads/utils/validation.test.ts | 31 +++++ apps/sim/lib/uploads/utils/validation.ts | 28 +++++ scripts/check-api-validation-contracts.ts | 4 +- 28 files changed, 1073 insertions(+), 69 deletions(-) create mode 100644 apps/sim/app/api/files/public/[token]/inline/route.test.ts create mode 100644 apps/sim/app/api/files/public/[token]/inline/route.ts create mode 100644 apps/sim/app/api/files/serve-inline-image.ts create mode 100644 apps/sim/app/api/workspaces/[id]/files/inline/route.test.ts create mode 100644 apps/sim/app/api/workspaces/[id]/files/inline/route.ts create mode 100644 apps/sim/lib/copilot/tools/server/files/embedded-image-refs.test.ts create mode 100644 apps/sim/lib/uploads/server/inline-image.test.ts create mode 100644 apps/sim/lib/uploads/server/inline-image.ts create mode 100644 apps/sim/lib/uploads/utils/embedded-image-ref.test.ts create mode 100644 apps/sim/lib/uploads/utils/embedded-image-ref.ts diff --git a/apps/sim/app/api/files/export/[id]/route.ts b/apps/sim/app/api/files/export/[id]/route.ts index 18c8aafb563..26dc06abe0d 100644 --- a/apps/sim/app/api/files/export/[id]/route.ts +++ b/apps/sim/app/api/files/export/[id]/route.ts @@ -7,6 +7,7 @@ import { NextResponse } from 'next/server' import { fileExportContract } from '@/lib/api/contracts/storage-transfer' import { parseRequest } from '@/lib/api/server' import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' +import { extractEmbeddedImageIds } from '@/lib/copilot/tools/server/files/embedded-image-refs' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import type { StorageContext } from '@/lib/uploads/config' import { USE_BLOB_STORAGE } from '@/lib/uploads/config' @@ -19,9 +20,6 @@ const logger = createLogger('FilesExportAPI') const MARKDOWN_MIME_TYPES = new Set(['text/markdown', 'text/x-markdown']) const MARKDOWN_EXTENSIONS = new Set(['md', 'markdown']) -const VIEW_URL_RE = - /\/api\/files\/view\/([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})/gi -const MAX_EMBEDDED_IMAGES = 50 function isMarkdown(originalName: string, contentType: string): boolean { if (MARKDOWN_MIME_TYPES.has(contentType)) return true @@ -82,10 +80,7 @@ export const GET = withRouteHandler( }) let mdContent = mdBuffer.toString('utf-8') - const imageIds = [...new Set([...mdContent.matchAll(VIEW_URL_RE)].map((m) => m[1]))].slice( - 0, - MAX_EMBEDDED_IMAGES - ) + const imageIds = extractEmbeddedImageIds(mdContent) logger.info('Exporting markdown', { id, imageCount: imageIds.length }) @@ -139,10 +134,11 @@ export const GET = withRouteHandler( for (const [imageId, asset] of assetMap) { const escapedId = imageId.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') const replacement = `./assets/${asset.filename}` - mdContent = mdContent.replace( - new RegExp(`/api/files/view/${escapedId}`, 'g'), - () => replacement - ) + // Rewrite both embed spellings the extractor resolves to this id — the view URL and the in-app + // `/workspace//files/` path — so a bundled asset never leaves a broken link in the export. + mdContent = mdContent + .replace(new RegExp(`/api/files/view/${escapedId}`, 'g'), () => replacement) + .replace(new RegExp(`/workspace/[A-Za-z0-9-]+/files/${escapedId}`, 'g'), () => replacement) } const zip = new JSZip() diff --git a/apps/sim/app/api/files/public/[token]/inline/route.test.ts b/apps/sim/app/api/files/public/[token]/inline/route.test.ts new file mode 100644 index 00000000000..3f2b654bda0 --- /dev/null +++ b/apps/sim/app/api/files/public/[token]/inline/route.test.ts @@ -0,0 +1,116 @@ +/** + * @vitest-environment node + */ +import { NextRequest } from 'next/server' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockResolveShare, mockRateLimit, mockValidateAuth, mockDownloadFile, mockResolveImage } = + vi.hoisted(() => ({ + mockResolveShare: vi.fn(), + mockRateLimit: vi.fn(), + mockValidateAuth: vi.fn(), + mockDownloadFile: vi.fn(), + mockResolveImage: vi.fn(), + })) + +vi.mock('@/lib/public-shares/share-manager', () => ({ + resolveActiveShareByToken: mockResolveShare, +})) +vi.mock('@/lib/public-shares/rate-limit', () => ({ enforcePublicFileRateLimit: mockRateLimit })) +vi.mock('@/lib/core/security/deployment-auth', () => ({ validateDeploymentAuth: mockValidateAuth })) +vi.mock('@/lib/uploads/core/storage-service', () => ({ downloadFile: mockDownloadFile })) +vi.mock('@/lib/uploads/server/inline-image', () => ({ + resolveWorkspaceInlineImage: mockResolveImage, +})) + +import { GET } from '@/app/api/files/public/[token]/inline/route' + +const TOKEN = 'tok_share_123456' +const DOC_KEY = 'workspace/ws-1/doc.md' +const IMG_KEY = 'workspace/ws-1/photo.png' +const FILE_ID = 'wf_YwDXi8eWOkTxn0sbgChlB' +const PNG = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00]) + +const params = { params: Promise.resolve({ token: TOKEN }) } +const req = (q: string) => new NextRequest(`http://localhost/api/files/public/${TOKEN}/inline?${q}`) + +const share = { + share: { id: 'sh_1', token: TOKEN, authType: 'public' }, + file: { id: 'wf_doc', key: DOC_KEY, workspaceId: 'ws-1', originalName: 'doc.md' }, + workspaceName: 'Acme', + ownerName: 'Jane', +} + +/** doc bytes embed the image via the view form; image bytes are a real PNG */ +function downloadByKey(docContent = `![a](/api/files/view/${FILE_ID})`) { + return ({ key }: { key: string }) => + Promise.resolve(key === DOC_KEY ? Buffer.from(docContent, 'utf-8') : PNG) +} + +describe('GET /api/files/public/[token]/inline', () => { + beforeEach(() => { + vi.clearAllMocks() + mockRateLimit.mockResolvedValue(null) + mockResolveShare.mockResolvedValue(share) + mockValidateAuth.mockResolvedValue({ authorized: true }) + mockResolveImage.mockResolvedValue({ + key: IMG_KEY, + contentType: 'image/png', + filename: 'photo.png', + }) + mockDownloadFile.mockImplementation(downloadByKey()) + }) + + it('serves a same-workspace image referenced by the doc, typed from its bytes', async () => { + const res = await GET(req(`fileId=${FILE_ID}`), params) + expect(res.status).toBe(200) + expect(res.headers.get('content-type')).toBe('image/png') + }) + + it('serves a key-referenced image', async () => { + mockDownloadFile.mockImplementation( + downloadByKey(`![a](/api/files/serve/${encodeURIComponent(IMG_KEY)}?context=workspace)`) + ) + const res = await GET(req(`key=${encodeURIComponent(IMG_KEY)}`), params) + expect(res.status).toBe(200) + }) + + it('404s when the reference is not embedded in the shared document', async () => { + mockDownloadFile.mockImplementation(downloadByKey('no images here')) + const res = await GET(req(`fileId=${FILE_ID}`), params) + expect(res.status).toBe(404) + expect(mockResolveImage).not.toHaveBeenCalled() + }) + + it('404s when the referenced file is not in the document workspace', async () => { + mockResolveImage.mockResolvedValue(null) + const res = await GET(req(`fileId=${FILE_ID}`), params) + expect(res.status).toBe(404) + }) + + it('404s when the bytes are not a renderable image', async () => { + mockDownloadFile.mockImplementation(({ key }: { key: string }) => + Promise.resolve( + key === DOC_KEY + ? Buffer.from(`![a](/api/files/view/${FILE_ID})`, 'utf-8') + : Buffer.from('', 'utf-8') + ) + ) + const res = await GET(req(`fileId=${FILE_ID}`), params) + expect(res.status).toBe(404) + }) + + it('401s and never reads storage when the share is unauthorized', async () => { + mockValidateAuth.mockResolvedValue({ authorized: false, error: 'auth_required_password' }) + const res = await GET(req(`fileId=${FILE_ID}`), params) + expect(res.status).toBe(401) + expect(mockDownloadFile).not.toHaveBeenCalled() + }) + + it('404s for an unknown or inactive token', async () => { + mockResolveShare.mockResolvedValue(null) + const res = await GET(req(`fileId=${FILE_ID}`), params) + expect(res.status).toBe(404) + expect(mockDownloadFile).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/app/api/files/public/[token]/inline/route.ts b/apps/sim/app/api/files/public/[token]/inline/route.ts new file mode 100644 index 00000000000..87c343a26a8 --- /dev/null +++ b/apps/sim/app/api/files/public/[token]/inline/route.ts @@ -0,0 +1,99 @@ +import { createLogger } from '@sim/logger' +import type { NextRequest } from 'next/server' +import { NextResponse } from 'next/server' +import { getPublicInlineFileContract } from '@/lib/api/contracts/public-shares' +import { parseRequest } from '@/lib/api/server' +import { + extractEmbeddedImageIds, + extractEmbeddedImageKeys, +} from '@/lib/copilot/tools/server/files/embedded-image-refs' +import { validateDeploymentAuth } from '@/lib/core/security/deployment-auth' +import { generateRequestId } from '@/lib/core/utils/request' +import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import { enforcePublicFileRateLimit } from '@/lib/public-shares/rate-limit' +import { resolveActiveShareByToken } from '@/lib/public-shares/share-manager' +import { downloadFile } from '@/lib/uploads/core/storage-service' +import { resolveWorkspaceInlineImage } from '@/lib/uploads/server/inline-image' +import { serveInlineImage } from '@/app/api/files/serve-inline-image' +import { createErrorResponse, FileNotFoundError } from '@/app/api/files/utils' + +export const dynamic = 'force-dynamic' + +const logger = createLogger('PublicInlineFileAPI') + +/** + * GET /api/files/public/[token]/inline?key=|fileId= + * + * Cascades a markdown document's public share to the images it embeds, so a logged-out viewer sees them + * instead of broken icons. The share grants the document bytes; this route extends that grant to the + * document's referenced images only, behind three gates that together hold the security boundary: + * + * 1. Referenced-by-doc — the requested key/id must appear in the shared document's current bytes. The + * token is a capability for the document and its embeds, never an arbitrary workspace file. + * 2. Same-workspace — the referenced file must be a `workspace` file in the document's own workspace + * ({@link resolveWorkspaceInlineImage}). This blocks any cross-workspace reference (which an author + * can write but must never resolve) from loading. + * 3. Content-truth — the served content type is sniffed from the bytes, not the client-declared type, + * and only genuine raster images are served. A file spoofing `image/png` while holding HTML/SVG is + * refused rather than rendered inline. + */ +export const GET = withRouteHandler( + async (request: NextRequest, context: { params: Promise<{ token: string }> }) => { + const requestId = generateRequestId() + + try { + const limited = await enforcePublicFileRateLimit(request, 'content') + if (limited) return limited + + const parsed = await parseRequest(getPublicInlineFileContract, request, context) + if (!parsed.success) return parsed.response + const { token } = parsed.data.params + const ref = parsed.data.query + + const resolved = await resolveActiveShareByToken(token) + if (!resolved) { + throw new FileNotFoundError('Not found') + } + + const auth = await validateDeploymentAuth( + requestId, + resolved.share, + request, + undefined, + 'file' + ) + if (!auth.authorized) { + return NextResponse.json({ error: auth.error ?? 'auth_required_password' }, { status: 401 }) + } + + const { file: doc } = resolved + if (!doc.workspaceId) { + throw new FileNotFoundError('Not found') + } + + // Referenced-by-doc gate: the share grants exactly the images the document embeds. + const docText = (await downloadFile({ key: doc.key, context: 'workspace' })).toString('utf-8') + const referenced = ref.fileId + ? extractEmbeddedImageIds(docText).includes(ref.fileId) + : extractEmbeddedImageKeys(docText).includes(ref.key as string) + if (!referenced) { + throw new FileNotFoundError('Not found') + } + + // Same-workspace gate: resolve scoped to the document's own workspace. + const image = await resolveWorkspaceInlineImage(doc.workspaceId, ref) + if (!image) { + throw new FileNotFoundError('Not found') + } + + // Content-truth gate (`sniff`): render only genuine raster image bytes. + return await serveInlineImage(image, { sniff: true }) + } catch (error) { + if (error instanceof FileNotFoundError) { + return createErrorResponse(error) + } + logger.error('Error serving public inline image:', error) + return createErrorResponse(error instanceof Error ? error : new Error('Failed to serve file')) + } + } +) diff --git a/apps/sim/app/api/files/serve-inline-image.ts b/apps/sim/app/api/files/serve-inline-image.ts new file mode 100644 index 00000000000..88c3383d961 --- /dev/null +++ b/apps/sim/app/api/files/serve-inline-image.ts @@ -0,0 +1,44 @@ +import { createLogger } from '@sim/logger' +import type { NextResponse } from 'next/server' +import { downloadFile } from '@/lib/uploads/core/storage-service' +import type { ResolvedInlineImage } from '@/lib/uploads/server/inline-image' +import { sniffImageContentType } from '@/lib/uploads/utils/validation' +import { createFileResponse, FileNotFoundError } from '@/app/api/files/utils' + +const logger = createLogger('InlineImageServe') + +/** + * A shared/edited/deleted file must never serve stale bytes from its fixed inline URL, so every inline + * image revalidates on each request. + */ +const INLINE_CACHE_CONTROL = 'private, no-cache, must-revalidate' + +/** + * Download and respond with an already-workspace-scoped inline image — the single serving tail for both + * the in-app and public inline routes. When `sniff` is set (public shares, a less-trusted audience), the + * served content type is derived from the bytes and non-raster content is refused with 404; otherwise the + * stored content type is served, matching the in-app serve route. + */ +export async function serveInlineImage( + image: ResolvedInlineImage, + { sniff }: { sniff: boolean } +): Promise { + const buffer = await downloadFile({ key: image.key, context: 'workspace' }) + + let contentType = image.contentType + if (sniff) { + const sniffed = sniffImageContentType(buffer) + if (!sniffed) { + logger.warn('Embedded reference is not a renderable image', { key: image.key }) + throw new FileNotFoundError('Not found') + } + contentType = sniffed + } + + return createFileResponse({ + buffer, + contentType, + filename: image.filename, + cacheControl: INLINE_CACHE_CONTROL, + }) +} diff --git a/apps/sim/app/api/workspaces/[id]/files/inline/route.test.ts b/apps/sim/app/api/workspaces/[id]/files/inline/route.test.ts new file mode 100644 index 00000000000..3bb2a8a06ba --- /dev/null +++ b/apps/sim/app/api/workspaces/[id]/files/inline/route.test.ts @@ -0,0 +1,77 @@ +/** + * @vitest-environment node + */ +import { NextRequest } from 'next/server' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockGetSession, mockGetPerms, mockResolveImage, mockDownloadFile } = vi.hoisted(() => ({ + mockGetSession: vi.fn(), + mockGetPerms: vi.fn(), + mockResolveImage: vi.fn(), + mockDownloadFile: vi.fn(), +})) + +vi.mock('@/lib/auth', () => ({ + auth: { api: { getSession: vi.fn() } }, + getSession: mockGetSession, +})) +vi.mock('@/lib/workspaces/permissions/utils', () => ({ getUserEntityPermissions: mockGetPerms })) +vi.mock('@/lib/uploads/server/inline-image', () => ({ + resolveWorkspaceInlineImage: mockResolveImage, +})) +vi.mock('@/lib/uploads/core/storage-service', () => ({ downloadFile: mockDownloadFile })) + +import { GET } from '@/app/api/workspaces/[id]/files/inline/route' + +const PNG = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00]) +const params = { params: Promise.resolve({ id: 'ws-1' }) } +const req = (q: string) => new NextRequest(`http://localhost/api/workspaces/ws-1/files/inline?${q}`) + +describe('GET /api/workspaces/[id]/files/inline', () => { + beforeEach(() => { + vi.clearAllMocks() + mockGetSession.mockResolvedValue({ user: { id: 'u1' } }) + mockGetPerms.mockResolvedValue('read') + mockResolveImage.mockResolvedValue({ + key: 'workspace/ws-1/x-photo.png', + contentType: 'image/png', + filename: 'photo.png', + }) + mockDownloadFile.mockResolvedValue(PNG) + }) + + it('serves a workspace-scoped image by fileId', async () => { + const res = await GET(req('fileId=wf_abc'), params) + expect(res.status).toBe(200) + expect(mockResolveImage).toHaveBeenCalledWith('ws-1', { fileId: 'wf_abc' }) + }) + + it('serves a workspace-scoped image by key', async () => { + const res = await GET(req(`key=${encodeURIComponent('workspace/ws-1/x-photo.png')}`), params) + expect(res.status).toBe(200) + }) + + it('404s when the reference does not resolve in the workspace (cross-workspace)', async () => { + mockResolveImage.mockResolvedValue(null) + const res = await GET(req('fileId=wf_other'), params) + expect(res.status).toBe(404) + }) + + it('404s without workspace membership, before resolving the file', async () => { + mockGetPerms.mockResolvedValue(null) + const res = await GET(req('fileId=wf_abc'), params) + expect(res.status).toBe(404) + expect(mockResolveImage).not.toHaveBeenCalled() + }) + + it('401s without a session', async () => { + mockGetSession.mockResolvedValue(null) + const res = await GET(req('fileId=wf_abc'), params) + expect(res.status).toBe(401) + }) + + it('400s when neither key nor fileId is provided', async () => { + const res = await GET(req(''), params) + expect(res.status).toBe(400) + }) +}) diff --git a/apps/sim/app/api/workspaces/[id]/files/inline/route.ts b/apps/sim/app/api/workspaces/[id]/files/inline/route.ts new file mode 100644 index 00000000000..245fb5731d8 --- /dev/null +++ b/apps/sim/app/api/workspaces/[id]/files/inline/route.ts @@ -0,0 +1,59 @@ +import { createLogger } from '@sim/logger' +import type { NextRequest } from 'next/server' +import { NextResponse } from 'next/server' +import { getInlineWorkspaceFileContract } from '@/lib/api/contracts/workspace-files' +import { parseRequest } from '@/lib/api/server' +import { getSession } from '@/lib/auth' +import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import { resolveWorkspaceInlineImage } from '@/lib/uploads/server/inline-image' +import { getUserEntityPermissions } from '@/lib/workspaces/permissions/utils' +import { serveInlineImage } from '@/app/api/files/serve-inline-image' +import { createErrorResponse, FileNotFoundError } from '@/app/api/files/utils' + +export const dynamic = 'force-dynamic' + +const logger = createLogger('WorkspaceInlineFileAPI') + +/** + * GET /api/workspaces/[id]/files/inline?key=|fileId= + * + * Serves an image embedded in a workspace markdown document, **scoped to the workspace in the path**. + * The markdown editor rewrites its embedded `/api/files/serve/` and `/api/files/view/` srcs to + * this route so a referenced file resolves only within the document's workspace — a cross-workspace + * reference returns 404 and does not render, even for a viewer who belongs to the other workspace. Read + * access to the workspace is required; disposition/content-type handling mirrors the serve route. + */ +export const GET = withRouteHandler( + async (request: NextRequest, context: { params: Promise<{ id: string }> }) => { + try { + const parsed = await parseRequest(getInlineWorkspaceFileContract, request, context) + if (!parsed.success) return parsed.response + const { id: workspaceId } = parsed.data.params + const ref = parsed.data.query + + const session = await getSession() + if (!session?.user?.id) { + return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) + } + + // Authorize before disclosing anything; deny with 404 so a non-member can't probe existence. + const permission = await getUserEntityPermissions(session.user.id, 'workspace', workspaceId) + if (!permission) { + throw new FileNotFoundError('Not found') + } + + const image = await resolveWorkspaceInlineImage(workspaceId, ref) + if (!image) { + throw new FileNotFoundError('Not found') + } + + return await serveInlineImage(image, { sniff: false }) + } catch (error) { + if (error instanceof FileNotFoundError) { + return createErrorResponse(error) + } + logger.error('Error serving workspace inline image:', error) + return createErrorResponse(error instanceof Error ? error : new Error('Failed to serve file')) + } + } +) diff --git a/apps/sim/app/f/[token]/public-file-view.tsx b/apps/sim/app/f/[token]/public-file-view.tsx index f27b63df65d..360119e4945 100644 --- a/apps/sim/app/f/[token]/public-file-view.tsx +++ b/apps/sim/app/f/[token]/public-file-view.tsx @@ -9,7 +9,7 @@ import type { WorkspaceFileRecord } from '@/lib/uploads/contexts/workspace' import { buildProvenance } from '@/app/f/[token]/utils' import { FileViewer } from '@/app/workspace/[workspaceId]/files/components/file-viewer' import { useBrandConfig } from '@/ee/whitelabeling' -import { type FileContentSource, FileContentSourceProvider } from '@/hooks/use-file-content-source' +import { createPublicFileContentSource } from '@/hooks/use-file-content-source' interface PublicFileViewProps { token: string @@ -41,7 +41,12 @@ export function PublicFileView({ // `updatedAt` fold in the content version so the React Query caches (keyed on the // storage key + `updatedAt`) refetch when the shared file changes — even when its // size is unchanged. - const source = useMemo(() => ({ buildUrl: () => contentUrl }), [contentUrl]) + // Embedded images route through the token-scoped cascade endpoint, which serves them only when the + // shared document actually references them and they live in its workspace. + const source = useMemo( + () => createPublicFileContentSource(token, contentUrl), + [token, contentUrl] + ) const file = useMemo( () => ({ id: token, @@ -116,9 +121,13 @@ export function PublicFileView({
- - - +
) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/file-viewer.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/file-viewer.tsx index 2e1d4d834ea..3d4f4b5d3e9 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/file-viewer.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/file-viewer.tsx @@ -6,6 +6,11 @@ import dynamic from 'next/dynamic' import type { WorkspaceFileRecord } from '@/lib/uploads/contexts/workspace' import { getFileExtension } from '@/lib/uploads/utils/file-utils' import { useWorkspaceFileBinary, useWorkspaceFileContent } from '@/hooks/queries/workspace-files' +import { + createWorkspaceFileContentSource, + type FileContentSource, + FileContentSourceProvider, +} from '@/hooks/use-file-content-source' import { CsvTablePreview } from './csv-table-preview' import { DocxPreview } from './docx-preview' import { resolveFileCategory } from './file-category' @@ -78,6 +83,12 @@ export type PreviewMode = 'editor' | 'split' | 'preview' interface FileViewerProps { file: WorkspaceFileRecord workspaceId: string + /** + * Content source for this view. Defaults to a workspace-scoped source derived from `workspaceId`; + * the public share page passes a token-scoped source. Provided to descendants (renderers, embedded + * images) via {@link FileContentSourceProvider}. + */ + contentSource?: FileContentSource canEdit: boolean /** * Render a read-only preview with no editing affordances. Text files render @@ -97,7 +108,20 @@ interface FileViewerProps { previewContextKey?: string } -export function FileViewer({ +export function FileViewer(props: FileViewerProps) { + const { contentSource, workspaceId } = props + const source = useMemo( + () => contentSource ?? createWorkspaceFileContentSource(workspaceId), + [contentSource, workspaceId] + ) + return ( + + + + ) +} + +function FileViewerContent({ file, workspaceId, canEdit, diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image.test.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image.test.ts index 41e2f888408..a2879b6da6f 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image.test.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image.test.ts @@ -2,26 +2,41 @@ * @vitest-environment jsdom */ import { describe, expect, it } from 'vitest' -import { resolveDisplaySrc } from './image' +import { + createPublicFileContentSource, + createWorkspaceFileContentSource, +} from '@/hooks/use-file-content-source' -describe('resolveDisplaySrc', () => { - it('rewrites an in-app workspace file path to its serving endpoint (display only)', () => { - expect(resolveDisplaySrc('/workspace/W1/files/F123')).toBe('/api/files/view/F123') - expect(resolveDisplaySrc('/workspace/any-ws-id/files/abc-def')).toBe('/api/files/view/abc-def') +const KEY = 'workspace/W1/1700000000000-deadbeefdeadbeef-photo.png' +const ENCODED = encodeURIComponent(KEY) + +describe('content-source resolveImageSrc', () => { + it('in-app source rewrites embeds to the workspace-scoped inline route', () => { + const src = createWorkspaceFileContentSource('ws-1') + expect(src.resolveImageSrc(`/api/files/serve/${ENCODED}?context=workspace`)).toBe( + `/api/workspaces/ws-1/files/inline?key=${encodeURIComponent(KEY)}` + ) + expect(src.resolveImageSrc('/api/files/view/wf_abc')).toBe( + '/api/workspaces/ws-1/files/inline?fileId=wf_abc' + ) }) - it('leaves absolute and non-workspace URLs untouched', () => { - expect(resolveDisplaySrc('https://cdn.example.com/a.png')).toBe('https://cdn.example.com/a.png') - expect(resolveDisplaySrc('http://localhost/workspace/W1/files/F1')).toBe( - 'http://localhost/workspace/W1/files/F1' + it('public source rewrites embeds to the token-scoped inline route', () => { + const src = createPublicFileContentSource('tok_1', '/api/files/public/tok_1/content') + expect(src.resolveImageSrc('/api/files/view/wf_abc')).toBe( + '/api/files/public/tok_1/inline?fileId=wf_abc' ) - expect(resolveDisplaySrc('/other/path/files/x')).toBe('/other/path/files/x') - expect(resolveDisplaySrc('relative/image.png')).toBe('relative/image.png') }) - it('passes through empty/undefined and unparseable values', () => { - expect(resolveDisplaySrc(undefined)).toBeUndefined() - expect(resolveDisplaySrc('')).toBe('') - expect(resolveDisplaySrc('/workspace/W1/files/')).toBe('/workspace/W1/files/') + it('passes external/data srcs through unchanged in both sources', () => { + const ws = createWorkspaceFileContentSource('ws-1') + const pub = createPublicFileContentSource('tok_1', '/c') + expect(ws.resolveImageSrc('https://cdn.example.com/a.png')).toBe( + 'https://cdn.example.com/a.png' + ) + expect(pub.resolveImageSrc('https://cdn.example.com/a.png')).toBe( + 'https://cdn.example.com/a.png' + ) + expect(ws.resolveImageSrc(undefined)).toBeUndefined() }) }) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image.tsx index 8e76a4244bb..5809af9625a 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image.tsx @@ -3,6 +3,7 @@ import type { JSONContent } from '@tiptap/core' import { Image } from '@tiptap/extension-image' import type { ReactNodeViewProps } from '@tiptap/react' import { NodeViewWrapper, ReactNodeViewRenderer } from '@tiptap/react' +import { useFileContentSource } from '@/hooks/use-file-content-source' import { normalizeLinkHref } from './markdown-fidelity' const MIN_WIDTH = 64 @@ -26,24 +27,6 @@ function escapeAttr(value: string): string { .replace(/>/g, '>') } -/** - * Rewrite an in-app workspace file path (`/workspace/{id}/files/{fileId}`) to its serving endpoint - * (`/api/files/view/{fileId}`) for display only — the stored `src` attribute keeps the original path - * so markdown round-trips unchanged. Absolute and non-workspace URLs pass through untouched. - */ -export function resolveDisplaySrc(src: string | undefined): string | undefined { - if (!src) return src - try { - const parsed = new URL(src, 'http://placeholder') - if (parsed.origin !== 'http://placeholder') return src - const [, seg1, , seg3, fileId] = parsed.pathname.split('/') - if (seg1 === 'workspace' && seg3 === 'files' && fileId) return `/api/files/view/${fileId}` - } catch { - // not a parseable URL — render as-is - } - return src -} - /** * Serialize an image to markdown when it has no explicit size, and to an HTML `` tag when * it does — standard markdown has no width syntax, so a resized image must round-trip as HTML to @@ -174,6 +157,7 @@ export const MarkdownImage = Image.extend({ * commits the new pixel width to the `width` attribute, which serializes to ``. */ function ResizableImageView({ node, updateAttributes, selected, editor }: ReactNodeViewProps) { + const source = useFileContentSource() const imageRef = useRef(null) const dragAbortRef = useRef(null) const [dragging, setDragging] = useState(false) @@ -232,7 +216,7 @@ function ResizableImageView({ node, updateAttributes, selected, editor }: ReactN const image = ( {attrs.alt(null) + // The `/Image` slash command opens this hidden picker; `pendingImagePosRef` holds the caret position + // captured when the command ran, so the upload inserts where `/Image` was typed. + const imageInputRef = useRef(null) + const pendingImagePosRef = useRef(null) + // Upload then insert each image at `at` (paste caret / drop point), sequentially; held in a ref so handlers reach the latest. const insertImagesRef = useRef<(images: File[], at: number) => Promise>(() => Promise.resolve() @@ -293,6 +298,19 @@ export function LoadedRichMarkdownEditor({ }) editorInstanceRef.current = editor + // Wire the `/Image` slash command to the hidden picker (per-editor storage, since the extension set is + // shared across instances). Reads only refs, so the handler stays stable across the editor's life. + useEffect(() => { + if (!editor) return + editor.storage.slashCommand.insertImage = (at: number) => { + pendingImagePosRef.current = at + imageInputRef.current?.click() + } + return () => { + editor.storage.slashCommand.insertImage = null + } + }, [editor]) + const wasStreamingRef = useRef(streamingAtMountRef.current) const pendingStreamBodyRef = useRef(null) @@ -386,6 +404,22 @@ export function LoadedRichMarkdownEditor({ > {editor && } {editor && } + { + const input = event.currentTarget + const images = Array.from(input.files ?? []).filter((f) => f.type.startsWith('image/')) + const at = + pendingImagePosRef.current ?? editorInstanceRef.current?.state.selection.from ?? 0 + pendingImagePosRef.current = null + input.value = '' + if (images.length > 0) void insertImagesRef.current(images, at) + }} + /> { @@ -48,4 +49,37 @@ describe('SLASH_COMMANDS registry', () => { const titles = SLASH_COMMANDS.map((c) => c.title) expect(new Set(titles).size).toBe(titles.length) }) + + it('Image command replaces the trigger and hands the caret to the host insertImage handler', () => { + const insertImage = vi.fn() + const deleteRange = vi.fn(() => chain) + const chain = { focus: () => chain, deleteRange, run: () => true } + const editor = { + chain: () => chain, + storage: { slashCommand: { insertImage } }, + state: { selection: { from: 7 } }, + } as unknown as Editor + + const image = SLASH_COMMANDS.find((c) => c.title === 'Image') + expect(image).toBeDefined() + image?.run({ editor, range: { from: 5, to: 6 } as Range }) + + expect(deleteRange).toHaveBeenCalledWith({ from: 5, to: 6 }) + expect(insertImage).toHaveBeenCalledWith(7) + }) + + it('Image command is a no-op when no handler is wired', () => { + const chain = { focus: () => chain, deleteRange: () => chain, run: () => true } + const editor = { + chain: () => chain, + storage: { slashCommand: { insertImage: null } }, + state: { selection: { from: 0 } }, + } as unknown as Editor + expect(() => + SLASH_COMMANDS.find((c) => c.title === 'Image')?.run({ + editor, + range: { from: 0, to: 1 } as Range, + }) + ).not.toThrow() + }) }) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/slash-command/commands.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/slash-command/commands.ts index acf945017d9..a3bdd960bc8 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/slash-command/commands.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/slash-command/commands.ts @@ -4,6 +4,7 @@ import { Heading1, Heading2, Heading3, + Image as ImageIcon, List, ListChecks, ListOrdered, @@ -19,6 +20,15 @@ export interface SlashCommandContext { range: Range } +/** + * Per-editor storage on the `slashCommand` extension. The host editor component sets `insertImage` + * after mount; it opens an image file picker and uploads + inserts the chosen image(s) at `at`. Null + * in headless/read-only contexts, where the Image command is a no-op. + */ +export interface SlashCommandStorage { + insertImage: ((at: number) => void) | null +} + export interface SlashCommandItem { title: string /** Group heading the item is shown under in the menu. */ @@ -131,6 +141,18 @@ export const SLASH_COMMANDS: readonly SlashCommandItem[] = [ aliases: ['hr', 'horizontal rule', 'separator'], run: ({ editor, range }) => editor.chain().focus().deleteRange(range).setHorizontalRule().run(), }, + { + title: 'Image', + group: 'Media', + icon: ImageIcon, + aliases: ['picture', 'photo', 'upload', 'img'], + run: ({ editor, range }) => { + // Replace the typed `/query`, then hand off to the host component's picker, which uploads and + // inserts the image at the caret (the same path as paste/drop). No-op when no handler is wired. + editor.chain().focus().deleteRange(range).run() + editor.storage.slashCommand.insertImage?.(editor.state.selection.from) + }, + }, ] /** diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/slash-command/slash-command.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/slash-command/slash-command.ts index 2a5118ec1cf..1ceff0c9eed 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/slash-command/slash-command.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/slash-command/slash-command.ts @@ -3,9 +3,20 @@ import type { Editor } from '@tiptap/core' import { Extension } from '@tiptap/core' import { ReactRenderer } from '@tiptap/react' import Suggestion, { type SuggestionOptions, type SuggestionProps } from '@tiptap/suggestion' -import { filterSlashCommands, type SlashCommandContext, type SlashCommandItem } from './commands' +import { + filterSlashCommands, + type SlashCommandContext, + type SlashCommandItem, + type SlashCommandStorage, +} from './commands' import { SlashCommandList, type SlashCommandListHandle } from './slash-command-list' +declare module '@tiptap/core' { + interface Storage { + slashCommand: SlashCommandStorage + } +} + type SlashSuggestionProps = SuggestionProps function positionPopup(element: HTMLElement, getRect: SlashSuggestionProps['clientRect']) { @@ -76,9 +87,13 @@ function renderSlashSuggestion(): ReturnType, SlashCommandStorage>({ name: 'slashCommand', + addStorage() { + return { insertImage: null } + }, + addProseMirrorPlugins() { return [ Suggestion({ diff --git a/apps/sim/hooks/use-file-content-source.tsx b/apps/sim/hooks/use-file-content-source.tsx index ee09819d7c8..bd9552d8e7e 100644 --- a/apps/sim/hooks/use-file-content-source.tsx +++ b/apps/sim/hooks/use-file-content-source.tsx @@ -1,6 +1,10 @@ 'use client' import { createContext, useContext } from 'react' +import { + type EmbeddedFileRef, + extractEmbeddedFileRef, +} from '@/lib/uploads/utils/embedded-image-ref' export interface FileContentUrlOptions { /** Request the uncompiled source instead of the rendered/compiled bytes. */ @@ -11,6 +15,12 @@ export interface FileContentUrlOptions { bust?: boolean } +function inlineRefQuery(ref: NonNullable): string { + return 'key' in ref + ? `key=${encodeURIComponent(ref.key)}` + : `fileId=${encodeURIComponent(ref.fileId)}` +} + /** * Seam for "where do a file's bytes come from". The in-app viewer resolves the * auth-gated workspace serve URL; the public share page swaps in a token-scoped @@ -19,18 +29,67 @@ export interface FileContentUrlOptions { */ export interface FileContentSource { buildUrl: (key: string, opts?: FileContentUrlOptions) => string + /** + * Map an embedded image `src` to a display URL scoped to the current context: the in-app source + * points at the workspace-scoped inline route, the public source at the token-scoped cascade route. + * Non-workspace srcs (external, `data:`, public assets) pass through unchanged. + */ + resolveImageSrc: (src: string | undefined) => string | undefined +} + +function buildServeUrl(key: string, opts?: FileContentUrlOptions): string { + const base = `/api/files/serve/${encodeURIComponent(key)}?context=workspace` + const params: string[] = [] + if (opts?.version != null) params.push(`v=${encodeURIComponent(String(opts.version))}`) + else if (opts?.bust) params.push(`t=${Date.now()}`) + if (opts?.raw) params.push('raw=1') + return params.length > 0 ? `${base}&${params.join('&')}` : base +} + +/** Build a source whose embeds resolve through `inlineBase` (the workspace- or token-scoped inline route). */ +function inlineImageSource( + buildUrl: FileContentSource['buildUrl'], + inlineBase: string +): FileContentSource { + return { + buildUrl, + resolveImageSrc: (src) => { + if (!src) return src + const ref = extractEmbeddedFileRef(src) + return ref ? `${inlineBase}?${inlineRefQuery(ref)}` : src + }, + } +} + +/** + * In-app source scoped to one workspace. Direct file bytes come from the workspace serve URL; embedded + * images route through `/api/workspaces/{workspaceId}/files/inline`, which resolves a reference only + * within this workspace — a cross-workspace embed 404s and does not render. + */ +export function createWorkspaceFileContentSource(workspaceId: string): FileContentSource { + return inlineImageSource(buildServeUrl, `/api/workspaces/${workspaceId}/files/inline`) +} + +/** + * Public share source. Direct file bytes come from the token content URL; embedded images route through + * `/api/files/public/{token}/inline`, which serves them only when referenced by the shared document and + * in its workspace. + */ +export function createPublicFileContentSource( + token: string, + contentUrl: string +): FileContentSource { + return inlineImageSource(() => contentUrl, `/api/files/public/${token}/inline`) } -/** Default source: the auth-gated workspace serve URL (the historical behavior). */ +/** + * Context default for components rendered outside a {@link FileContentSourceProvider}: serve URLs for + * direct bytes, embeds passed through unchanged. The file viewer always provides a workspace- or + * token-scoped source, so embeds resolve through the scoped inline routes there. + */ export const workspaceFileContentSource: FileContentSource = { - buildUrl: (key, opts) => { - const base = `/api/files/serve/${encodeURIComponent(key)}?context=workspace` - const params: string[] = [] - if (opts?.version != null) params.push(`v=${encodeURIComponent(String(opts.version))}`) - else if (opts?.bust) params.push(`t=${Date.now()}`) - if (opts?.raw) params.push('raw=1') - return params.length > 0 ? `${base}&${params.join('&')}` : base - }, + buildUrl: buildServeUrl, + resolveImageSrc: (src) => src, } const FileContentSourceContext = createContext(workspaceFileContentSource) diff --git a/apps/sim/lib/api/contracts/primitives.ts b/apps/sim/lib/api/contracts/primitives.ts index 3f5391b5408..7e73175abc3 100644 --- a/apps/sim/lib/api/contracts/primitives.ts +++ b/apps/sim/lib/api/contracts/primitives.ts @@ -50,6 +50,33 @@ export const organizationIdSchema = z.string().min(1, 'Organization ID is requir */ export const workflowIdSchema = z.string().min(1, 'Workflow ID is required') +/** + * A `workspace_files.id` value. The column is a free-form `text` primary key, so + * ids come in two shapes: UUID v4 (legacy rows and the `insertFileMetadata` + * default) and the current `wf_` form minted by the workspace upload + * path. Both are drawn from `[A-Za-z0-9_-]`, so accept that charset rather than a + * UUID-only schema — a `.uuid()` constraint here silently 400s every `wf_` file. + */ +export const workspaceFileIdSchema = z + .string() + .min(1, 'File ID is required') + .max(128, 'File ID is too long') + .regex(/^[A-Za-z0-9_-]+$/, 'Invalid file id') + +/** + * Reference to an image embedded in a document: either a workspace storage `key` + * (serve-URL embeds) or a workspace file `id` (view-URL embeds) — exactly one. Shared + * by the in-app and public inline-image routes, which resolve it within a workspace. + */ +export const inlineFileRefQuerySchema = z + .object({ + key: z.string().min(1).max(512).optional(), + fileId: workspaceFileIdSchema.optional(), + }) + .refine((q) => (q.key ? 1 : 0) + (q.fileId ? 1 : 0) === 1, { + message: 'Provide exactly one of `key` or `fileId`', + }) + /** * Boolean query-string primitive that correctly handles the literal strings * `"true"` / `"false"` (case-insensitive) in addition to real booleans. diff --git a/apps/sim/lib/api/contracts/public-shares.ts b/apps/sim/lib/api/contracts/public-shares.ts index aa623324558..a839d372ead 100644 --- a/apps/sim/lib/api/contracts/public-shares.ts +++ b/apps/sim/lib/api/contracts/public-shares.ts @@ -1,5 +1,5 @@ import { z } from 'zod' -import { workspaceIdSchema } from '@/lib/api/contracts/primitives' +import { inlineFileRefQuerySchema, workspaceIdSchema } from '@/lib/api/contracts/primitives' import { defineRouteContract } from '@/lib/api/contracts/types' export const shareResourceTypeSchema = z.enum(['file', 'folder']) @@ -126,6 +126,22 @@ export const getPublicFileContentContract = defineRouteContract({ }, }) +/** + * Binary stream of an image embedded in a shared document. Authorized by the parent + * document's active share — the route serves the bytes only when the reference is + * actually embedded in the shared document AND the file lives in the same workspace, + * and only when the bytes are a renderable raster image. + */ +export const getPublicInlineFileContract = defineRouteContract({ + method: 'GET', + path: '/api/files/public/[token]/inline', + params: publicFileTokenParamsSchema, + query: inlineFileRefQuerySchema, + response: { + mode: 'binary', + }, +}) + const authenticatePublicFileBodySchema = z.object({ password: z.string().min(1, 'Password is required').max(1024, 'Password is too long'), }) diff --git a/apps/sim/lib/api/contracts/storage-transfer.ts b/apps/sim/lib/api/contracts/storage-transfer.ts index 57de6d75b29..e5fef522c20 100644 --- a/apps/sim/lib/api/contracts/storage-transfer.ts +++ b/apps/sim/lib/api/contracts/storage-transfer.ts @@ -3,6 +3,7 @@ import { batchPresignedUploadResponseSchema, presignedUploadResponseSchema, } from '@/lib/api/contracts/file-uploads' +import { workspaceFileIdSchema } from '@/lib/api/contracts/primitives' import { type ContractBodyInput, type ContractJsonResponse, @@ -465,11 +466,11 @@ export const fileServeQuerySchema = z.object({ }) export const fileViewParamsSchema = z.object({ - id: z.string().uuid('File ID must be a valid UUID'), + id: workspaceFileIdSchema, }) export const fileExportParamsSchema = z.object({ - id: z.string().uuid('File ID must be a valid UUID'), + id: workspaceFileIdSchema, }) export const boxUploadContract = defineRouteContract({ diff --git a/apps/sim/lib/api/contracts/workspace-files.ts b/apps/sim/lib/api/contracts/workspace-files.ts index c7f1f6d5366..b47b5710528 100644 --- a/apps/sim/lib/api/contracts/workspace-files.ts +++ b/apps/sim/lib/api/contracts/workspace-files.ts @@ -1,4 +1,5 @@ import { z } from 'zod' +import { inlineFileRefQuerySchema } from '@/lib/api/contracts/primitives' import { shareRecordSchema } from '@/lib/api/contracts/public-shares' import { defineRouteContract } from '@/lib/api/contracts/types' @@ -16,6 +17,21 @@ export const listWorkspaceFilesQuerySchema = z.object({ scope: workspaceFileScopeSchema.default('active'), }) +/** + * Binary stream of an image embedded in a workspace markdown document, scoped to the + * workspace in the path. The route serves the bytes only when the referenced file is a + * `workspace` file belonging to `[id]` — cross-workspace references do not resolve. + */ +export const getInlineWorkspaceFileContract = defineRouteContract({ + method: 'GET', + path: '/api/workspaces/[id]/files/inline', + params: workspaceFilesParamsSchema, + query: inlineFileRefQuerySchema, + response: { + mode: 'binary', + }, +}) + const workspaceFileNameSchema = z .string({ error: 'Name is required' }) .trim() diff --git a/apps/sim/lib/copilot/tools/server/files/embedded-image-refs.test.ts b/apps/sim/lib/copilot/tools/server/files/embedded-image-refs.test.ts new file mode 100644 index 00000000000..d4c33793f3f --- /dev/null +++ b/apps/sim/lib/copilot/tools/server/files/embedded-image-refs.test.ts @@ -0,0 +1,43 @@ +import { describe, expect, it } from 'vitest' +import { + extractEmbeddedImageIds, + extractEmbeddedImageKeys, +} from '@/lib/copilot/tools/server/files/embedded-image-refs' + +const KEY = 'workspace/W1/1700000000000-deadbeefdeadbeef-photo.png' + +describe('extractEmbeddedImageIds', () => { + it('extracts unique ids from view-url and in-app-path embeds (wf_ and uuid)', () => { + const a = 'wf_YwDXi8eWOkTxn0sbgChlB' + const b = '4bdaf6c4-072e-464e-891d-b6af3b5fe2cc' + const content = `![x](/api/files/view/${a}) ![y](/workspace/W1/files/${b}) ![dup](/api/files/view/${a})` + expect(extractEmbeddedImageIds(content).sort()).toEqual([b, a].sort()) + }) + + it('ignores serve-url, external, and plain content', () => { + expect( + extractEmbeddedImageIds(`![a](/api/files/serve/${encodeURIComponent(KEY)}) plain`) + ).toEqual([]) + }) + + it('caps the result at 50 ids', () => { + const content = Array.from( + { length: 60 }, + (_, i) => `/api/files/view/wf_${String(i).padStart(6, '0')}` + ).join(' ') + expect(extractEmbeddedImageIds(content)).toHaveLength(50) + }) +}) + +describe('extractEmbeddedImageKeys', () => { + it('extracts decoded workspace keys from serve-url embeds (encoded + s3/blob prefixed)', () => { + const content = `![a](/api/files/serve/${encodeURIComponent(KEY)}?context=workspace) ![b](/api/files/serve/s3/${encodeURIComponent(KEY)})` + expect(extractEmbeddedImageKeys(content)).toEqual([KEY]) + }) + + it('drops non-workspace keys (e.g. public profile pictures) and view-url embeds', () => { + const content = + '![a](/api/files/serve/profile-pictures%2Fu1%2Favatar.png) ![b](/api/files/view/wf_abc)' + expect(extractEmbeddedImageKeys(content)).toEqual([]) + }) +}) diff --git a/apps/sim/lib/copilot/tools/server/files/embedded-image-refs.ts b/apps/sim/lib/copilot/tools/server/files/embedded-image-refs.ts index fb7a0eb5967..f5f7b9fbbad 100644 --- a/apps/sim/lib/copilot/tools/server/files/embedded-image-refs.ts +++ b/apps/sim/lib/copilot/tools/server/files/embedded-image-refs.ts @@ -1,8 +1,26 @@ import { getFileMetadataById } from '@/lib/uploads/server/metadata' +import { extractEmbeddedFileRefs } from '@/lib/uploads/utils/embedded-image-ref' -/** The canonical embed form the file agent writes for workspace images: `/api/files/view/`. */ +/** View-URL embed (`/api/files/view/`) — the only form the file agent writes; see {@link findUnembeddableImageRefs}. */ const VIEW_EMBED_RE = /\/api\/files\/view\/([A-Za-z0-9_-]+)/g +/** + * De-duplicated workspace file **ids** embedded in `content` (view URL or in-app workspace path). + * Shares the {@link extractEmbeddedFileRefs} grammar with the frontend renderer so the referenced-by-doc + * gate authorizes exactly what the client links. Resolution and access are checked by the caller. + */ +export function extractEmbeddedImageIds(content: string): string[] { + return extractEmbeddedFileRefs(content).ids +} + +/** + * De-duplicated workspace storage **keys** (`workspace//…`) embedded in `content` via the serve URL. + * Same shared grammar as {@link extractEmbeddedImageIds}. + */ +export function extractEmbeddedImageKeys(content: string): string[] { + return extractEmbeddedFileRefs(content).keys +} + /** * Returns the ids of `/api/files/view/` image embeds in `content` that will not render or survive a * workspace export. An embed is valid only when its id resolves to a workspace file in this same diff --git a/apps/sim/lib/uploads/server/inline-image.test.ts b/apps/sim/lib/uploads/server/inline-image.test.ts new file mode 100644 index 00000000000..ba774a3e8f1 --- /dev/null +++ b/apps/sim/lib/uploads/server/inline-image.test.ts @@ -0,0 +1,63 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockGetWorkspaceFile, mockGetFileMetadataByKey } = vi.hoisted(() => ({ + mockGetWorkspaceFile: vi.fn(), + mockGetFileMetadataByKey: vi.fn(), +})) + +vi.mock('@/lib/uploads/contexts/workspace', () => ({ getWorkspaceFile: mockGetWorkspaceFile })) +vi.mock('@/lib/uploads/server/metadata', () => ({ getFileMetadataByKey: mockGetFileMetadataByKey })) + +import { resolveWorkspaceInlineImage } from '@/lib/uploads/server/inline-image' + +describe('resolveWorkspaceInlineImage', () => { + beforeEach(() => vi.clearAllMocks()) + + it('resolves by fileId scoped to the workspace (getWorkspaceFile already enforces scope)', async () => { + mockGetWorkspaceFile.mockResolvedValue({ + key: 'workspace/ws-1/x.png', + type: 'image/png', + name: 'x.png', + }) + const out = await resolveWorkspaceInlineImage('ws-1', { fileId: 'wf_a' }) + expect(mockGetWorkspaceFile).toHaveBeenCalledWith('ws-1', 'wf_a') + expect(out).toEqual({ + key: 'workspace/ws-1/x.png', + contentType: 'image/png', + filename: 'x.png', + }) + }) + + it('returns null when getWorkspaceFile finds nothing (cross-workspace / deleted / non-workspace)', async () => { + mockGetWorkspaceFile.mockResolvedValue(null) + expect(await resolveWorkspaceInlineImage('ws-1', { fileId: 'wf_a' })).toBeNull() + }) + + it('resolves by key only when the row belongs to the workspace', async () => { + mockGetFileMetadataByKey.mockResolvedValue({ + key: 'workspace/ws-1/x.png', + workspaceId: 'ws-1', + contentType: 'image/png', + originalName: 'x.png', + }) + const out = await resolveWorkspaceInlineImage('ws-1', { key: 'workspace/ws-1/x.png' }) + expect(out).toEqual({ + key: 'workspace/ws-1/x.png', + contentType: 'image/png', + filename: 'x.png', + }) + }) + + it('returns null when the keyed row belongs to a different workspace', async () => { + mockGetFileMetadataByKey.mockResolvedValue({ + key: 'workspace/ws-2/x.png', + workspaceId: 'ws-2', + contentType: 'image/png', + originalName: 'x.png', + }) + expect(await resolveWorkspaceInlineImage('ws-1', { key: 'workspace/ws-2/x.png' })).toBeNull() + }) +}) diff --git a/apps/sim/lib/uploads/server/inline-image.ts b/apps/sim/lib/uploads/server/inline-image.ts new file mode 100644 index 00000000000..b44b9e08e4a --- /dev/null +++ b/apps/sim/lib/uploads/server/inline-image.ts @@ -0,0 +1,41 @@ +import { getWorkspaceFile } from '@/lib/uploads/contexts/workspace' +import { getFileMetadataByKey } from '@/lib/uploads/server/metadata' + +/** + * A markdown-embedded image reference: either a workspace file `fileId` (view-URL embeds) or a + * workspace storage `key` (serve-URL embeds). Exactly one is set — enforced at the route boundary. + */ +export interface InlineImageRef { + key?: string + fileId?: string +} + +/** The fields a serve handler needs to return an embedded image. */ +export interface ResolvedInlineImage { + key: string + contentType: string + filename: string +} + +/** + * Resolve an embedded-image reference to its storage key + metadata, **scoped to `workspaceId`**. + * Returns null whenever the reference is not a live `workspace` file in that workspace — a + * cross-workspace, non-workspace, missing, or deleted file. This is the single workspace-scope gate + * shared by the in-app inline route and the public-share cascade, mirroring how the user-facing file + * view resolves a file within its workspace ({@link getWorkspaceFile}). + */ +export async function resolveWorkspaceInlineImage( + workspaceId: string, + ref: InlineImageRef +): Promise { + if (ref.fileId) { + const file = await getWorkspaceFile(workspaceId, ref.fileId) + return file ? { key: file.key, contentType: file.type, filename: file.name } : null + } + if (ref.key) { + const record = await getFileMetadataByKey(ref.key, 'workspace') + if (!record || record.workspaceId !== workspaceId) return null + return { key: record.key, contentType: record.contentType, filename: record.originalName } + } + return null +} diff --git a/apps/sim/lib/uploads/utils/embedded-image-ref.test.ts b/apps/sim/lib/uploads/utils/embedded-image-ref.test.ts new file mode 100644 index 00000000000..c8773883b70 --- /dev/null +++ b/apps/sim/lib/uploads/utils/embedded-image-ref.test.ts @@ -0,0 +1,60 @@ +import { describe, expect, it } from 'vitest' +import { + extractEmbeddedFileRef, + extractEmbeddedFileRefs, +} from '@/lib/uploads/utils/embedded-image-ref' + +const KEY = 'workspace/W1/1700000000000-deadbeefdeadbeef-photo.png' +const ENCODED = encodeURIComponent(KEY) + +describe('extractEmbeddedFileRef', () => { + it('parses serve-url embeds (encoded, raw, and s3/blob prefixed) to the workspace key', () => { + expect(extractEmbeddedFileRef(`/api/files/serve/${ENCODED}?context=workspace`)).toEqual({ + key: KEY, + }) + expect(extractEmbeddedFileRef(`/api/files/serve/s3/${ENCODED}`)).toEqual({ key: KEY }) + expect(extractEmbeddedFileRef(`/api/files/serve/blob/${ENCODED}`)).toEqual({ key: KEY }) + }) + + it('parses view-url and in-app-path embeds to the file id', () => { + expect(extractEmbeddedFileRef('/api/files/view/wf_YwDXi8eWOkTxn0sbgChlB')).toEqual({ + fileId: 'wf_YwDXi8eWOkTxn0sbgChlB', + }) + expect(extractEmbeddedFileRef('/workspace/W1/files/wf_abc')).toEqual({ fileId: 'wf_abc' }) + }) + + it('returns null for external, data, and non-workspace serve urls', () => { + expect(extractEmbeddedFileRef('https://cdn.example.com/a.png')).toBeNull() + expect(extractEmbeddedFileRef('data:image/png;base64,AAAA')).toBeNull() + expect(extractEmbeddedFileRef('/api/files/serve/profile-pictures%2Fu1%2Favatar.png')).toBeNull() + }) +}) + +describe('extractEmbeddedFileRefs', () => { + it('collects de-duplicated keys and ids from a document via the shared parser', () => { + const content = ` + ![a](/api/files/serve/${ENCODED}?context=workspace) + ![b](/api/files/view/wf_abc) + ![c](/workspace/W1/files/4bdaf6c4-072e-464e-891d-b6af3b5fe2cc) + ![dup](/api/files/serve/s3/${ENCODED}) + ![ext](https://cdn.example.com/x.png) + ![pub](/api/files/serve/profile-pictures%2Fu1%2Favatar.png) + ` + const { keys, ids } = extractEmbeddedFileRefs(content) + expect(keys).toEqual([KEY]) + expect(ids.sort()).toEqual(['4bdaf6c4-072e-464e-891d-b6af3b5fe2cc', 'wf_abc'].sort()) + }) + + it('caps total references (keys + ids) at 50 combined', () => { + const ids = Array.from( + { length: 40 }, + (_, i) => `/api/files/view/wf_${String(i).padStart(6, '0')}` + ) + const keys = Array.from( + { length: 40 }, + (_, i) => `/api/files/serve/${encodeURIComponent(`workspace/W1/k${i}.png`)}` + ) + const { keys: k, ids: d } = extractEmbeddedFileRefs([...ids, ...keys].join(' ')) + expect(k.length + d.length).toBe(50) + }) +}) diff --git a/apps/sim/lib/uploads/utils/embedded-image-ref.ts b/apps/sim/lib/uploads/utils/embedded-image-ref.ts new file mode 100644 index 00000000000..0161af11ea7 --- /dev/null +++ b/apps/sim/lib/uploads/utils/embedded-image-ref.ts @@ -0,0 +1,73 @@ +/** + * The grammar of a markdown-embedded workspace image reference, shared by the frontend renderer + * (which rewrites one `src` at a time) and the server (which scans a whole document for the + * referenced-by-doc gate and the export bundler). Both go through {@link extractEmbeddedFileRef} so + * the set the client links and the set the server authorizes can never drift apart. + * + * Pure and isomorphic — no DOM, Node, or DB imports — so it is safe to import from both client and + * server code. + */ + +/** A reference parsed from an embed `src`: a workspace storage key, a workspace file id, or neither. */ +export type EmbeddedFileRef = { key: string } | { fileId: string } | null + +/** Hard cap on embedded images resolved from one document — bounds export bundles and the cascade. */ +export const MAX_EMBEDDED_IMAGES = 50 + +/** + * Candidate embed URL substrings in document text: a serve URL, a view URL, or the in-app workspace + * path. The captured run stops at whitespace/quote/paren/angle/query so authoritative parsing is left + * to {@link extractEmbeddedFileRef}. + */ +const EMBED_URL_RE = + /(?:\/api\/files\/(?:serve|view)\/|\/workspace\/[A-Za-z0-9-]+\/files\/)[^\s)"'<>?]*/g + +/** + * Parse a single embed `src` into the workspace file it references, normalizing the spellings the + * editor and file agent produce: `/api/files/serve/` (incl. `s3/`/`blob/` prefixes), `/api/files/view/`, + * and the in-app path `/workspace//files/`. Returns null for absolute, `data:`, or non-workspace + * URLs (e.g. public `profile-pictures/` assets), which render as-is. + */ +export function extractEmbeddedFileRef(src: string): EmbeddedFileRef { + try { + const parsed = new URL(src, 'http://placeholder') + if (parsed.origin !== 'http://placeholder') return null + const segs = parsed.pathname.split('/') + if (segs[1] === 'api' && segs[2] === 'files' && segs[3] === 'serve') { + let keySegs = segs.slice(4) + if (keySegs[0] === 's3' || keySegs[0] === 'blob') keySegs = keySegs.slice(1) + const raw = keySegs.join('/') + if (!raw) return null + const key = decodeURIComponent(raw) + return key.startsWith('workspace/') ? { key } : null + } + if (segs[1] === 'api' && segs[2] === 'files' && segs[3] === 'view' && segs[4]) { + return { fileId: segs[4] } + } + if (segs[1] === 'workspace' && segs[3] === 'files' && segs[4]) { + return { fileId: segs[4] } + } + return null + } catch { + return null + } +} + +/** + * The de-duplicated keys and ids embedded in `content`, bounded to {@link MAX_EMBEDDED_IMAGES} unique + * references **combined** (keys + ids). Every candidate URL is interpreted by {@link extractEmbeddedFileRef}, + * so this is exactly the set the frontend rewrites — the server's referenced-by-doc gate and the export + * bundler share one grammar. + */ +export function extractEmbeddedFileRefs(content: string): { keys: string[]; ids: string[] } { + const keys = new Set() + const ids = new Set() + for (const match of content.matchAll(EMBED_URL_RE)) { + const ref = extractEmbeddedFileRef(match[0]) + if (!ref) continue + if ('key' in ref) keys.add(ref.key) + else ids.add(ref.fileId) + if (keys.size + ids.size >= MAX_EMBEDDED_IMAGES) break + } + return { keys: [...keys], ids: [...ids] } +} diff --git a/apps/sim/lib/uploads/utils/validation.test.ts b/apps/sim/lib/uploads/utils/validation.test.ts index f5db99cbd09..9d5d31ea1d6 100644 --- a/apps/sim/lib/uploads/utils/validation.test.ts +++ b/apps/sim/lib/uploads/utils/validation.test.ts @@ -1,9 +1,40 @@ import { describe, expect, it } from 'vitest' import { SUPPORTED_ATTACHMENT_EXTENSIONS, + sniffImageContentType, validateAttachmentFileType, } from '@/lib/uploads/utils/validation' +describe('sniffImageContentType', () => { + const png = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00]) + const jpeg = Buffer.from([0xff, 0xd8, 0xff, 0xe0, 0x00]) + const gif87 = Buffer.from('GIF87a....', 'latin1') + const gif89 = Buffer.from('GIF89a....', 'latin1') + const webp = Buffer.concat([ + Buffer.from('RIFF', 'latin1'), + Buffer.from([0x00, 0x00, 0x00, 0x00]), + Buffer.from('WEBP', 'latin1'), + ]) + + it('detects real raster image formats from magic bytes', () => { + expect(sniffImageContentType(png)).toBe('image/png') + expect(sniffImageContentType(jpeg)).toBe('image/jpeg') + expect(sniffImageContentType(gif87)).toBe('image/gif') + expect(sniffImageContentType(gif89)).toBe('image/gif') + expect(sniffImageContentType(webp)).toBe('image/webp') + }) + + it('rejects non-image content, including image-shaped strings and SVG', () => { + expect( + sniffImageContentType(Buffer.from('', 'utf-8')) + ).toBeNull() + expect(sniffImageContentType(Buffer.from('', 'utf-8'))).toBeNull() + expect(sniffImageContentType(Buffer.from('RIFFxxxxAVI ', 'latin1'))).toBeNull() + expect(sniffImageContentType(Buffer.alloc(0))).toBeNull() + expect(sniffImageContentType(Buffer.from([0x89, 0x50]))).toBeNull() + }) +}) + describe('validateAttachmentFileType', () => { it('accepts image files (png, jpg, gif, webp, svg)', () => { expect(validateAttachmentFileType('screenshot.png')).toBeNull() diff --git a/apps/sim/lib/uploads/utils/validation.ts b/apps/sim/lib/uploads/utils/validation.ts index 4f46d67516a..b4e27684f63 100644 --- a/apps/sim/lib/uploads/utils/validation.ts +++ b/apps/sim/lib/uploads/utils/validation.ts @@ -335,6 +335,34 @@ export function isValidPng(buffer: Buffer): boolean { return buffer.length >= 8 && buffer.subarray(0, 8).equals(PNG_MAGIC_BYTES) } +/** + * Detect a renderable raster image from its leading bytes, returning the canonical MIME type or + * `null` when the content is not one of the inline-renderable image formats (PNG, JPEG, GIF, WebP). + * + * The stored `contentType` is client-declared and never sniffed at upload time, so any path that + * renders a file inline for a less-trusted audience (e.g. images embedded in a public share) must + * derive the served type from the bytes themselves — a file claiming `image/png` could be HTML, SVG, + * or a script. SVG is deliberately excluded: it can carry script and is not a raster format. + */ +export function sniffImageContentType(buffer: Buffer): string | null { + if (isValidPng(buffer)) return 'image/png' + if (buffer.length >= 3 && buffer[0] === 0xff && buffer[1] === 0xd8 && buffer[2] === 0xff) { + return 'image/jpeg' + } + if (buffer.length >= 6) { + const header = buffer.toString('latin1', 0, 6) + if (header === 'GIF87a' || header === 'GIF89a') return 'image/gif' + } + if ( + buffer.length >= 12 && + buffer.toString('latin1', 0, 4) === 'RIFF' && + buffer.toString('latin1', 8, 12) === 'WEBP' + ) { + return 'image/webp' + } + return null +} + export function validateMediaFileType( fileName: string, mimeType: string diff --git a/scripts/check-api-validation-contracts.ts b/scripts/check-api-validation-contracts.ts index 17f0a25fa29..a45c2d84914 100644 --- a/scripts/check-api-validation-contracts.ts +++ b/scripts/check-api-validation-contracts.ts @@ -9,8 +9,8 @@ const QUERY_HOOKS_DIR = path.join(ROOT, 'apps/sim/hooks/queries') const SELECTOR_HOOKS_DIR = path.join(ROOT, 'apps/sim/hooks/selectors') const BASELINE = { - totalRoutes: 860, - zodRoutes: 860, + totalRoutes: 862, + zodRoutes: 862, nonZodRoutes: 0, } as const From 6260eda22668dc8fced33452365ac8a6605d149d Mon Sep 17 00:00:00 2001 From: Waleed Date: Wed, 24 Jun 2026 18:46:18 -0700 Subject: [PATCH 12/15] fix(ssr): harden credential query-key factory + fetchers against the 'use client' stub bug (#5206) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(ssr): move credential query-key factory + fetchers to non-client modules Preventively closes the same 'use client' SSR client-reference-stub class that crashed the tables page. Server-evaluated modules (the credential block def, the workflow-comparison helpers) imported workspaceCredentialKeys / fetchWorkspaceCredentialList / fetchCredentialSetById from 'use client' hook modules, where they resolve to client-reference stubs on the server (a future server call path would throw 'X is not a function'). Extract them into non-client hooks/queries/utils/{credential-keys, fetch-workspace-credentials,fetch-credential-set}.ts (mirroring folder-keys.ts / fetch-workflow-envelope.ts) and import from there. No behavior change — these values were only ever called from browser paths. * docs+ci: codify the 'use client' server-import rule + add check:client-boundary Document the Next.js rule that server code can only render a 'use client' export as a component, never call it (server imports resolve to client-reference stubs that throw — the tables-page crash). Add the rule to .claude/rules/sim-queries.md + a cross-ref in sim-architecture.md. Add scripts/check-client-boundary-imports.ts (wired into CI as check:client-boundary) that flags any value import from a 'use client' module in a server-evaluated, non-JSX surface (prefetch / route handler / trigger / block definition), so this class can't silently recur. Escape hatch: // client-boundary-allow: . --- .claude/rules/sim-architecture.md | 4 + .claude/rules/sim-queries.md | 11 + .github/workflows/test-build.yml | 3 + .../secrets-manager/secrets-manager.tsx | 7 +- apps/sim/blocks/blocks/credential.ts | 3 +- apps/sim/hooks/queries/credential-sets.ts | 14 +- apps/sim/hooks/queries/credentials.ts | 34 +-- apps/sim/hooks/queries/invitations.ts | 2 +- apps/sim/hooks/queries/organization.ts | 2 +- .../hooks/queries/utils/credential-keys.ts | 24 ++ .../queries/utils/fetch-credential-set.ts | 21 ++ .../utils/fetch-workspace-credentials.ts | 20 ++ .../comparison/format-description.test.ts | 2 +- .../workflows/comparison/resolve-values.ts | 2 +- package.json | 1 + scripts/check-client-boundary-imports.ts | 233 ++++++++++++++++++ 16 files changed, 328 insertions(+), 55 deletions(-) create mode 100644 apps/sim/hooks/queries/utils/credential-keys.ts create mode 100644 apps/sim/hooks/queries/utils/fetch-credential-set.ts create mode 100644 apps/sim/hooks/queries/utils/fetch-workspace-credentials.ts create mode 100644 scripts/check-client-boundary-imports.ts diff --git a/.claude/rules/sim-architecture.md b/.claude/rules/sim-architecture.md index bc52fd37001..a0cfbfcd050 100644 --- a/.claude/rules/sim-architecture.md +++ b/.claude/rules/sim-architecture.md @@ -38,6 +38,10 @@ packages/ # @sim/* — audit, auth, db, logger, realtime-protocol - `apps/* → packages/*` only. Packages never import from `apps/*`. - `apps/realtime` avoids Next.js, React, the block/tool registry, provider SDKs, and the executor; never add `@/lib/webhooks/providers/*`, `@/executor/*`, `@/blocks/*`, or `@/tools/*` imports to any package it consumes. CI enforces this via `scripts/check-monorepo-boundaries.ts` and `scripts/check-realtime-prune-graph.ts`. +## The `'use client'` server boundary + +Every export of a `'use client'` module becomes a *client reference* on the server — server-evaluated code (RSC pages/layouts, `prefetch.ts`, route handlers, block definitions, triggers) can only *render* it as a component or pass it as a prop, never *call* it (doing so throws at runtime, e.g. `tableKeys.list is not a function`; `next build` does not catch it). Keep server-importable query primitives (key factories, fetchers, mappers, constants) in non-`'use client'` modules — see `.claude/rules/sim-queries.md`. Enforced by `scripts/check-client-boundary-imports.ts`. + ## Feature Organization Features live under `app/workspace/[workspaceId]/`: diff --git a/.claude/rules/sim-queries.md b/.claude/rules/sim-queries.md index 25707c740af..d1db9437ff0 100644 --- a/.claude/rules/sim-queries.md +++ b/.claude/rules/sim-queries.md @@ -27,6 +27,17 @@ Never use inline query keys — always use the factory. **Every identifier the `queryFn` forwards into the fetch MUST appear in the `queryKey`.** (Query-machinery identifiers — `signal`, `pageParam` — are exempt; they aren't fetch-scoping args.) If the fetch is scoped by `workspaceId`, `cursor`, `limit`, an org id, etc., those values must be part of the key — otherwise distinct fetch args share one cache entry (a cross-tenant / per-param cache collision). The lone exception is a globally-unique id used as the key while a second fetch arg is only an authz scope that cannot collide; annotate those with `// rq-lint-allow: `. Enforced by the `key-fetch-arg-drift` check in `scripts/check-react-query-patterns.ts`. +## Server-importable query primitives must NOT live in a `'use client'` module + +Next.js rewrites **every** export of a `'use client'` module into a *client reference* in the server bundle. Server-evaluated code — RSC `page.tsx`/`layout.tsx`, `prefetch.ts`, route handlers, **block definitions**, triggers/workers — can only *render* such an export as a component or pass it as a prop; **calling** one throws at runtime (`Attempted to call X from the server but X is on the client` — for an object export it surfaces as `X.list is not a function`). `next build` does **not** catch this — only SSR/runtime does. + +So any **query-key factory, standalone `requestJson` fetcher, mapper, or constant** that a server module imports must live in a **non-`'use client'`** module: + +- key factories → `hooks/queries/utils/-keys.ts` (see `folder-keys.ts`, `table-keys.ts`, `credential-keys.ts`) +- standalone fetchers/mappers → `hooks/queries/utils/fetch-*.ts` / `*-list-query.ts` (see `fetch-workflow-envelope.ts`, `fetch-credential-set.ts`) + +The `'use client'` hook module then imports these back for its hooks. **Never** define a server-imported factory/fetcher directly in a `'use client'` hooks file — it crashes SSR (this caused the tables-page crash). Enforced for prefetch/route/trigger/block files by `scripts/check-client-boundary-imports.ts` (`bun run check:client-boundary`, run in CI). Escape hatch for a genuinely browser-only path: `// client-boundary-allow: ` on the line above the import. + ## File Structure ```typescript diff --git a/.github/workflows/test-build.yml b/.github/workflows/test-build.yml index 40f83645146..c4ffd7449ea 100644 --- a/.github/workflows/test-build.yml +++ b/.github/workflows/test-build.yml @@ -122,6 +122,9 @@ jobs: - name: React Query pattern audit run: bun run check:react-query + - name: Client boundary import audit + run: bun run check:client-boundary + - name: Verify realtime prune graph run: bun run check:realtime-prune diff --git a/apps/sim/app/workspace/[workspaceId]/settings/components/secrets/components/secrets-manager/secrets-manager.tsx b/apps/sim/app/workspace/[workspaceId]/settings/components/secrets/components/secrets-manager/secrets-manager.tsx index e2581174df8..ffb2becd85a 100644 --- a/apps/sim/app/workspace/[workspaceId]/settings/components/secrets/components/secrets-manager/secrets-manager.tsx +++ b/apps/sim/app/workspace/[workspaceId]/settings/components/secrets/components/secrets-manager/secrets-manager.tsx @@ -17,11 +17,7 @@ import type { WorkspaceEnvironmentData } from '@/lib/environment/api' import { UnsavedChangesModal } from '@/app/workspace/[workspaceId]/components/credential-detail' import { SecretValueField } from '@/app/workspace/[workspaceId]/settings/components/secrets/components/secret-value-field' import { isValidEnvVarName } from '@/executor/constants' -import { - useWorkspaceCredentials, - type WorkspaceCredential, - workspaceCredentialKeys, -} from '@/hooks/queries/credentials' +import { useWorkspaceCredentials, type WorkspaceCredential } from '@/hooks/queries/credentials' import { usePersonalEnvironment, useRemoveWorkspaceEnvironment, @@ -29,6 +25,7 @@ import { useUpsertWorkspaceEnvironment, useWorkspaceEnvironment, } from '@/hooks/queries/environment' +import { workspaceCredentialKeys } from '@/hooks/queries/utils/credential-keys' import { useWorkspacePermissionsQuery } from '@/hooks/queries/workspace' import { useSettingsDirtyStore } from '@/stores/settings/dirty/store' diff --git a/apps/sim/blocks/blocks/credential.ts b/apps/sim/blocks/blocks/credential.ts index fc324d721a4..830a8b65b65 100644 --- a/apps/sim/blocks/blocks/credential.ts +++ b/apps/sim/blocks/blocks/credential.ts @@ -2,7 +2,8 @@ import { CredentialIcon } from '@/components/icons' import { getServiceConfigByProviderId } from '@/lib/oauth/utils' import { getQueryClient } from '@/app/_shell/providers/get-query-client' import type { BlockConfig } from '@/blocks/types' -import { fetchWorkspaceCredentialList, workspaceCredentialKeys } from '@/hooks/queries/credentials' +import { workspaceCredentialKeys } from '@/hooks/queries/utils/credential-keys' +import { fetchWorkspaceCredentialList } from '@/hooks/queries/utils/fetch-workspace-credentials' import { useWorkflowRegistry } from '@/stores/workflows/registry/store' interface CredentialBlockOutput { diff --git a/apps/sim/hooks/queries/credential-sets.ts b/apps/sim/hooks/queries/credential-sets.ts index d828e42544b..2a1086444a5 100644 --- a/apps/sim/hooks/queries/credential-sets.ts +++ b/apps/sim/hooks/queries/credential-sets.ts @@ -19,7 +19,6 @@ import { createCredentialSetContract, createCredentialSetInvitationContract, deleteCredentialSetContract, - getCredentialSetContract, leaveCredentialSetContract, listCredentialSetInvitationDetailsContract, listCredentialSetInvitationsContract, @@ -29,6 +28,7 @@ import { removeCredentialSetMemberContract, resendCredentialSetInvitationContract, } from '@/lib/api/contracts' +import { fetchCredentialSetById } from '@/hooks/queries/utils/fetch-credential-set' export type { CreateCredentialSetData, @@ -76,18 +76,6 @@ export function useCredentialSets(organizationId?: string, enabled = true) { }) } -export async function fetchCredentialSetById( - id: string, - signal?: AbortSignal -): Promise { - if (!id) return null - const data = await requestJson(getCredentialSetContract, { - params: { id }, - signal, - }) - return data.credentialSet ?? null -} - export function useCredentialSetDetail(id?: string, enabled = true) { return useQuery({ queryKey: credentialSetKeys.detail(id), diff --git a/apps/sim/hooks/queries/credentials.ts b/apps/sim/hooks/queries/credentials.ts index a2dda84d906..9e20147faab 100644 --- a/apps/sim/hooks/queries/credentials.ts +++ b/apps/sim/hooks/queries/credentials.ts @@ -20,6 +20,8 @@ import { type WorkspaceCredentialType, } from '@/lib/api/contracts' import { environmentKeys } from '@/hooks/queries/environment' +import { workspaceCredentialKeys } from '@/hooks/queries/utils/credential-keys' +import { fetchWorkspaceCredentialList } from '@/hooks/queries/utils/fetch-workspace-credentials' /** * Key prefix for OAuth credential queries. @@ -34,38 +36,6 @@ export type { WorkspaceCredentialType, } -export const workspaceCredentialKeys = { - all: ['workspaceCredentials'] as const, - lists: () => [...workspaceCredentialKeys.all, 'list'] as const, - list: (workspaceId?: string, type?: string, providerId?: string) => - [ - ...workspaceCredentialKeys.lists(), - workspaceId ?? 'none', - type ?? 'all', - providerId ?? 'all', - ] as const, - details: () => [...workspaceCredentialKeys.all, 'detail'] as const, - detail: (credentialId?: string) => - [...workspaceCredentialKeys.details(), credentialId ?? 'none'] as const, - members: (credentialId?: string) => - [...workspaceCredentialKeys.detail(credentialId), 'members'] as const, -} - -/** - * Fetch workspace credential list from API. - * Used by the prefetch function for hover-based cache warming. - */ -export async function fetchWorkspaceCredentialList( - workspaceId: string, - signal?: AbortSignal -): Promise { - const data = await requestJson(listWorkspaceCredentialsContract, { - query: { workspaceId }, - signal, - }) - return data.credentials ?? [] -} - /** * Prefetch workspace credentials into a QueryClient cache. * Use on hover to warm data before navigation. diff --git a/apps/sim/hooks/queries/invitations.ts b/apps/sim/hooks/queries/invitations.ts index 17e0bfe3963..86a772a5e70 100644 --- a/apps/sim/hooks/queries/invitations.ts +++ b/apps/sim/hooks/queries/invitations.ts @@ -11,8 +11,8 @@ import { resendInvitationContract, } from '@/lib/api/contracts/invitations' import { updateWorkspacePermissionsContract } from '@/lib/api/contracts/workspaces' -import { workspaceCredentialKeys } from '@/hooks/queries/credentials' import { organizationKeys } from '@/hooks/queries/organization' +import { workspaceCredentialKeys } from '@/hooks/queries/utils/credential-keys' import { workspaceKeys } from '@/hooks/queries/workspace' export const invitationKeys = { diff --git a/apps/sim/hooks/queries/organization.ts b/apps/sim/hooks/queries/organization.ts index 11352d88b97..f005c7eaeed 100644 --- a/apps/sim/hooks/queries/organization.ts +++ b/apps/sim/hooks/queries/organization.ts @@ -42,8 +42,8 @@ import { import { client } from '@/lib/auth/auth-client' import { isEnterprise, isPaid, isTeam } from '@/lib/billing/plan-helpers' import { hasPaidSubscriptionStatus } from '@/lib/billing/subscriptions/utils' -import { workspaceCredentialKeys } from '@/hooks/queries/credentials' import { subscriptionKeys } from '@/hooks/queries/subscription' +import { workspaceCredentialKeys } from '@/hooks/queries/utils/credential-keys' import { workspaceKeys } from '@/hooks/queries/workspace' const logger = createLogger('OrganizationQueries') diff --git a/apps/sim/hooks/queries/utils/credential-keys.ts b/apps/sim/hooks/queries/utils/credential-keys.ts new file mode 100644 index 00000000000..728cbd34ceb --- /dev/null +++ b/apps/sim/hooks/queries/utils/credential-keys.ts @@ -0,0 +1,24 @@ +/** + * React Query key factory for workspace credentials. + * + * Lives in this standalone (non-`'use client'`) module — like + * {@link file://./folder-keys.ts} — so server-evaluated code (block + * definitions, server prefetch) can import it without pulling client-reference + * stubs from the `'use client'` `@/hooks/queries/credentials` module. + */ +export const workspaceCredentialKeys = { + all: ['workspaceCredentials'] as const, + lists: () => [...workspaceCredentialKeys.all, 'list'] as const, + list: (workspaceId?: string, type?: string, providerId?: string) => + [ + ...workspaceCredentialKeys.lists(), + workspaceId ?? 'none', + type ?? 'all', + providerId ?? 'all', + ] as const, + details: () => [...workspaceCredentialKeys.all, 'detail'] as const, + detail: (credentialId?: string) => + [...workspaceCredentialKeys.details(), credentialId ?? 'none'] as const, + members: (credentialId?: string) => + [...workspaceCredentialKeys.detail(credentialId), 'members'] as const, +} diff --git a/apps/sim/hooks/queries/utils/fetch-credential-set.ts b/apps/sim/hooks/queries/utils/fetch-credential-set.ts new file mode 100644 index 00000000000..c9523d53633 --- /dev/null +++ b/apps/sim/hooks/queries/utils/fetch-credential-set.ts @@ -0,0 +1,21 @@ +import { requestJson } from '@/lib/api/client/request' +import { type CredentialSet, getCredentialSetContract } from '@/lib/api/contracts' + +/** + * Fetches a credential set by id (returns `null` for an empty id). + * + * Lives in this standalone (non-`'use client'`) module so server-reachable + * workflow-comparison helpers can import it without pulling client-reference + * stubs from the `'use client'` `@/hooks/queries/credential-sets` module. + */ +export async function fetchCredentialSetById( + id: string, + signal?: AbortSignal +): Promise { + if (!id) return null + const data = await requestJson(getCredentialSetContract, { + params: { id }, + signal, + }) + return data.credentialSet ?? null +} diff --git a/apps/sim/hooks/queries/utils/fetch-workspace-credentials.ts b/apps/sim/hooks/queries/utils/fetch-workspace-credentials.ts new file mode 100644 index 00000000000..9fd8efd7b6f --- /dev/null +++ b/apps/sim/hooks/queries/utils/fetch-workspace-credentials.ts @@ -0,0 +1,20 @@ +import { requestJson } from '@/lib/api/client/request' +import { listWorkspaceCredentialsContract, type WorkspaceCredential } from '@/lib/api/contracts' + +/** + * Fetches the workspace credential list. + * + * Lives in this standalone (non-`'use client'`) module so block definitions and + * server prefetch can import it without pulling client-reference stubs from the + * `'use client'` `@/hooks/queries/credentials` module. + */ +export async function fetchWorkspaceCredentialList( + workspaceId: string, + signal?: AbortSignal +): Promise { + const data = await requestJson(listWorkspaceCredentialsContract, { + query: { workspaceId }, + signal, + }) + return data.credentials ?? [] +} diff --git a/apps/sim/lib/workflows/comparison/format-description.test.ts b/apps/sim/lib/workflows/comparison/format-description.test.ts index f186a9d5a4f..22fb5b2fa05 100644 --- a/apps/sim/lib/workflows/comparison/format-description.test.ts +++ b/apps/sim/lib/workflows/comparison/format-description.test.ts @@ -36,7 +36,7 @@ vi.mock('@/lib/workflows/subblocks/context', () => ({ buildSelectorContextFromBlock: vi.fn(() => ({})), })) -vi.mock('@/hooks/queries/credential-sets', () => ({ +vi.mock('@/hooks/queries/utils/fetch-credential-set', () => ({ fetchCredentialSetById: vi.fn(), })) diff --git a/apps/sim/lib/workflows/comparison/resolve-values.ts b/apps/sim/lib/workflows/comparison/resolve-values.ts index bc9d6c7a753..fa9dbb37f9a 100644 --- a/apps/sim/lib/workflows/comparison/resolve-values.ts +++ b/apps/sim/lib/workflows/comparison/resolve-values.ts @@ -4,8 +4,8 @@ import { buildSelectorContextFromBlock } from '@/lib/workflows/subblocks/context import { getBlock } from '@/blocks/registry' import { SELECTOR_TYPES_HYDRATION_REQUIRED, type SubBlockConfig } from '@/blocks/types' import { CREDENTIAL_SET, isUuid } from '@/executor/constants' -import { fetchCredentialSetById } from '@/hooks/queries/credential-sets' import { fetchOAuthCredentialDetail } from '@/hooks/queries/oauth/oauth-credentials' +import { fetchCredentialSetById } from '@/hooks/queries/utils/fetch-credential-set' import { getSelectorDefinition, loadAllSelectorOptions } from '@/hooks/selectors/registry' import { resolveSelectorForSubBlock } from '@/hooks/selectors/resolution' import type { SelectorContext, SelectorKey } from '@/hooks/selectors/types' diff --git a/package.json b/package.json index ce15ab78107..54f25a8da65 100644 --- a/package.json +++ b/package.json @@ -28,6 +28,7 @@ "check:realtime-prune": "bun run scripts/check-realtime-prune-graph.ts", "check:zustand-v5": "bun run scripts/check-zustand-v5-selectors.ts", "check:react-query": "bun run scripts/check-react-query-patterns.ts --check", + "check:client-boundary": "bun run scripts/check-client-boundary-imports.ts --check", "check:utils": "bun run scripts/check-utils-enforcement.ts", "check:migrations": "bun run scripts/check-migrations-safety.ts", "mship-contracts:generate": "bun run scripts/sync-mothership-stream-contract.ts", diff --git a/scripts/check-client-boundary-imports.ts b/scripts/check-client-boundary-imports.ts new file mode 100644 index 00000000000..6d0ec2d6758 --- /dev/null +++ b/scripts/check-client-boundary-imports.ts @@ -0,0 +1,233 @@ +#!/usr/bin/env bun +/** + * Guards against the Next.js `'use client'` server-import foot-gun. + * + * Next.js rewrites EVERY export of a `'use client'` module into a client + * reference in the server bundle. Server-evaluated code can only *render* such + * an export as a component or pass it as a prop — *calling* one throws at + * runtime ("Attempted to call X from the server but X is on the client"). The + * crash for an object export looks like `tableKeys.list is not a function`. + * `next build` does NOT catch this; only SSR/runtime does. + * + * This script flags any **value** import (not `import type`) that resolves to a + * `'use client'` module from a server-evaluated, non-JSX surface — the places + * that never legitimately render a client component and so only ever import a + * client module to (illegally) call its values: + * + * - `apps/sim/app/** /prefetch*.ts` (RSC server prefetch) + * - `apps/sim/app/api/** /route.ts(x)` (route handlers) + * - `apps/sim/triggers/**` (trigger.dev tasks/pollers/webhooks) + * - `apps/sim/blocks/**` (block definitions — evaluated server-side) + * + * Fix: move the imported query-key factory / standalone fetcher / mapper / + * constant into a non-`'use client'` module (e.g. `hooks/queries/utils/*-keys.ts` + * or `hooks/queries/utils/fetch-*.ts`) and import it from there. See the rule in + * `.claude/rules/sim-queries.md`. + * + * Escape hatch: `// client-boundary-allow: ` on the line directly above + * the import (reason required). Use only for a genuinely browser-only code path. + * + * Usage: + * bun run scripts/check-client-boundary-imports.ts # report + * bun run scripts/check-client-boundary-imports.ts --check # CI gate (fail on any) + */ +import { readdir, readFile } from 'node:fs/promises' +import path from 'node:path' + +const ROOT = path.resolve(import.meta.dir, '..') +const APP_DIR = path.join(ROOT, 'apps/sim') + +/** Server-evaluated, non-JSX surfaces. A file matches if its path passes one. */ +function isServerSurface(rel: string): boolean { + if (/(^|\/)prefetch[^/]*\.ts$/.test(rel)) return true + if (/^app\/api\/.+\/route\.tsx?$/.test(rel)) return true + if (/^triggers\//.test(rel)) return true + if (/^blocks\//.test(rel)) return true + return false +} + +const SOURCE_EXTENSIONS = ['.ts', '.tsx'] +const ALLOW_DIRECTIVE = 'client-boundary-allow' + +async function listFiles(dir: string): Promise { + const out: string[] = [] + let entries: Awaited> + try { + entries = await readdir(dir, { withFileTypes: true }) + } catch { + return out + } + for (const entry of entries) { + const full = path.join(dir, entry.name) + if (entry.isDirectory()) { + if (entry.name === 'node_modules' || entry.name === '.next') continue + out.push(...(await listFiles(full))) + } else if (SOURCE_EXTENSIONS.includes(path.extname(entry.name))) { + out.push(full) + } + } + return out +} + +const useClientCache = new Map() + +async function isUseClientModule(absFile: string): Promise { + const cached = useClientCache.get(absFile) + if (cached !== undefined) return cached + let content: string + try { + content = await readFile(absFile, 'utf8') + } catch { + useClientCache.set(absFile, false) + return false + } + // The directive must be the first statement (comments/blank lines may precede it). + let isClient = false + for (const raw of content.split('\n')) { + const line = raw.trim() + if (line === '' || line.startsWith('//') || line.startsWith('/*') || line.startsWith('*')) { + continue + } + isClient = line === "'use client'" || line === '"use client"' + break + } + useClientCache.set(absFile, isClient) + return isClient +} + +/** Resolve an import specifier to an absolute source file, or null if external/unresolved. */ +async function resolveSpecifier(spec: string, fromFile: string): Promise { + let base: string + if (spec.startsWith('@/')) { + base = path.join(APP_DIR, spec.slice(2)) + } else if (spec.startsWith('./') || spec.startsWith('../')) { + base = path.resolve(path.dirname(fromFile), spec) + } else { + return null // external package + } + const candidates = [ + base, + ...SOURCE_EXTENSIONS.map((ext) => base + ext), + ...SOURCE_EXTENSIONS.map((ext) => path.join(base, `index${ext}`)), + ] + for (const candidate of candidates) { + if (!SOURCE_EXTENSIONS.includes(path.extname(candidate))) continue + try { + await readFile(candidate, 'utf8') + return candidate + } catch {} + } + return null +} + +interface ImportInfo { + line: number + specifier: string + clause: string +} + +/** Parse `import ... from '...'` statements, skipping side-effect-only imports. */ +function parseImports(content: string): ImportInfo[] { + const lines = content.split('\n') + const imports: ImportInfo[] = [] + const re = /^\s*import\s+([\s\S]*?)\s+from\s+['"]([^'"]+)['"]/ + for (let i = 0; i < lines.length; i++) { + if (!/^\s*import\b/.test(lines[i]) || !lines[i].includes('import')) continue + // Join up to 12 following lines to capture multi-line import clauses. + const block = lines.slice(i, i + 12).join('\n') + const match = re.exec(block) + if (!match) continue + imports.push({ line: i + 1, clause: match[1], specifier: match[2] }) + } + return imports +} + +/** True when the import brings in at least one runtime VALUE (not purely types). */ +function importsAValue(clause: string): boolean { + const trimmed = clause.trim() + if (trimmed.startsWith('type ')) return false // `import type { ... }` / `import type X` + const braceStart = trimmed.indexOf('{') + // A default or namespace binding outside the braces is always a value. + const beforeBrace = braceStart === -1 ? trimmed : trimmed.slice(0, braceStart) + if (beforeBrace.replace(/[,\s]/g, '').length > 0) return true + if (braceStart === -1) return true + const inner = trimmed.slice(braceStart + 1, trimmed.lastIndexOf('}')) + // A named import is a value unless every member is `type`-prefixed. + return inner + .split(',') + .map((s) => s.trim()) + .filter(Boolean) + .some((member) => !member.startsWith('type ')) +} + +function hasAllowDirective(content: string, importLine: number): boolean { + const lines = content.split('\n') + for (let i = importLine - 2; i >= 0 && i >= importLine - 5; i--) { + const line = lines[i]?.trim() ?? '' + if (line === '' || line.startsWith('//') || line.startsWith('*') || line.startsWith('/*')) { + if (line.includes(ALLOW_DIRECTIVE)) { + const reason = + line + .split(ALLOW_DIRECTIVE)[1] + ?.replace(/^[:\s]+/, '') + .trim() ?? '' + return reason.length > 0 + } + continue + } + break + } + return false +} + +interface Violation { + file: string + line: number + specifier: string +} + +async function main() { + const checkMode = process.argv.includes('--check') + const allFiles = await listFiles(APP_DIR) + const violations: Violation[] = [] + + for (const absFile of allFiles) { + const rel = path.relative(APP_DIR, absFile) + if (!isServerSurface(rel)) continue + // A server file that is itself `'use client'` is a client component — out of scope. + if (await isUseClientModule(absFile)) continue + + const content = await readFile(absFile, 'utf8') + for (const imp of parseImports(content)) { + if (!importsAValue(imp.clause)) continue + const resolved = await resolveSpecifier(imp.specifier, absFile) + if (!resolved) continue + if (!(await isUseClientModule(resolved))) continue + if (hasAllowDirective(content, imp.line)) continue + violations.push({ file: rel, line: imp.line, specifier: imp.specifier }) + } + } + + if (violations.length === 0) { + console.log( + "✓ Client-boundary import check passed (no server file imports a value from a 'use client' module)." + ) + return + } + + console.error( + `\n✗ ${violations.length} server file(s) import a runtime value from a 'use client' module.\n` + + ` On the server these resolve to client-reference stubs and throw when called (e.g. 'X.list is not a function').\n` + + ` Move the imported factory/fetcher/constant into a non-'use client' module (hooks/queries/utils/*-keys.ts or fetch-*.ts).\n` + + ` See .claude/rules/sim-queries.md. Escape hatch: // ${ALLOW_DIRECTIVE}: above the import.\n` + ) + for (const v of violations) { + console.error(` ${v.file}:${v.line} imports from '${v.specifier}'`) + } + if (checkMode) process.exit(1) +} + +main().catch((error) => { + console.error(error) + process.exit(1) +}) From 9bd8f149a52bf36b57e0df93391f3a21f45a45f2 Mon Sep 17 00:00:00 2001 From: Waleed Date: Wed, 24 Jun 2026 20:38:04 -0700 Subject: [PATCH 13/15] fix(workspace): add granular error boundaries to 7 more workspace segments (#5207) Adds error.tsx (reusing the shared ErrorState) to home, integrations, knowledge/[id], skills, settings, scheduled-tasks, and chat/[chatId] so a crash in any of these panels stays scoped to the panel and offers a retry, instead of bubbling to the generic workspace-level boundary. --- .../[workspaceId]/chat/[chatId]/error.tsx | 15 +++++++++++++++ .../app/workspace/[workspaceId]/home/error.tsx | 15 +++++++++++++++ .../[workspaceId]/integrations/error.tsx | 15 +++++++++++++++ .../[workspaceId]/knowledge/[id]/error.tsx | 15 +++++++++++++++ .../[workspaceId]/scheduled-tasks/error.tsx | 15 +++++++++++++++ .../workspace/[workspaceId]/settings/error.tsx | 15 +++++++++++++++ .../app/workspace/[workspaceId]/skills/error.tsx | 15 +++++++++++++++ 7 files changed, 105 insertions(+) create mode 100644 apps/sim/app/workspace/[workspaceId]/chat/[chatId]/error.tsx create mode 100644 apps/sim/app/workspace/[workspaceId]/home/error.tsx create mode 100644 apps/sim/app/workspace/[workspaceId]/integrations/error.tsx create mode 100644 apps/sim/app/workspace/[workspaceId]/knowledge/[id]/error.tsx create mode 100644 apps/sim/app/workspace/[workspaceId]/scheduled-tasks/error.tsx create mode 100644 apps/sim/app/workspace/[workspaceId]/settings/error.tsx create mode 100644 apps/sim/app/workspace/[workspaceId]/skills/error.tsx diff --git a/apps/sim/app/workspace/[workspaceId]/chat/[chatId]/error.tsx b/apps/sim/app/workspace/[workspaceId]/chat/[chatId]/error.tsx new file mode 100644 index 00000000000..1ebfc47facb --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/chat/[chatId]/error.tsx @@ -0,0 +1,15 @@ +'use client' + +import { type ErrorBoundaryProps, ErrorState } from '@/app/workspace/[workspaceId]/components' + +export default function ChatError({ error, reset }: ErrorBoundaryProps) { + return ( + + ) +} diff --git a/apps/sim/app/workspace/[workspaceId]/home/error.tsx b/apps/sim/app/workspace/[workspaceId]/home/error.tsx new file mode 100644 index 00000000000..03d205fdcd0 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/home/error.tsx @@ -0,0 +1,15 @@ +'use client' + +import { type ErrorBoundaryProps, ErrorState } from '@/app/workspace/[workspaceId]/components' + +export default function HomeError({ error, reset }: ErrorBoundaryProps) { + return ( + + ) +} diff --git a/apps/sim/app/workspace/[workspaceId]/integrations/error.tsx b/apps/sim/app/workspace/[workspaceId]/integrations/error.tsx new file mode 100644 index 00000000000..706d7eea055 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/integrations/error.tsx @@ -0,0 +1,15 @@ +'use client' + +import { type ErrorBoundaryProps, ErrorState } from '@/app/workspace/[workspaceId]/components' + +export default function IntegrationsError({ error, reset }: ErrorBoundaryProps) { + return ( + + ) +} diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/error.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/error.tsx new file mode 100644 index 00000000000..91dbaadb2c5 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/error.tsx @@ -0,0 +1,15 @@ +'use client' + +import { type ErrorBoundaryProps, ErrorState } from '@/app/workspace/[workspaceId]/components' + +export default function KnowledgeBaseError({ error, reset }: ErrorBoundaryProps) { + return ( + + ) +} diff --git a/apps/sim/app/workspace/[workspaceId]/scheduled-tasks/error.tsx b/apps/sim/app/workspace/[workspaceId]/scheduled-tasks/error.tsx new file mode 100644 index 00000000000..4fc75c7b937 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/scheduled-tasks/error.tsx @@ -0,0 +1,15 @@ +'use client' + +import { type ErrorBoundaryProps, ErrorState } from '@/app/workspace/[workspaceId]/components' + +export default function ScheduledTasksError({ error, reset }: ErrorBoundaryProps) { + return ( + + ) +} diff --git a/apps/sim/app/workspace/[workspaceId]/settings/error.tsx b/apps/sim/app/workspace/[workspaceId]/settings/error.tsx new file mode 100644 index 00000000000..02a18fd3362 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/settings/error.tsx @@ -0,0 +1,15 @@ +'use client' + +import { type ErrorBoundaryProps, ErrorState } from '@/app/workspace/[workspaceId]/components' + +export default function SettingsError({ error, reset }: ErrorBoundaryProps) { + return ( + + ) +} diff --git a/apps/sim/app/workspace/[workspaceId]/skills/error.tsx b/apps/sim/app/workspace/[workspaceId]/skills/error.tsx new file mode 100644 index 00000000000..9a860682257 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/skills/error.tsx @@ -0,0 +1,15 @@ +'use client' + +import { type ErrorBoundaryProps, ErrorState } from '@/app/workspace/[workspaceId]/components' + +export default function SkillsError({ error, reset }: ErrorBoundaryProps) { + return ( + + ) +} From e748a64ff4f3a83675882a529f5b355f668fa50d Mon Sep 17 00:00:00 2001 From: Waleed Date: Wed, 24 Jun 2026 20:43:52 -0700 Subject: [PATCH 14/15] refactor(realtime): type the socket event-handler boundary with @sim/realtime-protocol (#5208) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * refactor(realtime): type the socket event-handler boundary with @sim/realtime-protocol Replace the (data: any) event-handler types in socket-provider.tsx with precise broadcast types that mirror the exact payloads emitted by the realtime Socket.IO server (apps/realtime/src/handlers/** and rooms/**). Add @sim/realtime-protocol/events with the canonical wire types for the broadcast/confirmation events the server emits: WorkflowOperationBroadcast, SubblockUpdateBroadcast, VariableUpdateBroadcast, CursorUpdateBroadcast, SelectionUpdateBroadcast, the four workflow-lifecycle broadcasts, and OperationConfirmed/Failed. Typing change only; zero runtime/logic changes. Store-internal any (rehydrate state, subblock map, emit payloads) is left untouched as out of scope. * fix(realtime): type cursor-update broadcast cursor as nullable The client emits 'cursor-update' with { cursor: null } when a remote user's cursor leaves the canvas, and the server re-broadcasts it verbatim, so receivers genuinely get cursor: null. Type CursorUpdateBroadcast.cursor as CursorPosition | null to match the wire. (selection stays non-null — it signals absence via type: 'none', never null.) --- .../workspace/providers/socket-provider.tsx | 110 +++++++++------- packages/realtime-protocol/package.json | 4 + packages/realtime-protocol/src/events.ts | 122 ++++++++++++++++++ 3 files changed, 189 insertions(+), 47 deletions(-) create mode 100644 packages/realtime-protocol/src/events.ts diff --git a/apps/sim/app/workspace/providers/socket-provider.tsx b/apps/sim/app/workspace/providers/socket-provider.tsx index 47587fe4f0e..3e5d6fa9040 100644 --- a/apps/sim/app/workspace/providers/socket-provider.tsx +++ b/apps/sim/app/workspace/providers/socket-provider.tsx @@ -11,6 +11,19 @@ import { useState, } from 'react' import { createLogger } from '@sim/logger' +import type { + CursorUpdateBroadcast, + OperationConfirmedBroadcast, + OperationFailedBroadcast, + SelectionUpdateBroadcast, + SubblockUpdateBroadcast, + VariableUpdateBroadcast, + WorkflowDeletedBroadcast, + WorkflowDeployedBroadcast, + WorkflowOperationBroadcast, + WorkflowRevertedBroadcast, + WorkflowUpdatedBroadcast, +} from '@sim/realtime-protocol/events' import { generateId } from '@sim/utils/id' import { backoffWithJitter } from '@sim/utils/retry' import { useParams } from 'next/navigation' @@ -92,18 +105,18 @@ interface SocketContextType { emitCursorUpdate: (cursor: { x: number; y: number } | null) => void emitSelectionUpdate: (selection: { type: 'block' | 'edge' | 'none'; id?: string }) => void - onWorkflowOperation: (handler: (data: any) => void) => void - onSubblockUpdate: (handler: (data: any) => void) => void - onVariableUpdate: (handler: (data: any) => void) => void - - onCursorUpdate: (handler: (data: any) => void) => void - onSelectionUpdate: (handler: (data: any) => void) => void - onWorkflowDeleted: (handler: (data: any) => void) => void - onWorkflowReverted: (handler: (data: any) => void) => void - onWorkflowUpdated: (handler: (data: any) => void) => void - onWorkflowDeployed: (handler: (data: any) => void) => void - onOperationConfirmed: (handler: (data: any) => void) => void - onOperationFailed: (handler: (data: any) => void) => void + onWorkflowOperation: (handler: (data: WorkflowOperationBroadcast) => void) => void + onSubblockUpdate: (handler: (data: SubblockUpdateBroadcast) => void) => void + onVariableUpdate: (handler: (data: VariableUpdateBroadcast) => void) => void + + onCursorUpdate: (handler: (data: CursorUpdateBroadcast) => void) => void + onSelectionUpdate: (handler: (data: SelectionUpdateBroadcast) => void) => void + onWorkflowDeleted: (handler: (data: WorkflowDeletedBroadcast) => void) => void + onWorkflowReverted: (handler: (data: WorkflowRevertedBroadcast) => void) => void + onWorkflowUpdated: (handler: (data: WorkflowUpdatedBroadcast) => void) => void + onWorkflowDeployed: (handler: (data: WorkflowDeployedBroadcast) => void) => void + onOperationConfirmed: (handler: (data: OperationConfirmedBroadcast) => void) => void + onOperationFailed: (handler: (data: OperationFailedBroadcast) => void) => void } const SocketContext = createContext({ @@ -173,17 +186,17 @@ export function SocketProvider({ children, user }: SocketProviderProps) { explicitWorkflowIdRef.current = explicitWorkflowId const eventHandlers = useRef<{ - workflowOperation?: (data: any) => void - subblockUpdate?: (data: any) => void - variableUpdate?: (data: any) => void - cursorUpdate?: (data: any) => void - selectionUpdate?: (data: any) => void - workflowDeleted?: (data: any) => void - workflowReverted?: (data: any) => void - workflowUpdated?: (data: any) => void - workflowDeployed?: (data: any) => void - operationConfirmed?: (data: any) => void - operationFailed?: (data: any) => void + workflowOperation?: (data: WorkflowOperationBroadcast) => void + subblockUpdate?: (data: SubblockUpdateBroadcast) => void + variableUpdate?: (data: VariableUpdateBroadcast) => void + cursorUpdate?: (data: CursorUpdateBroadcast) => void + selectionUpdate?: (data: SelectionUpdateBroadcast) => void + workflowDeleted?: (data: WorkflowDeletedBroadcast) => void + workflowReverted?: (data: WorkflowRevertedBroadcast) => void + workflowUpdated?: (data: WorkflowUpdatedBroadcast) => void + workflowDeployed?: (data: WorkflowDeployedBroadcast) => void + operationConfirmed?: (data: OperationConfirmedBroadcast) => void + operationFailed?: (data: OperationFailedBroadcast) => void }>({}) const positionUpdateTimeouts = useRef>(new Map()) @@ -555,19 +568,19 @@ export function SocketProvider({ children, user }: SocketProviderProps) { executeJoinCommands(result.commands) }) - socketInstance.on('workflow-operation', (data) => { + socketInstance.on('workflow-operation', (data: WorkflowOperationBroadcast) => { eventHandlers.current.workflowOperation?.(data) }) - socketInstance.on('subblock-update', (data) => { + socketInstance.on('subblock-update', (data: SubblockUpdateBroadcast) => { eventHandlers.current.subblockUpdate?.(data) }) - socketInstance.on('variable-update', (data) => { + socketInstance.on('variable-update', (data: VariableUpdateBroadcast) => { eventHandlers.current.variableUpdate?.(data) }) - socketInstance.on('workflow-deleted', (data) => { + socketInstance.on('workflow-deleted', (data: WorkflowDeletedBroadcast) => { logger.warn(`Workflow ${data.workflowId} has been deleted`) const result = joinControllerRef.current.handleWorkflowDeleted(data.workflowId) if (result.shouldClearCurrent) { @@ -577,17 +590,17 @@ export function SocketProvider({ children, user }: SocketProviderProps) { eventHandlers.current.workflowDeleted?.(data) }) - socketInstance.on('workflow-reverted', (data) => { + socketInstance.on('workflow-reverted', (data: WorkflowRevertedBroadcast) => { logger.info(`Workflow ${data.workflowId} has been reverted to deployed state`) eventHandlers.current.workflowReverted?.(data) }) - socketInstance.on('workflow-updated', (data) => { + socketInstance.on('workflow-updated', (data: WorkflowUpdatedBroadcast) => { logger.info(`Workflow ${data.workflowId} has been updated externally`) eventHandlers.current.workflowUpdated?.(data) }) - socketInstance.on('workflow-deployed', (data) => { + socketInstance.on('workflow-deployed', (data: WorkflowDeployedBroadcast) => { logger.info(`Workflow ${data.workflowId} deployment state changed`) eventHandlers.current.workflowDeployed?.(data) }) @@ -647,17 +660,17 @@ export function SocketProvider({ children, user }: SocketProviderProps) { return true } - socketInstance.on('operation-confirmed', (data) => { + socketInstance.on('operation-confirmed', (data: OperationConfirmedBroadcast) => { logger.debug('Operation confirmed', { operationId: data.operationId }) eventHandlers.current.operationConfirmed?.(data) }) - socketInstance.on('operation-failed', (data) => { + socketInstance.on('operation-failed', (data: OperationFailedBroadcast) => { logger.warn('Operation failed', { operationId: data.operationId, error: data.error }) eventHandlers.current.operationFailed?.(data) }) - socketInstance.on('cursor-update', (data) => { + socketInstance.on('cursor-update', (data: CursorUpdateBroadcast) => { if (!isWorkflowVisible()) { return } @@ -675,7 +688,7 @@ export function SocketProvider({ children, user }: SocketProviderProps) { eventHandlers.current.cursorUpdate?.(data) }) - socketInstance.on('selection-update', (data) => { + socketInstance.on('selection-update', (data: SelectionUpdateBroadcast) => { if (!isWorkflowVisible()) { return } @@ -1045,47 +1058,50 @@ export function SocketProvider({ children, user }: SocketProviderProps) { [socket, currentWorkflowId, isWorkflowVisible] ) - const onWorkflowOperation = useCallback((handler: (data: any) => void) => { + const onWorkflowOperation = useCallback((handler: (data: WorkflowOperationBroadcast) => void) => { eventHandlers.current.workflowOperation = handler }, []) - const onSubblockUpdate = useCallback((handler: (data: any) => void) => { + const onSubblockUpdate = useCallback((handler: (data: SubblockUpdateBroadcast) => void) => { eventHandlers.current.subblockUpdate = handler }, []) - const onVariableUpdate = useCallback((handler: (data: any) => void) => { + const onVariableUpdate = useCallback((handler: (data: VariableUpdateBroadcast) => void) => { eventHandlers.current.variableUpdate = handler }, []) - const onCursorUpdate = useCallback((handler: (data: any) => void) => { + const onCursorUpdate = useCallback((handler: (data: CursorUpdateBroadcast) => void) => { eventHandlers.current.cursorUpdate = handler }, []) - const onSelectionUpdate = useCallback((handler: (data: any) => void) => { + const onSelectionUpdate = useCallback((handler: (data: SelectionUpdateBroadcast) => void) => { eventHandlers.current.selectionUpdate = handler }, []) - const onWorkflowDeleted = useCallback((handler: (data: any) => void) => { + const onWorkflowDeleted = useCallback((handler: (data: WorkflowDeletedBroadcast) => void) => { eventHandlers.current.workflowDeleted = handler }, []) - const onWorkflowReverted = useCallback((handler: (data: any) => void) => { + const onWorkflowReverted = useCallback((handler: (data: WorkflowRevertedBroadcast) => void) => { eventHandlers.current.workflowReverted = handler }, []) - const onWorkflowUpdated = useCallback((handler: (data: any) => void) => { + const onWorkflowUpdated = useCallback((handler: (data: WorkflowUpdatedBroadcast) => void) => { eventHandlers.current.workflowUpdated = handler }, []) - const onWorkflowDeployed = useCallback((handler: (data: any) => void) => { + const onWorkflowDeployed = useCallback((handler: (data: WorkflowDeployedBroadcast) => void) => { eventHandlers.current.workflowDeployed = handler }, []) - const onOperationConfirmed = useCallback((handler: (data: any) => void) => { - eventHandlers.current.operationConfirmed = handler - }, []) + const onOperationConfirmed = useCallback( + (handler: (data: OperationConfirmedBroadcast) => void) => { + eventHandlers.current.operationConfirmed = handler + }, + [] + ) - const onOperationFailed = useCallback((handler: (data: any) => void) => { + const onOperationFailed = useCallback((handler: (data: OperationFailedBroadcast) => void) => { eventHandlers.current.operationFailed = handler }, []) diff --git a/packages/realtime-protocol/package.json b/packages/realtime-protocol/package.json index 1bf2411e848..4f026e9d2ac 100644 --- a/packages/realtime-protocol/package.json +++ b/packages/realtime-protocol/package.json @@ -14,6 +14,10 @@ "types": "./src/constants.ts", "default": "./src/constants.ts" }, + "./events": { + "types": "./src/events.ts", + "default": "./src/events.ts" + }, "./schemas": { "types": "./src/schemas.ts", "default": "./src/schemas.ts" diff --git a/packages/realtime-protocol/src/events.ts b/packages/realtime-protocol/src/events.ts new file mode 100644 index 00000000000..95f630de80e --- /dev/null +++ b/packages/realtime-protocol/src/events.ts @@ -0,0 +1,122 @@ +import type { OperationTarget, SocketOperation } from './constants' + +/** + * Wire types for the broadcast/confirmation events the realtime Socket.IO server + * emits to clients. These mirror the exact object literals emitted by + * `apps/realtime/src/handlers/**` and `apps/realtime/src/rooms/**`, and are the + * canonical types consumed by the client socket transport + * (`apps/sim/app/workspace/providers/socket-provider.tsx`). + * + * Payload bodies that the transport forwards opaquely are typed `unknown` rather + * than a concrete operation union, because the transport never narrows them — the + * collaborative-workflow consumer dispatches on `operation`/`target` itself. + */ + +/** A live-presence cursor position broadcast over the socket. */ +export interface CursorPosition { + x: number + y: number +} + +/** A live-presence selection broadcast over the socket. */ +export interface PresenceSelection { + type: 'block' | 'edge' | 'none' + id?: string +} + +/** + * `workflow-operation` broadcast. The server re-broadcasts the originating + * operation envelope plus sender identity and operation metadata. + */ +export interface WorkflowOperationBroadcast { + operation: SocketOperation | string + target: OperationTarget | string + payload: unknown + timestamp: number + senderId: string + userId: string + userName: string + metadata: { + workflowId: string + operationId: string + isPositionUpdate?: boolean + isBatchPositionUpdate?: boolean + } +} + +/** `subblock-update` broadcast. */ +export interface SubblockUpdateBroadcast { + workflowId: string + blockId: string + subblockId: string + value: unknown + timestamp: number +} + +/** `variable-update` broadcast. */ +export interface VariableUpdateBroadcast { + workflowId: string + variableId: string + field: string + value: unknown + timestamp: number +} + +/** `cursor-update` presence broadcast for a single remote user. */ +export interface CursorUpdateBroadcast { + socketId: string + userId: string + userName: string + avatarUrl?: string | null + /** `null` when the remote user's cursor leaves the canvas (the client emits `{ cursor: null }`). */ + cursor: CursorPosition | null +} + +/** `selection-update` presence broadcast for a single remote user. */ +export interface SelectionUpdateBroadcast { + socketId: string + userId: string + userName: string + avatarUrl?: string | null + selection: PresenceSelection +} + +/** `workflow-deleted` lifecycle broadcast. */ +export interface WorkflowDeletedBroadcast { + workflowId: string + message: string + timestamp: number +} + +/** `workflow-reverted` lifecycle broadcast. */ +export interface WorkflowRevertedBroadcast { + workflowId: string + message: string + timestamp: number +} + +/** `workflow-updated` lifecycle broadcast. */ +export interface WorkflowUpdatedBroadcast { + workflowId: string + message: string + timestamp: number +} + +/** `workflow-deployed` lifecycle broadcast. */ +export interface WorkflowDeployedBroadcast { + workflowId: string + timestamp: number +} + +/** `operation-confirmed` ack for a previously-emitted operation. */ +export interface OperationConfirmedBroadcast { + operationId: string + serverTimestamp: number +} + +/** `operation-failed` rejection for a previously-emitted operation. */ +export interface OperationFailedBroadcast { + operationId: string + error: string + retryable?: boolean +} From 34d32b97ddc35d63b91ea1fc6e9acec3a0e7a437 Mon Sep 17 00:00:00 2001 From: Waleed Date: Thu, 25 Jun 2026 10:37:05 -0700 Subject: [PATCH 15/15] feat(salesforce): add Tooling API schema tools (custom field/object) + metadata query (#5209) * feat(salesforce): add Tooling API schema tools (custom field/object) + metadata query Add salesforce_create_custom_field, salesforce_update_custom_field, salesforce_delete_custom_field, salesforce_create_custom_object, and salesforce_tooling_query so the connector can make schema/metadata changes (e.g. create a custom field on Account). Previously the integration only did record CRUD via the REST Data API. Existing `api` OAuth scope covers the Tooling API; metadata creation is profile-permission gated, so no scope change. Also: fix Opportunity closeDate being wrongly required on update_opportunity, make list_reports/list_dashboards descriptions honest (recently-viewed scope), and document run_report's includeDetails default. * improvement(salesforce): non-destructive custom field update + align metadata param types - update_custom_field now does a read-modify-write (GET existing Metadata, overlay only provided changes, PATCH) so omitted properties are preserved instead of being reset by the Tooling API's full-metadata PATCH; no more fabricated label or injected create-time defaults on update - fieldType is now optional on update (kept from the existing field unless changed) - widen length/precision/scale/visibleLines param types to number | string to match the tool param configs (type: number) * improvement(salesforce): preserve picklist values and clear stale metadata on field type change - custom field update now unions provided picklist values with the field's existing values instead of replacing the whole valueSet (no data loss) - when fieldType changes on update, drop the prior type's type-specific metadata (length/precision/scale/visibleLines/valueSet/defaultValue/unique/ externalId) and backfill the new type's required defaults * improvement(salesforce): scope custom field update to attributes, never the type update_custom_field no longer changes a field's data type: Salesforce treats a type change as a separate conversion operation, and a stale forwarded fieldType could otherwise trigger an unintended destructive migration. The merge keeps the field's existing type and overlays only the other provided properties, dropping the type-change/stale-metadata-stripping logic entirely. --- .../docs/en/integrations/salesforce.mdx | 150 ++++++++++- apps/sim/blocks/blocks/salesforce.ts | 211 +++++++++++++++- apps/sim/lib/integrations/integrations.json | 28 ++- apps/sim/tools/registry.ts | 10 + .../tools/salesforce/create_custom_field.ts | 198 +++++++++++++++ .../tools/salesforce/create_custom_object.ts | 148 +++++++++++ .../tools/salesforce/delete_custom_field.ts | 87 +++++++ apps/sim/tools/salesforce/index.ts | 5 + apps/sim/tools/salesforce/list_dashboards.ts | 6 +- apps/sim/tools/salesforce/list_reports.ts | 6 +- apps/sim/tools/salesforce/run_report.ts | 2 + apps/sim/tools/salesforce/tooling_query.ts | 104 ++++++++ apps/sim/tools/salesforce/types.ts | 184 ++++++++++++++ .../tools/salesforce/update_custom_field.ts | 236 ++++++++++++++++++ apps/sim/tools/salesforce/utils.ts | 227 +++++++++++++++++ 15 files changed, 1585 insertions(+), 17 deletions(-) create mode 100644 apps/sim/tools/salesforce/create_custom_field.ts create mode 100644 apps/sim/tools/salesforce/create_custom_object.ts create mode 100644 apps/sim/tools/salesforce/delete_custom_field.ts create mode 100644 apps/sim/tools/salesforce/tooling_query.ts create mode 100644 apps/sim/tools/salesforce/update_custom_field.ts diff --git a/apps/docs/content/docs/en/integrations/salesforce.mdx b/apps/docs/content/docs/en/integrations/salesforce.mdx index 1125dc36cdd..76e5729536c 100644 --- a/apps/docs/content/docs/en/integrations/salesforce.mdx +++ b/apps/docs/content/docs/en/integrations/salesforce.mdx @@ -27,7 +27,7 @@ The Salesforce tool is ideal for workflows where your agents need to streamline ## Usage Instructions -Integrate Salesforce into your workflow. Manage accounts, contacts, leads, opportunities, cases, and tasks with powerful automation capabilities. +Integrate Salesforce into your workflow. Manage accounts, contacts, leads, opportunities, cases, and tasks, run reports and SOQL queries, and manage org schema by creating custom fields and objects via the Tooling API. @@ -717,7 +717,7 @@ Delete a task ### `salesforce_list_reports` -Get a list of reports accessible by the current user +Get a list of up to 200 recently viewed reports for the current user #### Input @@ -814,7 +814,7 @@ Get a list of available report types ### `salesforce_list_dashboards` -Get a list of dashboards accessible by the current user +Get a list of recently used dashboards for the current user #### Input @@ -1029,6 +1029,150 @@ Get a list of all available Salesforce objects | ↳ `totalReturned` | number | Number of objects returned | | ↳ `success` | boolean | Salesforce operation success | +### `salesforce_create_custom_field` + +Create a custom field on a Salesforce object (e.g., Account) using the Tooling API + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `idToken` | string | No | No description | +| `instanceUrl` | string | No | No description | +| `objectName` | string | Yes | API name of the object to add the field to \(e.g., Account, Contact, Lead, MyObject__c\) | +| `fieldName` | string | Yes | API name of the new field; the __c suffix is added automatically \(e.g., Region\) | +| `label` | string | No | Display label shown in the UI \(defaults to the field name when omitted\) | +| `fieldType` | string | Yes | Field data type: Text, TextArea, LongTextArea, Html, Number, Currency, Percent, Checkbox, Date, DateTime, Time, Phone, Email, Url, Picklist, or MultiselectPicklist | +| `length` | number | No | Maximum length for Text \(1-255\), LongTextArea, Html, or MultiselectPicklist fields | +| `precision` | number | No | Total number of digits for Number, Currency, or Percent fields \(1-18\) | +| `scale` | number | No | Number of digits to the right of the decimal for numeric fields | +| `visibleLines` | number | No | Number of visible lines for LongTextArea, Html, or MultiselectPicklist fields | +| `required` | boolean | No | Whether the field is required on record create/edit | +| `unique` | boolean | No | Whether the field enforces unique values | +| `externalId` | boolean | No | Whether the field is an external ID \(for Text, Number, or Email fields\) | +| `defaultValue` | string | No | Default value; for Checkbox fields use true or false | +| `description` | string | No | Internal description of the field | +| `inlineHelpText` | string | No | Help text shown next to the field in the UI | +| `picklistValues` | string | No | Comma-separated values for Picklist or MultiselectPicklist fields | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `success` | boolean | Operation success status | +| `output` | object | Created custom field metadata | +| ↳ `id` | string | Tooling API Id of the newly created custom field | +| ↳ `fullName` | string | Full API name of the field, including object \(e.g., Account.Region__c\) | +| ↳ `success` | boolean | Whether the create operation was successful | +| ↳ `created` | boolean | Whether the field was created \(always true on success\) | + +### `salesforce_update_custom_field` + +Update an existing custom field on a Salesforce object using the Tooling API + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `idToken` | string | No | No description | +| `instanceUrl` | string | No | No description | +| `fieldId` | string | Yes | Tooling API Id of the custom field to update \(find it via the Tooling Query tool\) | +| `label` | string | No | Display label shown in the UI | +| `length` | number | No | Maximum length for Text, LongTextArea, Html, or MultiselectPicklist fields | +| `precision` | number | No | Total number of digits for Number, Currency, or Percent fields | +| `scale` | number | No | Number of digits to the right of the decimal for numeric fields | +| `visibleLines` | number | No | Number of visible lines for LongTextArea, Html, or MultiselectPicklist fields | +| `required` | boolean | No | Whether the field is required on record create/edit | +| `unique` | boolean | No | Whether the field enforces unique values | +| `externalId` | boolean | No | Whether the field is an external ID | +| `defaultValue` | string | No | Default value; for Checkbox fields use true or false | +| `description` | string | No | Internal description of the field | +| `inlineHelpText` | string | No | Help text shown next to the field in the UI | +| `picklistValues` | string | No | Comma-separated values to add to a Picklist or MultiselectPicklist field \(existing values are kept\) | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `success` | boolean | Operation success status | +| `output` | object | Updated custom field metadata | +| ↳ `id` | string | Tooling API Id of the updated custom field | +| ↳ `updated` | boolean | Whether the field was updated \(always true on success\) | + +### `salesforce_delete_custom_field` + +Delete a custom field from a Salesforce object using the Tooling API + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `idToken` | string | No | No description | +| `instanceUrl` | string | No | No description | +| `fieldId` | string | Yes | Tooling API Id of the custom field to delete \(find it via the Tooling Query tool\) | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `success` | boolean | Operation success status | +| `output` | object | Deleted custom field metadata | +| ↳ `id` | string | Tooling API Id of the deleted custom field | +| ↳ `deleted` | boolean | Whether the field was deleted \(always true on success\) | + +### `salesforce_create_custom_object` + +Create a custom object in Salesforce using the Tooling API + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `idToken` | string | No | No description | +| `instanceUrl` | string | No | No description | +| `objectName` | string | Yes | API name of the new object; the __c suffix is added automatically \(e.g., Project\) | +| `label` | string | Yes | Singular display label for the object \(e.g., Project\) | +| `pluralLabel` | string | Yes | Plural display label for the object \(e.g., Projects\) | +| `nameFieldLabel` | string | No | Label for the standard Name field \(defaults to "<label> Name"\) | +| `description` | string | No | Internal description of the object | +| `sharingModel` | string | No | Org-wide sharing model: ReadWrite, Read, Private, or ControlledByParent \(default ReadWrite\) | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `success` | boolean | Operation success status | +| `output` | object | Created custom object metadata | +| ↳ `id` | string | Tooling API Id of the newly created custom object | +| ↳ `fullName` | string | Full API name of the object \(e.g., Project__c\) | +| ↳ `success` | boolean | Whether the create operation was successful | +| ↳ `created` | boolean | Whether the object was created \(always true on success\) | + +### `salesforce_tooling_query` + +Execute a SOQL query against the Tooling API to inspect metadata objects + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `idToken` | string | No | No description | +| `instanceUrl` | string | No | No description | +| `query` | string | Yes | Tooling SOQL query \(e.g., SELECT Id, DeveloperName FROM CustomField WHERE TableEnumOrId = 'Account'\) | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `success` | boolean | Operation success status | +| `output` | object | Tooling query results | +| ↳ `records` | array | Array of Tooling API records matching the query | +| ↳ `query` | string | The executed Tooling SOQL query | +| ↳ `metadata` | object | Response metadata | +| ↳ `totalReturned` | number | Number of records returned in this response | +| ↳ `hasMore` | boolean | Whether more records exist \(inverse of done\) | +| ↳ `success` | boolean | Salesforce operation success | + ## Triggers diff --git a/apps/sim/blocks/blocks/salesforce.ts b/apps/sim/blocks/blocks/salesforce.ts index 11cbc2b58d1..100ac47cad6 100644 --- a/apps/sim/blocks/blocks/salesforce.ts +++ b/apps/sim/blocks/blocks/salesforce.ts @@ -11,7 +11,7 @@ export const SalesforceBlock: BlockConfig = { description: 'Interact with Salesforce CRM', authMode: AuthMode.OAuth, longDescription: - 'Integrate Salesforce into your workflow. Manage accounts, contacts, leads, opportunities, cases, and tasks with powerful automation capabilities.', + 'Integrate Salesforce into your workflow. Manage accounts, contacts, leads, opportunities, cases, and tasks, run reports and SOQL queries, and manage org schema by creating custom fields and objects via the Tooling API.', docsLink: 'https://docs.sim.ai/integrations/salesforce', category: 'tools', integrationType: IntegrationType.Sales, @@ -69,6 +69,11 @@ export const SalesforceBlock: BlockConfig = { { label: 'Get More Query Results', id: 'query_more' }, { label: 'Describe Object', id: 'describe_object' }, { label: 'List Objects', id: 'list_objects' }, + { label: 'Create Custom Field', id: 'create_custom_field' }, + { label: 'Update Custom Field', id: 'update_custom_field' }, + { label: 'Delete Custom Field', id: 'delete_custom_field' }, + { label: 'Create Custom Object', id: 'create_custom_object' }, + { label: 'Run Tooling Query', id: 'tooling_query' }, ], value: () => 'get_accounts', }, @@ -445,7 +450,7 @@ export const SalesforceBlock: BlockConfig = { type: 'short-input', placeholder: 'YYYY-MM-DD (required for create)', condition: { field: 'operation', value: ['create_opportunity', 'update_opportunity'] }, - required: true, + required: { field: 'operation', value: ['create_opportunity'] }, wandConfig: { enabled: true, prompt: `Generate a date in YYYY-MM-DD format based on the user's description. @@ -608,8 +613,8 @@ Return ONLY the date string in YYYY-MM-DD format - no explanations, no quotes, n title: 'SOQL Query', type: 'long-input', placeholder: 'SELECT Id, Name FROM Account LIMIT 10', - condition: { field: 'operation', value: ['query'] }, - required: true, + condition: { field: 'operation', value: ['query', 'tooling_query'] }, + required: { field: 'operation', value: ['query', 'tooling_query'] }, }, { id: 'nextRecordsUrl', @@ -624,8 +629,14 @@ Return ONLY the date string in YYYY-MM-DD format - no explanations, no quotes, n title: 'Object Name', type: 'short-input', placeholder: 'API name (e.g., Account, Lead, Custom_Object__c)', - condition: { field: 'operation', value: ['describe_object'] }, - required: true, + condition: { + field: 'operation', + value: ['describe_object', 'create_custom_field', 'create_custom_object'], + }, + required: { + field: 'operation', + value: ['describe_object', 'create_custom_field', 'create_custom_object'], + }, }, // Long-input fields at the bottom { @@ -649,9 +660,182 @@ Return ONLY the date string in YYYY-MM-DD format - no explanations, no quotes, n 'update_case', 'create_task', 'update_task', + 'create_custom_field', + 'update_custom_field', + 'create_custom_object', ], }, }, + // Schema / metadata fields (Tooling API) + { + id: 'fieldName', + title: 'Field Name', + type: 'short-input', + placeholder: 'API name without __c (e.g., Region)', + condition: { field: 'operation', value: ['create_custom_field'] }, + required: { field: 'operation', value: ['create_custom_field'] }, + }, + { + id: 'fieldId', + title: 'Field ID', + type: 'short-input', + placeholder: 'Tooling API Id (find via Run Tooling Query)', + condition: { field: 'operation', value: ['update_custom_field', 'delete_custom_field'] }, + required: { field: 'operation', value: ['update_custom_field', 'delete_custom_field'] }, + }, + { + id: 'fieldType', + title: 'Field Type', + type: 'dropdown', + options: [ + { label: 'Text', id: 'Text' }, + { label: 'Text Area', id: 'TextArea' }, + { label: 'Text Area (Long)', id: 'LongTextArea' }, + { label: 'Rich Text Area', id: 'Html' }, + { label: 'Number', id: 'Number' }, + { label: 'Currency', id: 'Currency' }, + { label: 'Percent', id: 'Percent' }, + { label: 'Checkbox', id: 'Checkbox' }, + { label: 'Date', id: 'Date' }, + { label: 'Date/Time', id: 'DateTime' }, + { label: 'Time', id: 'Time' }, + { label: 'Phone', id: 'Phone' }, + { label: 'Email', id: 'Email' }, + { label: 'URL', id: 'Url' }, + { label: 'Picklist', id: 'Picklist' }, + { label: 'Picklist (Multi-Select)', id: 'MultiselectPicklist' }, + ], + condition: { field: 'operation', value: ['create_custom_field'] }, + required: { field: 'operation', value: ['create_custom_field'] }, + }, + { + id: 'label', + title: 'Label', + type: 'short-input', + placeholder: 'Display label', + condition: { + field: 'operation', + value: ['create_custom_field', 'update_custom_field', 'create_custom_object'], + }, + required: { field: 'operation', value: ['create_custom_object'] }, + }, + { + id: 'pluralLabel', + title: 'Plural Label', + type: 'short-input', + placeholder: 'Plural display label (e.g., Projects)', + condition: { field: 'operation', value: ['create_custom_object'] }, + required: { field: 'operation', value: ['create_custom_object'] }, + }, + { + id: 'picklistValues', + title: 'Picklist Values', + type: 'short-input', + placeholder: 'Comma-separated values (e.g., Low, Medium, High)', + condition: { field: 'operation', value: ['create_custom_field', 'update_custom_field'] }, + }, + { + id: 'length', + title: 'Length', + type: 'short-input', + placeholder: 'Max length for Text/LongTextArea/Html', + mode: 'advanced', + condition: { field: 'operation', value: ['create_custom_field', 'update_custom_field'] }, + }, + { + id: 'precision', + title: 'Precision', + type: 'short-input', + placeholder: 'Total digits for Number/Currency/Percent', + mode: 'advanced', + condition: { field: 'operation', value: ['create_custom_field', 'update_custom_field'] }, + }, + { + id: 'scale', + title: 'Scale', + type: 'short-input', + placeholder: 'Decimal places for numeric fields', + mode: 'advanced', + condition: { field: 'operation', value: ['create_custom_field', 'update_custom_field'] }, + }, + { + id: 'visibleLines', + title: 'Visible Lines', + type: 'short-input', + placeholder: 'Lines for LongTextArea/Html/MultiselectPicklist', + mode: 'advanced', + condition: { field: 'operation', value: ['create_custom_field', 'update_custom_field'] }, + }, + { + id: 'defaultValue', + title: 'Default Value', + type: 'short-input', + placeholder: 'Default value (true/false for Checkbox)', + mode: 'advanced', + condition: { field: 'operation', value: ['create_custom_field', 'update_custom_field'] }, + }, + { + id: 'inlineHelpText', + title: 'Help Text', + type: 'short-input', + placeholder: 'Help text shown next to the field', + mode: 'advanced', + condition: { field: 'operation', value: ['create_custom_field', 'update_custom_field'] }, + }, + { + id: 'required', + title: 'Required', + type: 'dropdown', + options: [ + { label: 'Yes', id: 'true' }, + { label: 'No', id: 'false' }, + ], + mode: 'advanced', + condition: { field: 'operation', value: ['create_custom_field', 'update_custom_field'] }, + }, + { + id: 'unique', + title: 'Unique', + type: 'dropdown', + options: [ + { label: 'Yes', id: 'true' }, + { label: 'No', id: 'false' }, + ], + mode: 'advanced', + condition: { field: 'operation', value: ['create_custom_field', 'update_custom_field'] }, + }, + { + id: 'externalId', + title: 'External ID', + type: 'dropdown', + options: [ + { label: 'Yes', id: 'true' }, + { label: 'No', id: 'false' }, + ], + mode: 'advanced', + condition: { field: 'operation', value: ['create_custom_field', 'update_custom_field'] }, + }, + { + id: 'nameFieldLabel', + title: 'Name Field Label', + type: 'short-input', + placeholder: 'Label for the Name field (defaults to "