simstudioai
diff --git a/‎apps/sim/app/api/tools/sftp/utils.ts‎
Lines changed: 15 additions & 2 deletions b/‎apps/sim/app/api/tools/sftp/utils.ts‎
Lines changed: 15 additions & 2 deletions
diff --git a/‎apps/sim/connectors/google-slides/google-slides.ts‎
Lines changed: 19 additions & 14 deletions b/‎apps/sim/connectors/google-slides/google-slides.ts‎
Lines changed: 19 additions & 14 deletions
diff --git a/‎apps/sim/connectors/google-vault/google-vault.ts‎
Lines changed: 117 additions & 66 deletions b/‎apps/sim/connectors/google-vault/google-vault.ts‎
Lines changed: 117 additions & 66 deletions
@@ -181,10 +181,23 @@ export async function createSftpConnection(config: SftpConnectionConfig): Promis
       connectConfig.timeout = config.timeout
     }
 
-    const expectedFingerprint = config.hostFingerprint?.trim()
-      ? normalizeSha256Fingerprint(config.hostFingerprint)
+    const suppliedFingerprint = config.hostFingerprint?.trim()
+    const expectedFingerprint = suppliedFingerprint
+      ? normalizeSha256Fingerprint(suppliedFingerprint)
       : undefined
 
+    /**
+     * Fail closed rather than silently skipping verification. A value that is
+     * non-blank but normalizes away (`SHA256:`, `=`) would otherwise leave no
+     * `hostVerifier` installed, trusting whatever host answers — the opposite
+     * of what supplying a fingerprint asks for.
+     */
+    if (suppliedFingerprint && !expectedFingerprint) {
+      throw new Error(
+        'Host key fingerprint is not a valid SHA-256 fingerprint. Expected the base64 form printed by `ssh-keyscan <host> | ssh-keygen -lf -`.'
+      )
+    }
+
     /**
      * Set when the pinned fingerprint does not match. ssh2 reports the
      * rejection through a generic `'error'` event, so the precise cause is
 
@@ -14,6 +14,9 @@ const logger = createLogger('GoogleSlidesConnector')
 
 const PRESENTATION_MIME_TYPE = 'application/vnd.google-apps.presentation'
 
+/** Reason recorded for a presentation whose slides contain no extractable text. */
+const NO_TEXT = 'No extractable text'
+
 /** Guards against a pathological (or cyclic) group nesting depth. */
 const MAX_GROUP_DEPTH = 12
 
@@ -373,21 +376,23 @@ export const googleSlidesConnector: ConnectorConfig = {
     if (file.trashed) return null
     if (file.mimeType !== PRESENTATION_MIME_TYPE) return null
 
-    try {
-      const content = await fetchPresentationContent(
-        accessToken,
-        file.id,
-        shouldIncludeSpeakerNotes(sourceConfig)
-      )
-      if (!content.trim()) return null
-
-      return { ...fileToStub(file), content, contentDeferred: false }
-    } catch (error) {
-      logger.warn(`Failed to extract content from presentation: ${file.name} (${file.id})`, {
-        error: toError(error).message,
-      })
-      return null
+    const content = await fetchPresentationContent(
+      accessToken,
+      file.id,
+      shouldIncludeSpeakerNotes(sourceConfig)
+    )
+
+    /**
+     * An image-only deck carries no extractable text. Surfacing it as a skipped
+     * row keeps it visible in the knowledge base UI — returning `null` would
+     * make the engine drop the document with no reason recorded, so the
+     * presentation would simply be missing and re-fetched on every sync.
+     */
+    if (!content.trim()) {
+      return { ...fileToStub(file), content: '', contentDeferred: false, skippedReason: NO_TEXT }
     }
+
+    return { ...fileToStub(file), content, contentDeferred: false }
   },
 
   validateConfig: async (
 
@@ -12,6 +12,26 @@ const VAULT_API_BASE = 'https://vault.googleapis.com/v1'
 /** Vault caps both `matters.list` and `matters.holds.list` page sizes at 100. */
 const PAGE_SIZE = 100
 
+/**
+ * Matters whose child resources are enumerated in a single `listDocuments` call.
+ *
+ * The sync engine allows a bounded number of `listDocuments` calls per run, so
+ * spending one call per matter per child kind would truncate the listing after a
+ * few hundred matters. Draining a batch of matters per call keeps the call count
+ * proportional to `matters / BATCH` instead of `matters × kinds`.
+ */
+const CHILD_MATTER_BATCH = 8
+
+/** Concurrent child listings issued within one batch. */
+const CHILD_CONCURRENCY = 4
+
+/**
+ * Upper bound on child pages drained for a single matter/kind pair. Vault returns at
+ * most 100 children per page, so this covers 5,000 holds or saved queries in one
+ * matter; exceeding it marks the listing capped rather than looping unbounded.
+ */
+const MAX_CHILD_PAGES = 50
+
 /**
  * Google Vault matter, as returned by `matters.list`/`matters.get` with `view=FULL`.
  * @see https://developers.google.com/workspace/vault/reference/rest/v1/matters
@@ -73,6 +93,10 @@ interface VaultQuery {
   accountInfo?: { emails?: string[] }
   orgUnitInfo?: { orgUnitId?: string }
   sharedDriveInfo?: { sharedDriveIds?: string[] }
+  teamDriveInfo?: { teamDriveIds?: string[] }
+  hangoutsChatInfo?: { roomId?: string[] }
+  sitesUrlInfo?: { urls?: string[] }
+  driveDocumentInfo?: { documentIds?: { ids?: string[] } }
 }
 
 /**
@@ -96,6 +120,9 @@ type VaultChildKind = 'holds' | 'savedQueries'
  * A single sync interleaves two levels of pagination: a page of matters, then the
  * child resources of every matter on that page. The cursor carries the matter IDs of
  * the current page so a resumed call never has to re-derive them.
+ *
+ * Child pagination is fully drained inside one call per batch of matters, so the
+ * cursor only has to remember how far through the matter page it has walked.
  */
 interface VaultCursor {
   phase: 'matters' | 'children'
@@ -104,9 +131,8 @@ interface VaultCursor {
   /** Page token for the matters page that follows the one currently being walked. */
   nextMattersPageToken?: string
   matterIds?: string[]
+  /** Index into `matterIds` of the first matter in the next batch to process. */
   matterIndex?: number
-  kindIndex?: number
-  childPageToken?: string
 }
 
 function encodeCursor(cursor: VaultCursor): string {
@@ -249,7 +275,14 @@ function renderSavedQuery(savedQuery: VaultSavedQuery, matterId: string): string
   appendLine(lines, 'Time zone', query?.timeZone)
   appendLine(lines, 'Accounts', query?.accountInfo?.emails?.join(', '))
   appendLine(lines, 'Organizational unit', query?.orgUnitInfo?.orgUnitId)
-  appendLine(lines, 'Shared drives', query?.sharedDriveInfo?.sharedDriveIds?.join(', '))
+  appendLine(
+    lines,
+    'Shared drives',
+    (query?.sharedDriveInfo?.sharedDriveIds ?? query?.teamDriveInfo?.teamDriveIds)?.join(', ')
+  )
+  appendLine(lines, 'Chat spaces', query?.hangoutsChatInfo?.roomId?.join(', '))
+  appendLine(lines, 'Site URLs', query?.sitesUrlInfo?.urls?.join(', '))
+  appendLine(lines, 'Drive documents', query?.driveDocumentInfo?.documentIds?.ids?.join(', '))
 
   return lines.join('\n')
 }
@@ -312,6 +345,8 @@ function holdToDocument(hold: VaultHold, matterId: string): ExternalDocument {
 /**
  * Builds a saved query document. Saved queries are immutable once created (the API
  * exposes only create, get, list, and delete), so `createTime` identifies the version.
+ * The `v2` token in the hash tracks the rendering itself: because the source can never
+ * change, a rendering change would otherwise never re-index existing documents.
  */
 function savedQueryToDocument(savedQuery: VaultSavedQuery, matterId: string): ExternalDocument {
   const savedQueryId = savedQuery.savedQueryId ?? ''
@@ -320,7 +355,7 @@ function savedQueryToDocument(savedQuery: VaultSavedQuery, matterId: string): Ex
     title: savedQuery.displayName || `Saved query ${savedQueryId}`,
     content: renderSavedQuery(savedQuery, matterId),
     mimeType: 'text/plain',
-    contentHash: `gvault:savedquery:${matterId}:${savedQueryId}:${savedQuery.createTime ?? ''}`,
+    contentHash: `gvault:savedquery:v2:${matterId}:${savedQueryId}:${savedQuery.createTime ?? ''}`,
     metadata: {
       resourceType: 'savedQuery',
       matterId,
@@ -409,32 +444,74 @@ async function fetchChildPage(
 }
 
 /**
- * Computes the cursor state that follows the page just emitted: continue the current
- * child listing, advance to the next kind, the next matter, the next matters page, or
- * finish.
+ * Drains every page of one matter's child listing of the given kind.
+ *
+ * `capped` is true when the listing was cut short — by the page bound or by a request
+ * failure (a matter the caller cannot read, or a transient error). The caller turns
+ * that into `syncContext.listingCapped` so deletion reconciliation is skipped for the
+ * run rather than hard-deleting documents that still exist at the source.
  */
-function advanceCursor(
-  state: VaultCursor,
-  kinds: VaultChildKind[],
-  childPageToken?: string
-): VaultCursor | undefined {
-  const matterIds = state.matterIds ?? []
-  const matterIndex = state.matterIndex ?? 0
-  const kindIndex = state.kindIndex ?? 0
-
-  if (childPageToken) {
-    return { ...state, childPageToken }
-  }
-  if (kindIndex + 1 < kinds.length) {
-    return { ...state, kindIndex: kindIndex + 1, childPageToken: undefined }
+async function fetchAllChildren(
+  accessToken: string,
+  matterId: string,
+  kind: VaultChildKind
+): Promise<{ documents: ExternalDocument[]; capped: boolean }> {
+  const documents: ExternalDocument[] = []
+  let pageToken: string | undefined
+
+  try {
+    for (let page = 0; page < MAX_CHILD_PAGES; page++) {
+      const result = await fetchChildPage(accessToken, matterId, kind, pageToken)
+      documents.push(...result.documents)
+      if (!result.nextPageToken) return { documents, capped: false }
+      pageToken = result.nextPageToken
+    }
+  } catch (error) {
+    logger.warn(`Failed to list ${kind} for Vault matter ${matterId}`, {
+      error: toError(error).message,
+    })
+    return { documents, capped: true }
   }
-  if (matterIndex + 1 < matterIds.length) {
-    return { ...state, matterIndex: matterIndex + 1, kindIndex: 0, childPageToken: undefined }
+
+  logger.warn(`Stopped listing ${kind} for Vault matter ${matterId} at the page bound`, {
+    maxChildPages: MAX_CHILD_PAGES,
+  })
+  return { documents, capped: true }
+}
+
+/**
+ * Lists every enabled child resource for a batch of matters, bounded concurrency.
+ *
+ * One `listDocuments` call covers a whole batch, which keeps the number of calls a
+ * sync needs proportional to the matter count rather than to `matters × kinds ×
+ * child pages`.
+ */
+async function fetchChildrenForMatters(
+  accessToken: string,
+  matterIds: string[],
+  kinds: VaultChildKind[]
+): Promise<{ documents: ExternalDocument[]; capped: boolean }> {
+  const tasks: { matterId: string; kind: VaultChildKind }[] = []
+  for (const matterId of matterIds) {
+    for (const kind of kinds) tasks.push({ matterId, kind })
   }
-  if (state.nextMattersPageToken) {
-    return { phase: 'matters', mattersPageToken: state.nextMattersPageToken }
+
+  const documents: ExternalDocument[] = []
+  let capped = false
+
+  for (let index = 0; index < tasks.length; index += CHILD_CONCURRENCY) {
+    const results = await Promise.all(
+      tasks
+        .slice(index, index + CHILD_CONCURRENCY)
+        .map((task) => fetchAllChildren(accessToken, task.matterId, task.kind))
+    )
+    for (const result of results) {
+      documents.push(...result.documents)
+      if (result.capped) capped = true
+    }
   }
-  return undefined
+
+  return { documents, capped }
 }
 
 export const googleVaultConnector: ConnectorConfig = {
@@ -469,53 +546,27 @@ export const googleVaultConnector: ConnectorConfig = {
           phase: 'children',
           matterIds,
           matterIndex: 0,
-          kindIndex: 0,
           nextMattersPageToken: nextPageToken,
         }
       } else if (nextPageToken) {
         nextState = { phase: 'matters', mattersPageToken: nextPageToken }
       }
     } else {
       const matterIds = state.matterIds ?? []
-      const matterIndex = state.matterIndex ?? 0
-      const matterId = matterIds[matterIndex]
-      const kind = kinds[state.kindIndex ?? 0]
-
-      if (!matterId || kinds.length === 0) {
-        nextState = state.nextMattersPageToken
-          ? { phase: 'matters', mattersPageToken: state.nextMattersPageToken }
-          : undefined
-      } else if (!kind) {
-        /**
-         * The enabled child kinds shrank while a cursor was in flight (the user
-         * turned holds or saved queries off mid-sync), so the remaining children of
-         * this matter are not listed. Skipping them silently would let deletion
-         * reconciliation hard-delete still-existing documents.
-         */
-        if (syncContext) syncContext.listingCapped = true
-        nextState =
-          matterIndex + 1 < matterIds.length
-            ? { ...state, matterIndex: matterIndex + 1, kindIndex: 0, childPageToken: undefined }
-            : state.nextMattersPageToken
-              ? { phase: 'matters', mattersPageToken: state.nextMattersPageToken }
-              : undefined
-      } else {
-        try {
-          const page = await fetchChildPage(accessToken, matterId, kind, state.childPageToken)
-          pageDocuments = page.documents
-          nextState = advanceCursor(state, kinds, page.nextPageToken)
-        } catch (error) {
-          /**
-           * A matter the caller cannot read (or a transient failure) must not make the
-           * engine believe those documents are gone — flag the listing as incomplete so
-           * deletion reconciliation is skipped for this run, then continue.
-           */
-          logger.warn(`Failed to list ${kind} for Vault matter ${matterId}`, {
-            error: toError(error).message,
-          })
-          if (syncContext) syncContext.listingCapped = true
-          nextState = advanceCursor(state, kinds, undefined)
-        }
+      const batchStart = state.matterIndex ?? 0
+      const batch = matterIds.slice(batchStart, batchStart + CHILD_MATTER_BATCH)
+
+      if (batch.length > 0 && kinds.length > 0) {
+        const children = await fetchChildrenForMatters(accessToken, batch, kinds)
+        pageDocuments = children.documents
+        if (children.capped && syncContext) syncContext.listingCapped = true
+      }
+
+      const nextMatterIndex = batchStart + batch.length
+      if (nextMatterIndex < matterIds.length && kinds.length > 0) {
+        nextState = { ...state, matterIndex: nextMatterIndex }
+      } else if (state.nextMattersPageToken) {
+        nextState = { phase: 'matters', mattersPageToken: state.nextMattersPageToken }
       }
     }