feat(connectors): full resync re-hydrates rendered content (transclusions)

waleedlatif1 · waleedlatif1 · commit 5789a6a4b873 · 2026-07-17T14:34:04.000-07:00
Version-based change detection can't see when a Confluence page's rendered view
changes because an *included* page was edited (the container's version doesn't
bump). Add a 'Full resync' path so that drift can be recovered:

- ConnectorMeta.rehydrateOnFullSync flag (set for Confluence)
- on fullSync, classifyExternalDoc promotes unchanged deferred docs to update and
  the hydration guard re-indexes unconditionally, so rendered content is refreshed
- fullSync threaded through the manual sync contract (query param), route, and hook
- 'Sync now' / 'Full resync' dropdown on the connector card

Incremental syncs stay hash-gated and cheap; only the deliberate full resync pays
the re-index cost.
diff --git a/apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/sync/route.test.ts b/apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/sync/route.test.ts
@@ -138,6 +138,50 @@ describe('Connector Manual Sync API Route', () => {
     expect(mockDispatchSync).toHaveBeenCalledWith('conn-456', {
       billingAttribution,
       requestId: 'test-req-id',
+      fullSync: false,
+    })
+  })
+
+  it('dispatches a full resync when fullSync=true is set', async () => {
+    const billingAttribution = {
+      actorUserId: 'external-admin',
+      workspaceId: 'ws-1',
+      organizationId: null,
+      billedAccountUserId: 'owner-1',
+      billingEntity: { type: 'user' as const, id: 'owner-1' },
+      billingPeriod: {
+        start: '2026-07-01T00:00:00.000Z',
+        end: '2026-08-01T00:00:00.000Z',
+      },
+      payerSubscription: null,
+    }
+    hybridAuthMockFns.mockCheckSessionOrInternalAuth.mockResolvedValue({
+      success: true,
+      authType: 'session',
+      userId: 'external-admin',
+      userName: 'Test',
+      userEmail: 'test@test.com',
+    })
+    mockCheckWriteAccess.mockResolvedValue({
+      hasAccess: true,
+      knowledgeBase: { workspaceId: 'ws-1', name: 'Test KB' },
+    })
+    mockDbChain.limit.mockResolvedValueOnce([{ id: 'conn-456', status: 'active' }])
+    mockResolveBillingAttribution.mockResolvedValue(billingAttribution)
+
+    const req = createMockRequest(
+      'POST',
+      undefined,
+      {},
+      'http://localhost:3000/api/knowledge/kb-123/connectors/conn-456/sync?fullSync=true'
+    )
+    const response = await POST(req as never, { params: mockParams })
+
+    expect(response.status).toBe(200)
+    expect(mockDispatchSync).toHaveBeenCalledWith('conn-456', {
+      billingAttribution,
+      requestId: 'test-req-id',
+      fullSync: true,
     })
   })
 })
diff --git a/apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/sync/route.ts b/apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/sync/route.ts
@@ -29,6 +29,7 @@ export const POST = withRouteHandler(async (request: NextRequest, context: Route
   const parsed = await parseRequest(triggerKnowledgeConnectorSyncContract, request, context)
   if (!parsed.success) return parsed.response
   const { id: knowledgeBaseId, connectorId } = parsed.data.params
+  const fullSync = parsed.data.query?.fullSync === 'true'
 
   try {
     const auth = await checkSessionOrInternalAuth(request, { requireWorkflowId: false })
@@ -81,7 +82,9 @@ export const POST = withRouteHandler(async (request: NextRequest, context: Route
             workspaceId: kbWorkspaceId,
           })
 
-    logger.info(`[${requestId}] Manual sync triggered for connector ${connectorId}`)
+    logger.info(
+      `[${requestId}] Manual ${fullSync ? 'full ' : ''}sync triggered for connector ${connectorId}`
+    )
 
     captureServerEvent(
       auth.userId,
@@ -109,12 +112,12 @@ export const POST = withRouteHandler(async (request: NextRequest, context: Route
         knowledgeBaseName: writeCheck.knowledgeBase.name,
         connectorType: connectorRows[0].connectorType,
         connectorStatus: connectorRows[0].status,
-        syncType: 'manual',
+        syncType: fullSync ? 'manual-full' : 'manual',
       },
       request,
     })
 
-    dispatchSync(connectorId, { billingAttribution, requestId }).catch((error) => {
+    dispatchSync(connectorId, { billingAttribution, requestId, fullSync }).catch((error) => {
       logger.error(
         `[${requestId}] Failed to dispatch manual sync for connector ${connectorId}`,
         error
diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connectors-section/connectors-section.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connectors-section/connectors-section.tsx
@@ -1,7 +1,19 @@
 'use client'
 
 import { useCallback, useEffect, useId, useMemo, useRef, useState } from 'react'
-import { Badge, Button, Checkbox, ChipConfirmModal, cn, Loader, Tooltip } from '@sim/emcn'
+import {
+  Badge,
+  Button,
+  Checkbox,
+  ChipConfirmModal,
+  cn,
+  DropdownMenu,
+  DropdownMenuContent,
+  DropdownMenuItem,
+  DropdownMenuTrigger,
+  Loader,
+  Tooltip,
+} from '@sim/emcn'
 import { createLogger } from '@sim/logger'
 import { format, formatDistanceToNow, isPast } from 'date-fns'
 import {
@@ -115,14 +127,14 @@ export function ConnectorsSection({
   }, [])
 
   const handleSync = useCallback(
-    (connectorId: string) => {
+    (connectorId: string, fullSync = false) => {
       if (isSyncOnCooldown(connectorId)) return
 
       syncTriggeredAt.current[connectorId] = Date.now()
       addToSet(setSyncingIds, connectorId)
 
       triggerSync(
-        { knowledgeBaseId, connectorId },
+        { knowledgeBaseId, connectorId, fullSync },
         {
           onSuccess: () => {
             setError(null)
@@ -214,7 +226,7 @@ export function ConnectorsSection({
               isSyncPending={syncingIds.has(connector.id)}
               isUpdating={updatingIds.has(connector.id)}
               syncCooldown={isSyncOnCooldown(connector.id)}
-              onSync={() => handleSync(connector.id)}
+              onSync={(fullSync) => handleSync(connector.id, fullSync)}
               onTogglePause={() => handleTogglePause(connector)}
               onEdit={() => setEditingConnector(connector)}
               onDelete={() => setDeleteTarget(connector.id)}
@@ -273,7 +285,7 @@ interface ConnectorCardProps {
   isSyncPending: boolean
   isUpdating: boolean
   syncCooldown: boolean
-  onSync: () => void
+  onSync: (fullSync?: boolean) => void
   onEdit: () => void
   onTogglePause: () => void
   onDelete: () => void
@@ -409,28 +421,39 @@ function ConnectorCard({
         <div className='flex flex-shrink-0 items-center gap-0.5'>
           {canEdit && (
             <>
-              <Tooltip.Root>
-                <Tooltip.Trigger asChild>
-                  <Button
-                    variant='ghost'
-                    className={CONNECTOR_ACTION_BUTTON_CLASSES}
-                    onClick={onSync}
-                    disabled={
-                      connector.status === 'syncing' ||
-                      connector.status === 'disabled' ||
-                      isSyncPending ||
-                      syncCooldown
-                    }
-                  >
-                    <RefreshCw
-                      className={cn('size-3.5', connector.status === 'syncing' && 'animate-spin')}
-                    />
-                  </Button>
-                </Tooltip.Trigger>
-                <Tooltip.Content>
-                  {syncCooldown ? 'Sync recently triggered' : 'Sync now'}
-                </Tooltip.Content>
-              </Tooltip.Root>
+              <DropdownMenu>
+                <Tooltip.Root>
+                  <Tooltip.Trigger asChild>
+                    <DropdownMenuTrigger asChild>
+                      <Button
+                        variant='ghost'
+                        aria-label='Sync options'
+                        className={CONNECTOR_ACTION_BUTTON_CLASSES}
+                        disabled={
+                          connector.status === 'syncing' ||
+                          connector.status === 'disabled' ||
+                          isSyncPending ||
+                          syncCooldown
+                        }
+                      >
+                        <RefreshCw
+                          className={cn(
+                            'size-3.5',
+                            connector.status === 'syncing' && 'animate-spin'
+                          )}
+                        />
+                      </Button>
+                    </DropdownMenuTrigger>
+                  </Tooltip.Trigger>
+                  <Tooltip.Content>
+                    {syncCooldown ? 'Sync recently triggered' : 'Sync'}
+                  </Tooltip.Content>
+                </Tooltip.Root>
+                <DropdownMenuContent align='end'>
+                  <DropdownMenuItem onSelect={() => onSync(false)}>Sync now</DropdownMenuItem>
+                  <DropdownMenuItem onSelect={() => onSync(true)}>Full resync</DropdownMenuItem>
+                </DropdownMenuContent>
+              </DropdownMenu>
 
               <Tooltip.Root>
                 <Tooltip.Trigger asChild>
diff --git a/apps/sim/connectors/confluence/confluence.ts b/apps/sim/connectors/confluence/confluence.ts
@@ -156,6 +156,14 @@ function cqlResultToStub(item: Record<string, unknown>, domain: string): Externa
 export const confluenceConnector: ConnectorConfig = {
   ...confluenceConnectorMeta,
 
+  /**
+   * Confluence pages can transclude other pages (Include Page / Excerpt macros).
+   * Editing an included page changes a container page's rendered `view` without
+   * bumping the container's version, so its version-based hash can't detect the
+   * change. A full resync re-hydrates and re-indexes to pick up that drift.
+   */
+  rehydrateOnFullSync: true,
+
   listDocuments: async (
     accessToken: string,
     sourceConfig: Record<string, unknown>,
diff --git a/apps/sim/connectors/types.ts b/apps/sim/connectors/types.ts
@@ -127,6 +127,20 @@ export interface ConnectorMeta {
    */
   supportsIncrementalSync?: boolean
 
+  /**
+   * Whether this connector's extracted content can change without the source item's
+   * own change-detection hash changing — e.g. a Confluence page that transcludes
+   * another page via the Include Page / Excerpt macro: editing the included page
+   * changes the container's rendered `view` output without bumping the container's
+   * version, so its version-based `contentHash` stays identical.
+   *
+   * Incremental syncs remain hash-gated (cheap). On an explicit **full resync**
+   * (`fullSync`), the engine re-hydrates and re-indexes these connectors' documents
+   * even when their hash is unchanged, so transcluded/rendered-dependency changes are
+   * picked up. Only the deliberate full resync pays this re-index cost.
+   */
+  rehydrateOnFullSync?: boolean
+
   /**
    * Tag definitions this connector populates. Shown in the add-connector modal
    * as opt-out checkboxes. On connector creation, tag definitions are auto-created
diff --git a/apps/sim/hooks/queries/kb/connectors.ts b/apps/sim/hooks/queries/kb/connectors.ts
@@ -204,11 +204,18 @@ export function useDeleteConnector() {
 interface TriggerSyncParams {
   knowledgeBaseId: string
   connectorId: string
+  /** Force a full resync (re-hydrate + re-index rendered content). Defaults to incremental. */
+  fullSync?: boolean
 }
 
-async function triggerSync({ knowledgeBaseId, connectorId }: TriggerSyncParams): Promise<void> {
+async function triggerSync({
+  knowledgeBaseId,
+  connectorId,
+  fullSync,
+}: TriggerSyncParams): Promise<void> {
   await requestJson(triggerKnowledgeConnectorSyncContract, {
     params: { id: knowledgeBaseId, connectorId },
+    query: fullSync ? { fullSync: 'true' } : {},
   })
 }
 
diff --git a/apps/sim/lib/api/contracts/knowledge/connectors.ts b/apps/sim/lib/api/contracts/knowledge/connectors.ts
@@ -150,10 +150,22 @@ export const deleteKnowledgeConnectorContract = defineRouteContract({
   },
 })
 
+export const triggerKnowledgeConnectorSyncQuerySchema = z.object({
+  /**
+   * Force a full resync: re-list everything and, for connectors whose rendered
+   * content can drift without a hash change (e.g. Confluence transclusions),
+   * re-hydrate and re-index every document rather than only hash-changed ones.
+   * Omitted performs the normal incremental, hash-gated sync. Modeled as a string
+   * literal (not a boolean) so it round-trips cleanly through the query string.
+   */
+  fullSync: z.enum(['true', 'false']).optional(),
+})
+
 export const triggerKnowledgeConnectorSyncContract = defineRouteContract({
   method: 'POST',
   path: '/api/knowledge/[id]/connectors/[connectorId]/sync',
   params: knowledgeConnectorParamsSchema,
+  query: triggerKnowledgeConnectorSyncQuerySchema,
   response: {
     mode: 'json',
     schema: z.object({
diff --git a/apps/sim/lib/knowledge/connectors/sync-engine.test.ts b/apps/sim/lib/knowledge/connectors/sync-engine.test.ts
@@ -227,6 +227,24 @@ describe('classifyExternalDoc', () => {
       type: 'unchanged',
     })
   })
+
+  it('forces re-hydration of an unchanged deferred doc when forceRehydrate is set', async () => {
+    const { classifyExternalDoc } = await import('@/lib/knowledge/connectors/sync-engine')
+    const deferred = { ...base, content: '', contentDeferred: true }
+    // Same hash → normally unchanged, but forceRehydrate promotes it to update.
+    expect(classifyExternalDoc(deferred, { id: 'doc-1', contentHash: 'h1' }, true)).toEqual({
+      type: 'update',
+      existingId: 'doc-1',
+    })
+  })
+
+  it('does not force re-hydration of a non-deferred doc (content already final)', async () => {
+    const { classifyExternalDoc } = await import('@/lib/knowledge/connectors/sync-engine')
+    // Ready (non-deferred) content with an unchanged hash stays unchanged even under forceRehydrate.
+    expect(classifyExternalDoc(base, { id: 'doc-1', contentHash: 'h1' }, true)).toEqual({
+      type: 'unchanged',
+    })
+  })
 })
 
 describe('chunkOpsByByteBudget', () => {
diff --git a/apps/sim/lib/knowledge/connectors/sync-engine.ts b/apps/sim/lib/knowledge/connectors/sync-engine.ts
@@ -84,10 +84,17 @@ type DocClassification =
  *   is already indexed is kept as-is (last-known-good) rather than downgraded.
  * - `drop`: empty, non-deferred content that cannot be indexed.
  * - `add` / `update` / `unchanged`: normal content reconciliation by content hash.
+ *
+ * `forceRehydrate` (set on a full resync of a `rehydrateOnFullSync` connector) promotes
+ * an otherwise-`unchanged` deferred document to `update` so its content is re-fetched —
+ * needed when rendered content can drift without the hash changing (e.g. Confluence
+ * transclusions). Non-deferred docs already carry final content from listing, so they
+ * are left `unchanged` (re-indexing identical content would be pointless).
  */
 export function classifyExternalDoc(
   extDoc: Pick<ExternalDocument, 'content' | 'contentDeferred' | 'contentHash' | 'skippedReason'>,
-  existing: { id: string; contentHash: string | null } | undefined
+  existing: { id: string; contentHash: string | null } | undefined,
+  forceRehydrate = false
 ): DocClassification {
   if (extDoc.skippedReason) {
     return existing ? { type: 'unchanged' } : { type: 'skip' }
@@ -101,6 +108,9 @@ export function classifyExternalDoc(
   if (existing.contentHash !== extDoc.contentHash) {
     return { type: 'update', existingId: existing.id }
   }
+  if (forceRehydrate && extDoc.contentDeferred) {
+    return { type: 'update', existingId: existing.id }
+  }
   return { type: 'unchanged' }
 }
 
@@ -427,6 +437,13 @@ export async function executeSync(
     const lastSyncAt =
       isIncremental && connector.lastSyncAt ? new Date(connector.lastSyncAt) : undefined
 
+    /**
+     * On an explicit full resync, re-hydrate and re-index connectors whose rendered
+     * content can drift without a hash change (transclusions) — see
+     * `ConnectorMeta.rehydrateOnFullSync`. Incremental syncs stay hash-gated.
+     */
+    const forceRehydrate = Boolean(options?.fullSync && connectorConfig.rehydrateOnFullSync)
+
     for (let pageNum = 0; hasMore && pageNum < MAX_PAGES; pageNum++) {
       if (pageNum > 0 && connectorConfig.auth.mode === 'oauth') {
         accessToken = await resolveAccessToken(connector, connectorConfig, userId)
@@ -551,7 +568,7 @@ export async function executeSync(
       }
 
       const existing = existingByExternalId.get(extDoc.externalId)
-      const classification = classifyExternalDoc(extDoc, existing)
+      const classification = classifyExternalDoc(extDoc, existing, forceRehydrate)
 
       switch (classification.type) {
         case 'skip':
@@ -635,8 +652,15 @@ export async function executeSync(
               return null
             }
             const hydratedHash = fullDoc.contentHash ?? op.extDoc.contentHash
+            /**
+             * Normally an update whose hydrated hash matches the stored hash is a
+             * no-op (content unchanged). On a forced re-hydration the hash is
+             * version-based and cannot reflect the rendered-dependency change we are
+             * refreshing for, so re-index unconditionally instead of skipping.
+             */
             if (
               op.type === 'update' &&
+              !forceRehydrate &&
               existingByExternalId.get(op.extDoc.externalId)?.contentHash === hydratedHash
             ) {
               result.docsUnchanged++
