fix(connectors): purge archived/deleted source items across seven more KB connectors

The sync engine only purges a knowledge-base document when its source item is
absent from a full-sync listing, so any connector that keeps listing
archived/trashed/canceled items never drops them. An audit of all 51 connectors
found seven with this bug:

- asana: list only non-archived projects (the API returns both when `archived`
  is omitted), so tasks under archived projects stop being re-listed
- google-sheets: skip a spreadsheet Drive reports as trashed, which stays
  readable by id for 30 days before the Sheets call starts 404ing
- incidentio: exclude canceled incidents by default (cancelling is incident.io's
  documented stand-in for deletion), with an explicit opt-in to sync them
- outlook: exclude Deleted Items from the all-mail listing, which Graph
  otherwise includes
- servicenow: drop retired knowledge articles, which the Table API returns with
  no implicit state filter
- webflow: drop archived CMS items, which the staged items endpoint always
  returns and offers no way to filter
- youtube: drop playlist entries whose video was deleted or made private, which
  the API keeps returning as placeholder items

Every exclusion keys off an explicit non-current signal and fails open on a
missing field or a failed metadata read, since wrongly excluding a live item
would hard-delete it. Explicit user filter selections are still honoured
verbatim; the new defaults apply only when nothing is configured.

Also flag truncated listings as capped in asana, outlook, and servicenow. All
three silently cut a listing short at their configured item cap without setting
`syncContext.listingCapped`, so reconciliation read the untraversed tail as
deleted at the source and hard-deleted it.

Author: waleedlatif1

apps/sim/connectors/asana/asana.test.ts

@@ -23,7 +23,7 @@ interface AsanaPageResponse {
 /**
  * Minimal Asana task shape used by this connector.
  */
-interface AsanaTask {
+export interface AsanaTask {
   gid: string
   name: string
   notes?: string
@@ -33,6 +33,7 @@ interface AsanaTask {
   assignee?: { name: string }
   tags?: { name: string }[]
   permalink_url?: string
+  projects?: AsanaProject[]
 }
 
 /**
@@ -46,9 +47,125 @@ interface AsanaWorkspace {
 /**
  * Asana project shape.
  */
-interface AsanaProject {
+export interface AsanaProject {
   gid: string
   name: string
+  archived?: boolean
+}
+
+/**
+ * Optional fields requested when enumerating workspace projects. `archived` is
+ * not returned by default and is needed to defensively re-check the server-side
+ * `archived=false` filter.
+ */
+const PROJECT_OPT_FIELDS = 'gid,name,archived'
+
+/**
+ * Optional fields requested when re-fetching a single task. Adds the task's
+ * parent projects (with their archived flag) on top of the listing fields so
+ * `isTaskUnderActiveProject` can run on the rehydrate path. `opt_fields`
+ * supports dot paths, so `projects.archived` expands the compact project stubs
+ * with the field the listing filter relies on.
+ */
+const TASK_DETAIL_OPT_FIELDS = `${TASK_OPT_FIELDS},projects.archived`
+
+/**
+ * Builds the workspace projects listing path.
+ *
+ * `GET /projects` only filters on `archived` when the parameter is supplied —
+ * omitting it returns archived AND active projects. Archived projects are
+ * hidden from view in Asana, yet their tasks keep appearing in the connector's
+ * full listing, so those tasks would never fall out via deletion reconciliation
+ * (which hard-deletes only stored documents absent from the full listing) and
+ * would linger in the knowledge base forever.
+ *
+ * `opt_fields` is appended raw rather than through `URLSearchParams` so its
+ * separators stay literal commas, matching every other task call in this
+ * connector and Asana's own documented examples. Only the caller-supplied
+ * values (workspace gid, pagination offset) go through `URLSearchParams`, which
+ * is where escaping actually matters.
+ */
+export function buildProjectsPath(workspaceGid: string, offset?: string): string {
+  const params = new URLSearchParams({
+    workspace: workspaceGid,
+    archived: 'false',
+    limit: '100',
+  })
+  if (offset) params.append('offset', offset)
+  return `/projects?${params.toString()}&opt_fields=${PROJECT_OPT_FIELDS}`
+}
+
+/**
+ * Keeps only projects that are still active. Asana regressed the server-side
+ * `archived=false` filter once before (fixed 2024-11-06), so results are
+ * re-checked client side. Fails open: a project is dropped only on an explicit
+ * `archived === true`, never on a missing or non-boolean field.
+ */
+export function isActiveProject(project: AsanaProject): boolean {
+  return project.archived !== true
+}
+
+/**
+ * Mirrors `isActiveProject` on the single-task rehydrate path so a task whose
+ * every parent project is archived cannot be resurrected after the listing
+ * dropped it. Fails open in every ambiguous case: a task with no `projects`
+ * field (the field is optional and only returned when requested), an empty
+ * array, or any entry whose `archived` is missing or non-boolean is kept. A
+ * task that still sits in at least one active project is kept, matching the
+ * listing, which reaches it through that active project.
+ */
+export function isTaskUnderActiveProject(task: AsanaTask): boolean {
+  const projects = task.projects
+  if (!Array.isArray(projects) || projects.length === 0) return true
+  return !projects.every((project) => project?.archived === true)
+}
+
+/**
+ * Outcome of applying the `maxTasks` cap to one listing page.
+ */
+export interface TaskCapDecision {
+  /** How many of this page's documents to keep. */
+  keepCount: number
+  /** True once the cap is reached, so pagination must stop. */
+  hitLimit: boolean
+  /**
+   * True when the cap made the listing knowingly incomplete — either documents
+   * on this page were dropped, or pages beyond the cap were left unread. The
+   * caller flags `syncContext.listingCapped` so the sync engine refuses to
+   * reconcile deletions against a partial listing and hard-delete everything
+   * past the cap.
+   */
+  truncated: boolean
+}
+
+/**
+ * Decides how the `maxTasks` cap applies to a listing page.
+ *
+ * `pageDocumentCount` and `previouslyFetched` are pre-filter counts of returned
+ * tasks — the cap is never derived from post-filter array lengths. Reaching the
+ * cap exactly as the source runs out (`morePagesAvailable === false` and nothing
+ * dropped) leaves the listing complete, so it is not reported as truncated.
+ */
+export function decideTaskCap(
+  maxTasks: number,
+  previouslyFetched: number,
+  pageDocumentCount: number,
+  morePagesAvailable: boolean
+): TaskCapDecision {
+  if (!(maxTasks > 0)) {
+    return { keepCount: pageDocumentCount, hitLimit: false, truncated: false }
+  }
+
+  const remaining = Math.max(maxTasks - previouslyFetched, 0)
+  const keepCount = Math.min(pageDocumentCount, remaining)
+  const droppedFromPage = keepCount < pageDocumentCount
+  const hitLimit = previouslyFetched + keepCount >= maxTasks
+
+  return {
+    keepCount,
+    hitLimit,
+    truncated: droppedFromPage || (hitLimit && morePagesAvailable),
+  }
 }
 
 /**
@@ -106,7 +223,18 @@ function buildTaskContent(task: AsanaTask): string {
 }
 
 /**
- * Fetches all project GIDs in a workspace, used when no specific project is configured.
+ * Fetches all active project GIDs in a workspace, used when no specific project
+ * is configured. Archived projects are excluded so their tasks drop out of the
+ * full listing and get purged by deletion reconciliation. A project the user
+ * pinned explicitly via the `project` config field keeps syncing even once
+ * archived — that is a deliberate user choice, not a stale listing.
+ *
+ * This is a one-time destructive change on rollout: the first full sync after
+ * deploy stops listing tasks that live only under archived projects, so
+ * deletion reconciliation removes their stored documents. That is the intended
+ * correction — those documents track content the workspace already archived —
+ * but it must be called out in the release notes, since re-indexing them
+ * requires unarchiving the project in Asana.
  */
 async function listWorkspaceProjects(
   accessToken: string,
@@ -117,12 +245,11 @@ async function listWorkspaceProjects(
 
   // eslint-disable-next-line no-constant-condition
   while (true) {
-    const offsetParam = offset ? `&offset=${offset}` : ''
     const result = await asanaGet<{ data: AsanaProject[]; next_page: { offset: string } | null }>(
       accessToken,
-      `/projects?workspace=${workspaceGid}&limit=100${offsetParam}`
+      buildProjectsPath(workspaceGid, offset)
     )
-    projects.push(...result.data)
+    projects.push(...result.data.filter(isActiveProject))
     if (!result.next_page) break
     offset = result.next_page.offset
   }
@@ -235,18 +362,13 @@ export const asanaConnector: ConnectorConfig = {
     }
 
     const previouslyFetched = (syncContext?.totalDocsFetched as number) ?? 0
-    if (maxTasks > 0) {
-      const remaining = maxTasks - previouslyFetched
-      if (documents.length > remaining) {
-        documents.splice(remaining)
-      }
-    }
+    const cap = decideTaskCap(maxTasks, previouslyFetched, documents.length, hasMore)
+    if (cap.keepCount < documents.length) documents.splice(cap.keepCount)
 
-    const totalFetched = previouslyFetched + documents.length
-    if (syncContext) syncContext.totalDocsFetched = totalFetched
-    const hitLimit = maxTasks > 0 && totalFetched >= maxTasks
+    if (syncContext) syncContext.totalDocsFetched = previouslyFetched + documents.length
+    if (cap.truncated && syncContext) syncContext.listingCapped = true
 
-    if (hitLimit) {
+    if (cap.hitLimit) {
       hasMore = false
       nextCursor = undefined
     }
@@ -266,12 +388,17 @@ export const asanaConnector: ConnectorConfig = {
     try {
       const result = await asanaGet<{ data: AsanaTask }>(
         accessToken,
-        `/tasks/${externalId}?opt_fields=${TASK_OPT_FIELDS}`
+        `/tasks/${externalId}?opt_fields=${TASK_DETAIL_OPT_FIELDS}`
       )
       const task = result.data
 
       if (!task) return null
 
+      if (!isTaskUnderActiveProject(task)) {
+        logger.info('Skipping Asana task whose projects are all archived', { externalId })
+        return null
+      }
+
       const content = buildTaskContent(task)
       const tagNames = task.tags?.map((t) => t.name).filter(Boolean) || []