diff --git a/apps/sim/connectors/confluence/confluence.test.ts b/apps/sim/connectors/confluence/confluence.test.ts index 706023ccb93..25b3dfff5cb 100644 --- a/apps/sim/connectors/confluence/confluence.test.ts +++ b/apps/sim/connectors/confluence/confluence.test.ts @@ -2,7 +2,12 @@ * @vitest-environment node */ import { describe, expect, it } from 'vitest' -import { escapeCql, isCurrentContent } from '@/connectors/confluence/confluence' +import { + escapeCql, + isCurrentContent, + preserveConfluenceCallouts, +} from '@/connectors/confluence/confluence' +import { htmlToPlainText } from '@/connectors/utils' describe('escapeCql', () => { it.concurrent('returns plain strings unchanged', () => { @@ -48,3 +53,248 @@ describe('isCurrentContent', () => { expect(isCurrentContent({ id: '1', status: 'deleted' })).toBe(false) }) }) + +describe('preserveConfluenceCallouts', () => { + it.concurrent('handles empty content', () => { + expect(preserveConfluenceCallouts('')).toBe('') + }) + + it.concurrent('leaves content with no macros unchanged', () => { + const html = '

Just a normal paragraph.

' + expect(preserveConfluenceCallouts(html)).toContain('Just a normal paragraph.') + }) + + it.concurrent('labels a built-in warning macro and keeps its body', () => { + const html = + '
' + + '' + + '

Do NOT use this form for GitLab access.

' + + '
' + const result = preserveConfluenceCallouts(html) + expect(result).toContain('[WARNING]') + expect(result).toContain('Do NOT use this form for GitLab access.') + }) + + it.concurrent('labels a built-in info macro', () => { + const html = + '
' + + '

Heads up.

' + + '
' + expect(preserveConfluenceCallouts(html)).toContain('[INFO] Heads up.') + }) + + it.concurrent('labels a built-in note macro', () => { + const html = + '
' + + '

See also.

' + + '
' + expect(preserveConfluenceCallouts(html)).toContain('[NOTE] See also.') + }) + + it.concurrent('labels a built-in tip macro', () => { + const html = + '
' + + '

Pro tip.

' + + '
' + expect(preserveConfluenceCallouts(html)).toContain('[TIP] Pro tip.') + }) + + it.concurrent('labels a generic custom-colored Panel macro using its header title', () => { + const html = + '
' + + '
Do NOT use this form for:
' + + '

GitLab access requests go to the private channel instead.

' + + '
' + const result = preserveConfluenceCallouts(html) + expect(result).toContain('[CALLOUT: Do NOT use this form for:]') + expect(result).toContain('GitLab access requests go to the private channel instead.') + }) + + it.concurrent('preserves word boundaries between a block header and its own content', () => { + const html = + '
Warning:
' + + '

See replacement form.

' + const result = preserveConfluenceCallouts(html) + expect(result).toContain('[CALLOUT: Warning:] See replacement form.') + }) + + it.concurrent( + 'keeps a rich header with a real source space intact, without adding a second one', + () => { + const html = + '
Warning: Do not use
' + + '

See replacement form.

' + const result = preserveConfluenceCallouts(html) + expect(result).toContain('[CALLOUT: Warning: Do not use]') + } + ) + + it.concurrent('falls back to a bare CALLOUT label when a Panel macro has no header text', () => { + const html = + '

Untitled panel body.

' + const result = preserveConfluenceCallouts(html) + expect(result).toContain('[CALLOUT]') + expect(result).toContain('Untitled panel body.') + }) + + it.concurrent( + 'keeps the exclusion marker attached to its content through htmlToPlainText, even across surrounding whitespace collapse', + () => { + const html = + '

Intro paragraph.

\n\n' + + '
' + + '

Do NOT use this form for:

' + + '
' + + '
\n\n' + + '

Trailing paragraph.

' + const plainText = htmlToPlainText(preserveConfluenceCallouts(html)) + expect(plainText).toContain('[WARNING] Do NOT use this form for: GitLab') + expect(plainText).toContain('Intro paragraph.') + expect(plainText).toContain('Trailing paragraph.') + } + ) + + it.concurrent( + 'does not fuse adjacent paragraph and list-item text together (word-boundary regression)', + () => { + const html = + '
' + + '
' + + '

Do NOT use this form for:

' + + '' + + '
' + const result = preserveConfluenceCallouts(html) + expect(result).not.toContain('for:GitLab') + expect(result).not.toContain('GitLabServiceNow') + expect(result).toContain('Do NOT use this form for: GitLab ServiceNow') + } + ) + + it.concurrent( + 'preserves word boundaries across multiple paragraphs in a generic Panel macro', + () => { + const html = + '
' + + '

First sentence.

Second sentence.

' + + '
' + const result = preserveConfluenceCallouts(html) + expect(result).toContain('First sentence. Second sentence.') + expect(result).not.toContain('sentence.Second') + } + ) + + it.concurrent( + 'does not duplicate or fuse text from a nested list inside a callout body (nesting regression)', + () => { + const html = + '
' + + '
' + + '' + + '
' + const result = preserveConfluenceCallouts(html) + // Each nested
  • 's text must appear exactly once, not duplicated by the + // outer
  • also being matched and its .text() recursing into it. + const occurrences = (result.match(/Nested item A/g) ?? []).length + expect(occurrences).toBe(1) + expect(result).not.toContain('Nested item ANested item B') + expect(result).toContain('Outer item Nested item A Nested item B Outer item two') + } + ) + + it.concurrent('does not fuse text from a blockquote nested inside a table cell', () => { + const html = + '
    ' + + '
    Cell text

    quoted text

    after quote
    ' + + '
    ' + const result = preserveConfluenceCallouts(html) + expect(result).not.toContain('quotedtext') + expect(result).not.toContain('textafter') + expect(result).toContain('Cell text quoted text after quote') + }) + + it.concurrent( + 'does not inject an artificial space into inline-formatted text mid-word (inline vs. block regression)', + () => { + const html = + '
    ' + + '

    This is unbelieveable.

    ' + + '
    ' + const result = preserveConfluenceCallouts(html) + expect(result).not.toContain('un believe able') + expect(result).toContain('This is unbelieveable.') + } + ) + + it.concurrent('does not inject a space before punctuation carried by an inline tag', () => { + const html = + '
    ' + + '

    Do not proceed!

    ' + + '
    ' + const result = preserveConfluenceCallouts(html) + expect(result).not.toContain('proceed !') + expect(result).toContain('[WARNING] Do not proceed!') + }) + + it.concurrent('keeps natural word spacing when inline tags wrap a whole word', () => { + const html = + '
    ' + + '

    Do NOT use this form.

    ' + + '
    ' + const result = preserveConfluenceCallouts(html) + expect(result).toContain('Do NOT use this form.') + }) + + it.concurrent( + 'labels a nested panel-in-panel with both its own and its parent label (nesting regression)', + () => { + const html = + '
    Outer
    ' + + '
    Inner
    ' + + '

    inner body

    ' + + '
    ' + const result = preserveConfluenceCallouts(html) + expect(result).toContain('[CALLOUT: Outer]') + expect(result).toContain('[CALLOUT: Inner] inner body') + } + ) + + it.concurrent( + 'labels a nested info-macro inside a panel with its own type instead of dropping it', + () => { + const html = + '
    ' + + '
    ' + + '

    Do not use this.

    ' + + '
    ' + const result = preserveConfluenceCallouts(html) + expect(result).toContain('[WARNING] Do not use this.') + } + ) + + it.concurrent( + "does not let an untitled outer panel adopt a nested panel's header as its own", + () => { + const html = + '
    ' + + '
    Inner title
    ' + + '

    inner body

    ' + + '
    ' + const result = preserveConfluenceCallouts(html) + // The outer panel has no header of its own — it must fall back to a + // bare [CALLOUT], not steal "Inner title" from the nested panel. + expect(result).toContain('[CALLOUT] [CALLOUT: Inner title] inner body') + } + ) + + it.concurrent('does not fuse text on either side of a
    line break', () => { + const html = + '
    ' + + '

    Do NOT use this form for:
    GitLab

    ' + + '
    ' + const result = preserveConfluenceCallouts(html) + expect(result).not.toContain('for:GitLab') + expect(result).toContain('[WARNING] Do NOT use this form for: GitLab') + }) +}) diff --git a/apps/sim/connectors/confluence/confluence.ts b/apps/sim/connectors/confluence/confluence.ts index 47cf4f7a626..470cdc8ab68 100644 --- a/apps/sim/connectors/confluence/confluence.ts +++ b/apps/sim/connectors/confluence/confluence.ts @@ -1,5 +1,6 @@ import { createLogger } from '@sim/logger' import { toError } from '@sim/utils/errors' +import * as cheerio from 'cheerio' import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { confluenceConnectorMeta } from '@/connectors/confluence/meta' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' @@ -8,6 +9,155 @@ import { getConfluenceCloudId, normalizeConfluenceDomainHost } from '@/tools/con const logger = createLogger('ConfluenceConnector') +/** Label prefixes for Confluence's built-in Info/Note/Warning/Tip macros, by their rendered CSS suffix. */ +const CALLOUT_LABELS: Record = { + information: '[INFO]', + note: '[NOTE]', + warning: '[WARNING]', + tip: '[TIP]', + error: '[ERROR]', +} + +/** + * Inline formatting tags whose text flows directly into their surrounding + * sentence with no implied word break — e.g. `unbelieveable` must stay + * `unbelievable`, and `Hello!` must stay `Hello!`, not gain an + * artificial space. Anything not in this set (p, li, td, div, headings, br, + * etc.) is treated as a block boundary that always implies a break, even when + * the source HTML has no literal whitespace there. + */ +const INLINE_FORMATTING_TAGS = new Set([ + 'b', + 'strong', + 'i', + 'em', + 'u', + 's', + 'strike', + 'del', + 'ins', + 'sup', + 'sub', + 'small', + 'mark', + 'code', + 'span', + 'a', + 'abbr', + 'cite', + 'q', + 'kbd', + 'var', + 'samp', + 'time', +]) + +/** + * Cheerio's `.text()` concatenates every descendant text node with no + * separator at all, so pulling a macro body's text in one call fuses adjacent + * blocks together (e.g. a `

    ...for:

    ` immediately followed by + * `
  • GitLab
    • ` becomes `for:GitLab`, corrupting the very word boundaries + * RAG chunking depends on). Simply joining every text node with a space isn't + * right either — that would corrupt genuinely inline-formatted text the same + * way. This walks the DOM, accumulating text through inline tags without a + * separator (preserving exact source adjacency) and flushing to a new segment + * at every other tag boundary (a block always implies a break, regardless of + * source whitespace) — matching how `html-parser.ts` already walks HTML for a + * related reason elsewhere in this codebase, extended with the inline/block + * distinction real Confluence rich text requires. + */ +function extractBlockJoinedText($: cheerio.CheerioAPI, $el: cheerio.Cheerio): string { + const parts: string[] = [] + let current = '' + + const flush = () => { + const text = current.trim() + if (text) parts.push(text) + current = '' + } + + const visit = ($node: cheerio.Cheerio) => { + $node.contents().each((_, child) => { + if (child.type === 'text') { + current += $(child).text() + } else if (child.type === 'tag') { + const tag = child.tagName?.toLowerCase() + if (tag && INLINE_FORMATTING_TAGS.has(tag)) { + visit($(child)) + } else { + flush() + visit($(child)) + flush() + } + } + }) + } + + visit($el) + flush() + return parts.join(' ').trim() +} + +/** Matches either flavor of panel/macro this function rewrites. */ +const MACRO_SELECTOR = 'div.confluence-information-macro, div.panel' + +/** + * Confluence's rendered `view` HTML wraps Info/Note/Warning/Tip macros in + * `confluence-information-macro confluence-information-macro-{type}` divs, and + * the customizable Panel macro in `.panel` > `.panelHeader` + `.panelContent` + * divs. `htmlToPlainText`'s blind tag-stripping discards the divs' classes along + * with the tags, so a red "do not use" warning panel becomes indistinguishable + * from a plain paragraph once flattened — and its trailing whitespace collapse + * would erase any newline-based separation too. Each detected panel is rewritten + * into a single bracketed label plus its own text so the callout semantic + * survives both the tag strip and the whitespace collapse. + * + * A panel can itself contain another panel or macro (e.g. a nested Note inside + * a Warning panel). Processing matches in document order — outermost first — + * would read a not-yet-converted nested macro as plain body text before it + * ever got its own label, silently dropping the inner callout's semantic, and + * `.find('.panelHeader')` would then risk pulling a nested panel's header up + * as if it were the outer panel's own title. Converting only "leaf" macros + * (ones with no remaining nested macro/panel inside them) and repeating until + * none are left processes innermost-first, so a nested macro is already a + * bracketed `

    ` by the time its parent's body/header text is read — at which + * point it correctly reads as plain text carrying its own label. + */ +export function preserveConfluenceCallouts(html: string): string { + if (!html) return html + + const $ = cheerio.load(html) + + let progressed = true + while (progressed) { + progressed = false + const leaves = $(MACRO_SELECTOR).filter((_, el) => $(el).find(MACRO_SELECTOR).length === 0) + if (leaves.length === 0) break + + leaves.each((_, el) => { + const $el = $(el) + if ($el.hasClass('confluence-information-macro')) { + const type = ($el.attr('class') ?? '') + .match(/confluence-information-macro-(\w+)/)?.[1] + ?.toLowerCase() + const label = (type && CALLOUT_LABELS[type]) || CALLOUT_LABELS.information + const macroBody = $el.find('.confluence-information-macro-body').first() + const body = extractBlockJoinedText($, macroBody.length > 0 ? macroBody : $el) + $el.replaceWith($('

    ').text(`${label} ${body}`)) + } else { + const headerText = extractBlockJoinedText($, $el.find('.panelHeader').first()) + const panelContent = $el.find('.panelContent').first() + const bodyText = extractBlockJoinedText($, panelContent.length > 0 ? panelContent : $el) + const label = headerText ? `[CALLOUT: ${headerText}]` : '[CALLOUT]' + $el.replaceWith($('

    ').text(`${label} ${bodyText}`)) + } + progressed = true + }) + } + + return $.html() +} + /** * Escapes a value for use inside CQL double-quoted strings. */ @@ -108,10 +258,12 @@ async function fetchLabelsForPages( * invalidates every previously-synced Confluence document so a one-time * re-hydration picks up content newly reachable by the current extraction * (e.g. the switch from `storage` to rendered `view`, which expands Include - * Page / Excerpt macros). Without it, already-indexed pages whose version is - * unchanged classify as `unchanged` and keep their stale (empty) content. + * Page / Excerpt macros; or `preserveConfluenceCallouts`, which stops + * flattening panel/info/note/warning/tip macros into indistinguishable plain + * text). Without it, already-indexed pages whose version is unchanged + * classify as `unchanged` and keep their stale (pre-fix) content. */ -const CONTENT_REPRESENTATION = 'view' +const CONTENT_REPRESENTATION = 'view-callouts' /** * Produces a canonical metadata stub with a deterministic contentHash that @@ -288,7 +440,7 @@ export const confluenceConnector: ConnectorConfig = { const body = page.body as Record | undefined const view = body?.view as Record | undefined const rawContent = (view?.value as string) || '' - const plainText = htmlToPlainText(rawContent) + const plainText = htmlToPlainText(preserveConfluenceCallouts(rawContent)) const labelMap = await fetchLabelsForPages(cloudId, accessToken, [String(page.id)]) const labels = labelMap.get(String(page.id)) ?? []