fix(files): stop rejecting ordinary HTML documents at the parser limits (#6446)

waleedlatif1 · web-flow · commit 28f5e505eb29 · 2026-08-08T14:07:42.000-07:00
The markup-token cap rejected a 30,000-row table export at 3.6 MB. Doubled
it to a value measured safe, and left the byte cap where it is - that one is
load-bearing, so the reasoning is now recorded alongside it.
diff --git a/apps/sim/lib/file-parsers/html-parser.test.ts b/apps/sim/lib/file-parsers/html-parser.test.ts
@@ -11,25 +11,39 @@ const parser = new HtmlParser()
 
 describe('HtmlParser', () => {
   describe('resource limits', () => {
+    /**
+     * Pinned by value: a 64 MB body aborts the process, so raising the cap
+     * toward the shared upload limit must fail here, not in production.
+     */
     it('rejects a document above the input byte cap', async () => {
-      const sparse = Buffer.concat([
-        Buffer.from('<html><body><p>'),
-        Buffer.alloc(32 * 1024 * 1024, 0x61),
-        Buffer.from('</p></body></html>'),
-      ])
-
-      await expect(parser.parseBuffer(sparse)).rejects.toThrow(
-        /above the maximum of 33554432 bytes/
-      )
+      const oversized = Buffer.alloc(32 * 1024 * 1024 + 1)
+
+      const error = await parser.parseBuffer(oversized).catch((e) => e)
+
+      expect(error).toBeInstanceOf(HtmlComplexityError)
+      expect(error.message).toMatch(/above the maximum of 33554432 bytes/)
     })
 
     it('rejects a tag-dense document above the markup-token cap', async () => {
-      const dense = Buffer.from(`<html><body>${'<p>a</p>'.repeat(300_000)}</body></html>`)
+      const dense = Buffer.from(`<html><body>${'<p>a</p>'.repeat(600_000)}</body></html>`)
 
       const error = await parser.parseBuffer(dense).catch((e) => e)
 
       expect(error).toBeInstanceOf(HtmlComplexityError)
-      expect(error.message).toMatch(/exceeds the maximum of 500000 markup tokens/)
+      expect(error.message).toMatch(/exceeds the maximum of 1000000 markup tokens/)
+    })
+
+    /**
+     * A 30,000-row by 8-column export is ~540k tokens in 3.6 MB, an ordinary
+     * document that an earlier, tighter token cap rejected.
+     */
+    it('accepts a realistic large table export', async () => {
+      const row = `<tr>${'<td>value</td>'.repeat(8)}</tr>`
+      const buffer = Buffer.from(`<html><body><table>${row.repeat(30_000)}</table></body></html>`)
+
+      const result = await parser.parseBuffer(buffer)
+
+      expect(result.content).toContain('| value |')
     })
 
     it('accepts a byte-heavy document whose markup stays under the token cap', async () => {
@@ -42,13 +56,11 @@ describe('HtmlParser', () => {
     })
 
     /**
-     * Deep nesting overflows the stack inside cheerio's own recursive `.text()`,
-     * which the pre-parse caps cannot predict. It still has to be classified as
-     * a resource rejection so callers fail closed rather than fall back to
-     * storing the document as raw text.
+     * `parseFile` must not wrap the rejection in a generic error, or the route
+     * stops recognising it and falls back to storing the document as raw text.
      */
     it('preserves the error type through parseFile so callers still fail closed', async () => {
-      const dense = `<html><body>${'<p>a</p>'.repeat(300_000)}</body></html>`
+      const dense = `<html><body>${'<p>a</p>'.repeat(600_000)}</body></html>`
       const path = join(tmpdir(), `html-parser-limits-${process.pid}.html`)
       await writeFile(path, dense)
 
@@ -59,6 +71,11 @@ describe('HtmlParser', () => {
       }
     })
 
+    /**
+     * Deep nesting overflows the stack inside cheerio's own recursive `.text()`,
+     * which the pre-parse caps cannot predict. It still has to be classified as
+     * a resource rejection so callers fail closed.
+     */
     it('classifies a deep-nesting stack overflow as a complexity rejection', async () => {
       const depth = 15_000
       const buffer = Buffer.from(
diff --git a/apps/sim/lib/file-parsers/html-parser.ts b/apps/sim/lib/file-parsers/html-parser.ts
@@ -8,15 +8,25 @@ import { sanitizeTextForUTF8 } from '@/lib/file-parsers/utils'
 const logger = createLogger('HtmlParser')
 
 /**
- * `cheerio.load` retains ~530 bytes of DOM per markup token (`<`) — measured on
- * cheerio 1.1.2 at 0.2M/1M/2M tokens (101/504/1008 MB), flat across all three —
- * so this bounds one document's tree at roughly 256 MB.
+ * Bounds the DOM tree, which costs ~500 bytes per markup token (`<`) on cheerio
+ * 1.1.2: measured at 0.2M/1M/2M tokens as 101/504/1008 MB retained, linear
+ * across all three.
+ *
+ * A 50,000-row by 8-column table export is ~900k tokens in only 8.7 MB, so a
+ * tighter cap rejects ordinary exports; 999k tokens in a 10 MB body parses
+ * inside a 2 GB heap.
  */
-const MAX_HTML_MARKUP_TOKENS = 500_000
+const MAX_HTML_MARKUP_TOKENS = 1_000_000
 
 /**
- * Backstop for markup sparse enough to pass the token cap: bounds the UTF-16
- * copy `buffer.toString` allocates and the text nodes the DOM keeps.
+ * Bounds the body, which governs peak memory: extraction materialises the text
+ * several times over (UTF-16 buffer copy, per-node strings, the joined output).
+ *
+ * Measured against a 2 GB heap with the token cap saturated: 32 MB parses,
+ * 48 MB parses, 64 MB aborts the process. Deliberately NOT raised to the
+ * shared 100 MB document limit — that limit governs what may be uploaded, and
+ * a 100 MB body aborts here even with few tokens. Any change to this number
+ * needs the same abort test, not a consistency argument.
  */
 const MAX_HTML_INPUT_BYTES = 32 * 1024 * 1024
 
