From 53933e92e0cb307d385af6af62f455b709de4f59 Mon Sep 17 00:00:00 2001 From: Kresna <13603341+slaveofcode@users.noreply.github.com> Date: Mon, 17 Aug 2026 20:50:33 +0700 Subject: [PATCH] fix(doc-viewer): render tables row-per-line + real .doc regression tests Tested against real Word 97-2003 .doc files: extraction is correct for plain text, Unicode (smart quotes/accents/em-dash), long docs, tables and lists. Improved cleanup so table rows (a run of 2+ cell/row marks) become line breaks with tab-separated cells and leading tabs stripped, instead of collapsing onto one line. Adds two neutral binary .doc fixtures + tests. --- src/tools/documents/__fixtures__/sample.doc | Bin 0 -> 19456 bytes src/tools/documents/__fixtures__/table.doc | Bin 0 -> 19456 bytes src/tools/documents/doc.fixtures.test.ts | 31 ++++++++++++++++++++ src/tools/documents/doc.lib.ts | 10 +++++-- 4 files changed, 39 insertions(+), 2 deletions(-) create mode 100644 src/tools/documents/__fixtures__/sample.doc create mode 100644 src/tools/documents/__fixtures__/table.doc create mode 100644 src/tools/documents/doc.fixtures.test.ts diff --git a/src/tools/documents/__fixtures__/sample.doc b/src/tools/documents/__fixtures__/sample.doc new file mode 100644 index 0000000000000000000000000000000000000000..643838ddfeb00440729ef379443809efb596586e GIT binary patch literal 19456 zcmeHPOH30{6uoVsQUobb5y8)h-$Y|DYN81-;0F$3pXYP z6ILchqj4*`aOY>kLRPpW(TMI?NImDx3~jN<=t6M-w?ey79y(c9x~1tP>qOEOEMR*9dmi|; zBi(9Wq0G9O{~*8MD`oi;15ZF*o5-rRL=kw0dGUTyQUsO{0(Ej#+VSba)B6(A9_Y=I zkIzMEl{V>x-A<#>VyNAV)IyyuX_7wafTa_by=c)0y$SYClv>g)QS=s-YbNbRZTVx7lJxF`aJeZ4EmK>H! z*$$mE>_l7E&zCb8nK?^pQgTO;%V=_*^D`L@LMAOf?_7OA&V!HDmnQtYsZ4@kMXwb; zdH$Onl2_@!oHZOA%bCu#@*4^U3!fBEt__z~m!&Lz&d*P=Y9><<@MX$Eb)D%ED3Sun z#bcli(vLGAeh!FKe~bGmaSU)4epHX~)wn0;#x!Cc9ej*5IFFK!x=_}y{Z~m%_4oLQ zRH3Luj>G$Ef#22#50gG$1N8a+VXg7mV!w1XWGp;Zu}_%X(+M-E9$7P24m0wzeewG;guL_bOUbatPX8f5@&uAn6W-0yMp z8i3P=k=VxTjBRKB2Id+=+a|QOjN^L^ZZko(u>Q)N+odx70TXZaak>+y+jo$u8v_+X zJOv_RfpD_jwz?K8Ak!|+PBRsr5f}}Qt$HrcWl)MLJvZ`Pk@sdr;%kZPl;gRrR#XHO z0YyL&Py`eKML-cy1QdZ4h`=m%)LFUib}q3%S|)Ej_&#&4HT>vKj_ls~>?8LmrC^qk zx$)yBW9kPIB}G6HPy`eKML-cy1QY>9KoL*`6ahs*5m>Sa{BHm6AAdW3e{XsC?mg`P z_sl%j{r{4^j5;wzKoL*`6ahs*5l{pa0YyL&Py`eKMc}sx@OOOvh93a)U17dO&o|`} z`fvW%Aq1HVUJYh+GovLD@{ec%ySxyx2wV(~fElTd2qakt=Kn&hhh)f6Id}tjBX|>d zGnnP%iX@$RrGJM20~#24Ysm=&HN+5fFo3J!ok}Fz=gyY*P%y;1No35VQfrIUbQ`jR zj%gp_jH3vSXh+b(-_RP|z3%vwUivVv!Sm%6kR9{fjOdiqcxJnF@w80*K7NPXIV|47 zso{*Dji_lxQQNW519A#ip#^iXBW@VR!6@d1X&1&P;{5JpF literal 0 HcmV?d00001 diff --git a/src/tools/documents/__fixtures__/table.doc b/src/tools/documents/__fixtures__/table.doc new file mode 100644 index 0000000000000000000000000000000000000000..ddaba516d368ea2dd2a314f56c35ad6d708a663d GIT binary patch literal 19456 zcmeHPU1(fI6h3#;WOvi%*Sf8#KWSY2!BRHb6d#H-q()nzO`3qd$>y&K?4~8zT8j!H zRH!~xd?=_WE#}1*QQC_1L1QTeMKK61RD5U@#7e=nAf-aH{l3|~duKPh$p(T=>Y3%c zbI$zSGxN>dduML$$<&W^SB`z&FfDoE87Y&QR8T6cbO-W!SFaU$7&6mlQmGV^8v%$K zmJtKjCcfd&AtdX+HnCHOL3KFroBbly(uZI&3LHWrmv| z5sqh6H+FLi#VoVBP)3ooFOa?pI0xAEZb1JNxKm*omsF$Av;Hf@dp%X9CCb1hw2Sje zgEFvu80eJ!5|t52NI#A-If$qDZpiy&uSugWAznS9)`Vs=*<$rk8z8M{nCbW zkD~S{Qa72j7UZ#RP?{Qp(heIVrUcuuA2wq+Qvx>A`CZU@agKDm3YF+aA9#*F)M-O) z+8aXoJ?Ia6FpM*>4|h?(bG`}a0A2!mfMH++cpVrA-Ui+SjsquwFM+Rs3&8il@4z)6 z!1V&q0C4SaBl*Yl<)6MgclOlDi4Tt+d29T&M10`o#pG_)t{r943AF84i@WbK(Q52= zwgjB{^xS^sl-?Y0=E%jli{?fIxf0?!kK=ET_hIPvn12G@9{2Om|FwtlA6xm$R<7F0 zZ?p1R4O_tq)C;4Y=t?U@LSd&hEK)#v++?l^JAq5>*!IZQm1CjF z^<(H|z;SX=YFZ&HUfSZ@LsQGbPP&K1)?KTZlbFjA{wql#n6>)5?!xr)h#Z?h7F z^R}Y$A~WEP4i}L%ho*dWha$HySXCDS;`>FWup?Ofd4t)GvFE*7!~c>2r(SBL65IVD z#6C<1{x;zj_;Ru2tUTDM+L9g{7{ z2+20Aod&QH0NCGt-s={E~}t&)DjB2G0tLKMa(h> z>YUgSTzT`O>o<1|)}DALAWuAc>N58$c+GCMh|K3E zkQRhX0NMIx_4v{us}~nIQpZgM%22(nCnf9k;cdLlr^JCXq>Vl=gI#dkjMz%Oo{R zH=G-K;NH-mTc|{>8nf56&$yIo=974Syn^r|dQX4fZHRk~r!zf}XM3QeOV2(whz~x3 z-v2$BU7|=z#(sI8fAw})A24mAdr1PWAM{r)9v*{zGR|wa)czaOS^N1t&-M47uRTx0 a{zCcZH$>*+A;ub%0cAiLPzLS@1GfRBYO0k0 literal 0 HcmV?d00001 diff --git a/src/tools/documents/doc.fixtures.test.ts b/src/tools/documents/doc.fixtures.test.ts new file mode 100644 index 0000000..5c6321e --- /dev/null +++ b/src/tools/documents/doc.fixtures.test.ts @@ -0,0 +1,31 @@ +import { describe, it, expect } from 'vitest'; +import { readFileSync } from 'node:fs'; +import { resolve } from 'node:path'; +import { extractDocText } from './doc.lib'; + +// Real Word 97-2003 binary .doc files (generated with macOS `textutil`, neutral +// synthetic content) — regression coverage for the OLE + piece-table extractor. +const load = (name: string) => + new Uint8Array(readFileSync(resolve(process.cwd(), 'src/tools/documents/__fixtures__', name))); + +describe('extractDocText on real .doc fixtures', () => { + it('extracts plain and Unicode text from a simple .doc', () => { + const text = extractDocText(load('sample.doc')); + expect(text).toContain('Hello World'); + expect(text).toContain('This is a test document.'); + expect(text).toContain('Café résumé'); // accented characters + expect(text).toContain('90% done'); + expect(text).toContain('Line A'); + expect(text).toContain('Line B'); + }); + + it('extracts a table as rows (tab-separated cells) plus a bulleted list', () => { + const text = extractDocText(load('table.doc')); + expect(text).toContain('Quarterly Report'); + expect(text).toContain('Item\tQ1\tQ2'); + expect(text).toContain('Revenue\t100\t150'); + expect(text).toContain('Costs\t40\t55'); + expect(text).toContain('First bullet'); + expect(text).toContain('Closing line.'); + }); +}); diff --git a/src/tools/documents/doc.lib.ts b/src/tools/documents/doc.lib.ts index b90dc83..8b44dd6 100644 --- a/src/tools/documents/doc.lib.ts +++ b/src/tools/documents/doc.lib.ts @@ -106,12 +106,18 @@ function decodePieces(wd: Uint8Array, table: Uint8Array, pcdt: Pcdt, ccpText: nu return out; } -/** Tidy extracted text: collapse runs of blank lines and trailing spaces. */ +/** + * Tidy extracted text. Table cells are separated by single tab marks; a run of + * two or more marks (a cell mark plus the row-end mark) is a row boundary, so + * those become line breaks. Leading tabs and blank-line runs are trimmed. + */ export function cleanDocText(s: string): string { return s .replace(/\r\n?/g, '\n') - .replace(/[ \t]+\n/g, '\n') .replace(/\uFEFF/g, '') + .replace(/\t{2,}/g, '\n') // consecutive cell/row marks \u2192 new row + .replace(/[ \t]+\n/g, '\n') + .replace(/\n[ \t]+/g, '\n') // strip leading tabs/spaces on a line .replace(/\n{3,}/g, '\n\n') .trim(); }