diff --git a/src/islands/documents/PdfToDocx.tsx b/src/islands/documents/PdfToDocx.tsx index 3d564e9..72fbac9 100644 --- a/src/islands/documents/PdfToDocx.tsx +++ b/src/islands/documents/PdfToDocx.tsx @@ -4,7 +4,7 @@ import { Dropzone } from '@/components/ui/Dropzone'; import { Alert } from '@/components/ui/Alert'; import { ProgressBar } from '@/components/ui/ProgressBar'; import { downloadService } from '@/services/download.service'; -import { reconstruct, textDensity, type TextItem, type DocParagraph } from '@/tools/documents/pdf-docx.lib'; +import { reconstruct, reconstructBlocks, textDensity, type TextItem, type DocBlock } from '@/tools/documents/pdf-docx.lib'; import type { Lang } from '@/i18n/config'; const OCR_MIN_CHARS = 8; // pages with fewer real characters are treated as scanned @@ -21,7 +21,7 @@ const TR: Record `Reading text — page ${p} of ${n}…`, ocr: (p, n) => `Reading (OCR) — page ${p} of ${n}…`, building: 'Building the Word document…', another: 'Convert another', - note: 'The .docx contains editable, reflowable text (paragraphs and headings), not a pixel-perfect copy of the PDF layout. Exact positioning, tables and columns are not preserved.', + note: 'The .docx has editable text: headings, paragraphs and reconstructed tables. Table detection is best-effort — complex or borderless tables and multi-column layouts may still need cleanup.', errRead: 'Could not open this file — is it a valid PDF?', errConvert: 'Sorry, converting this PDF failed.', }, id: { @@ -31,7 +31,7 @@ const TR: Record `Membaca teks — halaman ${p} dari ${n}…`, ocr: (p, n) => `Membaca (OCR) — halaman ${p} dari ${n}…`, building: 'Menyusun dokumen Word…', another: 'Konversi yang lain', - note: 'Berkas .docx berisi teks yang dapat diedit dan disusun ulang (paragraf dan judul), bukan salinan tata letak PDF yang sempurna. Posisi persis, tabel, dan kolom tidak dipertahankan.', + note: 'Berkas .docx berisi teks yang dapat diedit: judul, paragraf, dan tabel yang direkonstruksi. Deteksi tabel bersifat best-effort — tabel rumit atau tanpa garis dan tata letak multi-kolom mungkin masih perlu dirapikan.', errRead: 'Tidak dapat membuka berkas ini — apakah PDF yang valid?', errConvert: 'Maaf, konversi PDF ini gagal.', }, }; @@ -58,7 +58,7 @@ export default function PdfToDocx({ lang = 'en' }: { lang?: Lang }) { try { const pdf = await loadingTask.promise; const total = pdf.numPages; - const pages: DocParagraph[][] = []; + const pages: DocBlock[][] = []; for (let p = 1; p <= total; p++) { const page = await pdf.getPage(p); @@ -77,7 +77,7 @@ export default function PdfToDocx({ lang = 'en' }: { lang?: Lang }) { pages.push(await ocrPage(page)); } else { setStatus(t.reading(p, total)); - pages.push(reconstruct(items)); + pages.push(reconstructBlocks(items)); } page.cleanup(); setProgress(Math.round((p / total) * 90)); @@ -133,7 +133,7 @@ export default function PdfToDocx({ lang = 'en' }: { lang?: Lang }) { } // Render a page to a canvas and reconstruct paragraphs from on-device OCR. -async function ocrPage(page: import('pdfjs-dist').PDFPageProxy): Promise { +async function ocrPage(page: import('pdfjs-dist').PDFPageProxy): Promise { const viewport = page.getViewport({ scale: 2 }); const canvas = document.createElement('canvas'); canvas.width = Math.floor(viewport.width); @@ -147,21 +147,35 @@ async function ocrPage(page: import('pdfjs-dist').PDFPageProxy): Promise ({ text: l.text, x: l.box.x, y: l.box.y, width: l.box.width, height: l.box.height })); - return reconstruct(items); + // OCR gives lines, not reliable column geometry, so keep the paragraph path. + return reconstruct(items).map((p) => ({ type: 'paragraph', ...p })); } -async function buildDocx(pages: DocParagraph[][]): Promise { - const { Document, Packer, Paragraph, HeadingLevel } = await import('docx'); - const children: InstanceType[] = []; - pages.forEach((paras, pageIdx) => { - paras.forEach((par, i) => { - children.push(new Paragraph({ - text: par.text, - heading: par.heading === 1 ? HeadingLevel.HEADING_1 : par.heading === 2 ? HeadingLevel.HEADING_2 : undefined, - pageBreakBefore: pageIdx > 0 && i === 0 ? true : undefined, - })); +async function buildDocx(pages: DocBlock[][]): Promise { + const { Document, Packer, Paragraph, HeadingLevel, Table, TableRow, TableCell, WidthType } = await import('docx'); + const children: (InstanceType | InstanceType)[] = []; + + pages.forEach((blocks, pageIdx) => { + blocks.forEach((block, i) => { + const first = pageIdx > 0 && i === 0; + if (block.type === 'table') { + children.push(new Table({ + width: { size: 100, type: WidthType.PERCENTAGE }, + rows: block.rows.map((row) => new TableRow({ + children: row.map((cell) => new TableCell({ children: [new Paragraph({ text: cell })] })), + })), + })); + children.push(new Paragraph({ text: '' })); // spacing after the table + } else { + children.push(new Paragraph({ + text: block.text, + heading: block.heading === 1 ? HeadingLevel.HEADING_1 : block.heading === 2 ? HeadingLevel.HEADING_2 : undefined, + pageBreakBefore: first ? true : undefined, + })); + } }); }); + if (children.length === 0) children.push(new Paragraph({ text: '' })); const doc = new Document({ sections: [{ children }] }); return Packer.toBlob(doc); diff --git a/src/tools/documents/pdf-docx.e2e.test.ts b/src/tools/documents/pdf-docx.e2e.test.ts new file mode 100644 index 0000000..1fd5370 --- /dev/null +++ b/src/tools/documents/pdf-docx.e2e.test.ts @@ -0,0 +1,38 @@ +import { describe, it, expect } from 'vitest'; +import { reconstructBlocks, type TextItem } from './pdf-docx.lib'; + +// Full pipeline: draw a PDF (pdf-lib) → extract positioned text (pdf.js) → +// reconstruct blocks. Guards against regressions like pdf.js's synthetic +// whitespace items merging table cells. +describe('pdf → docx reconstruction (end to end)', () => { + it('reconstructs a drawn table as a real table, prose as paragraphs', async () => { + const { PDFDocument, StandardFonts } = await import('pdf-lib'); + const doc = await PDFDocument.create(); + const page = doc.addPage([612, 792]); + const font = await doc.embedFont(StandardFonts.Helvetica); + const draw = (text: string, x: number, y: number, size = 12) => page.drawText(text, { x, y, size, font }); + draw('A short introductory paragraph of text on its own.', 50, 715); + const rows = [['Item', 'Q1', 'Q2'], ['Revenue', '100', '150'], ['Costs', '40', '55']]; + const cx = [50, 260, 400]; + rows.forEach((r, ri) => r.forEach((c, ci) => draw(c, cx[ci], 670 - ri * 22))); + const bytes = await doc.save(); + + const pdfjs = await import('pdfjs-dist/legacy/build/pdf.mjs'); + const pdf = await pdfjs.getDocument({ data: bytes, isEvalSupported: false, useWorkerFetch: false }).promise; + const p = await pdf.getPage(1); + const viewport = p.getViewport({ scale: 1 }); + const tc = await p.getTextContent(); + const items: TextItem[] = tc.items + .filter((i): i is Extract => 'str' in i) + .map((i) => { + const tr = pdfjs.Util.transform(viewport.transform, i.transform); + return { text: i.str, x: tr[4], y: tr[5], width: i.width, height: Math.hypot(tr[2], tr[3]) || 10 }; + }); + + const blocks = reconstructBlocks(items); + const table = blocks.find((b) => b.type === 'table'); + expect(table).toBeDefined(); + if (table && table.type === 'table') expect(table.rows).toEqual(rows); + expect(blocks.some((b) => b.type === 'paragraph' && b.text.includes('introductory'))).toBe(true); + }); +}); diff --git a/src/tools/documents/pdf-docx.lib.test.ts b/src/tools/documents/pdf-docx.lib.test.ts index f331629..ead4e96 100644 --- a/src/tools/documents/pdf-docx.lib.test.ts +++ b/src/tools/documents/pdf-docx.lib.test.ts @@ -1,5 +1,5 @@ import { describe, it, expect } from 'vitest'; -import { groupLines, paragraphsFromLines, reconstruct, textDensity, type TextItem } from './pdf-docx.lib'; +import { groupLines, paragraphsFromLines, reconstruct, textDensity, segmentsOf, clusterColumns, reconstructBlocks, type TextItem } from './pdf-docx.lib'; const item = (text: string, x: number, y: number, width: number, height = 10): TextItem => ({ text, x, y, width, height }); @@ -69,3 +69,52 @@ describe('reconstruct + textDensity', () => { expect(textDensity([])).toBe(0); }); }); + +describe('segmentsOf', () => { + it('splits a row into cells on wide column gaps but keeps words together', () => { + // "Name Field" as one cell (small gap), then a big gap, then "Value" + const items = [ + item('Name', 10, 100, 30), item('Field', 46, 100, 30), + item('Value', 200, 100, 40), + ]; + const segs = segmentsOf(items); + expect(segs.map(s => s.text)).toEqual(['Name Field', 'Value']); + }); +}); + +describe('clusterColumns', () => { + it('groups nearby x positions into column centres', () => { + expect(clusterColumns([10, 11, 100, 101, 200], 5)).toEqual([10.5, 100.5, 200]); + }); +}); + +describe('reconstructBlocks', () => { + const cell = (text: string, x: number, y: number) => item(text, x, y, 40, 10); + + it('emits a table for aligned multi-column rows', () => { + const items: TextItem[] = [ + cell('Item', 10, 100), cell('Q1', 200, 100), cell('Q2', 300, 100), + cell('Rev', 10, 120), cell('100', 200, 120), cell('150', 300, 120), + cell('Cost', 10, 140), cell('40', 200, 140), cell('55', 300, 140), + ]; + const blocks = reconstructBlocks(items); + const table = blocks.find(b => b.type === 'table'); + expect(table).toBeDefined(); + if (table && table.type === 'table') { + expect(table.rows).toEqual([ + ['Item', 'Q1', 'Q2'], + ['Rev', '100', '150'], + ['Cost', '40', '55'], + ]); + } + }); + + it('keeps ordinary prose as paragraphs', () => { + const items: TextItem[] = [ + item('This is a normal sentence of text.', 10, 100, 300, 10), + item('Another line right below it.', 10, 118, 260, 10), + ]; + const blocks = reconstructBlocks(items); + expect(blocks.every(b => b.type === 'paragraph')).toBe(true); + }); +}); diff --git a/src/tools/documents/pdf-docx.lib.ts b/src/tools/documents/pdf-docx.lib.ts index b8e0e13..ae23729 100644 --- a/src/tools/documents/pdf-docx.lib.ts +++ b/src/tools/documents/pdf-docx.lib.ts @@ -119,6 +119,144 @@ export function reconstruct(items: TextItem[]): DocParagraph[] { return paragraphsFromLines(groupLines(items)); } +/* ------------------------------------------------------------------ * + * Table-aware reconstruction: detect grid-like regions (rows whose text + * splits into columns that line up across several rows) and emit them as + * tables, with everything else falling back to paragraphs. + * ------------------------------------------------------------------ */ + +export interface Segment { x: number; text: string; } +interface ItemRow { y: number; height: number; segs: Segment[]; } + +export type DocBlock = + | { type: 'paragraph'; text: string; heading: 0 | 1 | 2 } + | { type: 'table'; rows: string[][] }; + +/** Group items into rows, keeping each row's items (not just joined text). */ +function groupItemRows(items: TextItem[], yTolRatio = 0.5): TextItem[][] { + // Drop whitespace-only items — pdf.js emits synthetic space items between + // cells that would otherwise bridge column gaps and merge table cells. + const valid = items.filter((it) => it.text.trim().length > 0); + if (!valid.length) return []; + const medH = median(valid.map((i) => i.height).filter((h) => h > 0)) || 1; + const tol = medH * yTolRatio; + const sorted = [...valid].sort((a, b) => a.y - b.y || a.x - b.x); + const groups: TextItem[][] = []; + for (const it of sorted) { + const last = groups[groups.length - 1]; + const lastY = last ? last.reduce((s, g) => s + g.y, 0) / last.length : 0; + if (last && Math.abs(it.y - lastY) <= tol) last.push(it); + else groups.push([it]); + } + return groups.map((g) => [...g].sort((a, b) => a.x - b.x)); +} + +/** Split a row's items into cell segments, breaking on wide (column) gaps. */ +export function segmentsOf(lineItems: TextItem[], gapFactor = 1): Segment[] { + const sorted = [...lineItems].sort((a, b) => a.x - b.x); + const segs: Segment[] = []; + let text = ''; + let x = 0; + let prevEnd: number | null = null; + for (const it of sorted) { + if (it.text.trim().length === 0) continue; // skip synthetic whitespace items + if (prevEnd !== null && it.x - prevEnd > it.height * gapFactor) { + if (text.trim()) segs.push({ x, text: text.trim() }); + text = ''; + } + if (text === '') x = it.x; + else if (prevEnd !== null && it.x - prevEnd > it.height * 0.2 && !/\s$/.test(text)) text += ' '; + text += it.text; + prevEnd = it.x + it.width; + } + if (text.trim()) segs.push({ x, text: text.trim() }); + return segs; +} + +/** Cluster x positions into column centres (1-D agglomerative by tolerance). */ +export function clusterColumns(xs: number[], tol: number): number[] { + const sorted = [...xs].sort((a, b) => a - b); + const cols: number[][] = []; + for (const v of sorted) { + const last = cols[cols.length - 1]; + const c = last ? last.reduce((s, n) => s + n, 0) / last.length : 0; + if (last && Math.abs(v - c) <= tol) last.push(v); + else cols.push([v]); + } + return cols.map((c) => c.reduce((s, n) => s + n, 0) / c.length); +} + +const nearestCol = (x: number, cols: number[]): number => { + let best = 0; + let bestD = Infinity; + for (let i = 0; i < cols.length; i++) { + const d = Math.abs(x - cols[i]); + if (d < bestD) { bestD = d; best = i; } + } + return best; +}; + +/** + * Reconstruct a page's text items into a mix of paragraph and table blocks. + * A table is a run of ≥2 adjacent rows that each split into ≥2 aligned columns. + */ +export function reconstructBlocks(items: TextItem[]): DocBlock[] { + const itemRows = groupItemRows(items); + if (!itemRows.length) return []; + const medH = median(itemRows.flat().map((i) => i.height).filter((h) => h > 0)) || 1; + + const rows: ItemRow[] = itemRows.map((g) => ({ + y: g.reduce((s, i) => s + i.y, 0) / g.length, + height: Math.max(...g.map((i) => i.height)), + segs: segmentsOf(g), + })); + + const blocks: DocBlock[] = []; + let paraLines: DocLine[] = []; + const flushParas = () => { + if (!paraLines.length) return; + for (const p of paragraphsFromLines(paraLines)) blocks.push({ type: 'paragraph', ...p }); + paraLines = []; + }; + const asLine = (r: ItemRow): DocLine => ({ + text: r.segs.map((s) => s.text).join(' ').replace(/\s+/g, ' ').trim(), + x: r.segs.length ? r.segs[0].x : 0, + y: r.y, + height: r.height, + }); + + let i = 0; + while (i < rows.length) { + // A table candidate starts on a multi-column row. + if (rows[i].segs.length >= 2) { + let j = i + 1; + while (j < rows.length && rows[j].segs.length >= 2 && rows[j].y - rows[j - 1].y <= medH * 2.5) j++; + const region = rows.slice(i, j); + if (region.length >= 2) { + const cols = clusterColumns(region.flatMap((r) => r.segs.map((s) => s.x)), medH * 1.5); + if (cols.length >= 2) { + flushParas(); + const grid = region.map((r) => { + const cells = Array(cols.length).fill(''); + for (const seg of r.segs) { + const c = nearestCol(seg.x, cols); + cells[c] = cells[c] ? `${cells[c]} ${seg.text}` : seg.text; + } + return cells; + }); + blocks.push({ type: 'table', rows: grid }); + i = j; + continue; + } + } + } + paraLines.push(asLine(rows[i])); + i++; + } + flushParas(); + return blocks; +} + /** Count of non-whitespace characters — used to decide if a page needs OCR. */ export function textDensity(items: TextItem[]): number { return items.reduce((n, it) => n + it.text.replace(/\s/g, '').length, 0);