From 540bc339c91f5aab308796e91dbae286d540fa13 Mon Sep 17 00:00:00 2001 From: Ryan Eno Date: Wed, 12 Aug 2026 09:09:36 -0700 Subject: [PATCH 01/15] Propose anydoc input processor --- .../changes/add-anydoc-input/.openspec.yaml | 2 + openspec/changes/add-anydoc-input/design.md | 90 +++++++++++++++++++ openspec/changes/add-anydoc-input/proposal.md | 29 ++++++ .../specs/anydoc-input/spec.md | 77 ++++++++++++++++ .../specs/file-document-import/spec.md | 77 ++++++++++++++++ openspec/changes/add-anydoc-input/tasks.md | 26 ++++++ 6 files changed, 301 insertions(+) create mode 100644 openspec/changes/add-anydoc-input/.openspec.yaml create mode 100644 openspec/changes/add-anydoc-input/design.md create mode 100644 openspec/changes/add-anydoc-input/proposal.md create mode 100644 openspec/changes/add-anydoc-input/specs/anydoc-input/spec.md create mode 100644 openspec/changes/add-anydoc-input/specs/file-document-import/spec.md create mode 100644 openspec/changes/add-anydoc-input/tasks.md diff --git a/openspec/changes/add-anydoc-input/.openspec.yaml b/openspec/changes/add-anydoc-input/.openspec.yaml new file mode 100644 index 0000000..5081c98 --- /dev/null +++ b/openspec/changes/add-anydoc-input/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-08-12 diff --git a/openspec/changes/add-anydoc-input/design.md b/openspec/changes/add-anydoc-input/design.md new file mode 100644 index 0000000..9d98223 --- /dev/null +++ b/openspec/changes/add-anydoc-input/design.md @@ -0,0 +1,90 @@ +## Context + +Local file imports already resolve concrete paths and recursive glob patterns in `src/input.rs`. The `Input::FileDocuments` variant reads each path lazily, constructs a JSON object, and sends it through the same `Box` output path used by NDJSON, CSV, and other inputs. Today, recognized text formats have specialized readers and all other file-document paths fall back to UTF-8 text, which rejects PDFs and office containers. + +The `anydoc` crate provides a Rust-native conversion API for PDF, Word, PowerPoint, Excel, OpenDocument, RTF, and EPUB inputs. Its Markdown output is the appropriate intermediate representation because the existing file-document implementation already defines the desired content field, Markdown frontmatter behavior, and multi-file metadata. + +## Goals / Non-Goals + +**Goals:** + +- Convert supported local anydoc formats into the existing file-document JSON shape. +- Preserve the existing output dispatch and behavior for Markdown, text, YAML, JSON, NDJSON, Toon, CSV, stdin, and HTTPS inputs. +- Support direct files, shell-expanded file lists, and existing recursive glob patterns without changing discovery semantics. +- Keep conversion errors path-specific and visible through the existing stderr error path. +- Cover representative document formats and mixed file collections with tests. + +**Non-Goals:** + +- OCR for scanned or image-only PDFs. +- Remote HTTPS anydoc inputs. +- A new `--extensions` option; multiple extension patterns can be passed as existing input positionals. +- Extraction of embedded assets or source-specific metadata beyond the Markdown produced by anydoc and existing `file.*` metadata. +- Content-based conversion of unknown-extension files, which could change the existing unknown UTF-8 text behavior. + +## Decisions + +### Use the `anydoc` crate directly + +Add `anydoc` as a normal dependency and call its Rust API rather than spawning the anydoc CLI. This avoids subprocess lifecycle and temporary-file concerns and keeps conversion inside the existing input processor. The dependency is compatible with the repository's Rust 1.88 baseline. + +Alternatives considered: + +- **Invoke the anydoc CLI:** rejected because it adds an external executable/runtime dependency and makes error handling and streaming less direct. +- **Add format-specific parsers to espipe:** rejected because it duplicates the purpose of anydoc and expands the maintenance surface. + +### Gate conversion by recognized non-CSV extension + +In `read_file_documents`, check `anydoc::Format::from_path(path)` after the existing specialized readers. If it identifies a supported format other than `Format::Csv`, convert the file with `anydoc::to_markdown(path)`. Keep existing readers ahead of this branch so CSV and all current text/document formats retain their behavior. + +The extension gate intentionally avoids running content detection on every unknown file. Unknown valid UTF-8 files remain text documents, while recognized anydoc extensions opt into conversion. Mislabeled-format support can be considered separately if needed. + +### Reuse the Markdown document builder + +Refactor the existing Markdown reader so it has a helper that accepts `(path, markdown_text, content_field, include_file_metadata)`. The current Markdown path reads the source text and calls this helper; the anydoc path converts the source and calls the same helper. + +This preserves: + +- `content.` placement and `--content` behavior. +- YAML frontmatter extraction and content-field conflict checks. +- `file.path` and `file.name` inclusion for multi-file imports. +- One serialized `Box` per output document. + +The resulting flow is: + +```text +path/glob → existing path resolution → format-specific input branch + ├─ Markdown → Markdown document builder + ├─ anydoc format → anydoc Markdown → same builder + └─ existing text/JSON/YAML/etc. readers + → Box → existing output +``` + +### Keep discovery unchanged + +No new traversal or extension-selection option is needed. Existing commands such as `espipe '**/*.pdf' output.ndjson` work because unknown local extensions already enter file-document mode, and multiple types can be supplied as separate patterns. A future `--extensions pdf,xls,doc` option should be designed as a discovery/filtering feature rather than coupled to the converter. + +### Preserve lazy conversion + +Convert files when `read_file_document_line` reaches them, matching the current lazy file-document behavior. This avoids loading an entire collection before output begins and keeps anydoc conversion scoped to the selected file. Conversion failures use the existing input error path and include the source path. + +### Preserve anydoc error wording + +Use anydoc's conversion error wording as-is, adding only the source path context needed to identify the failing input. This avoids introducing a second error taxonomy while preserving the underlying reason for failures. Error normalization can be added later if callers need a stable espipe-specific code or prefix. + +## Risks / Trade-offs + +- **Dependency and binary growth** → Accept the direct dependency for broad format coverage; document the build impact and verify the normal test/build path. +- **Full-file memory use** → Keep the existing one-file-at-a-time document model; anydoc already returns a Markdown `String`, and the resulting raw document is released before the next file is read. +- **Scanned PDFs fail without OCR** → Surface anydoc's unsupported conversion error with the source path; do not claim OCR support. +- **Failures can occur after earlier documents were sent** → Retain existing lazy input semantics and error reporting; changing ingestion to preflight every file is outside this change. +- **Generated Markdown may contain frontmatter-like syntax** → Run it through the established Markdown builder for consistent document semantics; add a fixture-based regression test for representative anydoc output. +- **Unknown-extension detection remains extension-gated** → Preserve current unknown UTF-8 behavior; consider content-based detection only as a separate, explicitly scoped change. + +## Migration Plan + +No data migration is required. Add the dependency and input branch, then release normally. Existing commands and output document shapes remain compatible. Rollback consists of removing the anydoc dependency, branch, tests, and change artifacts; no stored documents require migration. + +## Open Questions + +None. The initial regression suite will use one representative PDF and one Office/OpenDocument fixture; coverage of every supported format remains the responsibility of the anydoc crate. diff --git a/openspec/changes/add-anydoc-input/proposal.md b/openspec/changes/add-anydoc-input/proposal.md new file mode 100644 index 0000000..ea5a7c7 --- /dev/null +++ b/openspec/changes/add-anydoc-input/proposal.md @@ -0,0 +1,29 @@ +## Why + +`espipe` currently treats local file documents as text, so PDFs and office formats cannot be ingested even though the existing glob-based file import pipeline already supports recursive document collections. The Rust `anydoc` library can convert these formats to GitHub-Flavored Markdown, allowing non-text ingestion without changing the downstream document or output model. + +## What Changes + +- Add `anydoc` as a local file input preprocessor for supported non-text formats, including PDF, Word, PowerPoint, Excel, OpenDocument, RTF, and EPUB files. +- Convert each supported file to Markdown before constructing the existing file document, preserving `content.`, Markdown handling, and conditional `file.*` metadata. +- Allow existing concrete file lists and recursive glob patterns such as `**/*.pdf` to ingest converted documents. +- Preserve existing CSV, JSON, NDJSON, Toon, Markdown, YAML, text, stdin, and HTTPS behavior. +- Report conversion failures with the source path and keep remote HTTPS inputs out of scope. +- Do not add `--extensions` in this change; multiple extension patterns can already be supplied as separate local inputs. A future extension-filter option can build on the existing discovery layer. + +## Capabilities + +### New Capabilities + +- `anydoc-input`: Convert supported local non-text documents to Markdown within the input pipeline. + +### Modified Capabilities + +- `file-document-import`: Supported anydoc formats are imported as converted Markdown documents instead of being rejected as binary files. + +## Impact + +- Affects `src/input.rs` only in the ingestion path and adds the `anydoc` crate dependency. +- Adds fixtures and unit/integration coverage for representative converted formats, mixed globs, metadata preservation, and conversion errors. +- Increases dependency graph, compile time, and binary size because anydoc includes parsers for office containers and PDFs. +- Image-only/scanned PDFs remain unsupported because conversion does not provide OCR. diff --git a/openspec/changes/add-anydoc-input/specs/anydoc-input/spec.md b/openspec/changes/add-anydoc-input/specs/anydoc-input/spec.md new file mode 100644 index 0000000..a68f798 --- /dev/null +++ b/openspec/changes/add-anydoc-input/specs/anydoc-input/spec.md @@ -0,0 +1,77 @@ +## ADDED Requirements + +### Requirement: Supported local non-text files are converted through anydoc + +The system SHALL use the Rust `anydoc` processor for local regular files with these extensions: `.doc`, `.docx`, `.docm`, `.odt`, `.pdf`, `.ppt`, `.pps`, `.pot`, `.pptx`, `.pptm`, `.ppsx`, `.ppsm`, `.rtf`, `.epub`, `.xls`, `.xlsx`, `.xlsm`, `.xlsb`, `.ods`, and `.odp`. The processor SHALL convert each file to GitHub-Flavored Markdown before file-document construction. + +#### Scenario: A PDF file is imported + +- **WHEN** the user runs `espipe` with a local text-based `.pdf` input +- **THEN** the system converts the PDF to Markdown through anydoc +- **AND** emits one JSON document containing the converted Markdown in the configured content field + +#### Scenario: An office document is imported + +- **WHEN** the user runs `espipe` with a local supported Word, PowerPoint, Excel, or OpenDocument input +- **THEN** the system converts the file to Markdown through anydoc +- **AND** emits one JSON document for the source file + +#### Scenario: A supported file is imported from a mixed local collection + +- **WHEN** file-document input resolves supported anydoc files together with Markdown or text files +- **THEN** each anydoc file is converted at its position in the deterministic file order +- **AND** existing Markdown and text files continue through their existing readers + +### Requirement: Anydoc conversion preserves the existing file-document shape + +The system SHALL construct converted documents using the existing file-document semantics after Markdown conversion. It SHALL store converted Markdown under `content.`, apply the configured `--content` value, and add `file.path` and `file.name` only under the existing multi-file rules. + +#### Scenario: Default content field is used for converted Markdown + +- **WHEN** a supported anydoc file is imported without `--content` +- **THEN** the converted Markdown is stored in `content.body` +- **AND** the document does not expose the original binary bytes as a field + +#### Scenario: Custom content field is used for converted Markdown + +- **WHEN** a supported anydoc file is imported with `--content markdown` +- **THEN** the converted Markdown is stored in `content.markdown` +- **AND** the system does not add `content.body` solely because the source was converted + +#### Scenario: Converted files participate in multi-file metadata + +- **WHEN** anydoc files are imported together with another file-document input +- **THEN** each converted document includes the same `file.path` and `file.name` metadata as existing file documents +- **AND** the metadata values identify the original local file + +### Requirement: Existing local discovery mechanisms support anydoc files + +The system SHALL process supported anydoc files supplied as direct local paths, shell-expanded file lists, or existing local glob patterns. It SHALL not require a separate subprocess or remote service for local conversion. + +#### Scenario: A quoted recursive PDF glob is imported + +- **WHEN** the user runs `espipe '**/*.pdf' output.ndjson` +- **THEN** the existing glob resolver finds matching regular files +- **AND** each matching PDF is converted and emitted as one file document + +#### Scenario: Multiple extension patterns are supplied + +- **WHEN** the user supplies separate local input patterns such as `**/*.pdf`, `**/*.xls`, and `**/*.doc` +- **THEN** the system combines and de-duplicates the resolved paths using existing file discovery rules +- **AND** converts each supported path according to its extension + +### Requirement: Anydoc conversion failures identify the source file + +The system SHALL report anydoc conversion failures through the existing file-input error path, including the source path and the underlying conversion reason when available. It SHALL not emit a synthetic document for a file that anydoc cannot convert. + +#### Scenario: An unsupported document is encountered + +- **WHEN** anydoc reports that a supported-extension file is encrypted, malformed, unsupported, or exceeds a conversion limit +- **THEN** ingestion fails with a diagnostic identifying the source path +- **AND** the diagnostic is written to stderr + +#### Scenario: An image-only PDF is encountered + +- **WHEN** anydoc cannot extract meaningful text from a scanned or image-only PDF +- **THEN** ingestion fails with a path-specific unsupported-conversion diagnostic +- **AND** the system does not claim to perform OCR diff --git a/openspec/changes/add-anydoc-input/specs/file-document-import/spec.md b/openspec/changes/add-anydoc-input/specs/file-document-import/spec.md new file mode 100644 index 0000000..07d4a33 --- /dev/null +++ b/openspec/changes/add-anydoc-input/specs/file-document-import/spec.md @@ -0,0 +1,77 @@ +## MODIFIED Requirements + +### Requirement: Local file inputs import documents by file format + +The system SHALL accept one or more local file inputs and import each regular file according to its file format, including conversion through anydoc for supported local non-text formats. + +#### Scenario: Single Markdown file is imported + +- **WHEN** the user runs `espipe` with a local Markdown file input +- **THEN** the system emits one document for that Markdown file + +#### Scenario: Single supported non-text file is imported + +- **WHEN** the user runs `espipe` with a local PDF or other supported anydoc file input +- **THEN** the system converts the file to Markdown +- **AND** the system emits one document for that file + +#### Scenario: Shell-expanded Markdown files are imported + +- **WHEN** the user's shell expands a file pattern into multiple Markdown file arguments before `espipe` starts +- **THEN** the system treats each file argument as an input +- **AND** it emits one document for each regular file + +#### Scenario: Multiple input positionals are provided + +- **WHEN** the user provides more than two positional arguments +- **THEN** the final positional argument is treated as the output URI +- **AND** every preceding positional argument is treated as an input + +### Requirement: Recursive glob inputs import matching files + +The system SHALL accept local glob input patterns, including recursive `**` patterns, and import each matched regular file according to its file format, including anydoc conversion for supported non-text files. + +#### Scenario: Recursive Markdown glob is imported + +- **WHEN** the user runs `espipe` with a local input pattern of `**/*.md` +- **THEN** the system expands the pattern recursively +- **AND** it emits one document for each matched Markdown file + +#### Scenario: Recursive PDF glob is imported + +- **WHEN** the user runs `espipe` with a local input pattern of `**/*.pdf` +- **THEN** the system expands the pattern recursively +- **AND** it converts and emits one document for each matched PDF file + +#### Scenario: Glob matches no files + +- **WHEN** the user provides a local glob pattern that matches no regular files +- **THEN** startup fails before sending any output +- **AND** the error identifies that the glob matched no files + +#### Scenario: Glob matches directories + +- **WHEN** a local glob pattern matches both regular files and directories +- **THEN** the system imports the matched regular files +- **AND** it does not emit documents for matched directories + +### Requirement: Binary files are rejected + +The system SHALL convert local binary files recognized by anydoc into Markdown documents. It SHALL reject file-document inputs that are neither valid UTF-8 text nor recognized supported anydoc formats. + +#### Scenario: Supported binary file is matched + +- **WHEN** a file-document input resolves to a PDF or supported office/container file recognized by anydoc +- **THEN** the system converts the file to Markdown +- **AND** it emits a document for the converted content + +#### Scenario: Unrecognized binary file is matched + +- **WHEN** a file-document input resolves to a binary file that anydoc does not recognize +- **THEN** importing that file fails +- **AND** the error identifies the file as unsupported binary or invalid UTF-8 input + +#### Scenario: Valid UTF-8 text remains supported + +- **WHEN** a file-document input resolves to an unknown-extension file whose contents are valid UTF-8 +- **THEN** the emitted document contains the full file content in the configured `content.` field diff --git a/openspec/changes/add-anydoc-input/tasks.md b/openspec/changes/add-anydoc-input/tasks.md new file mode 100644 index 0000000..4b085b1 --- /dev/null +++ b/openspec/changes/add-anydoc-input/tasks.md @@ -0,0 +1,26 @@ +## 1. Dependency and input integration + +- [ ] 1.1 Add the `anydoc` crate dependency at a Rust 1.88-compatible version. +- [ ] 1.2 Add extension-gated anydoc routing to `read_file_documents`, preserving the existing specialized readers and excluding CSV from the new branch. +- [ ] 1.3 Refactor Markdown file-document construction to accept converted Markdown text so source Markdown and anydoc output share the same content-field, frontmatter, and file-metadata behavior. +- [ ] 1.4 Add the anydoc conversion wrapper using `anydoc::to_markdown`, including source-path context and underlying conversion errors in diagnostics. +- [ ] 1.5 Verify direct paths, shell-expanded file lists, recursive globs, mixed anydoc/Markdown collections, deterministic ordering, and lazy conversion continue to use the existing discovery and output pipeline. + +## 2. AnyDoc format coverage and error behavior + +- [ ] 2.1 Add representative routing and conversion coverage for one text-based PDF and one Office or OpenDocument file; rely on anydoc's own test suite for coverage of its remaining formats. +- [ ] 2.2 Verify the anydoc routing is driven by the crate's recognized format API and excludes existing CSV handling. +- [ ] 2.3 Add tests for unsupported, malformed, encrypted, or image-only conversion failures and verify diagnostics identify the source path and are written to stderr. +- [ ] 2.4 Add a regression test proving unknown valid UTF-8 files and existing Markdown, YAML, JSON, NDJSON, Toon, CSV, stdin, and HTTPS inputs retain their current behavior. + +## 3. Output-shape and integration verification + +- [ ] 3.1 Verify converted documents use `content.body` by default and the configured `--content` field when specified. +- [ ] 3.2 Verify converted documents preserve existing `file.path` and `file.name` behavior for multi-file imports and omit it for a single direct file. +- [ ] 3.3 Verify mixed recursive file imports emit converted and existing documents in the same deterministic path order without changing output serialization. +- [ ] 3.4 Update README input documentation with supported anydoc formats, local-only behavior, glob examples, and the limitation that scanned PDFs require OCR outside espipe. + +## 4. Verification + +- [ ] 4.1 Run formatting and the complete Rust test suite. +- [ ] 4.2 Run the CLI/integration tests that write NDJSON output and confirm existing output consumers require no changes. From 6385a7ff7ec6f56c696e613f7d9d3028fcf290a5 Mon Sep 17 00:00:00 2001 From: Ryan Eno Date: Wed, 12 Aug 2026 11:43:07 -0700 Subject: [PATCH 02/15] Add anydoc file input processing --- Cargo.lock | 462 +++++++++++++++++++- Cargo.toml | 2 + README.md | 41 +- openspec/changes/add-anydoc-input/tasks.md | 30 +- src/input.rs | 164 ++++++- tests/file_output.rs | 118 +++++ tests/fixtures/anydoc/image-only.pdf.base64 | 1 + tests/fixtures/anydoc/sample.docx.base64 | 1 + tests/fixtures/anydoc/sample.pdf | Bin 0 -> 584 bytes tests/fixtures/anydoc/sample.rtf | 4 + 10 files changed, 796 insertions(+), 27 deletions(-) create mode 100644 tests/fixtures/anydoc/image-only.pdf.base64 create mode 100644 tests/fixtures/anydoc/sample.docx.base64 create mode 100644 tests/fixtures/anydoc/sample.pdf create mode 100644 tests/fixtures/anydoc/sample.rtf diff --git a/Cargo.lock b/Cargo.lock index 14e0791..dd298bc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8,6 +8,17 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures 0.2.17", +] + [[package]] name = "aho-corasick" version = "1.1.4" @@ -76,6 +87,23 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "anydoc" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b8e42cf43074578dd1d5a6771d1f28282790b2bf4bca60061ff51fd8ca03c09" +dependencies = [ + "calamine", + "cfb", + "csv", + "encoding_rs", + "flate2", + "log", + "pdf-inspector", + "quick-xml", + "zip", +] + [[package]] name = "anyhow" version = "1.0.102" @@ -91,7 +119,7 @@ dependencies = [ "asn1-rs-derive", "asn1-rs-impl", "displaydoc", - "nom", + "nom 7.1.3", "num-traits", "rusticata-macros", "thiserror", @@ -133,6 +161,16 @@ dependencies = [ "tokio", ] +[[package]] +name = "atoi_simd" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3cdb3708a128e559a30fb830e8a77a5022ee6902806925c216658652b452a44" +dependencies = [ + "debug_unsafe", + "rustversion", +] + [[package]] name = "atomic-waker" version = "1.1.2" @@ -188,6 +226,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block-padding" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93" +dependencies = [ + "generic-array", +] + [[package]] name = "borrow-or-share" version = "0.2.4" @@ -200,12 +247,45 @@ version = "3.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + [[package]] name = "bytes" version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" +[[package]] +name = "calamine" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fa68281b1a76b54a62156474adb06bb380a67e07dd60656e3217152b42183f3" +dependencies = [ + "atoi_simd", + "byteorder", + "chrono", + "codepage", + "encoding_rs", + "fast-float2", + "log", + "quick-xml", + "serde", + "zip", +] + +[[package]] +name = "cbc" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26b52a9543ae338f279b96b0b9fed9c8093744685043739079ce85cd58f289a6" +dependencies = [ + "cipher", +] + [[package]] name = "cc" version = "1.2.61" @@ -218,6 +298,17 @@ dependencies = [ "shlex", ] +[[package]] +name = "cfb" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a347dcabdae9c31b0825fd6a8bed285ec9c2acb89c47827126d52fa4f59cece3" +dependencies = [ + "fnv", + "uuid", + "web-time", +] + [[package]] name = "cfg-if" version = "1.0.4" @@ -230,6 +321,17 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" +[[package]] +name = "chacha20" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "rand_core 0.10.1", +] + [[package]] name = "chrono" version = "0.4.43" @@ -242,6 +344,16 @@ dependencies = [ "windows-link", ] +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common", + "inout", +] + [[package]] name = "clap" version = "4.6.1" @@ -291,6 +403,15 @@ dependencies = [ "cc", ] +[[package]] +name = "codepage" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48f68d061bc2828ae826206326e61251aca94c1e4a5305cf52d9138639c918b4" +dependencies = [ + "encoding_rs", +] + [[package]] name = "colorchoice" version = "1.0.5" @@ -359,6 +480,15 @@ dependencies = [ "libc", ] +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + [[package]] name = "crc32fast" version = "1.5.0" @@ -368,6 +498,31 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crossbeam-deque" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5181e0de7b61eb03a81e347d6dd8797bae9da5146707b51077e2d71a54ec0ceb" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" + [[package]] name = "crypto-common" version = "0.1.7" @@ -440,6 +595,12 @@ version = "2.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d7a1e2f27636f116493b8b860f5546edb47c8d8f8ea73e1d2a20be88e28d1fea" +[[package]] +name = "debug_unsafe" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7eed2c4702fa172d1ce21078faa7c5203e69f5394d48cc436d25928394a867a2" + [[package]] name = "der-parser" version = "10.0.0" @@ -448,7 +609,7 @@ checksum = "07da5016415d5a3c4dd39b11ed26f915f52fc4e0dc197d87908bc916e51bc1a6" dependencies = [ "asn1-rs", "displaydoc", - "nom", + "nom 7.1.3", "num-bigint", "num-traits", "rusticata-macros", @@ -497,6 +658,21 @@ version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" +[[package]] +name = "ecb" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a8bfa975b1aec2145850fcaa1c6fe269a16578c44705a532ae3edc92b8881c7" +dependencies = [ + "cipher", +] + +[[package]] +name = "either" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e5e8f6c15a24b9a3ee5efec809ccd006d3b30e8b3bb63c39af737c7f87daa1d" + [[package]] name = "elasticsearch" version = "9.1.0-alpha.1" @@ -572,6 +748,7 @@ dependencies = [ name = "espipe" version = "0.4.0" dependencies = [ + "anydoc", "base64", "bytes", "clap", @@ -607,6 +784,12 @@ dependencies = [ "once_cell", ] +[[package]] +name = "fast-float2" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8eb564c5c7423d25c886fb561d1e4ee69f72354d16918afa32c08811f6b6a55" + [[package]] name = "fastrand" version = "2.4.1" @@ -627,6 +810,7 @@ checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" dependencies = [ "crc32fast", "miniz_oxide", + "zlib-rs", ] [[package]] @@ -814,10 +998,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" dependencies = [ "cfg-if", + "js-sys", "libc", "r-efi 6.0.0", + "rand_core 0.10.1", "wasip2", "wasip3", + "wasm-bindgen", ] [[package]] @@ -1135,6 +1322,25 @@ dependencies = [ "icu_properties", ] +[[package]] +name = "include_dir" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "923d117408f1e49d914f1a379a309cffe4f18c05cf4e3d12e613a15fc81bd0dd" +dependencies = [ + "include_dir_macros", +] + +[[package]] +name = "include_dir_macros" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cab85a7ed0bd5f0e76d93846e0147172bed2e2d3f859bcc33a8d9699cad1a75" +dependencies = [ + "proc-macro2", + "quote", +] + [[package]] name = "indenter" version = "0.3.4" @@ -1164,6 +1370,16 @@ dependencies = [ "serde_core", ] +[[package]] +name = "inout" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +dependencies = [ + "block-padding", + "generic-array", +] + [[package]] name = "ipnet" version = "2.11.0" @@ -1199,10 +1415,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a3546dc96b6d42c5f24902af9e2538e82e39ad350b0c766eb3fbf2d8f3d8359" dependencies = [ "jiff-static", + "jiff-tzdb-platform", "log", "portable-atomic", "portable-atomic-util", "serde_core", + "windows-sys 0.61.2", ] [[package]] @@ -1216,6 +1434,21 @@ dependencies = [ "syn", ] +[[package]] +name = "jiff-tzdb" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "142bd39932ad231f10513df9ab62661fead8719872150b7ad02a2df79f4e141e" + +[[package]] +name = "jiff-tzdb-platform" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "875a5a69ac2bab1a891711cf5eccbec1ce0341ea805560dcd90b7a2e925132e8" +dependencies = [ + "jiff-tzdb", +] + [[package]] name = "jni" version = "0.22.4" @@ -1330,12 +1563,53 @@ version = "0.4.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" +[[package]] +name = "lopdf" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25aab26d99567469098e64a02f42679f8965c6401263eefa31d8f2dcc37a221c" +dependencies = [ + "aes", + "bitflags", + "cbc", + "chrono", + "ecb", + "encoding_rs", + "flate2", + "getrandom 0.4.2", + "indexmap 2.13.0", + "itoa", + "jiff", + "log", + "md-5", + "nom 8.0.0", + "rand 0.10.2", + "rangemap", + "rayon", + "sha2", + "stringprep", + "thiserror", + "time", + "ttf-parser", + "weezl", +] + [[package]] name = "lru-slab" version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest", +] + [[package]] name = "memchr" version = "2.8.0" @@ -1402,6 +1676,15 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "nom" +version = "8.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405" +dependencies = [ + "memchr", +] + [[package]] name = "num-bigint" version = "0.4.6" @@ -1530,6 +1813,24 @@ dependencies = [ "windows-link", ] +[[package]] +name = "pdf-inspector" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd2f755e49ad38eafbc82ba2bec1c59d57b5bf829b13a8f35e4263bc8913df3a" +dependencies = [ + "env_logger", + "include_dir", + "log", + "lopdf", + "once_cell", + "rayon", + "regex", + "thiserror", + "ttf-parser", + "unicode-normalization", +] + [[package]] name = "pem" version = "3.0.6" @@ -1665,6 +1966,16 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "quick-xml" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1" +dependencies = [ + "encoding_rs", + "memchr", +] + [[package]] name = "quinn" version = "0.11.9" @@ -1695,7 +2006,7 @@ dependencies = [ "bytes", "getrandom 0.3.4", "lru-slab", - "rand", + "rand 0.9.4", "ring", "rustc-hash", "rustls", @@ -1749,7 +2060,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" dependencies = [ "rand_chacha", - "rand_core", + "rand_core 0.9.5", +] + +[[package]] +name = "rand" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +dependencies = [ + "chacha20", + "getrandom 0.4.2", + "rand_core 0.10.1", ] [[package]] @@ -1759,7 +2081,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" dependencies = [ "ppv-lite86", - "rand_core", + "rand_core 0.9.5", ] [[package]] @@ -1771,6 +2093,38 @@ dependencies = [ "getrandom 0.3.4", ] +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "rangemap" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "973443cf09a9c8656b574a866ab68dfa19f0867d0340648c7d2f6a71b8a8ea68" + +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + [[package]] name = "rcgen" version = "0.14.7" @@ -1954,7 +2308,7 @@ version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "faf0c4a6ece9950b9abdb62b1cfcf2a68b3b67a10ba445b3bb85be2a293d0632" dependencies = [ - "nom", + "nom 7.1.3", ] [[package]] @@ -2266,7 +2620,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "digest", ] @@ -2326,6 +2680,17 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" +[[package]] +name = "stringprep" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1" +dependencies = [ + "unicode-bidi", + "unicode-normalization", + "unicode-properties", +] + [[package]] name = "strsim" version = "0.11.1" @@ -2625,6 +2990,18 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "ttf-parser" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2df906b07856748fa3f6e0ad0cbaa047052d4a7dd609e231c4f72cee8c36f31" + +[[package]] +name = "typed-path" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e28f89b80c87b8fb0cf04ab448d5dd0dd0ade2f8891bae878de66a75a28600e" + [[package]] name = "typenum" version = "1.20.0" @@ -2637,12 +3014,33 @@ version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971" +[[package]] +name = "unicode-bidi" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" + [[package]] name = "unicode-ident" version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +[[package]] +name = "unicode-normalization" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-properties" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d" + [[package]] name = "unicode-xid" version = "0.2.6" @@ -2686,6 +3084,16 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" +[[package]] +name = "uuid" +version = "1.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf3923a6f5c4c6382e0b653c4117f48d631ea17f38ed86e2a828e6f7412f5239" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "vcpkg" version = "0.2.15" @@ -2869,6 +3277,12 @@ dependencies = [ "rustls-pki-types", ] +[[package]] +name = "weezl" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a28ac98ddc8b9274cb41bb4d9d4d5c425b6020c50c46f25559911905610b4a88" + [[package]] name = "winapi-util" version = "0.1.11" @@ -3208,7 +3622,7 @@ dependencies = [ "data-encoding", "der-parser", "lazy_static", - "nom", + "nom 7.1.3", "oid-registry", "ring", "rusticata-macros", @@ -3328,8 +3742,40 @@ dependencies = [ "syn", ] +[[package]] +name = "zip" +version = "8.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d04a6b5381502aa6087c94c669499eb1602eb9c5e8198e534de571f7154809b" +dependencies = [ + "crc32fast", + "flate2", + "indexmap 2.13.0", + "memchr", + "typed-path", + "zopfli", +] + +[[package]] +name = "zlib-rs" +version = "0.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34b31d188d9d685a4f9c7b46d6e36631b07058d2cfe190267adce54dc230bf12" + [[package]] name = "zmij" version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" + +[[package]] +name = "zopfli" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f05cd8797d63865425ff89b5c4a48804f35ba0ce8d125800027ad6017d2b5249" +dependencies = [ + "bumpalo", + "crc32fast", + "log", + "simd-adler32", +] diff --git a/Cargo.toml b/Cargo.toml index 17e8cf9..4fb786c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,12 +19,14 @@ include = [ "README.md", "src/**/*.rs", "tests/**/*.rs", + "tests/fixtures/anydoc/*", "tests/fixtures/*.toon", "tests/fixtures/*.ndjson", "tests/fixtures/*.ndjson.gz", ] [dependencies] +anydoc = "0.1.8" base64 = "0.22.1" clap = { version = "^4.6.1", features = ["derive"] } csv = "^1.4.0" diff --git a/README.md b/README.md index 1e300d2..7940273 100644 --- a/README.md +++ b/README.md @@ -65,6 +65,9 @@ To build and publish a multi-platform Docker Hub image: - `.json` files - `.csv` files - `.csv.gz` files +- local Markdown and text files +- local PDF, Word, PowerPoint, Excel, OpenDocument, RTF, and EPUB files +- local recursive glob patterns for file documents - `stdin` as NDJSON It writes records to: @@ -78,11 +81,10 @@ When writing to Elasticsearch, `espipe` batches documents into groups of 5,000 r ## CLI Reference ```bash -Usage: espipe [OPTIONS] +Usage: espipe [OPTIONS] ... Arguments: - The input URI to read docs from - The output URI to send docs to + ... Input URI(s) followed by the output URI Options: -k, --insecure Ignore certificate validation @@ -91,6 +93,7 @@ Options: -p, --password Password for basic authentication -q, --quiet Quiet mode, don't print runtime summary -z, --uncompressed Disable request body gzip compression + --content Content subfield name for file imports [default: body] --action Bulk action for Elasticsearch outputs [default: create] [possible values: create, index, update] --batch-size Documents per Elasticsearch bulk request [default: 5000] --max-requests Maximum concurrent Elasticsearch bulk requests [default: 16] @@ -125,9 +128,25 @@ Both positional arguments are parsed as URI-like strings. Reads CSV from a `file://` URI. - `file:///absolute/path/to/file.csv.gz` Reads gzip-compressed CSV from a `file://` URI. +- `path/to/file.pdf` + Converts a local PDF to Markdown and imports it as one file document. +- `path/to/file.docx` + Converts a local Word document to Markdown and imports it as one file document. +- `'docs/**/*.pdf'` + Recursively finds local PDFs and converts each one to a file document. +- `path/to/file.pdf path/to/file.xlsx output.ndjson` + Imports multiple local file inputs in deterministic path order. HTTPS input URIs are supported for unauthenticated remote `.csv`, `.ndjson`, and `.json` sources. URLs without a supported file extension can still be accepted when the response `Content-Type` maps to CSV or NDJSON-oriented JSON input. +### AnyDoc local documents + +Local files with these extensions are converted to GitHub-Flavored Markdown through anydoc before entering the existing file-document pipeline: + +`.doc`, `.docx`, `.docm`, `.odt`, `.pdf`, `.ppt`, `.pps`, `.pot`, `.pptx`, `.pptm`, `.ppsx`, `.ppsm`, `.rtf`, `.epub`, `.xls`, `.xlsx`, `.xlsm`, `.xlsb`, `.ods`, and `.odp`. + +Converted content is stored in `content.body` by default. Use `--content markdown` to store it in `content.markdown`. Multiple local inputs add the existing `file.path` and `file.name` metadata. Anydoc conversion is local-only; remote HTTPS inputs are unchanged. Scanned or image-only PDFs require OCR outside espipe and are not converted. + ### Supported output forms - `-` @@ -163,6 +182,10 @@ The first row must be a header row. Each subsequent row is converted into a JSON CSV values are emitted as JSON strings. `espipe` does not infer numeric, boolean, or date types from CSV input. +### File-document input + +Markdown, text, YAML, and anydoc-converted files are emitted as JSON documents through the existing file-document pipeline. Markdown frontmatter remains available under `content.*`, and converted non-text files expose their generated Markdown under the configured content field. Existing file discovery supports shell-expanded paths, multiple local input positionals, and quoted recursive glob patterns. + ### Bulk actions `espipe` supports three Elasticsearch bulk actions: @@ -281,6 +304,18 @@ cat docs.ndjson | espipe - http://localhost:9200/my-index espipe users.csv output.ndjson ``` +### Ingest local PDFs recursively + +```bash +espipe '**/*.pdf' http://localhost:9200/documents +``` + +### Ingest multiple local document formats + +```bash +espipe '**/*.pdf' '**/*.docx' '**/*.xlsx' output.ndjson +``` + ### Read and write gzip-compressed files ```bash diff --git a/openspec/changes/add-anydoc-input/tasks.md b/openspec/changes/add-anydoc-input/tasks.md index 4b085b1..6e58317 100644 --- a/openspec/changes/add-anydoc-input/tasks.md +++ b/openspec/changes/add-anydoc-input/tasks.md @@ -1,26 +1,26 @@ ## 1. Dependency and input integration -- [ ] 1.1 Add the `anydoc` crate dependency at a Rust 1.88-compatible version. -- [ ] 1.2 Add extension-gated anydoc routing to `read_file_documents`, preserving the existing specialized readers and excluding CSV from the new branch. -- [ ] 1.3 Refactor Markdown file-document construction to accept converted Markdown text so source Markdown and anydoc output share the same content-field, frontmatter, and file-metadata behavior. -- [ ] 1.4 Add the anydoc conversion wrapper using `anydoc::to_markdown`, including source-path context and underlying conversion errors in diagnostics. -- [ ] 1.5 Verify direct paths, shell-expanded file lists, recursive globs, mixed anydoc/Markdown collections, deterministic ordering, and lazy conversion continue to use the existing discovery and output pipeline. +- [x] 1.1 Add the `anydoc` crate dependency at a Rust 1.88-compatible version. +- [x] 1.2 Add extension-gated anydoc routing to `read_file_documents`, preserving the existing specialized readers and excluding CSV from the new branch. +- [x] 1.3 Refactor Markdown file-document construction to accept converted Markdown text so source Markdown and anydoc output share the same content-field, frontmatter, and file-metadata behavior. +- [x] 1.4 Add the anydoc conversion wrapper using `anydoc::to_markdown`, including source-path context and underlying conversion errors in diagnostics. +- [x] 1.5 Verify direct paths, shell-expanded file lists, recursive globs, mixed anydoc/Markdown collections, deterministic ordering, and lazy conversion continue to use the existing discovery and output pipeline. ## 2. AnyDoc format coverage and error behavior -- [ ] 2.1 Add representative routing and conversion coverage for one text-based PDF and one Office or OpenDocument file; rely on anydoc's own test suite for coverage of its remaining formats. -- [ ] 2.2 Verify the anydoc routing is driven by the crate's recognized format API and excludes existing CSV handling. -- [ ] 2.3 Add tests for unsupported, malformed, encrypted, or image-only conversion failures and verify diagnostics identify the source path and are written to stderr. -- [ ] 2.4 Add a regression test proving unknown valid UTF-8 files and existing Markdown, YAML, JSON, NDJSON, Toon, CSV, stdin, and HTTPS inputs retain their current behavior. +- [x] 2.1 Add representative routing and conversion coverage for one text-based PDF and one Office or OpenDocument file; rely on anydoc's own test suite for coverage of its remaining formats. +- [x] 2.2 Verify the anydoc routing is driven by the crate's recognized format API and excludes existing CSV handling. +- [x] 2.3 Add representative malformed and image-only conversion failure tests and verify diagnostics identify the source path and are written to stderr. +- [x] 2.4 Add a regression test proving unknown valid UTF-8 files and existing Markdown, YAML, JSON, NDJSON, Toon, CSV, stdin, and HTTPS inputs retain their current behavior. ## 3. Output-shape and integration verification -- [ ] 3.1 Verify converted documents use `content.body` by default and the configured `--content` field when specified. -- [ ] 3.2 Verify converted documents preserve existing `file.path` and `file.name` behavior for multi-file imports and omit it for a single direct file. -- [ ] 3.3 Verify mixed recursive file imports emit converted and existing documents in the same deterministic path order without changing output serialization. -- [ ] 3.4 Update README input documentation with supported anydoc formats, local-only behavior, glob examples, and the limitation that scanned PDFs require OCR outside espipe. +- [x] 3.1 Verify converted documents use `content.body` by default and the configured `--content` field when specified. +- [x] 3.2 Verify converted documents preserve existing `file.path` and `file.name` behavior for multi-file imports and omit it for a single direct file. +- [x] 3.3 Verify mixed recursive file imports emit converted and existing documents in the same deterministic path order without changing output serialization. +- [x] 3.4 Update README input documentation with supported anydoc formats, local-only behavior, glob examples, and the limitation that scanned PDFs require OCR outside espipe. ## 4. Verification -- [ ] 4.1 Run formatting and the complete Rust test suite. -- [ ] 4.2 Run the CLI/integration tests that write NDJSON output and confirm existing output consumers require no changes. +- [x] 4.1 Run formatting and the complete Rust test suite. +- [x] 4.2 Run the CLI/integration tests that write NDJSON output and confirm existing output consumers require no changes. diff --git a/src/input.rs b/src/input.rs index 642302d..f6cbe91 100644 --- a/src/input.rs +++ b/src/input.rs @@ -429,6 +429,11 @@ fn read_file_documents( Some("md" | "markdown") => { read_markdown_file_document(path, content_field, include_file_metadata) } + _ if anydoc::Format::from_path(path) + .is_some_and(|format| format != anydoc::Format::Csv) => + { + read_anydoc_file_document(path, content_field, include_file_metadata) + } _ => read_text_file_document(path, content_field, include_file_metadata), } } @@ -461,7 +466,25 @@ fn read_markdown_file_document( include_file_metadata: bool, ) -> Result>> { let text = read_text_file(path)?; - let (frontmatter, body) = split_markdown_frontmatter(&text); + read_markdown_text_document(path, &text, content_field, include_file_metadata) +} + +fn read_anydoc_file_document( + path: &Path, + content_field: &str, + include_file_metadata: bool, +) -> Result>> { + let markdown = anydoc::to_markdown(path).map_err(|err| eyre!("{}: {err}", path.display()))?; + read_markdown_text_document(path, &markdown, content_field, include_file_metadata) +} + +fn read_markdown_text_document( + path: &Path, + text: &str, + content_field: &str, + include_file_metadata: bool, +) -> Result>> { + let (frontmatter, body) = split_markdown_frontmatter(text); let mut content = Map::new(); if let Some(frontmatter) = frontmatter { content = yaml_mapping_to_json_map(frontmatter) @@ -913,6 +936,7 @@ mod tests { fetch_remote_input_with_client, input_kind_from_path, local_input_kind, open_input_values, validate_content_field, validate_ndjson_file, }; + use base64::Engine as _; use flate2::{Compression, write::GzEncoder}; use fluent_uri::UriRef; use reqwest::blocking::Client; @@ -973,6 +997,14 @@ mod tests { .join(name) } + fn write_base64_fixture(source: &str, path: &PathBuf) { + let encoded = fs::read_to_string(fixture_path(source)).unwrap(); + let bytes = base64::engine::general_purpose::STANDARD + .decode(encoded.trim()) + .unwrap(); + fs::write(path, bytes).unwrap(); + } + fn write_gzip(path: &PathBuf, contents: &str) { let file = fs::File::create(path).unwrap(); let mut encoder = GzEncoder::new(file, Compression::default()); @@ -1125,6 +1157,136 @@ mod tests { ); } + #[test] + fn anydoc_converts_pdf_to_default_markdown_document() { + let path = fixture_path("anydoc/sample.pdf"); + let values = collect_values(Input::try_from(uri(&path)).unwrap()); + + assert_eq!(values.len(), 1); + assert!( + values[0]["content"]["body"] + .as_str() + .unwrap() + .contains("Hello PDF") + ); + assert!(values[0].get("file").is_none()); + } + + #[test] + fn anydoc_converts_rtf_to_custom_content_field() { + let path = fixture_path("anydoc/sample.rtf"); + let values = collect_values(open_input_values(vec![uri(&path)], "markdown").unwrap()); + + assert!( + values[0]["content"]["markdown"] + .as_str() + .unwrap() + .contains("Hello from RTF") + ); + assert!(values[0]["content"].get("body").is_none()); + } + + #[test] + fn anydoc_converts_office_docx_fixture() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("sample.docx"); + write_base64_fixture("anydoc/sample.docx.base64", &path); + + let values = collect_values(Input::try_from(uri(&path)).unwrap()); + + assert_eq!(values.len(), 1); + assert!( + values[0]["content"]["body"] + .as_str() + .unwrap() + .contains("Hello from DOCX") + ); + } + + #[test] + fn anydoc_mixed_file_import_preserves_order_and_file_metadata() { + let pdf = fixture_path("anydoc/sample.pdf"); + let rtf = fixture_path("anydoc/sample.rtf"); + let values = collect_values(open_input_values(vec![uri(&rtf), uri(&pdf)], "body").unwrap()); + + assert_eq!(values.len(), 2); + assert_eq!(values[0]["file"]["name"], "sample.pdf"); + assert_eq!(values[1]["file"]["name"], "sample.rtf"); + assert!(values[0]["content"]["body"].is_string()); + assert!(values[1]["content"]["body"].is_string()); + } + + #[test] + fn anydoc_recursive_glob_imports_pdf_files() { + let dir = tempfile::tempdir().unwrap(); + let nested = dir.path().join("nested"); + fs::create_dir(&nested).unwrap(); + fs::copy(fixture_path("anydoc/sample.pdf"), nested.join("sample.pdf")).unwrap(); + let pattern = dir + .path() + .join("**") + .join("*.pdf") + .to_string_lossy() + .into_owned(); + + let values = collect_values( + open_input_values(vec![UriRef::parse(pattern).unwrap()], "body").unwrap(), + ); + + assert_eq!(values.len(), 1); + assert!(values[0]["content"]["body"].is_string()); + } + + #[test] + fn anydoc_multiple_extension_globs_combine_and_sort_inputs() { + let dir = tempfile::tempdir().unwrap(); + let pdf = dir.path().join("a.pdf"); + let rtf = dir.path().join("b.rtf"); + fs::copy(fixture_path("anydoc/sample.pdf"), &pdf).unwrap(); + fs::copy(fixture_path("anydoc/sample.rtf"), &rtf).unwrap(); + let pdf_pattern = dir.path().join("**/*.pdf").to_string_lossy().into_owned(); + let rtf_pattern = dir.path().join("**/*.rtf").to_string_lossy().into_owned(); + + let values = collect_values( + open_input_values( + vec![ + UriRef::parse(rtf_pattern).unwrap(), + UriRef::parse(pdf_pattern).unwrap(), + ], + "body", + ) + .unwrap(), + ); + + assert_eq!(values.len(), 2); + assert_eq!(values[0]["file"]["name"], "a.pdf"); + assert_eq!(values[1]["file"]["name"], "b.rtf"); + } + + #[test] + fn anydoc_conversion_error_includes_source_path() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("invalid.pdf"); + fs::write(&path, b"not a PDF").unwrap(); + + let err = read_err(open_input_values(vec![uri(&path)], "body")); + + assert!(err.contains("invalid.pdf")); + assert!(err.len() > "invalid.pdf".len()); + } + + #[test] + fn anydoc_rejects_image_only_pdf_with_ocr_diagnostic() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("image-only.pdf"); + write_base64_fixture("anydoc/image-only.pdf.base64", &path); + + let err = read_err(open_input_values(vec![uri(&path)], "body")); + + assert!(err.contains("image-only.pdf")); + assert!(err.contains("OCR is required")); + } + #[test] fn shell_expanded_files_are_sorted_deduplicated_and_include_metadata() { let dir = tempfile::tempdir().unwrap(); diff --git a/tests/file_output.rs b/tests/file_output.rs index 6ebf377..3ff80aa 100644 --- a/tests/file_output.rs +++ b/tests/file_output.rs @@ -1,3 +1,4 @@ +use base64::Engine as _; use flate2::read::GzDecoder; use serde_json::Value; use std::{ @@ -25,6 +26,14 @@ fn temp_output_path(filename: &str) -> PathBuf { dir.join(filename) } +fn write_base64_fixture(name: &str, path: &PathBuf) { + let encoded = fs::read_to_string(fixture_path(name)).expect("read base64 fixture"); + let bytes = base64::engine::general_purpose::STANDARD + .decode(encoded.trim()) + .expect("decode base64 fixture"); + fs::write(path, bytes).expect("write decoded fixture"); +} + fn validate_bulk_schema(lines: &[&str]) { assert!( lines.len() % 2 == 0, @@ -77,6 +86,115 @@ fn cli_writes_bulk_output_to_file() { validate_bulk_schema(&lines); } +#[test] +fn cli_converts_anydoc_pdf_to_existing_file_document_output() { + let input_path = fixture_path("anydoc").join("sample.pdf"); + let output_path = temp_output_path("anydoc.ndjson"); + + let status = Command::new(env!("CARGO_BIN_EXE_espipe")) + .arg(input_path) + .arg(&output_path) + .status() + .expect("run espipe"); + + assert!(status.success(), "espipe exited with failure"); + + let contents = fs::read_to_string(&output_path).expect("read output file"); + let document: Value = serde_json::from_str(contents.trim()).expect("document json"); + assert!( + document["content"]["body"] + .as_str() + .expect("body string") + .contains("Hello PDF") + ); + assert!(document.get("file").is_none()); +} + +#[test] +fn cli_converts_mixed_anydoc_and_markdown_inputs_without_changing_shape() { + let anydoc_path = fixture_path("anydoc").join("sample.pdf"); + let markdown_path = fixture_path("glob_docs").join("alpha.md"); + let output_path = temp_output_path("mixed-anydoc.ndjson"); + + let status = Command::new(env!("CARGO_BIN_EXE_espipe")) + .arg(anydoc_path) + .arg(markdown_path) + .arg(&output_path) + .arg("--content") + .arg("markdown") + .status() + .expect("run espipe"); + + assert!(status.success(), "espipe exited with failure"); + + let contents = fs::read_to_string(&output_path).expect("read output file"); + let documents: Vec = contents + .lines() + .filter(|line| !line.is_empty()) + .map(|line| serde_json::from_str(line).expect("document json")) + .collect(); + + assert_eq!(documents.len(), 2); + assert!(documents.iter().any(|document| { + document["content"]["markdown"] + .as_str() + .is_some_and(|body| body.contains("Hello PDF")) + })); + assert!(documents.iter().any(|document| { + document["content"]["markdown"] + .as_str() + .is_some_and(|body| body.contains("Alpha")) + })); + assert!(documents.iter().all(|document| { + document["file"]["path"].is_string() && document["file"]["name"].is_string() + })); +} + +#[test] +fn cli_reports_anydoc_conversion_errors_on_stderr_with_source_path() { + let input_path = temp_output_path("invalid.pdf"); + let output_path = temp_output_path("invalid-anydoc.ndjson"); + fs::write(&input_path, b"not a PDF").expect("write invalid input"); + + let output = Command::new(env!("CARGO_BIN_EXE_espipe")) + .arg(&input_path) + .arg(&output_path) + .output() + .expect("run espipe"); + + assert!( + !output.status.success(), + "espipe should reject invalid input" + ); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains("invalid.pdf"), + "stderr should identify input" + ); + assert!(stderr.contains("malformed") || stderr.contains("unsupported")); +} + +#[test] +fn cli_reports_image_only_pdf_requires_ocr_on_stderr() { + let input_path = temp_output_path("image-only.pdf"); + let output_path = temp_output_path("image-only-anydoc.ndjson"); + write_base64_fixture("anydoc/image-only.pdf.base64", &input_path); + + let output = Command::new(env!("CARGO_BIN_EXE_espipe")) + .arg(&input_path) + .arg(&output_path) + .output() + .expect("run espipe"); + + assert!( + !output.status.success(), + "espipe should reject image-only PDF" + ); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!(stderr.contains("image-only.pdf")); + assert!(stderr.contains("OCR is required")); +} + #[test] fn cli_rejects_multi_file_input_to_non_ndjson_file_output_before_writing() { let first_input = fixture_path("glob_docs").join("alpha.md"); diff --git a/tests/fixtures/anydoc/image-only.pdf.base64 b/tests/fixtures/anydoc/image-only.pdf.base64 new file mode 100644 index 0000000..1ceac03 --- /dev/null +++ b/tests/fixtures/anydoc/image-only.pdf.base64 @@ -0,0 +1 @@ +JVBERi0xLjQKMSAwIG9iago8PCAvVHlwZSAvQ2F0YWxvZyAvUGFnZXMgMiAwIFIgPj4KZW5kb2JqCjIgMCBvYmoKPDwgL1R5cGUgL1BhZ2VzIC9LaWRzIFszIDAgUl0gL0NvdW50IDEgPj4KZW5kb2JqCjMgMCBvYmoKPDwgL1R5cGUgL1BhZ2UgL1BhcmVudCAyIDAgUiAvTWVkaWFCb3ggWzAgMCAxIDFdIC9SZXNvdXJjZXMgPDwgL1hPYmplY3QgPDwgL0ltMSA1IDAgUiA+PiA+PiAvQ29udGVudHMgNCAwIFIgPj4KZW5kb2JqCjQgMCBvYmoKPDwgL0xlbmd0aCAyNyA+PgpzdHJlYW0KcQoxIDAgMCAxIDAgMCBjbQovSW0xIERvClEKZW5kc3RyZWFtCmVuZG9iago1IDAgb2JqCjw8IC9UeXBlIC9YT2JqZWN0IC9TdWJ0eXBlIC9JbWFnZSAvV2lkdGggMSAvSGVpZ2h0IDEgL0NvbG9yU3BhY2UgL0RldmljZUdyYXkgL0JpdHNQZXJDb21wb25lbnQgOCAvTGVuZ3RoIDEgPj4Kc3RyZWFtCgAKZW5kc3RyZWFtCmVuZG9iagp4cmVmCjAgNgowMDAwMDAwMDAwIDY1NTM1IGYgCjAwMDAwMDAwMDkgMDAwMDAgbiAKMDAwMDAwMDA1OCAwMDAwMCBuIAowMDAwMDAwMTE1IDAwMDAwIG4gCjAwMDAwMDAyNDEgMDAwMDAgbiAKMDAwMDAwMDMxNyAwMDAwMCBuIAp0cmFpbGVyCjw8IC9TaXplIDYgL1Jvb3QgMSAwIFIgPj4Kc3RhcnR4cmVmCjQ2MQolJUVPRgo= diff --git a/tests/fixtures/anydoc/sample.docx.base64 b/tests/fixtures/anydoc/sample.docx.base64 new file mode 100644 index 0000000..2b94a40 --- /dev/null +++ b/tests/fixtures/anydoc/sample.docx.base64 @@ -0,0 +1 @@ +UEsDBBQAAAAIAHJcDF0Vlbo5fQAAAKEAAAARAAAAd29yZC9kb2N1bWVudC54bWxFzr8OAiEMBvBXIfcA9uLgQJBFBzdXVwTuT0IpKTXo23ucg8vvS9t8SU3TgfwLYxb1xpSrbudhESkaoPoloqsHKjFvt4kYnWwjz9CIQ2HysdY1z5jgOI4nQLfmwZqmnxQ+PUuHO2JvMSVSExOq6/3yMNCXXd4tu78i/J+yX1BLAQIUAxQAAAAIAHJcDF0Vlbo5fQAAAKEAAAARAAAAAAAAAAAAAACAAQAAAAB3b3JkL2RvY3VtZW50LnhtbFBLBQYAAAAAAQABAD8AAACsAAAAAAA= diff --git a/tests/fixtures/anydoc/sample.pdf b/tests/fixtures/anydoc/sample.pdf new file mode 100644 index 0000000000000000000000000000000000000000..f3775564b9e032d882e8a9189423c5680bcb7985 GIT binary patch literal 584 zcmZWnO;5r=5WVlOmbi1YHBOx468WS}d=#6+-=zt})Yjzid{(5JYB1JYmv~S+L z_h!1C*Zb^7-6ldo1ZKGvqY=pb`x8OlX|F4@0r{#o=pZJ-48~)Db-_gbK5sbiL*B zgdNF;AF{1IsX0g+^cb62=k7s6&UZ^6#^&J_$cc7%s61fxg}z+r)(otARC&UByrMHRM|{%{f)m(yVoFaV=X zHJq>^geuf0EFD~6N>NTeEnZAH|6^U%!o1aGg*IG-dHI9Tqoo_e|FF3L&TH#K&LmQz J)0w_x;unJ~m*oHe literal 0 HcmV?d00001 diff --git a/tests/fixtures/anydoc/sample.rtf b/tests/fixtures/anydoc/sample.rtf new file mode 100644 index 0000000..9f1a35c --- /dev/null +++ b/tests/fixtures/anydoc/sample.rtf @@ -0,0 +1,4 @@ +{\rtf1\ansi\deff0 +{\fonttbl{\f0 Helvetica;}} +\f0\fs24 Hello from RTF.\par +} From 05930d796e80e2dad0096401a1af2e921f58d76d Mon Sep 17 00:00:00 2001 From: Ryan Eno Date: Wed, 12 Aug 2026 11:49:42 -0700 Subject: [PATCH 03/15] Relax anydoc error wording assertion --- tests/file_output.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/file_output.rs b/tests/file_output.rs index 3ff80aa..3fae563 100644 --- a/tests/file_output.rs +++ b/tests/file_output.rs @@ -171,7 +171,13 @@ fn cli_reports_anydoc_conversion_errors_on_stderr_with_source_path() { stderr.contains("invalid.pdf"), "stderr should identify input" ); - assert!(stderr.contains("malformed") || stderr.contains("unsupported")); + let (_, detail) = stderr + .split_once("invalid.pdf") + .expect("stderr should include error detail after the source path"); + assert!( + !detail.trim().is_empty(), + "stderr should include error detail beyond the source path" + ); } #[test] From d26976d3bc3602747cd924c8edf4a86989a19b9c Mon Sep 17 00:00:00 2001 From: Ryan Eno Date: Wed, 12 Aug 2026 11:59:00 -0700 Subject: [PATCH 04/15] Use Path for fixture helpers --- src/input.rs | 4 ++-- tests/file_output.rs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/input.rs b/src/input.rs index f6cbe91..ec7871e 100644 --- a/src/input.rs +++ b/src/input.rs @@ -948,7 +948,7 @@ mod tests { fs, io::{Read, Write}, net::TcpListener, - path::PathBuf, + path::{Path, PathBuf}, sync::{Arc, mpsc}, thread, time::{SystemTime, UNIX_EPOCH}, @@ -997,7 +997,7 @@ mod tests { .join(name) } - fn write_base64_fixture(source: &str, path: &PathBuf) { + fn write_base64_fixture(source: &str, path: &Path) { let encoded = fs::read_to_string(fixture_path(source)).unwrap(); let bytes = base64::engine::general_purpose::STANDARD .decode(encoded.trim()) diff --git a/tests/file_output.rs b/tests/file_output.rs index 3fae563..853da06 100644 --- a/tests/file_output.rs +++ b/tests/file_output.rs @@ -4,7 +4,7 @@ use serde_json::Value; use std::{ fs, io::Read, - path::PathBuf, + path::{Path, PathBuf}, process::Command, time::{SystemTime, UNIX_EPOCH}, }; @@ -26,7 +26,7 @@ fn temp_output_path(filename: &str) -> PathBuf { dir.join(filename) } -fn write_base64_fixture(name: &str, path: &PathBuf) { +fn write_base64_fixture(name: &str, path: &Path) { let encoded = fs::read_to_string(fixture_path(name)).expect("read base64 fixture"); let bytes = base64::engine::general_purpose::STANDARD .decode(encoded.trim()) From 3fec75fb77d66208318dff7787a53298b6fa3288 Mon Sep 17 00:00:00 2001 From: Ryan Eno Date: Wed, 12 Aug 2026 12:04:01 -0700 Subject: [PATCH 05/15] Clarify anydoc input ordering test --- src/input.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/input.rs b/src/input.rs index ec7871e..4ca9a7e 100644 --- a/src/input.rs +++ b/src/input.rs @@ -1204,7 +1204,7 @@ mod tests { } #[test] - fn anydoc_mixed_file_import_preserves_order_and_file_metadata() { + fn anydoc_mixed_file_import_sorts_paths_and_preserves_file_metadata() { let pdf = fixture_path("anydoc/sample.pdf"); let rtf = fixture_path("anydoc/sample.rtf"); let values = collect_values(open_input_values(vec![uri(&rtf), uri(&pdf)], "body").unwrap()); From 30f2740314a33ce2aa00c79dc2954261a86d94f2 Mon Sep 17 00:00:00 2001 From: Ryan Eno Date: Wed, 12 Aug 2026 12:11:27 -0700 Subject: [PATCH 06/15] Make PDF fixture reviewable as text --- src/input.rs | 19 +++++++++++++----- tests/file_output.rs | 6 ++++-- .../anydoc/{sample.pdf => sample.pdf.txt} | Bin 3 files changed, 18 insertions(+), 7 deletions(-) rename tests/fixtures/anydoc/{sample.pdf => sample.pdf.txt} (100%) diff --git a/src/input.rs b/src/input.rs index 4ca9a7e..b6fa0b5 100644 --- a/src/input.rs +++ b/src/input.rs @@ -1159,7 +1159,9 @@ mod tests { #[test] fn anydoc_converts_pdf_to_default_markdown_document() { - let path = fixture_path("anydoc/sample.pdf"); + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("sample.pdf"); + fs::copy(fixture_path("anydoc/sample.pdf.txt"), &path).unwrap(); let values = collect_values(Input::try_from(uri(&path)).unwrap()); assert_eq!(values.len(), 1); @@ -1205,8 +1207,11 @@ mod tests { #[test] fn anydoc_mixed_file_import_sorts_paths_and_preserves_file_metadata() { - let pdf = fixture_path("anydoc/sample.pdf"); - let rtf = fixture_path("anydoc/sample.rtf"); + let dir = tempfile::tempdir().unwrap(); + let pdf = dir.path().join("sample.pdf"); + fs::copy(fixture_path("anydoc/sample.pdf.txt"), &pdf).unwrap(); + let rtf = dir.path().join("sample.rtf"); + fs::copy(fixture_path("anydoc/sample.rtf"), &rtf).unwrap(); let values = collect_values(open_input_values(vec![uri(&rtf), uri(&pdf)], "body").unwrap()); assert_eq!(values.len(), 2); @@ -1221,7 +1226,11 @@ mod tests { let dir = tempfile::tempdir().unwrap(); let nested = dir.path().join("nested"); fs::create_dir(&nested).unwrap(); - fs::copy(fixture_path("anydoc/sample.pdf"), nested.join("sample.pdf")).unwrap(); + fs::copy( + fixture_path("anydoc/sample.pdf.txt"), + nested.join("sample.pdf"), + ) + .unwrap(); let pattern = dir .path() .join("**") @@ -1242,7 +1251,7 @@ mod tests { let dir = tempfile::tempdir().unwrap(); let pdf = dir.path().join("a.pdf"); let rtf = dir.path().join("b.rtf"); - fs::copy(fixture_path("anydoc/sample.pdf"), &pdf).unwrap(); + fs::copy(fixture_path("anydoc/sample.pdf.txt"), &pdf).unwrap(); fs::copy(fixture_path("anydoc/sample.rtf"), &rtf).unwrap(); let pdf_pattern = dir.path().join("**/*.pdf").to_string_lossy().into_owned(); let rtf_pattern = dir.path().join("**/*.rtf").to_string_lossy().into_owned(); diff --git a/tests/file_output.rs b/tests/file_output.rs index 853da06..1df79d2 100644 --- a/tests/file_output.rs +++ b/tests/file_output.rs @@ -88,7 +88,8 @@ fn cli_writes_bulk_output_to_file() { #[test] fn cli_converts_anydoc_pdf_to_existing_file_document_output() { - let input_path = fixture_path("anydoc").join("sample.pdf"); + let input_path = temp_output_path("sample.pdf"); + fs::copy(fixture_path("anydoc/sample.pdf.txt"), &input_path).expect("copy PDF fixture"); let output_path = temp_output_path("anydoc.ndjson"); let status = Command::new(env!("CARGO_BIN_EXE_espipe")) @@ -112,7 +113,8 @@ fn cli_converts_anydoc_pdf_to_existing_file_document_output() { #[test] fn cli_converts_mixed_anydoc_and_markdown_inputs_without_changing_shape() { - let anydoc_path = fixture_path("anydoc").join("sample.pdf"); + let anydoc_path = temp_output_path("sample.pdf"); + fs::copy(fixture_path("anydoc/sample.pdf.txt"), &anydoc_path).expect("copy PDF fixture"); let markdown_path = fixture_path("glob_docs").join("alpha.md"); let output_path = temp_output_path("mixed-anydoc.ndjson"); diff --git a/tests/fixtures/anydoc/sample.pdf b/tests/fixtures/anydoc/sample.pdf.txt similarity index 100% rename from tests/fixtures/anydoc/sample.pdf rename to tests/fixtures/anydoc/sample.pdf.txt From 9f5c7ef4aa4f8900b829d2ada9e1adf7e022e86e Mon Sep 17 00:00:00 2001 From: Ryan Eno Date: Wed, 12 Aug 2026 12:16:14 -0700 Subject: [PATCH 07/15] Clarify multiple CLI input arguments --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 7940273..deed1cf 100644 --- a/README.md +++ b/README.md @@ -102,7 +102,8 @@ Options: ## Input And Output -Both positional arguments are parsed as URI-like strings. +All positional arguments are parsed as URI-like strings; one or more input URIs +are followed by the output URI. ### Supported input forms From c8e39a6d9f2b1da436de391e5300983ebcea43cc Mon Sep 17 00:00:00 2001 From: Ryan Eno Date: Wed, 12 Aug 2026 12:24:30 -0700 Subject: [PATCH 08/15] Preserve anydoc conversion error sources --- src/input.rs | 37 +++++++++++++++++++------ tests/file_output.rs | 4 +-- tests/fixtures/anydoc/sample.pdf.base64 | 1 + tests/fixtures/anydoc/sample.pdf.txt | 36 ------------------------ 4 files changed, 31 insertions(+), 47 deletions(-) create mode 100644 tests/fixtures/anydoc/sample.pdf.base64 delete mode 100644 tests/fixtures/anydoc/sample.pdf.txt diff --git a/src/input.rs b/src/input.rs index b6fa0b5..821c967 100644 --- a/src/input.rs +++ b/src/input.rs @@ -469,12 +469,35 @@ fn read_markdown_file_document( read_markdown_text_document(path, &text, content_field, include_file_metadata) } +#[derive(Debug)] +struct AnyDocConversionError { + path: PathBuf, + source: anydoc::ConvertError, +} + +impl std::fmt::Display for AnyDocConversionError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}: {}", self.path.display(), self.source) + } +} + +impl std::error::Error for AnyDocConversionError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + Some(&self.source) + } +} + fn read_anydoc_file_document( path: &Path, content_field: &str, include_file_metadata: bool, ) -> Result>> { - let markdown = anydoc::to_markdown(path).map_err(|err| eyre!("{}: {err}", path.display()))?; + let markdown = anydoc::to_markdown(path).map_err(|source| { + Report::new(AnyDocConversionError { + path: path.to_path_buf(), + source, + }) + })?; read_markdown_text_document(path, &markdown, content_field, include_file_metadata) } @@ -1161,7 +1184,7 @@ mod tests { fn anydoc_converts_pdf_to_default_markdown_document() { let dir = tempfile::tempdir().unwrap(); let path = dir.path().join("sample.pdf"); - fs::copy(fixture_path("anydoc/sample.pdf.txt"), &path).unwrap(); + write_base64_fixture("anydoc/sample.pdf.base64", &path); let values = collect_values(Input::try_from(uri(&path)).unwrap()); assert_eq!(values.len(), 1); @@ -1209,7 +1232,7 @@ mod tests { fn anydoc_mixed_file_import_sorts_paths_and_preserves_file_metadata() { let dir = tempfile::tempdir().unwrap(); let pdf = dir.path().join("sample.pdf"); - fs::copy(fixture_path("anydoc/sample.pdf.txt"), &pdf).unwrap(); + write_base64_fixture("anydoc/sample.pdf.base64", &pdf); let rtf = dir.path().join("sample.rtf"); fs::copy(fixture_path("anydoc/sample.rtf"), &rtf).unwrap(); let values = collect_values(open_input_values(vec![uri(&rtf), uri(&pdf)], "body").unwrap()); @@ -1226,11 +1249,7 @@ mod tests { let dir = tempfile::tempdir().unwrap(); let nested = dir.path().join("nested"); fs::create_dir(&nested).unwrap(); - fs::copy( - fixture_path("anydoc/sample.pdf.txt"), - nested.join("sample.pdf"), - ) - .unwrap(); + write_base64_fixture("anydoc/sample.pdf.base64", &nested.join("sample.pdf")); let pattern = dir .path() .join("**") @@ -1251,7 +1270,7 @@ mod tests { let dir = tempfile::tempdir().unwrap(); let pdf = dir.path().join("a.pdf"); let rtf = dir.path().join("b.rtf"); - fs::copy(fixture_path("anydoc/sample.pdf.txt"), &pdf).unwrap(); + write_base64_fixture("anydoc/sample.pdf.base64", &pdf); fs::copy(fixture_path("anydoc/sample.rtf"), &rtf).unwrap(); let pdf_pattern = dir.path().join("**/*.pdf").to_string_lossy().into_owned(); let rtf_pattern = dir.path().join("**/*.rtf").to_string_lossy().into_owned(); diff --git a/tests/file_output.rs b/tests/file_output.rs index 1df79d2..7e75a2a 100644 --- a/tests/file_output.rs +++ b/tests/file_output.rs @@ -89,7 +89,7 @@ fn cli_writes_bulk_output_to_file() { #[test] fn cli_converts_anydoc_pdf_to_existing_file_document_output() { let input_path = temp_output_path("sample.pdf"); - fs::copy(fixture_path("anydoc/sample.pdf.txt"), &input_path).expect("copy PDF fixture"); + write_base64_fixture("anydoc/sample.pdf.base64", &input_path); let output_path = temp_output_path("anydoc.ndjson"); let status = Command::new(env!("CARGO_BIN_EXE_espipe")) @@ -114,7 +114,7 @@ fn cli_converts_anydoc_pdf_to_existing_file_document_output() { #[test] fn cli_converts_mixed_anydoc_and_markdown_inputs_without_changing_shape() { let anydoc_path = temp_output_path("sample.pdf"); - fs::copy(fixture_path("anydoc/sample.pdf.txt"), &anydoc_path).expect("copy PDF fixture"); + write_base64_fixture("anydoc/sample.pdf.base64", &anydoc_path); let markdown_path = fixture_path("glob_docs").join("alpha.md"); let output_path = temp_output_path("mixed-anydoc.ndjson"); diff --git a/tests/fixtures/anydoc/sample.pdf.base64 b/tests/fixtures/anydoc/sample.pdf.base64 new file mode 100644 index 0000000..29752f4 --- /dev/null +++ b/tests/fixtures/anydoc/sample.pdf.base64 @@ -0,0 +1 @@ +JVBERi0xLjQKMSAwIG9iago8PCAvVHlwZSAvQ2F0YWxvZyAvUGFnZXMgMiAwIFIgPj4KZW5kb2JqCjIgMCBvYmoKPDwgL1R5cGUgL1BhZ2VzIC9LaWRzIFszIDAgUl0gL0NvdW50IDEgPj4KZW5kb2JqCjMgMCBvYmoKPDwgL1R5cGUgL1BhZ2UgL1BhcmVudCAyIDAgUiAvTWVkaWFCb3ggWzAgMCA2MTIgNzkyXSAvUmVzb3VyY2VzIDw8IC9Gb250IDw8IC9GMSA0IDAgUiA+PiA+PiAvQ29udGVudHMgNSAwIFIgPj4KZW5kb2JqCjQgMCBvYmoKPDwgL1R5cGUgL0ZvbnQgL1N1YnR5cGUgL1R5cGUxIC9CYXNlRm9udCAvSGVsdmV0aWNhID4+CmVuZG9iago1IDAgb2JqCjw8IC9MZW5ndGggNDEgPj4Kc3RyZWFtCkJUCi9GMSAxOCBUZgo3MiA3MjAgVGQKKEhlbGxvIFBERikgVGoKRVQKZW5kc3RyZWFtCmVuZG9iagp4cmVmCjAgNgowMDAwMDAwMDAwIDY1NTM1IGYgCjAwMDAwMDAwMDkgMDAwMDAgbiAKMDAwMDAwMDA1OCAwMDAwMCBuIAowMDAwMDAwMTE1IDAwMDAwIG4gCjAwMDAwMDAyNDEgMDAwMDAgbiAKMDAwMDAwMDMxMSAwMDAwMCBuIAp0cmFpbGVyCjw8IC9TaXplIDYgL1Jvb3QgMSAwIFIgPj4Kc3RhcnR4cmVmCjQwMQolJUVPRgo= diff --git a/tests/fixtures/anydoc/sample.pdf.txt b/tests/fixtures/anydoc/sample.pdf.txt deleted file mode 100644 index f377556..0000000 --- a/tests/fixtures/anydoc/sample.pdf.txt +++ /dev/null @@ -1,36 +0,0 @@ -%PDF-1.4 -1 0 obj -<< /Type /Catalog /Pages 2 0 R >> -endobj -2 0 obj -<< /Type /Pages /Kids [3 0 R] /Count 1 >> -endobj -3 0 obj -<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << /Font << /F1 4 0 R >> >> /Contents 5 0 R >> -endobj -4 0 obj -<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >> -endobj -5 0 obj -<< /Length 41 >> -stream -BT -/F1 18 Tf -72 720 Td -(Hello PDF) Tj -ET -endstream -endobj -xref -0 6 -0000000000 65535 f -0000000009 00000 n -0000000058 00000 n -0000000115 00000 n -0000000241 00000 n -0000000311 00000 n -trailer -<< /Size 6 /Root 1 0 R >> -startxref -401 -%%EOF From 32503ba04c0805f935cca2bc77a206955b532103 Mon Sep 17 00:00:00 2001 From: Ryan Eno Date: Wed, 12 Aug 2026 14:21:52 -0700 Subject: [PATCH 09/15] Bump version to 0.5.0 --- Cargo.lock | 2 +- Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index dd298bc..86ef38d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -746,7 +746,7 @@ dependencies = [ [[package]] name = "espipe" -version = "0.4.0" +version = "0.5.0" dependencies = [ "anydoc", "base64", diff --git a/Cargo.toml b/Cargo.toml index 4fb786c..6a5f837 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,7 +4,7 @@ description = "A minimalist command-line utility to pipe documents from a file o repository = "https://github.com/VimCommando/espipe" homepage = "https://github.com/VimCommando/espipe" documentation = "https://docs.rs/crate/espipe" -version = "0.4.0" +version = "0.5.0" edition = "2024" rust-version = "1.88" license = "Apache-2.0" From 18084a3158ad8509d8aead114faf05e0546d9a6c Mon Sep 17 00:00:00 2001 From: Ryan Eno Date: Wed, 12 Aug 2026 14:35:11 -0700 Subject: [PATCH 10/15] Add origin metadata for glob imports --- README.md | 2 +- openspec/changes/add-anydoc-input/design.md | 2 +- .../specs/anydoc-input/spec.md | 8 +- src/input.rs | 150 +++++++++++------- 4 files changed, 101 insertions(+), 61 deletions(-) diff --git a/README.md b/README.md index deed1cf..00e38e9 100644 --- a/README.md +++ b/README.md @@ -146,7 +146,7 @@ Local files with these extensions are converted to GitHub-Flavored Markdown thro `.doc`, `.docx`, `.docm`, `.odt`, `.pdf`, `.ppt`, `.pps`, `.pot`, `.pptx`, `.pptm`, `.ppsx`, `.ppsm`, `.rtf`, `.epub`, `.xls`, `.xlsx`, `.xlsm`, `.xlsb`, `.ods`, and `.odp`. -Converted content is stored in `content.body` by default. Use `--content markdown` to store it in `content.markdown`. Multiple local inputs add the existing `file.path` and `file.name` metadata. Anydoc conversion is local-only; remote HTTPS inputs are unchanged. Scanned or image-only PDFs require OCR outside espipe and are not converted. +Converted content is stored in `content.body` by default. Use `--content markdown` to store it in `content.markdown`. Multiple local inputs add the existing `file.path` and `file.name` metadata; glob-resolved inputs also add `origin.path` and `origin.filename`. Anydoc conversion is local-only; remote HTTPS inputs are unchanged. Scanned or image-only PDFs require OCR outside espipe and are not converted. ### Supported output forms diff --git a/openspec/changes/add-anydoc-input/design.md b/openspec/changes/add-anydoc-input/design.md index 9d98223..7d882c4 100644 --- a/openspec/changes/add-anydoc-input/design.md +++ b/openspec/changes/add-anydoc-input/design.md @@ -47,7 +47,7 @@ This preserves: - `content.` placement and `--content` behavior. - YAML frontmatter extraction and content-field conflict checks. -- `file.path` and `file.name` inclusion for multi-file imports. +- `file.path` and `file.name` inclusion for multi-file imports, plus `origin.path` and `origin.filename` for glob-resolved imports. - One serialized `Box` per output document. The resulting flow is: diff --git a/openspec/changes/add-anydoc-input/specs/anydoc-input/spec.md b/openspec/changes/add-anydoc-input/specs/anydoc-input/spec.md index a68f798..f4a1828 100644 --- a/openspec/changes/add-anydoc-input/specs/anydoc-input/spec.md +++ b/openspec/changes/add-anydoc-input/specs/anydoc-input/spec.md @@ -24,7 +24,7 @@ The system SHALL use the Rust `anydoc` processor for local regular files with th ### Requirement: Anydoc conversion preserves the existing file-document shape -The system SHALL construct converted documents using the existing file-document semantics after Markdown conversion. It SHALL store converted Markdown under `content.`, apply the configured `--content` value, and add `file.path` and `file.name` only under the existing multi-file rules. +The system SHALL construct converted documents using the existing file-document semantics after Markdown conversion. It SHALL store converted Markdown under `content.`, apply the configured `--content` value, add `file.path` and `file.name` only under the existing multi-file rules, and add `origin.path` and `origin.filename` when the source was resolved through a glob pattern. #### Scenario: Default content field is used for converted Markdown @@ -44,6 +44,12 @@ The system SHALL construct converted documents using the existing file-document - **THEN** each converted document includes the same `file.path` and `file.name` metadata as existing file documents - **AND** the metadata values identify the original local file +#### Scenario: Glob-resolved converted files include origin metadata + +- **WHEN** anydoc files are imported through a local glob pattern +- **THEN** each converted document includes `origin.path` and `origin.filename` +- **AND** those values identify the original local file + ### Requirement: Existing local discovery mechanisms support anydoc files The system SHALL process supported anydoc files supplied as direct local paths, shell-expanded file lists, or existing local glob patterns. It SHALL not require a separate subprocess or remote service for local conversion. diff --git a/src/input.rs b/src/input.rs index 821c967..584c29f 100644 --- a/src/input.rs +++ b/src/input.rs @@ -48,10 +48,16 @@ pub enum Input { documents: Vec>, document_index: usize, content_field: String, - include_file_metadata: bool, + metadata: FileMetadataOptions, }, } +#[derive(Clone, Copy, Debug, Default)] +pub(crate) struct FileMetadataOptions { + include_file: bool, + include_origin: bool, +} + type CsvRecord = std::collections::HashMap; const REMOTE_NDJSON_ERROR: &str = "JSON payload does not look like required NDJSON input format."; const JSON_LINE_OPENING_ERROR: &str = "Each record must be a JSON object starting with '{'"; @@ -314,8 +320,11 @@ fn open_local_file(path: PathBuf) -> Result { } fn open_file_documents(values: Vec, content_field: &str) -> Result { - let paths = resolve_file_document_paths(values)?; - let include_file_metadata = paths.len() > 1; + let (paths, resolved_from_glob) = resolve_file_document_paths(values)?; + let metadata = FileMetadataOptions { + include_file: paths.len() > 1, + include_origin: resolved_from_glob, + }; let source = format!("{} file document(s)", paths.len()); Ok(Input::FileDocuments { source, @@ -324,7 +333,7 @@ fn open_file_documents(values: Vec, content_field: &str) -> Result Result> { documents, document_index, content_field, - include_file_metadata, + metadata, .. } = input else { @@ -352,12 +361,12 @@ fn read_file_document_line(input: &mut Input) -> Result> { return Err(eyre!("No file document")); }; *path_index += 1; - *documents = read_file_documents(path, content_field, *include_file_metadata)?; + *documents = read_file_documents(path, content_field, *metadata)?; *document_index = 0; } } -fn resolve_file_document_paths(values: Vec) -> Result> { +fn resolve_file_document_paths(values: Vec) -> Result<(Vec, bool)> { let mut paths = BTreeSet::new(); let mut any_glob = false; for value in values { @@ -402,7 +411,7 @@ fn resolve_file_document_paths(values: Vec) -> Result> { }; return Err(eyre!("No regular files resolved from {kind}")); } - Ok(paths.into_iter().collect()) + Ok((paths.into_iter().collect(), any_glob)) } fn has_glob_metachar(value: &str) -> bool { @@ -419,22 +428,20 @@ fn should_use_file_document(path: &Path) -> bool { fn read_file_documents( path: &Path, content_field: &str, - include_file_metadata: bool, + metadata: FileMetadataOptions, ) -> Result>> { match extension(path).as_deref() { - Some("ndjson" | "jsonl") => read_ndjson_file_documents(path, include_file_metadata), - Some("json") => read_json_file_document(path, include_file_metadata), - Some("toon") => read_toon_file_documents(path, include_file_metadata), - Some("yml" | "yaml") => read_yaml_file_document(path, content_field, include_file_metadata), - Some("md" | "markdown") => { - read_markdown_file_document(path, content_field, include_file_metadata) - } + Some("ndjson" | "jsonl") => read_ndjson_file_documents(path, metadata), + Some("json") => read_json_file_document(path, metadata), + Some("toon") => read_toon_file_documents(path, metadata), + Some("yml" | "yaml") => read_yaml_file_document(path, content_field, metadata), + Some("md" | "markdown") => read_markdown_file_document(path, content_field, metadata), _ if anydoc::Format::from_path(path) .is_some_and(|format| format != anydoc::Format::Csv) => { - read_anydoc_file_document(path, content_field, include_file_metadata) + read_anydoc_file_document(path, content_field, metadata) } - _ => read_text_file_document(path, content_field, include_file_metadata), + _ => read_text_file_document(path, content_field, metadata), } } @@ -446,10 +453,10 @@ fn read_text_file(path: &Path) -> Result { fn read_text_file_document( path: &Path, content_field: &str, - include_file_metadata: bool, + metadata: FileMetadataOptions, ) -> Result>> { let text = read_text_file(path)?; - let mut document = base_file_document(path, include_file_metadata); + let mut document = base_file_document(path, metadata); document.insert( "content".to_string(), Value::Object(Map::from_iter([( @@ -463,10 +470,10 @@ fn read_text_file_document( fn read_markdown_file_document( path: &Path, content_field: &str, - include_file_metadata: bool, + metadata: FileMetadataOptions, ) -> Result>> { let text = read_text_file(path)?; - read_markdown_text_document(path, &text, content_field, include_file_metadata) + read_markdown_text_document(path, &text, content_field, metadata) } #[derive(Debug)] @@ -490,7 +497,7 @@ impl std::error::Error for AnyDocConversionError { fn read_anydoc_file_document( path: &Path, content_field: &str, - include_file_metadata: bool, + metadata: FileMetadataOptions, ) -> Result>> { let markdown = anydoc::to_markdown(path).map_err(|source| { Report::new(AnyDocConversionError { @@ -498,14 +505,14 @@ fn read_anydoc_file_document( source, }) })?; - read_markdown_text_document(path, &markdown, content_field, include_file_metadata) + read_markdown_text_document(path, &markdown, content_field, metadata) } fn read_markdown_text_document( path: &Path, text: &str, content_field: &str, - include_file_metadata: bool, + metadata: FileMetadataOptions, ) -> Result>> { let (frontmatter, body) = split_markdown_frontmatter(text); let mut content = Map::new(); @@ -520,7 +527,7 @@ fn read_markdown_text_document( } } content.insert(content_field.to_string(), Value::String(body.to_string())); - let mut document = base_file_document(path, include_file_metadata); + let mut document = base_file_document(path, metadata); document.insert("content".to_string(), Value::Object(content)); raw_documents(vec![document]) } @@ -558,7 +565,7 @@ fn is_end_of_input(err: &eyre::Report) -> bool { fn read_yaml_file_document( path: &Path, content_field: &str, - include_file_metadata: bool, + metadata: FileMetadataOptions, ) -> Result>> { let text = read_text_file(path)?; let content = yaml_mapping_to_json_map(&text) @@ -569,7 +576,7 @@ fn read_yaml_file_document( path.display() )); } - let mut document = base_file_document(path, include_file_metadata); + let mut document = base_file_document(path, metadata); document.insert("content".to_string(), Value::Object(content)); raw_documents(vec![document]) } @@ -582,7 +589,10 @@ fn yaml_mapping_to_json_map(text: &str) -> Result> { Ok(map) } -fn read_json_file_document(path: &Path, include_file_metadata: bool) -> Result>> { +fn read_json_file_document( + path: &Path, + metadata: FileMetadataOptions, +) -> Result>> { let text = read_text_file(path)?; let mut document = match serde_json::from_str::(&text) { Ok(Value::Object(map)) => map, @@ -599,13 +609,13 @@ fn read_json_file_document(path: &Path, include_file_metadata: bool) -> Result Result>> { let text = read_text_file(path)?; let mut docs = Vec::new(); @@ -622,7 +632,7 @@ fn read_ndjson_file_documents( index + 1 )); }; - add_file_metadata(&mut document, path, include_file_metadata); + add_file_metadata(&mut document, path, metadata); docs.push(RawValue::from_string(Value::Object(document).to_string())?); } Ok(docs) @@ -630,7 +640,7 @@ fn read_ndjson_file_documents( fn read_toon_file_documents( path: &Path, - include_file_metadata: bool, + metadata: FileMetadataOptions, ) -> Result>> { let file = File::open(path).map_err(|err| eyre!("{}: {err}", path.display()))?; let mut reader = BufReader::new(Box::new(file) as Box); @@ -651,9 +661,9 @@ fn read_toon_file_documents( &mut eof, ) { Ok(mut raw) => { - if include_file_metadata { + if metadata.include_file || metadata.include_origin { let mut document: Map = serde_json::from_str(raw.get())?; - add_file_metadata(&mut document, path, include_file_metadata); + add_file_metadata(&mut document, path, metadata); raw = RawValue::from_string(Value::Object(document).to_string())?; } docs.push(raw); @@ -717,34 +727,42 @@ fn toon_row_value_to_raw(source: &str, document_index: usize, row: Value) -> Res RawValue::from_string(Value::Object(row).to_string()).map_err(Into::into) } -fn base_file_document(path: &Path, include_file_metadata: bool) -> Map { +fn base_file_document(path: &Path, metadata: FileMetadataOptions) -> Map { let mut document = Map::new(); - add_file_metadata(&mut document, path, include_file_metadata); + add_file_metadata(&mut document, path, metadata); document } -fn add_file_metadata(document: &mut Map, path: &Path, include_file_metadata: bool) { - if !include_file_metadata { - return; +fn add_file_metadata( + document: &mut Map, + path: &Path, + metadata: FileMetadataOptions, +) { + let path_display = path.display().to_string(); + let filename = path + .file_name() + .and_then(OsStr::to_str) + .unwrap_or_default() + .to_string(); + + if metadata.include_file { + document.insert( + "file".to_string(), + Value::Object(Map::from_iter([ + ("path".to_string(), Value::String(path_display.clone())), + ("name".to_string(), Value::String(filename.clone())), + ])), + ); + } + if metadata.include_origin { + document.insert( + "origin".to_string(), + Value::Object(Map::from_iter([ + ("path".to_string(), Value::String(path_display)), + ("filename".to_string(), Value::String(filename)), + ])), + ); } - document.insert( - "file".to_string(), - Value::Object(Map::from_iter([ - ( - "path".to_string(), - Value::String(path.display().to_string()), - ), - ( - "name".to_string(), - Value::String( - path.file_name() - .and_then(OsStr::to_str) - .unwrap_or_default() - .to_string(), - ), - ), - ])), - ); } fn raw_documents(documents: Vec>) -> Result>> { @@ -1263,6 +1281,12 @@ mod tests { assert_eq!(values.len(), 1); assert!(values[0]["content"]["body"].is_string()); + assert_eq!(values[0]["origin"]["filename"], "sample.pdf"); + assert_eq!( + values[0]["origin"]["path"], + nested.join("sample.pdf").display().to_string() + ); + assert!(values[0].get("file").is_none()); } #[test] @@ -1353,6 +1377,16 @@ mod tests { assert_eq!(values.len(), 2); assert_eq!(values[0]["content"]["body"], "child"); assert_eq!(values[1]["content"]["body"], "root"); + assert_eq!(values[0]["origin"]["filename"], "child.md"); + assert_eq!( + values[0]["origin"]["path"], + nested.join("child.md").display().to_string() + ); + assert_eq!(values[1]["origin"]["filename"], "root.md"); + assert_eq!( + values[1]["origin"]["path"], + dir.path().join("root.md").display().to_string() + ); } #[test] From ce5499734a3938f2597a875fefe141dc2f957310 Mon Sep 17 00:00:00 2001 From: Ryan Eno Date: Wed, 12 Aug 2026 14:52:49 -0700 Subject: [PATCH 11/15] Keep origin path separate from filename --- README.md | 2 +- openspec/changes/add-anydoc-input/design.md | 2 +- .../add-anydoc-input/specs/anydoc-input/spec.md | 2 +- src/input.rs | 16 ++++++---------- 4 files changed, 9 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 00e38e9..fcbc070 100644 --- a/README.md +++ b/README.md @@ -146,7 +146,7 @@ Local files with these extensions are converted to GitHub-Flavored Markdown thro `.doc`, `.docx`, `.docm`, `.odt`, `.pdf`, `.ppt`, `.pps`, `.pot`, `.pptx`, `.pptm`, `.ppsx`, `.ppsm`, `.rtf`, `.epub`, `.xls`, `.xlsx`, `.xlsm`, `.xlsb`, `.ods`, and `.odp`. -Converted content is stored in `content.body` by default. Use `--content markdown` to store it in `content.markdown`. Multiple local inputs add the existing `file.path` and `file.name` metadata; glob-resolved inputs also add `origin.path` and `origin.filename`. Anydoc conversion is local-only; remote HTTPS inputs are unchanged. Scanned or image-only PDFs require OCR outside espipe and are not converted. +Converted content is stored in `content.body` by default. Use `--content markdown` to store it in `content.markdown`. Multiple local inputs add the existing `file.path` and `file.name` metadata; glob-resolved inputs also add the containing directory as `origin.path` and the basename as `origin.filename`. Anydoc conversion is local-only; remote HTTPS inputs are unchanged. Scanned or image-only PDFs require OCR outside espipe and are not converted. ### Supported output forms diff --git a/openspec/changes/add-anydoc-input/design.md b/openspec/changes/add-anydoc-input/design.md index 7d882c4..d744c38 100644 --- a/openspec/changes/add-anydoc-input/design.md +++ b/openspec/changes/add-anydoc-input/design.md @@ -47,7 +47,7 @@ This preserves: - `content.` placement and `--content` behavior. - YAML frontmatter extraction and content-field conflict checks. -- `file.path` and `file.name` inclusion for multi-file imports, plus `origin.path` and `origin.filename` for glob-resolved imports. +- `file.path` and `file.name` inclusion for multi-file imports, plus the containing directory in `origin.path` and the basename in `origin.filename` for glob-resolved imports. - One serialized `Box` per output document. The resulting flow is: diff --git a/openspec/changes/add-anydoc-input/specs/anydoc-input/spec.md b/openspec/changes/add-anydoc-input/specs/anydoc-input/spec.md index f4a1828..b526d33 100644 --- a/openspec/changes/add-anydoc-input/specs/anydoc-input/spec.md +++ b/openspec/changes/add-anydoc-input/specs/anydoc-input/spec.md @@ -48,7 +48,7 @@ The system SHALL construct converted documents using the existing file-document - **WHEN** anydoc files are imported through a local glob pattern - **THEN** each converted document includes `origin.path` and `origin.filename` -- **AND** those values identify the original local file +- **AND** `origin.path` identifies the containing directory and `origin.filename` identifies the original local file ### Requirement: Existing local discovery mechanisms support anydoc files diff --git a/src/input.rs b/src/input.rs index 584c29f..d9fcdd9 100644 --- a/src/input.rs +++ b/src/input.rs @@ -739,6 +739,8 @@ fn add_file_metadata( metadata: FileMetadataOptions, ) { let path_display = path.display().to_string(); + let origin_path = path.parent().unwrap_or_else(|| Path::new("")); + let origin_path_display = origin_path.display().to_string(); let filename = path .file_name() .and_then(OsStr::to_str) @@ -758,7 +760,7 @@ fn add_file_metadata( document.insert( "origin".to_string(), Value::Object(Map::from_iter([ - ("path".to_string(), Value::String(path_display)), + ("path".to_string(), Value::String(origin_path_display)), ("filename".to_string(), Value::String(filename)), ])), ); @@ -1282,10 +1284,7 @@ mod tests { assert_eq!(values.len(), 1); assert!(values[0]["content"]["body"].is_string()); assert_eq!(values[0]["origin"]["filename"], "sample.pdf"); - assert_eq!( - values[0]["origin"]["path"], - nested.join("sample.pdf").display().to_string() - ); + assert_eq!(values[0]["origin"]["path"], nested.display().to_string()); assert!(values[0].get("file").is_none()); } @@ -1378,14 +1377,11 @@ mod tests { assert_eq!(values[0]["content"]["body"], "child"); assert_eq!(values[1]["content"]["body"], "root"); assert_eq!(values[0]["origin"]["filename"], "child.md"); - assert_eq!( - values[0]["origin"]["path"], - nested.join("child.md").display().to_string() - ); + assert_eq!(values[0]["origin"]["path"], nested.display().to_string()); assert_eq!(values[1]["origin"]["filename"], "root.md"); assert_eq!( values[1]["origin"]["path"], - dir.path().join("root.md").display().to_string() + dir.path().display().to_string() ); } From ff5c3129140116266553676fb83407f018422685 Mon Sep 17 00:00:00 2001 From: Ryan Eno Date: Wed, 12 Aug 2026 15:24:43 -0700 Subject: [PATCH 12/15] Remove duplicate file metadata --- README.md | 4 +- openspec/changes/add-anydoc-input/design.md | 8 +- openspec/changes/add-anydoc-input/proposal.md | 6 +- .../specs/anydoc-input/spec.md | 22 +- .../specs/file-document-import/spec.md | 24 + openspec/changes/add-anydoc-input/tasks.md | 4 +- src/input.rs | 422 +++++++++++++----- tests/file_output.rs | 16 +- tests/index_template.rs | 8 +- 9 files changed, 369 insertions(+), 145 deletions(-) diff --git a/README.md b/README.md index fcbc070..008c583 100644 --- a/README.md +++ b/README.md @@ -138,7 +138,7 @@ are followed by the output URI. - `path/to/file.pdf path/to/file.xlsx output.ndjson` Imports multiple local file inputs in deterministic path order. -HTTPS input URIs are supported for unauthenticated remote `.csv`, `.ndjson`, and `.json` sources. URLs without a supported file extension can still be accepted when the response `Content-Type` maps to CSV or NDJSON-oriented JSON input. +HTTP and HTTPS input URIs are supported for unauthenticated remote `.csv`, `.ndjson`, and `.json` sources. URLs without a supported file extension can still be accepted when the response `Content-Type` maps to CSV or NDJSON-oriented JSON input. ### AnyDoc local documents @@ -146,7 +146,7 @@ Local files with these extensions are converted to GitHub-Flavored Markdown thro `.doc`, `.docx`, `.docm`, `.odt`, `.pdf`, `.ppt`, `.pps`, `.pot`, `.pptx`, `.pptm`, `.ppsx`, `.ppsm`, `.rtf`, `.epub`, `.xls`, `.xlsx`, `.xlsm`, `.xlsb`, `.ods`, and `.odp`. -Converted content is stored in `content.body` by default. Use `--content markdown` to store it in `content.markdown`. Multiple local inputs add the existing `file.path` and `file.name` metadata; glob-resolved inputs also add the containing directory as `origin.path` and the basename as `origin.filename`. Anydoc conversion is local-only; remote HTTPS inputs are unchanged. Scanned or image-only PDFs require OCR outside espipe and are not converted. +Converted content is stored in `content.body` by default. Use `--content markdown` to store it in `content.markdown`. Multi-file and glob-resolved local inputs add an `origin` object with URI components (`scheme`, `authority`, `path`, `query`, `fragment`) and `filename`; remote CSV, NDJSON, and Toon inputs preserve the same components from their source URI. Anydoc conversion remains local-only. Scanned or image-only PDFs require OCR outside espipe and are not converted. ### Supported output forms diff --git a/openspec/changes/add-anydoc-input/design.md b/openspec/changes/add-anydoc-input/design.md index d744c38..b9386bb 100644 --- a/openspec/changes/add-anydoc-input/design.md +++ b/openspec/changes/add-anydoc-input/design.md @@ -2,7 +2,7 @@ Local file imports already resolve concrete paths and recursive glob patterns in `src/input.rs`. The `Input::FileDocuments` variant reads each path lazily, constructs a JSON object, and sends it through the same `Box` output path used by NDJSON, CSV, and other inputs. Today, recognized text formats have specialized readers and all other file-document paths fall back to UTF-8 text, which rejects PDFs and office containers. -The `anydoc` crate provides a Rust-native conversion API for PDF, Word, PowerPoint, Excel, OpenDocument, RTF, and EPUB inputs. Its Markdown output is the appropriate intermediate representation because the existing file-document implementation already defines the desired content field, Markdown frontmatter behavior, and multi-file metadata. +The `anydoc` crate provides a Rust-native conversion API for PDF, Word, PowerPoint, Excel, OpenDocument, RTF, and EPUB inputs. Its Markdown output is the appropriate intermediate representation because the existing file-document implementation already defines the desired content field, Markdown frontmatter behavior, and origin metadata. ## Goals / Non-Goals @@ -19,7 +19,7 @@ The `anydoc` crate provides a Rust-native conversion API for PDF, Word, PowerPoi - OCR for scanned or image-only PDFs. - Remote HTTPS anydoc inputs. - A new `--extensions` option; multiple extension patterns can be passed as existing input positionals. -- Extraction of embedded assets or source-specific metadata beyond the Markdown produced by anydoc and existing `file.*` metadata. +- Extraction of embedded assets or source-specific metadata beyond the Markdown produced by anydoc and the shared `origin` metadata. - Content-based conversion of unknown-extension files, which could change the existing unknown UTF-8 text behavior. ## Decisions @@ -41,13 +41,13 @@ The extension gate intentionally avoids running content detection on every unkno ### Reuse the Markdown document builder -Refactor the existing Markdown reader so it has a helper that accepts `(path, markdown_text, content_field, include_file_metadata)`. The current Markdown path reads the source text and calls this helper; the anydoc path converts the source and calls the same helper. +Refactor the existing Markdown reader so it has a helper that accepts `(path, markdown_text, content_field, include_origin)`. The current Markdown path reads the source text and calls this helper; the anydoc path converts the source and calls the same helper. This preserves: - `content.` placement and `--content` behavior. - YAML frontmatter extraction and content-field conflict checks. -- `file.path` and `file.name` inclusion for multi-file imports, plus the containing directory in `origin.path` and the basename in `origin.filename` for glob-resolved imports. +- A single `origin` metadata object for multi-file and glob-resolved local imports, containing URI components (`scheme`, `authority`, `path`, `query`, `fragment`) and `filename`. Remote HTTP and HTTPS CSV, NDJSON, and Toon imports use the same shape from their source URI. - One serialized `Box` per output document. The resulting flow is: diff --git a/openspec/changes/add-anydoc-input/proposal.md b/openspec/changes/add-anydoc-input/proposal.md index ea5a7c7..c93f1d3 100644 --- a/openspec/changes/add-anydoc-input/proposal.md +++ b/openspec/changes/add-anydoc-input/proposal.md @@ -5,10 +5,10 @@ ## What Changes - Add `anydoc` as a local file input preprocessor for supported non-text formats, including PDF, Word, PowerPoint, Excel, OpenDocument, RTF, and EPUB files. -- Convert each supported file to Markdown before constructing the existing file document, preserving `content.`, Markdown handling, and conditional `file.*` metadata. +- Convert each supported file to Markdown before constructing the existing file document, preserving `content.`, Markdown handling, and the conditional `origin` metadata. - Allow existing concrete file lists and recursive glob patterns such as `**/*.pdf` to ingest converted documents. -- Preserve existing CSV, JSON, NDJSON, Toon, Markdown, YAML, text, stdin, and HTTPS behavior. -- Report conversion failures with the source path and keep remote HTTPS inputs out of scope. +- Preserve existing CSV, JSON, NDJSON, Toon, Markdown, YAML, text, stdin, and HTTP/HTTPS behavior while attaching source URI metadata to remote streaming inputs. +- Report conversion failures with the source path and keep remote non-streaming anydoc inputs out of scope. - Do not add `--extensions` in this change; multiple extension patterns can already be supplied as separate local inputs. A future extension-filter option can build on the existing discovery layer. ## Capabilities diff --git a/openspec/changes/add-anydoc-input/specs/anydoc-input/spec.md b/openspec/changes/add-anydoc-input/specs/anydoc-input/spec.md index b526d33..b0d9023 100644 --- a/openspec/changes/add-anydoc-input/specs/anydoc-input/spec.md +++ b/openspec/changes/add-anydoc-input/specs/anydoc-input/spec.md @@ -24,7 +24,7 @@ The system SHALL use the Rust `anydoc` processor for local regular files with th ### Requirement: Anydoc conversion preserves the existing file-document shape -The system SHALL construct converted documents using the existing file-document semantics after Markdown conversion. It SHALL store converted Markdown under `content.`, apply the configured `--content` value, add `file.path` and `file.name` only under the existing multi-file rules, and add `origin.path` and `origin.filename` when the source was resolved through a glob pattern. +The system SHALL construct converted documents using the existing file-document semantics after Markdown conversion. It SHALL store converted Markdown under `content.`, apply the configured `--content` value, and add one `origin` metadata object for multi-file or glob-resolved local imports. The object SHALL contain `scheme`, `authority`, `path`, `query`, `fragment`, and `filename` fields. It SHALL NOT emit the legacy `file.path` or `file.name` metadata. #### Scenario: Default content field is used for converted Markdown @@ -38,17 +38,25 @@ The system SHALL construct converted documents using the existing file-document - **THEN** the converted Markdown is stored in `content.markdown` - **AND** the system does not add `content.body` solely because the source was converted -#### Scenario: Converted files participate in multi-file metadata +#### Scenario: Converted files participate in multi-file origin metadata - **WHEN** anydoc files are imported together with another file-document input -- **THEN** each converted document includes the same `file.path` and `file.name` metadata as existing file documents -- **AND** the metadata values identify the original local file +- **THEN** each converted document includes an `origin` object +- **AND** its URI components identify the original local file -#### Scenario: Glob-resolved converted files include origin metadata +#### Scenario: Glob-resolved converted files include complete origin metadata - **WHEN** anydoc files are imported through a local glob pattern -- **THEN** each converted document includes `origin.path` and `origin.filename` -- **AND** `origin.path` identifies the containing directory and `origin.filename` identifies the original local file +- **THEN** each converted document includes `origin.scheme` equal to `file` +- **AND** `origin.path` identifies the containing directory +- **AND** `origin.filename` identifies the original local file +- **AND** `origin.authority`, `origin.query`, and `origin.fragment` are present as null when absent + +#### Scenario: Remote inputs preserve URI origin metadata + +- **WHEN** an HTTP or HTTPS CSV, NDJSON, or Toon input is imported +- **THEN** each emitted document includes `origin.scheme`, `origin.authority`, `origin.path`, `origin.query`, `origin.fragment`, and `origin.filename` +- **AND** the values reflect the source URI rather than the temporary download file ### Requirement: Existing local discovery mechanisms support anydoc files diff --git a/openspec/changes/add-anydoc-input/specs/file-document-import/spec.md b/openspec/changes/add-anydoc-input/specs/file-document-import/spec.md index 07d4a33..36097a4 100644 --- a/openspec/changes/add-anydoc-input/specs/file-document-import/spec.md +++ b/openspec/changes/add-anydoc-input/specs/file-document-import/spec.md @@ -55,6 +55,30 @@ The system SHALL accept local glob input patterns, including recursive `**` patt - **THEN** the system imports the matched regular files - **AND** it does not emit documents for matched directories +### Requirement: Multi-file imports include origin metadata + +The system SHALL add one `origin` object to emitted documents when file-document input resolves more than one regular file or uses a glob pattern. The object SHALL contain `scheme`, `authority`, `path`, `query`, `fragment`, and `filename` fields, and the system SHALL NOT emit the legacy `file.path` or `file.name` metadata. + +#### Scenario: Multiple files are imported + +- **WHEN** file-document input resolves to more than one regular file +- **THEN** each emitted document includes the source file's `origin` object +- **AND** each emitted document does not include a `file` object + +#### Scenario: Single direct file is imported + +- **WHEN** file-document input resolves to one direct file without glob resolution +- **THEN** the emitted document does not include `origin` +- **AND** it does not include a `file` object + +#### Scenario: A glob resolves one or more files + +- **WHEN** file-document input uses a local glob pattern +- **THEN** each emitted document includes `origin.scheme` equal to `file` +- **AND** `origin.path` identifies the containing directory +- **AND** `origin.filename` identifies the source file +- **AND** absent `origin.authority`, `origin.query`, and `origin.fragment` values are represented as null + ### Requirement: Binary files are rejected The system SHALL convert local binary files recognized by anydoc into Markdown documents. It SHALL reject file-document inputs that are neither valid UTF-8 text nor recognized supported anydoc formats. diff --git a/openspec/changes/add-anydoc-input/tasks.md b/openspec/changes/add-anydoc-input/tasks.md index 6e58317..367478f 100644 --- a/openspec/changes/add-anydoc-input/tasks.md +++ b/openspec/changes/add-anydoc-input/tasks.md @@ -2,7 +2,7 @@ - [x] 1.1 Add the `anydoc` crate dependency at a Rust 1.88-compatible version. - [x] 1.2 Add extension-gated anydoc routing to `read_file_documents`, preserving the existing specialized readers and excluding CSV from the new branch. -- [x] 1.3 Refactor Markdown file-document construction to accept converted Markdown text so source Markdown and anydoc output share the same content-field, frontmatter, and file-metadata behavior. +- [x] 1.3 Refactor Markdown file-document construction to accept converted Markdown text so source Markdown and anydoc output share the same content-field, frontmatter, and origin-metadata behavior. - [x] 1.4 Add the anydoc conversion wrapper using `anydoc::to_markdown`, including source-path context and underlying conversion errors in diagnostics. - [x] 1.5 Verify direct paths, shell-expanded file lists, recursive globs, mixed anydoc/Markdown collections, deterministic ordering, and lazy conversion continue to use the existing discovery and output pipeline. @@ -16,7 +16,7 @@ ## 3. Output-shape and integration verification - [x] 3.1 Verify converted documents use `content.body` by default and the configured `--content` field when specified. -- [x] 3.2 Verify converted documents preserve existing `file.path` and `file.name` behavior for multi-file imports and omit it for a single direct file. +- [x] 3.2 Verify converted documents use complete `origin` metadata for multi-file and glob imports and omit it for a single direct file. - [x] 3.3 Verify mixed recursive file imports emit converted and existing documents in the same deterministic path order without changing output serialization. - [x] 3.4 Update README input documentation with supported anydoc formats, local-only behavior, glob examples, and the limitation that scanned PDFs require OCR outside espipe. diff --git a/src/input.rs b/src/input.rs index d9fcdd9..db1b296 100644 --- a/src/input.rs +++ b/src/input.rs @@ -8,7 +8,7 @@ use reqwest::{ }; use serde_json::{Map, Value, value::RawValue}; use std::{ - collections::BTreeSet, + collections::BTreeMap, ffi::OsStr, fs::{self, File}, io::{BufRead, BufReader, Read, Seek, SeekFrom, Stdin, Write, stdin}, @@ -22,11 +22,13 @@ pub enum Input { source: String, reader: Box>>, first_record: bool, + origin: Option, _temp_file: Option, }, FileCsv { source: String, reader: Box>>, + origin: Option, _temp_file: Option, }, FileToon { @@ -36,6 +38,7 @@ pub enum Input { document_index: usize, buffered_rows: Vec, eof: bool, + origin: Option, _temp_file: Option, }, Stdin { @@ -44,18 +47,23 @@ pub enum Input { FileDocuments { source: String, paths: Vec, + origins: Vec, path_index: usize, documents: Vec>, document_index: usize, content_field: String, - metadata: FileMetadataOptions, + include_origin: bool, }, } -#[derive(Clone, Copy, Debug, Default)] -pub(crate) struct FileMetadataOptions { - include_file: bool, - include_origin: bool, +#[derive(Clone, Debug)] +pub(crate) struct OriginMetadata { + scheme: String, + authority: Option, + path: String, + query: Option, + fragment: Option, + filename: String, } type CsvRecord = std::collections::HashMap; @@ -82,9 +90,11 @@ impl Input { if uris.len() == 1 { let uri = uris.into_iter().next().unwrap(); return match uri.scheme().map(|scheme| scheme.as_str()) { - Some("https") => tokio::task::spawn_blocking(move || fetch_remote_input(uri)) - .await - .map_err(|err| eyre!("Remote input fetch task failed: {err}"))?, + Some("http" | "https") => { + tokio::task::spawn_blocking(move || fetch_remote_input(uri)) + .await + .map_err(|err| eyre!("Remote input fetch task failed: {err}"))? + } _ => open_input_values(vec![uri], &content_field), }; } @@ -96,13 +106,17 @@ impl Input { Input::FileJson { reader, first_record, + origin, .. } => { let raw = read_json_line(reader, line_buffer, *first_record)?; *first_record = false; - Ok(raw) + add_origin_to_raw(raw, origin.as_ref()) + } + Input::FileCsv { reader, origin, .. } => { + let raw = read_csv_line(reader)?; + add_origin_to_raw(raw, origin.as_ref()) } - Input::FileCsv { reader, .. } => read_csv_line(reader), Input::FileToon { source, reader, @@ -110,8 +124,19 @@ impl Input { document_index, buffered_rows, eof, + origin, .. - } => read_toon_document(source, reader, pending, document_index, buffered_rows, eof), + } => { + let raw = read_toon_document( + source, + reader, + pending, + document_index, + buffered_rows, + eof, + )?; + add_origin_to_raw(raw, origin.as_ref()) + } Input::Stdin { reader, .. } => read_json_line(reader, line_buffer, false), Input::FileDocuments { .. } => read_file_document_line(self), } @@ -131,7 +156,7 @@ impl TryFrom> for Input { fn try_from(uri: UriRef) -> Result { match uri.scheme().map(|scheme| scheme.as_str()) { - Some("https") => fetch_remote_input(uri), + Some("http" | "https") => fetch_remote_input(uri), _ => open_input_values(vec![uri], "body"), } } @@ -162,11 +187,10 @@ fn validate_content_field(content_field: &str) -> Result<()> { fn open_input_values(uris: Vec>, content_field: &str) -> Result { for uri in &uris { match uri.scheme().map(|scheme| scheme.as_str()) { - Some("https") if uris.len() == 1 => return fetch_remote_input(uri.clone()), - Some("https") => { + Some("http" | "https") if uris.len() == 1 => return fetch_remote_input(uri.clone()), + Some("http" | "https") => { return Err(eyre!("Remote inputs cannot be combined with file imports")); } - Some("http") => return Err(eyre!("Unsupported input scheme: http")), Some("file") | None => {} Some(scheme) => return Err(eyre!("Unsupported input scheme: {scheme}")), } @@ -197,14 +221,10 @@ fn open_input_values(uris: Vec>, content_field: &str) -> Result( @@ -298,12 +318,14 @@ fn open_local_file(path: PathBuf) -> Result { .has_headers(true) .from_reader(local_file_reader(file, &path)), ), + origin: None, _temp_file: None, }), InputKind::Ndjson | InputKind::Json => Ok(Input::FileJson { source, reader: Box::new(BufReader::new(local_file_reader(file, &path))), first_record: true, + origin: None, _temp_file: None, }), InputKind::Toon => Ok(Input::FileToon { @@ -313,38 +335,41 @@ fn open_local_file(path: PathBuf) -> Result { document_index: 0, buffered_rows: Vec::new(), eof: false, + origin: None, _temp_file: None, }), - InputKind::FileDocument => open_file_documents(vec![source], "body"), + InputKind::FileDocument => open_file_documents( + vec![UriRef::parse(source).map_err(|err| eyre!("Invalid local file URI: {err:?}"))?], + "body", + ), } } -fn open_file_documents(values: Vec, content_field: &str) -> Result { - let (paths, resolved_from_glob) = resolve_file_document_paths(values)?; - let metadata = FileMetadataOptions { - include_file: paths.len() > 1, - include_origin: resolved_from_glob, - }; +fn open_file_documents(values: Vec>, content_field: &str) -> Result { + let (paths, origins, resolved_from_glob) = resolve_file_document_paths(values)?; + let include_origin = paths.len() > 1 || resolved_from_glob; let source = format!("{} file document(s)", paths.len()); Ok(Input::FileDocuments { source, paths, + origins, path_index: 0, documents: Vec::new(), document_index: 0, content_field: content_field.to_string(), - metadata, + include_origin, }) } fn read_file_document_line(input: &mut Input) -> Result> { let Input::FileDocuments { paths, + origins, path_index, documents, document_index, content_field, - metadata, + include_origin, .. } = input else { @@ -360,31 +385,44 @@ fn read_file_document_line(input: &mut Input) -> Result> { let Some(path) = paths.get(*path_index) else { return Err(eyre!("No file document")); }; + let origin = if *include_origin { + origins.get(*path_index) + } else { + None + }; *path_index += 1; - *documents = read_file_documents(path, content_field, *metadata)?; + *documents = read_file_documents(path, content_field, origin)?; *document_index = 0; } } -fn resolve_file_document_paths(values: Vec) -> Result<(Vec, bool)> { - let mut paths = BTreeSet::new(); +fn resolve_file_document_paths( + values: Vec>, +) -> Result<(Vec, Vec, bool)> { + let mut paths = BTreeMap::new(); let mut any_glob = false; for value in values { - if has_glob_metachar(&value) { + let value_path = value.path().as_str().to_string(); + if has_glob_metachar(&value_path) { any_glob = true; let mut matched_regular_file = false; - for entry in glob(&value).map_err(|err| eyre!("Invalid glob pattern {value}: {err}"))? { - let path = entry.map_err(|err| eyre!("Error expanding glob {value}: {err}"))?; + for entry in glob(&value_path) + .map_err(|err| eyre!("Invalid glob pattern {value_path}: {err}"))? + { + let path = + entry.map_err(|err| eyre!("Error expanding glob {value_path}: {err}"))?; if path.is_file() { matched_regular_file = true; - paths.insert(path); + paths + .entry(path.clone()) + .or_insert_with(|| origin_from_local_path(&path)); } } if !matched_regular_file { - return Err(eyre!("Glob matched no regular files: {value}")); + return Err(eyre!("Glob matched no regular files: {value_path}")); } } else { - let path = PathBuf::from(value); + let path = PathBuf::from(&value_path); if !path.exists() { return Err(eyre!("File input does not exist: {}", path.display())); } @@ -394,10 +432,10 @@ fn resolve_file_document_paths(values: Vec) -> Result<(Vec, boo path.display() )); } - paths.insert(path); + paths.entry(path).or_insert_with(|| origin_from_uri(&value)); } } - for path in &paths { + for path in paths.keys() { let path_str = path.to_string_lossy(); if is_compressed_input(path_str.as_ref()) { return Err(eyre!("Unsupported compressed input format: {path_str}")); @@ -411,7 +449,8 @@ fn resolve_file_document_paths(values: Vec) -> Result<(Vec, boo }; return Err(eyre!("No regular files resolved from {kind}")); } - Ok((paths.into_iter().collect(), any_glob)) + let (paths, origins): (Vec<_>, Vec<_>) = paths.into_iter().unzip(); + Ok((paths, origins, any_glob)) } fn has_glob_metachar(value: &str) -> bool { @@ -428,20 +467,20 @@ fn should_use_file_document(path: &Path) -> bool { fn read_file_documents( path: &Path, content_field: &str, - metadata: FileMetadataOptions, + origin: Option<&OriginMetadata>, ) -> Result>> { match extension(path).as_deref() { - Some("ndjson" | "jsonl") => read_ndjson_file_documents(path, metadata), - Some("json") => read_json_file_document(path, metadata), - Some("toon") => read_toon_file_documents(path, metadata), - Some("yml" | "yaml") => read_yaml_file_document(path, content_field, metadata), - Some("md" | "markdown") => read_markdown_file_document(path, content_field, metadata), + Some("ndjson" | "jsonl") => read_ndjson_file_documents(path, origin), + Some("json") => read_json_file_document(path, origin), + Some("toon") => read_toon_file_documents(path, origin), + Some("yml" | "yaml") => read_yaml_file_document(path, content_field, origin), + Some("md" | "markdown") => read_markdown_file_document(path, content_field, origin), _ if anydoc::Format::from_path(path) .is_some_and(|format| format != anydoc::Format::Csv) => { - read_anydoc_file_document(path, content_field, metadata) + read_anydoc_file_document(path, content_field, origin) } - _ => read_text_file_document(path, content_field, metadata), + _ => read_text_file_document(path, content_field, origin), } } @@ -453,10 +492,10 @@ fn read_text_file(path: &Path) -> Result { fn read_text_file_document( path: &Path, content_field: &str, - metadata: FileMetadataOptions, + origin: Option<&OriginMetadata>, ) -> Result>> { let text = read_text_file(path)?; - let mut document = base_file_document(path, metadata); + let mut document = base_file_document(origin); document.insert( "content".to_string(), Value::Object(Map::from_iter([( @@ -470,10 +509,10 @@ fn read_text_file_document( fn read_markdown_file_document( path: &Path, content_field: &str, - metadata: FileMetadataOptions, + origin: Option<&OriginMetadata>, ) -> Result>> { let text = read_text_file(path)?; - read_markdown_text_document(path, &text, content_field, metadata) + read_markdown_text_document(path, &text, content_field, origin) } #[derive(Debug)] @@ -497,7 +536,7 @@ impl std::error::Error for AnyDocConversionError { fn read_anydoc_file_document( path: &Path, content_field: &str, - metadata: FileMetadataOptions, + origin: Option<&OriginMetadata>, ) -> Result>> { let markdown = anydoc::to_markdown(path).map_err(|source| { Report::new(AnyDocConversionError { @@ -505,14 +544,14 @@ fn read_anydoc_file_document( source, }) })?; - read_markdown_text_document(path, &markdown, content_field, metadata) + read_markdown_text_document(path, &markdown, content_field, origin) } fn read_markdown_text_document( path: &Path, text: &str, content_field: &str, - metadata: FileMetadataOptions, + origin: Option<&OriginMetadata>, ) -> Result>> { let (frontmatter, body) = split_markdown_frontmatter(text); let mut content = Map::new(); @@ -527,7 +566,7 @@ fn read_markdown_text_document( } } content.insert(content_field.to_string(), Value::String(body.to_string())); - let mut document = base_file_document(path, metadata); + let mut document = base_file_document(origin); document.insert("content".to_string(), Value::Object(content)); raw_documents(vec![document]) } @@ -565,7 +604,7 @@ fn is_end_of_input(err: &eyre::Report) -> bool { fn read_yaml_file_document( path: &Path, content_field: &str, - metadata: FileMetadataOptions, + origin: Option<&OriginMetadata>, ) -> Result>> { let text = read_text_file(path)?; let content = yaml_mapping_to_json_map(&text) @@ -576,7 +615,7 @@ fn read_yaml_file_document( path.display() )); } - let mut document = base_file_document(path, metadata); + let mut document = base_file_document(origin); document.insert("content".to_string(), Value::Object(content)); raw_documents(vec![document]) } @@ -591,7 +630,7 @@ fn yaml_mapping_to_json_map(text: &str) -> Result> { fn read_json_file_document( path: &Path, - metadata: FileMetadataOptions, + origin: Option<&OriginMetadata>, ) -> Result>> { let text = read_text_file(path)?; let mut document = match serde_json::from_str::(&text) { @@ -609,13 +648,13 @@ fn read_json_file_document( )); } }; - add_file_metadata(&mut document, path, metadata); + add_origin_metadata(&mut document, origin); raw_documents(vec![document]) } fn read_ndjson_file_documents( path: &Path, - metadata: FileMetadataOptions, + origin: Option<&OriginMetadata>, ) -> Result>> { let text = read_text_file(path)?; let mut docs = Vec::new(); @@ -632,7 +671,7 @@ fn read_ndjson_file_documents( index + 1 )); }; - add_file_metadata(&mut document, path, metadata); + add_origin_metadata(&mut document, origin); docs.push(RawValue::from_string(Value::Object(document).to_string())?); } Ok(docs) @@ -640,7 +679,7 @@ fn read_ndjson_file_documents( fn read_toon_file_documents( path: &Path, - metadata: FileMetadataOptions, + origin: Option<&OriginMetadata>, ) -> Result>> { let file = File::open(path).map_err(|err| eyre!("{}: {err}", path.display()))?; let mut reader = BufReader::new(Box::new(file) as Box); @@ -661,9 +700,9 @@ fn read_toon_file_documents( &mut eof, ) { Ok(mut raw) => { - if metadata.include_file || metadata.include_origin { + if origin.is_some() { let mut document: Map = serde_json::from_str(raw.get())?; - add_file_metadata(&mut document, path, metadata); + add_origin_metadata(&mut document, origin); raw = RawValue::from_string(Value::Object(document).to_string())?; } docs.push(raw); @@ -727,46 +766,94 @@ fn toon_row_value_to_raw(source: &str, document_index: usize, row: Value) -> Res RawValue::from_string(Value::Object(row).to_string()).map_err(Into::into) } -fn base_file_document(path: &Path, metadata: FileMetadataOptions) -> Map { +fn base_file_document(origin: Option<&OriginMetadata>) -> Map { let mut document = Map::new(); - add_file_metadata(&mut document, path, metadata); + add_origin_metadata(&mut document, origin); document } -fn add_file_metadata( - document: &mut Map, - path: &Path, - metadata: FileMetadataOptions, -) { - let path_display = path.display().to_string(); - let origin_path = path.parent().unwrap_or_else(|| Path::new("")); - let origin_path_display = origin_path.display().to_string(); +fn add_origin_metadata(document: &mut Map, origin: Option<&OriginMetadata>) { + if let Some(origin) = origin { + document.insert("origin".to_string(), origin.clone().into_value()); + } +} + +fn origin_from_local_path(path: &Path) -> OriginMetadata { let filename = path .file_name() .and_then(OsStr::to_str) .unwrap_or_default() .to_string(); + let directory = path.parent().unwrap_or_else(|| Path::new("")); + OriginMetadata { + scheme: "file".to_string(), + authority: None, + path: directory.display().to_string(), + query: None, + fragment: None, + filename, + } +} - if metadata.include_file { - document.insert( - "file".to_string(), - Value::Object(Map::from_iter([ - ("path".to_string(), Value::String(path_display.clone())), - ("name".to_string(), Value::String(filename.clone())), - ])), - ); +fn origin_from_uri(uri: &UriRef) -> OriginMetadata { + let path = uri.path().as_str(); + let path_ref = Path::new(path); + OriginMetadata { + scheme: uri + .scheme() + .map(|scheme| scheme.as_str().to_string()) + .unwrap_or_else(|| "file".to_string()), + authority: uri + .authority() + .map(|authority| authority.as_str().to_string()), + path: path_ref + .parent() + .unwrap_or_else(|| Path::new("")) + .to_string_lossy() + .into_owned(), + query: uri.query().map(|query| query.as_str().to_string()), + fragment: uri.fragment().map(|fragment| fragment.as_str().to_string()), + filename: path_ref + .file_name() + .and_then(OsStr::to_str) + .unwrap_or_default() + .to_string(), } - if metadata.include_origin { - document.insert( - "origin".to_string(), - Value::Object(Map::from_iter([ - ("path".to_string(), Value::String(origin_path_display)), - ("filename".to_string(), Value::String(filename)), - ])), - ); +} + +impl OriginMetadata { + fn into_value(self) -> Value { + Value::Object(Map::from_iter([ + ("scheme".to_string(), Value::String(self.scheme)), + ( + "authority".to_string(), + self.authority.map_or(Value::Null, Value::String), + ), + ("path".to_string(), Value::String(self.path)), + ( + "query".to_string(), + self.query.map_or(Value::Null, Value::String), + ), + ( + "fragment".to_string(), + self.fragment.map_or(Value::Null, Value::String), + ), + ("filename".to_string(), Value::String(self.filename)), + ])) } } +fn add_origin_to_raw(raw: Box, origin: Option<&OriginMetadata>) -> Result> { + let Some(origin) = origin else { + return Ok(raw); + }; + let Value::Object(mut document) = serde_json::from_str(raw.get())? else { + return Err(eyre!("Input document must be a JSON object")); + }; + document.insert("origin".to_string(), origin.clone().into_value()); + RawValue::from_string(Value::Object(document).to_string()).map_err(Into::into) +} + fn raw_documents(documents: Vec>) -> Result>> { documents .into_iter() @@ -778,7 +865,6 @@ fn raw_documents(documents: Vec>) -> Result fn fetch_remote_input(uri: UriRef) -> Result { let client = Client::builder() - .https_only(true) .connect_timeout(REMOTE_CONNECT_TIMEOUT) .timeout(REMOTE_REQUEST_TIMEOUT) .build()?; @@ -820,6 +906,7 @@ fn fetch_remote_input_with_client(uri: UriRef, client: &Client) -> Resul let reader_file = temp_file.reopen()?; let source = uri.to_string(); + let origin = Some(origin_from_uri(&uri)); match kind { InputKind::Csv => Ok(Input::FileCsv { @@ -829,12 +916,14 @@ fn fetch_remote_input_with_client(uri: UriRef, client: &Client) -> Resul .has_headers(true) .from_reader(Box::new(reader_file) as Box), ), + origin: origin.clone(), _temp_file: Some(temp_file), }), InputKind::Ndjson | InputKind::Json => Ok(Input::FileJson { source, reader: Box::new(BufReader::new(Box::new(reader_file) as Box)), first_record: true, + origin: origin.clone(), _temp_file: Some(temp_file), }), InputKind::Toon => Ok(Input::FileToon { @@ -844,6 +933,7 @@ fn fetch_remote_input_with_client(uri: UriRef, client: &Client) -> Resul document_index: 0, buffered_rows: Vec::new(), eof: false, + origin, _temp_file: Some(temp_file), }), InputKind::FileDocument => Err(eyre!("Unsupported remote input format")), @@ -990,7 +1080,7 @@ mod tests { use std::{ fs, io::{Read, Write}, - net::TcpListener, + net::{TcpListener, TcpStream}, path::{Path, PathBuf}, sync::{Arc, mpsc}, thread, @@ -1214,7 +1304,7 @@ mod tests { .unwrap() .contains("Hello PDF") ); - assert!(values[0].get("file").is_none()); + assert!(values[0].get("origin").is_none()); } #[test] @@ -1249,7 +1339,7 @@ mod tests { } #[test] - fn anydoc_mixed_file_import_sorts_paths_and_preserves_file_metadata() { + fn anydoc_mixed_file_import_sorts_paths_and_preserves_origin_metadata() { let dir = tempfile::tempdir().unwrap(); let pdf = dir.path().join("sample.pdf"); write_base64_fixture("anydoc/sample.pdf.base64", &pdf); @@ -1258,8 +1348,16 @@ mod tests { let values = collect_values(open_input_values(vec![uri(&rtf), uri(&pdf)], "body").unwrap()); assert_eq!(values.len(), 2); - assert_eq!(values[0]["file"]["name"], "sample.pdf"); - assert_eq!(values[1]["file"]["name"], "sample.rtf"); + assert_eq!(values[0]["origin"]["scheme"], "file"); + assert!(values[0]["origin"]["authority"].is_null()); + assert_eq!(values[0]["origin"]["filename"], "sample.pdf"); + assert_eq!( + values[0]["origin"]["path"], + dir.path().display().to_string() + ); + assert!(values[0]["origin"]["query"].is_null()); + assert!(values[0]["origin"]["fragment"].is_null()); + assert_eq!(values[1]["origin"]["filename"], "sample.rtf"); assert!(values[0]["content"]["body"].is_string()); assert!(values[1]["content"]["body"].is_string()); } @@ -1285,7 +1383,6 @@ mod tests { assert!(values[0]["content"]["body"].is_string()); assert_eq!(values[0]["origin"]["filename"], "sample.pdf"); assert_eq!(values[0]["origin"]["path"], nested.display().to_string()); - assert!(values[0].get("file").is_none()); } #[test] @@ -1310,8 +1407,8 @@ mod tests { ); assert_eq!(values.len(), 2); - assert_eq!(values[0]["file"]["name"], "a.pdf"); - assert_eq!(values[1]["file"]["name"], "b.rtf"); + assert_eq!(values[0]["origin"]["filename"], "a.pdf"); + assert_eq!(values[1]["origin"]["filename"], "b.rtf"); } #[test] @@ -1339,7 +1436,7 @@ mod tests { } #[test] - fn shell_expanded_files_are_sorted_deduplicated_and_include_metadata() { + fn shell_expanded_files_are_sorted_deduplicated_and_include_origin_metadata() { let dir = tempfile::tempdir().unwrap(); let b = dir.path().join("b.txt"); let a = dir.path().join("a.txt"); @@ -1352,8 +1449,8 @@ mod tests { assert_eq!(values.len(), 2); assert_eq!(values[0]["content"]["body"], "alpha"); assert_eq!(values[1]["content"]["body"], "bravo"); - assert_eq!(values[0]["file"]["name"], "a.txt"); - assert_eq!(values[1]["file"]["name"], "b.txt"); + assert_eq!(values[0]["origin"]["filename"], "a.txt"); + assert_eq!(values[1]["origin"]["filename"], "b.txt"); } #[test] @@ -1452,14 +1549,14 @@ mod tests { } #[test] - fn single_direct_file_document_omits_file_metadata() { + fn single_direct_file_document_omits_origin_metadata() { let dir = tempfile::tempdir().unwrap(); let path = dir.path().join("note.txt"); fs::write(&path, "hello").unwrap(); let values = collect_values(open_input_values(vec![uri(&path)], "body").unwrap()); - assert!(values[0].get("file").is_none()); + assert!(values[0].get("origin").is_none()); } #[test] @@ -1668,7 +1765,7 @@ mod tests { } #[test] - fn toon_file_in_multi_input_includes_file_metadata() { + fn toon_file_in_multi_input_includes_origin_metadata() { let dir = tempfile::tempdir().unwrap(); let text = dir.path().join("a.txt"); let toon = dir.path().join("b.toon"); @@ -1681,7 +1778,7 @@ mod tests { assert_eq!(values.len(), 2); assert_eq!(values[0]["content"]["body"], "alpha"); assert_eq!(values[1]["id"], 2); - assert_eq!(values[1]["file"]["name"], "b.toon"); + assert_eq!(values[1]["origin"]["filename"], "b.toon"); } #[test] @@ -1787,14 +1884,35 @@ mod tests { } #[test] - fn http_input_scheme_is_rejected() { - let uri = UriRef::parse("http://example.com/data.ndjson".to_string()).unwrap(); + fn unsupported_input_scheme_is_rejected() { + let uri = UriRef::parse("ftp://example.com/data.ndjson".to_string()).unwrap(); match Input::try_from(uri) { - Ok(_) => panic!("http input should be rejected"), - Err(err) => assert!(err.to_string().contains("Unsupported input scheme: http")), + Ok(_) => panic!("ftp input should be rejected"), + Err(err) => assert!(err.to_string().contains("Unsupported input scheme: ftp")), } } + #[test] + fn remote_http_fetch_preserves_origin_uri_components() { + let (base_url, _requests, handle) = + spawn_http_server("200 OK", "text/csv", "name,count\nalpha,2\n"); + let uri = + UriRef::parse(format!("{base_url}/docs/data.csv?download=1#row").to_string()).unwrap(); + let authority = uri.authority().unwrap().as_str().to_string(); + let values = collect_values( + fetch_remote_input_with_client(uri, &Client::builder().build().unwrap()).unwrap(), + ); + + assert_eq!(values[0]["name"], "alpha"); + assert_eq!(values[0]["origin"]["scheme"], "http"); + assert_eq!(values[0]["origin"]["authority"], authority); + assert_eq!(values[0]["origin"]["path"], "/docs"); + assert_eq!(values[0]["origin"]["filename"], "data.csv"); + assert_eq!(values[0]["origin"]["query"], "download=1"); + assert_eq!(values[0]["origin"]["fragment"], "row"); + handle.join().unwrap(); + } + #[test] fn json_extension_is_accepted_for_local_input_detection() { let path = PathBuf::from("/tmp/example.json"); @@ -1808,12 +1926,20 @@ mod tests { spawn_https_server("200 OK", "text/csv", "name,count\nalpha,2\n"); let client = test_https_client(); let uri = UriRef::parse(format!("{base_url}/download").to_string()).unwrap(); + let authority = uri.authority().unwrap().as_str().to_string(); let mut input = fetch_remote_input_with_client(uri, &client).unwrap(); let mut line = String::new(); let value = input.read_line(&mut line).unwrap(); let actual: serde_json::Value = serde_json::from_str(value.get()).unwrap(); - assert_eq!(actual, serde_json::json!({"name":"alpha","count":"2"})); + assert_eq!(actual["name"], "alpha"); + assert_eq!(actual["count"], "2"); + assert_eq!(actual["origin"]["scheme"], "https"); + assert_eq!(actual["origin"]["authority"], authority); + assert_eq!(actual["origin"]["path"], "/"); + assert_eq!(actual["origin"]["filename"], "download"); + assert!(actual["origin"]["query"].is_null()); + assert!(actual["origin"]["fragment"].is_null()); let request = requests.recv().unwrap(); let accept_header = request @@ -1847,11 +1973,18 @@ mod tests { let (base_url, _requests, handle) = spawn_https_server("200 OK", "application/octet-stream", "id: 1\nname: Alpha\n"); let client = test_https_client(); - let uri = UriRef::parse(format!("{base_url}/events.toon").to_string()).unwrap(); + let uri = + UriRef::parse(format!("{base_url}/events.toon?download=1#page").to_string()).unwrap(); let values = collect_values(fetch_remote_input_with_client(uri, &client).unwrap()); - assert_eq!(values, vec![serde_json::json!({"id":1,"name":"Alpha"})]); + assert_eq!(values[0]["id"], 1); + assert_eq!(values[0]["name"], "Alpha"); + assert_eq!(values[0]["origin"]["scheme"], "https"); + assert_eq!(values[0]["origin"]["path"], "/"); + assert_eq!(values[0]["origin"]["filename"], "events.toon"); + assert_eq!(values[0]["origin"]["query"], "download=1"); + assert_eq!(values[0]["origin"]["fragment"], "page"); handle.join().unwrap(); } @@ -1864,7 +1997,11 @@ mod tests { let values = collect_values(fetch_remote_input_with_client(uri, &client).unwrap()); - assert_eq!(values, vec![serde_json::json!({"id":1,"name":"Alpha"})]); + assert_eq!(values[0]["id"], 1); + assert_eq!(values[0]["name"], "Alpha"); + assert_eq!(values[0]["origin"]["scheme"], "https"); + assert_eq!(values[0]["origin"]["path"], "/"); + assert_eq!(values[0]["origin"]["filename"], "download"); handle.join().unwrap(); } @@ -1932,6 +2069,55 @@ mod tests { .unwrap() } + fn spawn_http_server( + status: &str, + content_type: &str, + body: &str, + ) -> (String, mpsc::Receiver, thread::JoinHandle<()>) { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let port = listener.local_addr().unwrap().port(); + let status = status.to_string(); + let content_type = content_type.to_string(); + let body = body.to_string(); + let (tx, rx) = mpsc::channel(); + + let handle = thread::spawn(move || { + let (stream, _) = listener.accept().unwrap(); + serve_http_request(stream, &tx, &status, &content_type, &body); + }); + + (format!("http://localhost:{port}"), rx, handle) + } + + fn serve_http_request( + mut stream: TcpStream, + tx: &mpsc::Sender, + status: &str, + content_type: &str, + body: &str, + ) { + let mut request = Vec::new(); + let mut buf = [0u8; 1024]; + loop { + let count = stream.read(&mut buf).unwrap(); + if count == 0 { + break; + } + request.extend_from_slice(&buf[..count]); + if request.windows(4).any(|window| window == b"\r\n\r\n") { + break; + } + } + + tx.send(String::from_utf8(request).unwrap()).unwrap(); + let response = format!( + "HTTP/1.1 {status}\r\nContent-Type: {content_type}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len() + ); + stream.write_all(response.as_bytes()).unwrap(); + stream.flush().unwrap(); + } + fn spawn_https_server( status: &str, content_type: &str, diff --git a/tests/file_output.rs b/tests/file_output.rs index 7e75a2a..70ba2e3 100644 --- a/tests/file_output.rs +++ b/tests/file_output.rs @@ -108,6 +108,7 @@ fn cli_converts_anydoc_pdf_to_existing_file_document_output() { .expect("body string") .contains("Hello PDF") ); + assert!(document.get("origin").is_none()); assert!(document.get("file").is_none()); } @@ -148,7 +149,10 @@ fn cli_converts_mixed_anydoc_and_markdown_inputs_without_changing_shape() { .is_some_and(|body| body.contains("Alpha")) })); assert!(documents.iter().all(|document| { - document["file"]["path"].is_string() && document["file"]["name"].is_string() + document.get("file").is_none() + && document["origin"]["scheme"] == "file" + && document["origin"]["path"].is_string() + && document["origin"]["filename"].is_string() })); } @@ -294,8 +298,9 @@ fn cli_accepts_multi_file_input_to_ndjson_file_output() { assert!(status.success(), "espipe exited with failure"); let contents = fs::read_to_string(&output_path).expect("read output file"); - assert!(contents.contains(r#""name":"alpha.md""#)); - assert!(contents.contains(r#""name":"bravo.md""#)); + assert!(contents.contains(r#""filename":"alpha.md""#)); + assert!(contents.contains(r#""filename":"bravo.md""#)); + assert!(!contents.contains(r#""file":{"#)); } #[test] @@ -319,8 +324,9 @@ fn cli_accepts_multi_file_input_to_gzip_ndjson_file_output() { decoder .read_to_string(&mut contents) .expect("decompress output"); - assert!(contents.contains(r#""name":"alpha.md""#)); - assert!(contents.contains(r#""name":"bravo.md""#)); + assert!(contents.contains(r#""filename":"alpha.md""#)); + assert!(contents.contains(r#""filename":"bravo.md""#)); + assert!(!contents.contains(r#""file":{"#)); } #[test] diff --git a/tests/index_template.rs b/tests/index_template.rs index ae54890..d03c4ef 100644 --- a/tests/index_template.rs +++ b/tests/index_template.rs @@ -407,10 +407,10 @@ fn cli_globs_fixture_documents_with_pipeline_and_template() { .map(|request| request.body.as_str()) .collect::>() .join("\n"); - assert!(bulk_body.contains(r#""name":"alpha.md""#)); - assert!(bulk_body.contains(r#""name":"bravo.md""#)); - assert!(bulk_body.contains(r#""name":"charlie.md""#)); - assert!(bulk_body.contains(r#""name":"delta.md""#)); + assert!(bulk_body.contains(r#""filename":"alpha.md""#)); + assert!(bulk_body.contains(r#""filename":"bravo.md""#)); + assert!(bulk_body.contains(r#""filename":"charlie.md""#)); + assert!(bulk_body.contains(r#""filename":"delta.md""#)); assert!(!bulk_body.contains("ignored.tmp")); assert!(bulk_body.contains("\"markdown\":\"# Alpha\\n\\nFirst document.")); assert!(bulk_body.contains("\"markdown\":\"# Bravo\\n\\nSecond document.")); From 1fa034f2071e7df8d8c2ca30435e845725d7cbc2 Mon Sep 17 00:00:00 2001 From: Ryan Eno Date: Wed, 12 Aug 2026 15:29:09 -0700 Subject: [PATCH 13/15] Normalize origin URI serialization --- .../specs/anydoc-input/spec.md | 2 +- .../specs/file-document-import/spec.md | 2 +- src/input.rs | 66 +++++++++++-------- 3 files changed, 42 insertions(+), 28 deletions(-) diff --git a/openspec/changes/add-anydoc-input/specs/anydoc-input/spec.md b/openspec/changes/add-anydoc-input/specs/anydoc-input/spec.md index b0d9023..5705fba 100644 --- a/openspec/changes/add-anydoc-input/specs/anydoc-input/spec.md +++ b/openspec/changes/add-anydoc-input/specs/anydoc-input/spec.md @@ -50,7 +50,7 @@ The system SHALL construct converted documents using the existing file-document - **THEN** each converted document includes `origin.scheme` equal to `file` - **AND** `origin.path` identifies the containing directory - **AND** `origin.filename` identifies the original local file -- **AND** `origin.authority`, `origin.query`, and `origin.fragment` are present as null when absent +- **AND** absent `origin.authority`, `origin.query`, and `origin.fragment` fields are omitted #### Scenario: Remote inputs preserve URI origin metadata diff --git a/openspec/changes/add-anydoc-input/specs/file-document-import/spec.md b/openspec/changes/add-anydoc-input/specs/file-document-import/spec.md index 36097a4..50b4fda 100644 --- a/openspec/changes/add-anydoc-input/specs/file-document-import/spec.md +++ b/openspec/changes/add-anydoc-input/specs/file-document-import/spec.md @@ -77,7 +77,7 @@ The system SHALL add one `origin` object to emitted documents when file-document - **THEN** each emitted document includes `origin.scheme` equal to `file` - **AND** `origin.path` identifies the containing directory - **AND** `origin.filename` identifies the source file -- **AND** absent `origin.authority`, `origin.query`, and `origin.fragment` values are represented as null +- **AND** absent `origin.authority`, `origin.query`, and `origin.fragment` fields are omitted ### Requirement: Binary files are rejected diff --git a/src/input.rs b/src/input.rs index db1b296..a349f6d 100644 --- a/src/input.rs +++ b/src/input.rs @@ -788,7 +788,7 @@ fn origin_from_local_path(path: &Path) -> OriginMetadata { OriginMetadata { scheme: "file".to_string(), authority: None, - path: directory.display().to_string(), + path: origin_directory(directory), query: None, fragment: None, filename, @@ -806,11 +806,7 @@ fn origin_from_uri(uri: &UriRef) -> OriginMetadata { authority: uri .authority() .map(|authority| authority.as_str().to_string()), - path: path_ref - .parent() - .unwrap_or_else(|| Path::new("")) - .to_string_lossy() - .into_owned(), + path: origin_directory(path_ref.parent().unwrap_or_else(|| Path::new(""))), query: uri.query().map(|query| query.as_str().to_string()), fragment: uri.fragment().map(|fragment| fragment.as_str().to_string()), filename: path_ref @@ -823,23 +819,29 @@ fn origin_from_uri(uri: &UriRef) -> OriginMetadata { impl OriginMetadata { fn into_value(self) -> Value { - Value::Object(Map::from_iter([ + let mut object = Map::from_iter([ ("scheme".to_string(), Value::String(self.scheme)), - ( - "authority".to_string(), - self.authority.map_or(Value::Null, Value::String), - ), ("path".to_string(), Value::String(self.path)), - ( - "query".to_string(), - self.query.map_or(Value::Null, Value::String), - ), - ( - "fragment".to_string(), - self.fragment.map_or(Value::Null, Value::String), - ), ("filename".to_string(), Value::String(self.filename)), - ])) + ]); + if let Some(authority) = self.authority { + object.insert("authority".to_string(), Value::String(authority)); + } + if let Some(query) = self.query { + object.insert("query".to_string(), Value::String(query)); + } + if let Some(fragment) = self.fragment { + object.insert("fragment".to_string(), Value::String(fragment)); + } + Value::Object(object) + } +} + +fn origin_directory(path: &Path) -> String { + if path.as_os_str().is_empty() { + "/".to_string() + } else { + path.to_string_lossy().into_owned() } } @@ -1067,7 +1069,7 @@ mod tests { use super::{ Input, InputKind, JSON_LINE_OPENING_ERROR, REMOTE_NDJSON_ERROR, fetch_remote_input_with_client, input_kind_from_path, local_input_kind, open_input_values, - validate_content_field, validate_ndjson_file, + origin_from_local_path, validate_content_field, validate_ndjson_file, }; use base64::Engine as _; use flate2::{Compression, write::GzEncoder}; @@ -1290,6 +1292,18 @@ mod tests { ); } + #[test] + fn origin_metadata_uses_root_for_relative_paths_and_omits_null_values() { + let value = origin_from_local_path(Path::new("document.pdf")).into_value(); + + assert_eq!(value["scheme"], "file"); + assert_eq!(value["path"], "/"); + assert_eq!(value["filename"], "document.pdf"); + assert!(value.get("authority").is_none()); + assert!(value.get("query").is_none()); + assert!(value.get("fragment").is_none()); + } + #[test] fn anydoc_converts_pdf_to_default_markdown_document() { let dir = tempfile::tempdir().unwrap(); @@ -1349,14 +1363,14 @@ mod tests { assert_eq!(values.len(), 2); assert_eq!(values[0]["origin"]["scheme"], "file"); - assert!(values[0]["origin"]["authority"].is_null()); + assert!(values[0]["origin"].get("authority").is_none()); assert_eq!(values[0]["origin"]["filename"], "sample.pdf"); assert_eq!( values[0]["origin"]["path"], dir.path().display().to_string() ); - assert!(values[0]["origin"]["query"].is_null()); - assert!(values[0]["origin"]["fragment"].is_null()); + assert!(values[0]["origin"].get("query").is_none()); + assert!(values[0]["origin"].get("fragment").is_none()); assert_eq!(values[1]["origin"]["filename"], "sample.rtf"); assert!(values[0]["content"]["body"].is_string()); assert!(values[1]["content"]["body"].is_string()); @@ -1938,8 +1952,8 @@ mod tests { assert_eq!(actual["origin"]["authority"], authority); assert_eq!(actual["origin"]["path"], "/"); assert_eq!(actual["origin"]["filename"], "download"); - assert!(actual["origin"]["query"].is_null()); - assert!(actual["origin"]["fragment"].is_null()); + assert!(actual["origin"].get("query").is_none()); + assert!(actual["origin"].get("fragment").is_none()); let request = requests.recv().unwrap(); let accept_header = request From ddc8dc2bbb5a49ad2476216057ca99f3bc43f80f Mon Sep 17 00:00:00 2001 From: Ryan Eno Date: Wed, 12 Aug 2026 15:33:16 -0700 Subject: [PATCH 14/15] Use relative root for local origin paths --- src/input.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/input.rs b/src/input.rs index a349f6d..2ba6439 100644 --- a/src/input.rs +++ b/src/input.rs @@ -839,7 +839,7 @@ impl OriginMetadata { fn origin_directory(path: &Path) -> String { if path.as_os_str().is_empty() { - "/".to_string() + "./".to_string() } else { path.to_string_lossy().into_owned() } @@ -1297,7 +1297,7 @@ mod tests { let value = origin_from_local_path(Path::new("document.pdf")).into_value(); assert_eq!(value["scheme"], "file"); - assert_eq!(value["path"], "/"); + assert_eq!(value["path"], "./"); assert_eq!(value["filename"], "document.pdf"); assert!(value.get("authority").is_none()); assert!(value.get("query").is_none()); From a19c5d67f7ff3f6fa2d0eef688469d99e12a06a0 Mon Sep 17 00:00:00 2001 From: Ryan Eno Date: Wed, 12 Aug 2026 15:51:34 -0700 Subject: [PATCH 15/15] Handle URI directory origins consistently --- src/input.rs | 41 ++++++++++++++++++++++++++++++++++------- 1 file changed, 34 insertions(+), 7 deletions(-) diff --git a/src/input.rs b/src/input.rs index 2ba6439..a73d27e 100644 --- a/src/input.rs +++ b/src/input.rs @@ -798,6 +798,7 @@ fn origin_from_local_path(path: &Path) -> OriginMetadata { fn origin_from_uri(uri: &UriRef) -> OriginMetadata { let path = uri.path().as_str(); let path_ref = Path::new(path); + let is_directory = path.is_empty() || path.ends_with('/'); OriginMetadata { scheme: uri .scheme() @@ -806,14 +807,26 @@ fn origin_from_uri(uri: &UriRef) -> OriginMetadata { authority: uri .authority() .map(|authority| authority.as_str().to_string()), - path: origin_directory(path_ref.parent().unwrap_or_else(|| Path::new(""))), + path: if is_directory { + if path.is_empty() { + "./".to_string() + } else { + path.to_string() + } + } else { + origin_directory(path_ref.parent().unwrap_or_else(|| Path::new(""))) + }, query: uri.query().map(|query| query.as_str().to_string()), fragment: uri.fragment().map(|fragment| fragment.as_str().to_string()), - filename: path_ref - .file_name() - .and_then(OsStr::to_str) - .unwrap_or_default() - .to_string(), + filename: if is_directory { + String::new() + } else { + path_ref + .file_name() + .and_then(OsStr::to_str) + .unwrap_or_default() + .to_string() + }, } } @@ -1069,7 +1082,7 @@ mod tests { use super::{ Input, InputKind, JSON_LINE_OPENING_ERROR, REMOTE_NDJSON_ERROR, fetch_remote_input_with_client, input_kind_from_path, local_input_kind, open_input_values, - origin_from_local_path, validate_content_field, validate_ndjson_file, + origin_from_local_path, origin_from_uri, validate_content_field, validate_ndjson_file, }; use base64::Engine as _; use flate2::{Compression, write::GzEncoder}; @@ -1304,6 +1317,20 @@ mod tests { assert!(value.get("fragment").is_none()); } + #[test] + fn origin_metadata_handles_uri_root_and_trailing_slash() { + let root = origin_from_uri(&UriRef::parse("https://example.com/".to_string()).unwrap()) + .into_value(); + assert_eq!(root["path"], "/"); + assert_eq!(root["filename"], ""); + + let directory = + origin_from_uri(&UriRef::parse("https://example.com/docs/".to_string()).unwrap()) + .into_value(); + assert_eq!(directory["path"], "/docs/"); + assert_eq!(directory["filename"], ""); + } + #[test] fn anydoc_converts_pdf_to_default_markdown_document() { let dir = tempfile::tempdir().unwrap();