diff --git a/Cargo.lock b/Cargo.lock index 14e0791..86ef38d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8,6 +8,17 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures 0.2.17", +] + [[package]] name = "aho-corasick" version = "1.1.4" @@ -76,6 +87,23 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "anydoc" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b8e42cf43074578dd1d5a6771d1f28282790b2bf4bca60061ff51fd8ca03c09" +dependencies = [ + "calamine", + "cfb", + "csv", + "encoding_rs", + "flate2", + "log", + "pdf-inspector", + "quick-xml", + "zip", +] + [[package]] name = "anyhow" version = "1.0.102" @@ -91,7 +119,7 @@ dependencies = [ "asn1-rs-derive", "asn1-rs-impl", "displaydoc", - "nom", + "nom 7.1.3", "num-traits", "rusticata-macros", "thiserror", @@ -133,6 +161,16 @@ dependencies = [ "tokio", ] +[[package]] +name = "atoi_simd" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3cdb3708a128e559a30fb830e8a77a5022ee6902806925c216658652b452a44" +dependencies = [ + "debug_unsafe", + "rustversion", +] + [[package]] name = "atomic-waker" version = "1.1.2" @@ -188,6 +226,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block-padding" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93" +dependencies = [ + "generic-array", +] + [[package]] name = "borrow-or-share" version = "0.2.4" @@ -200,12 +247,45 @@ version = "3.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + [[package]] name = "bytes" version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" +[[package]] +name = "calamine" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fa68281b1a76b54a62156474adb06bb380a67e07dd60656e3217152b42183f3" +dependencies = [ + "atoi_simd", + "byteorder", + "chrono", + "codepage", + "encoding_rs", + "fast-float2", + "log", + "quick-xml", + "serde", + "zip", +] + +[[package]] +name = "cbc" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26b52a9543ae338f279b96b0b9fed9c8093744685043739079ce85cd58f289a6" +dependencies = [ + "cipher", +] + [[package]] name = "cc" version = "1.2.61" @@ -218,6 +298,17 @@ dependencies = [ "shlex", ] +[[package]] +name = "cfb" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a347dcabdae9c31b0825fd6a8bed285ec9c2acb89c47827126d52fa4f59cece3" +dependencies = [ + "fnv", + "uuid", + "web-time", +] + [[package]] name = "cfg-if" version = "1.0.4" @@ -230,6 +321,17 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" +[[package]] +name = "chacha20" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "rand_core 0.10.1", +] + [[package]] name = "chrono" version = "0.4.43" @@ -242,6 +344,16 @@ dependencies = [ "windows-link", ] +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common", + "inout", +] + [[package]] name = "clap" version = "4.6.1" @@ -291,6 +403,15 @@ dependencies = [ "cc", ] +[[package]] +name = "codepage" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48f68d061bc2828ae826206326e61251aca94c1e4a5305cf52d9138639c918b4" +dependencies = [ + "encoding_rs", +] + [[package]] name = "colorchoice" version = "1.0.5" @@ -359,6 +480,15 @@ dependencies = [ "libc", ] +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + [[package]] name = "crc32fast" version = "1.5.0" @@ -368,6 +498,31 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crossbeam-deque" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5181e0de7b61eb03a81e347d6dd8797bae9da5146707b51077e2d71a54ec0ceb" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" + [[package]] name = "crypto-common" version = "0.1.7" @@ -440,6 +595,12 @@ version = "2.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d7a1e2f27636f116493b8b860f5546edb47c8d8f8ea73e1d2a20be88e28d1fea" +[[package]] +name = "debug_unsafe" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7eed2c4702fa172d1ce21078faa7c5203e69f5394d48cc436d25928394a867a2" + [[package]] name = "der-parser" version = "10.0.0" @@ -448,7 +609,7 @@ checksum = "07da5016415d5a3c4dd39b11ed26f915f52fc4e0dc197d87908bc916e51bc1a6" dependencies = [ "asn1-rs", "displaydoc", - "nom", + "nom 7.1.3", "num-bigint", "num-traits", "rusticata-macros", @@ -497,6 +658,21 @@ version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" +[[package]] +name = "ecb" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a8bfa975b1aec2145850fcaa1c6fe269a16578c44705a532ae3edc92b8881c7" +dependencies = [ + "cipher", +] + +[[package]] +name = "either" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e5e8f6c15a24b9a3ee5efec809ccd006d3b30e8b3bb63c39af737c7f87daa1d" + [[package]] name = "elasticsearch" version = "9.1.0-alpha.1" @@ -570,8 +746,9 @@ dependencies = [ [[package]] name = "espipe" -version = "0.4.0" +version = "0.5.0" dependencies = [ + "anydoc", "base64", "bytes", "clap", @@ -607,6 +784,12 @@ dependencies = [ "once_cell", ] +[[package]] +name = "fast-float2" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8eb564c5c7423d25c886fb561d1e4ee69f72354d16918afa32c08811f6b6a55" + [[package]] name = "fastrand" version = "2.4.1" @@ -627,6 +810,7 @@ checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" dependencies = [ "crc32fast", "miniz_oxide", + "zlib-rs", ] [[package]] @@ -814,10 +998,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" dependencies = [ "cfg-if", + "js-sys", "libc", "r-efi 6.0.0", + "rand_core 0.10.1", "wasip2", "wasip3", + "wasm-bindgen", ] [[package]] @@ -1135,6 +1322,25 @@ dependencies = [ "icu_properties", ] +[[package]] +name = "include_dir" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "923d117408f1e49d914f1a379a309cffe4f18c05cf4e3d12e613a15fc81bd0dd" +dependencies = [ + "include_dir_macros", +] + +[[package]] +name = "include_dir_macros" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cab85a7ed0bd5f0e76d93846e0147172bed2e2d3f859bcc33a8d9699cad1a75" +dependencies = [ + "proc-macro2", + "quote", +] + [[package]] name = "indenter" version = "0.3.4" @@ -1164,6 +1370,16 @@ dependencies = [ "serde_core", ] +[[package]] +name = "inout" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +dependencies = [ + "block-padding", + "generic-array", +] + [[package]] name = "ipnet" version = "2.11.0" @@ -1199,10 +1415,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a3546dc96b6d42c5f24902af9e2538e82e39ad350b0c766eb3fbf2d8f3d8359" dependencies = [ "jiff-static", + "jiff-tzdb-platform", "log", "portable-atomic", "portable-atomic-util", "serde_core", + "windows-sys 0.61.2", ] [[package]] @@ -1216,6 +1434,21 @@ dependencies = [ "syn", ] +[[package]] +name = "jiff-tzdb" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "142bd39932ad231f10513df9ab62661fead8719872150b7ad02a2df79f4e141e" + +[[package]] +name = "jiff-tzdb-platform" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "875a5a69ac2bab1a891711cf5eccbec1ce0341ea805560dcd90b7a2e925132e8" +dependencies = [ + "jiff-tzdb", +] + [[package]] name = "jni" version = "0.22.4" @@ -1330,12 +1563,53 @@ version = "0.4.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" +[[package]] +name = "lopdf" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25aab26d99567469098e64a02f42679f8965c6401263eefa31d8f2dcc37a221c" +dependencies = [ + "aes", + "bitflags", + "cbc", + "chrono", + "ecb", + "encoding_rs", + "flate2", + "getrandom 0.4.2", + "indexmap 2.13.0", + "itoa", + "jiff", + "log", + "md-5", + "nom 8.0.0", + "rand 0.10.2", + "rangemap", + "rayon", + "sha2", + "stringprep", + "thiserror", + "time", + "ttf-parser", + "weezl", +] + [[package]] name = "lru-slab" version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest", +] + [[package]] name = "memchr" version = "2.8.0" @@ -1402,6 +1676,15 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "nom" +version = "8.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405" +dependencies = [ + "memchr", +] + [[package]] name = "num-bigint" version = "0.4.6" @@ -1530,6 +1813,24 @@ dependencies = [ "windows-link", ] +[[package]] +name = "pdf-inspector" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd2f755e49ad38eafbc82ba2bec1c59d57b5bf829b13a8f35e4263bc8913df3a" +dependencies = [ + "env_logger", + "include_dir", + "log", + "lopdf", + "once_cell", + "rayon", + "regex", + "thiserror", + "ttf-parser", + "unicode-normalization", +] + [[package]] name = "pem" version = "3.0.6" @@ -1665,6 +1966,16 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "quick-xml" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1" +dependencies = [ + "encoding_rs", + "memchr", +] + [[package]] name = "quinn" version = "0.11.9" @@ -1695,7 +2006,7 @@ dependencies = [ "bytes", "getrandom 0.3.4", "lru-slab", - "rand", + "rand 0.9.4", "ring", "rustc-hash", "rustls", @@ -1749,7 +2060,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" dependencies = [ "rand_chacha", - "rand_core", + "rand_core 0.9.5", +] + +[[package]] +name = "rand" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +dependencies = [ + "chacha20", + "getrandom 0.4.2", + "rand_core 0.10.1", ] [[package]] @@ -1759,7 +2081,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" dependencies = [ "ppv-lite86", - "rand_core", + "rand_core 0.9.5", ] [[package]] @@ -1771,6 +2093,38 @@ dependencies = [ "getrandom 0.3.4", ] +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "rangemap" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "973443cf09a9c8656b574a866ab68dfa19f0867d0340648c7d2f6a71b8a8ea68" + +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + [[package]] name = "rcgen" version = "0.14.7" @@ -1954,7 +2308,7 @@ version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "faf0c4a6ece9950b9abdb62b1cfcf2a68b3b67a10ba445b3bb85be2a293d0632" dependencies = [ - "nom", + "nom 7.1.3", ] [[package]] @@ -2266,7 +2620,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "digest", ] @@ -2326,6 +2680,17 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" +[[package]] +name = "stringprep" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1" +dependencies = [ + "unicode-bidi", + "unicode-normalization", + "unicode-properties", +] + [[package]] name = "strsim" version = "0.11.1" @@ -2625,6 +2990,18 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "ttf-parser" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2df906b07856748fa3f6e0ad0cbaa047052d4a7dd609e231c4f72cee8c36f31" + +[[package]] +name = "typed-path" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e28f89b80c87b8fb0cf04ab448d5dd0dd0ade2f8891bae878de66a75a28600e" + [[package]] name = "typenum" version = "1.20.0" @@ -2637,12 +3014,33 @@ version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971" +[[package]] +name = "unicode-bidi" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" + [[package]] name = "unicode-ident" version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +[[package]] +name = "unicode-normalization" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-properties" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d" + [[package]] name = "unicode-xid" version = "0.2.6" @@ -2686,6 +3084,16 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" +[[package]] +name = "uuid" +version = "1.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf3923a6f5c4c6382e0b653c4117f48d631ea17f38ed86e2a828e6f7412f5239" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "vcpkg" version = "0.2.15" @@ -2869,6 +3277,12 @@ dependencies = [ "rustls-pki-types", ] +[[package]] +name = "weezl" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a28ac98ddc8b9274cb41bb4d9d4d5c425b6020c50c46f25559911905610b4a88" + [[package]] name = "winapi-util" version = "0.1.11" @@ -3208,7 +3622,7 @@ dependencies = [ "data-encoding", "der-parser", "lazy_static", - "nom", + "nom 7.1.3", "oid-registry", "ring", "rusticata-macros", @@ -3328,8 +3742,40 @@ dependencies = [ "syn", ] +[[package]] +name = "zip" +version = "8.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d04a6b5381502aa6087c94c669499eb1602eb9c5e8198e534de571f7154809b" +dependencies = [ + "crc32fast", + "flate2", + "indexmap 2.13.0", + "memchr", + "typed-path", + "zopfli", +] + +[[package]] +name = "zlib-rs" +version = "0.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34b31d188d9d685a4f9c7b46d6e36631b07058d2cfe190267adce54dc230bf12" + [[package]] name = "zmij" version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" + +[[package]] +name = "zopfli" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f05cd8797d63865425ff89b5c4a48804f35ba0ce8d125800027ad6017d2b5249" +dependencies = [ + "bumpalo", + "crc32fast", + "log", + "simd-adler32", +] diff --git a/Cargo.toml b/Cargo.toml index 17e8cf9..6a5f837 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,7 +4,7 @@ description = "A minimalist command-line utility to pipe documents from a file o repository = "https://github.com/VimCommando/espipe" homepage = "https://github.com/VimCommando/espipe" documentation = "https://docs.rs/crate/espipe" -version = "0.4.0" +version = "0.5.0" edition = "2024" rust-version = "1.88" license = "Apache-2.0" @@ -19,12 +19,14 @@ include = [ "README.md", "src/**/*.rs", "tests/**/*.rs", + "tests/fixtures/anydoc/*", "tests/fixtures/*.toon", "tests/fixtures/*.ndjson", "tests/fixtures/*.ndjson.gz", ] [dependencies] +anydoc = "0.1.8" base64 = "0.22.1" clap = { version = "^4.6.1", features = ["derive"] } csv = "^1.4.0" diff --git a/README.md b/README.md index 1e300d2..008c583 100644 --- a/README.md +++ b/README.md @@ -65,6 +65,9 @@ To build and publish a multi-platform Docker Hub image: - `.json` files - `.csv` files - `.csv.gz` files +- local Markdown and text files +- local PDF, Word, PowerPoint, Excel, OpenDocument, RTF, and EPUB files +- local recursive glob patterns for file documents - `stdin` as NDJSON It writes records to: @@ -78,11 +81,10 @@ When writing to Elasticsearch, `espipe` batches documents into groups of 5,000 r ## CLI Reference ```bash -Usage: espipe [OPTIONS] +Usage: espipe [OPTIONS] ... Arguments: - The input URI to read docs from - The output URI to send docs to + ... Input URI(s) followed by the output URI Options: -k, --insecure Ignore certificate validation @@ -91,6 +93,7 @@ Options: -p, --password Password for basic authentication -q, --quiet Quiet mode, don't print runtime summary -z, --uncompressed Disable request body gzip compression + --content Content subfield name for file imports [default: body] --action Bulk action for Elasticsearch outputs [default: create] [possible values: create, index, update] --batch-size Documents per Elasticsearch bulk request [default: 5000] --max-requests Maximum concurrent Elasticsearch bulk requests [default: 16] @@ -99,7 +102,8 @@ Options: ## Input And Output -Both positional arguments are parsed as URI-like strings. +All positional arguments are parsed as URI-like strings; one or more input URIs +are followed by the output URI. ### Supported input forms @@ -125,8 +129,24 @@ Both positional arguments are parsed as URI-like strings. Reads CSV from a `file://` URI. - `file:///absolute/path/to/file.csv.gz` Reads gzip-compressed CSV from a `file://` URI. +- `path/to/file.pdf` + Converts a local PDF to Markdown and imports it as one file document. +- `path/to/file.docx` + Converts a local Word document to Markdown and imports it as one file document. +- `'docs/**/*.pdf'` + Recursively finds local PDFs and converts each one to a file document. +- `path/to/file.pdf path/to/file.xlsx output.ndjson` + Imports multiple local file inputs in deterministic path order. -HTTPS input URIs are supported for unauthenticated remote `.csv`, `.ndjson`, and `.json` sources. URLs without a supported file extension can still be accepted when the response `Content-Type` maps to CSV or NDJSON-oriented JSON input. +HTTP and HTTPS input URIs are supported for unauthenticated remote `.csv`, `.ndjson`, and `.json` sources. URLs without a supported file extension can still be accepted when the response `Content-Type` maps to CSV or NDJSON-oriented JSON input. + +### AnyDoc local documents + +Local files with these extensions are converted to GitHub-Flavored Markdown through anydoc before entering the existing file-document pipeline: + +`.doc`, `.docx`, `.docm`, `.odt`, `.pdf`, `.ppt`, `.pps`, `.pot`, `.pptx`, `.pptm`, `.ppsx`, `.ppsm`, `.rtf`, `.epub`, `.xls`, `.xlsx`, `.xlsm`, `.xlsb`, `.ods`, and `.odp`. + +Converted content is stored in `content.body` by default. Use `--content markdown` to store it in `content.markdown`. Multi-file and glob-resolved local inputs add an `origin` object with URI components (`scheme`, `authority`, `path`, `query`, `fragment`) and `filename`; remote CSV, NDJSON, and Toon inputs preserve the same components from their source URI. Anydoc conversion remains local-only. Scanned or image-only PDFs require OCR outside espipe and are not converted. ### Supported output forms @@ -163,6 +183,10 @@ The first row must be a header row. Each subsequent row is converted into a JSON CSV values are emitted as JSON strings. `espipe` does not infer numeric, boolean, or date types from CSV input. +### File-document input + +Markdown, text, YAML, and anydoc-converted files are emitted as JSON documents through the existing file-document pipeline. Markdown frontmatter remains available under `content.*`, and converted non-text files expose their generated Markdown under the configured content field. Existing file discovery supports shell-expanded paths, multiple local input positionals, and quoted recursive glob patterns. + ### Bulk actions `espipe` supports three Elasticsearch bulk actions: @@ -281,6 +305,18 @@ cat docs.ndjson | espipe - http://localhost:9200/my-index espipe users.csv output.ndjson ``` +### Ingest local PDFs recursively + +```bash +espipe '**/*.pdf' http://localhost:9200/documents +``` + +### Ingest multiple local document formats + +```bash +espipe '**/*.pdf' '**/*.docx' '**/*.xlsx' output.ndjson +``` + ### Read and write gzip-compressed files ```bash diff --git a/openspec/changes/add-anydoc-input/.openspec.yaml b/openspec/changes/add-anydoc-input/.openspec.yaml new file mode 100644 index 0000000..5081c98 --- /dev/null +++ b/openspec/changes/add-anydoc-input/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-08-12 diff --git a/openspec/changes/add-anydoc-input/design.md b/openspec/changes/add-anydoc-input/design.md new file mode 100644 index 0000000..b9386bb --- /dev/null +++ b/openspec/changes/add-anydoc-input/design.md @@ -0,0 +1,90 @@ +## Context + +Local file imports already resolve concrete paths and recursive glob patterns in `src/input.rs`. The `Input::FileDocuments` variant reads each path lazily, constructs a JSON object, and sends it through the same `Box` output path used by NDJSON, CSV, and other inputs. Today, recognized text formats have specialized readers and all other file-document paths fall back to UTF-8 text, which rejects PDFs and office containers. + +The `anydoc` crate provides a Rust-native conversion API for PDF, Word, PowerPoint, Excel, OpenDocument, RTF, and EPUB inputs. Its Markdown output is the appropriate intermediate representation because the existing file-document implementation already defines the desired content field, Markdown frontmatter behavior, and origin metadata. + +## Goals / Non-Goals + +**Goals:** + +- Convert supported local anydoc formats into the existing file-document JSON shape. +- Preserve the existing output dispatch and behavior for Markdown, text, YAML, JSON, NDJSON, Toon, CSV, stdin, and HTTPS inputs. +- Support direct files, shell-expanded file lists, and existing recursive glob patterns without changing discovery semantics. +- Keep conversion errors path-specific and visible through the existing stderr error path. +- Cover representative document formats and mixed file collections with tests. + +**Non-Goals:** + +- OCR for scanned or image-only PDFs. +- Remote HTTPS anydoc inputs. +- A new `--extensions` option; multiple extension patterns can be passed as existing input positionals. +- Extraction of embedded assets or source-specific metadata beyond the Markdown produced by anydoc and the shared `origin` metadata. +- Content-based conversion of unknown-extension files, which could change the existing unknown UTF-8 text behavior. + +## Decisions + +### Use the `anydoc` crate directly + +Add `anydoc` as a normal dependency and call its Rust API rather than spawning the anydoc CLI. This avoids subprocess lifecycle and temporary-file concerns and keeps conversion inside the existing input processor. The dependency is compatible with the repository's Rust 1.88 baseline. + +Alternatives considered: + +- **Invoke the anydoc CLI:** rejected because it adds an external executable/runtime dependency and makes error handling and streaming less direct. +- **Add format-specific parsers to espipe:** rejected because it duplicates the purpose of anydoc and expands the maintenance surface. + +### Gate conversion by recognized non-CSV extension + +In `read_file_documents`, check `anydoc::Format::from_path(path)` after the existing specialized readers. If it identifies a supported format other than `Format::Csv`, convert the file with `anydoc::to_markdown(path)`. Keep existing readers ahead of this branch so CSV and all current text/document formats retain their behavior. + +The extension gate intentionally avoids running content detection on every unknown file. Unknown valid UTF-8 files remain text documents, while recognized anydoc extensions opt into conversion. Mislabeled-format support can be considered separately if needed. + +### Reuse the Markdown document builder + +Refactor the existing Markdown reader so it has a helper that accepts `(path, markdown_text, content_field, include_origin)`. The current Markdown path reads the source text and calls this helper; the anydoc path converts the source and calls the same helper. + +This preserves: + +- `content.` placement and `--content` behavior. +- YAML frontmatter extraction and content-field conflict checks. +- A single `origin` metadata object for multi-file and glob-resolved local imports, containing URI components (`scheme`, `authority`, `path`, `query`, `fragment`) and `filename`. Remote HTTP and HTTPS CSV, NDJSON, and Toon imports use the same shape from their source URI. +- One serialized `Box` per output document. + +The resulting flow is: + +```text +path/glob → existing path resolution → format-specific input branch + ├─ Markdown → Markdown document builder + ├─ anydoc format → anydoc Markdown → same builder + └─ existing text/JSON/YAML/etc. readers + → Box → existing output +``` + +### Keep discovery unchanged + +No new traversal or extension-selection option is needed. Existing commands such as `espipe '**/*.pdf' output.ndjson` work because unknown local extensions already enter file-document mode, and multiple types can be supplied as separate patterns. A future `--extensions pdf,xls,doc` option should be designed as a discovery/filtering feature rather than coupled to the converter. + +### Preserve lazy conversion + +Convert files when `read_file_document_line` reaches them, matching the current lazy file-document behavior. This avoids loading an entire collection before output begins and keeps anydoc conversion scoped to the selected file. Conversion failures use the existing input error path and include the source path. + +### Preserve anydoc error wording + +Use anydoc's conversion error wording as-is, adding only the source path context needed to identify the failing input. This avoids introducing a second error taxonomy while preserving the underlying reason for failures. Error normalization can be added later if callers need a stable espipe-specific code or prefix. + +## Risks / Trade-offs + +- **Dependency and binary growth** → Accept the direct dependency for broad format coverage; document the build impact and verify the normal test/build path. +- **Full-file memory use** → Keep the existing one-file-at-a-time document model; anydoc already returns a Markdown `String`, and the resulting raw document is released before the next file is read. +- **Scanned PDFs fail without OCR** → Surface anydoc's unsupported conversion error with the source path; do not claim OCR support. +- **Failures can occur after earlier documents were sent** → Retain existing lazy input semantics and error reporting; changing ingestion to preflight every file is outside this change. +- **Generated Markdown may contain frontmatter-like syntax** → Run it through the established Markdown builder for consistent document semantics; add a fixture-based regression test for representative anydoc output. +- **Unknown-extension detection remains extension-gated** → Preserve current unknown UTF-8 behavior; consider content-based detection only as a separate, explicitly scoped change. + +## Migration Plan + +No data migration is required. Add the dependency and input branch, then release normally. Existing commands and output document shapes remain compatible. Rollback consists of removing the anydoc dependency, branch, tests, and change artifacts; no stored documents require migration. + +## Open Questions + +None. The initial regression suite will use one representative PDF and one Office/OpenDocument fixture; coverage of every supported format remains the responsibility of the anydoc crate. diff --git a/openspec/changes/add-anydoc-input/proposal.md b/openspec/changes/add-anydoc-input/proposal.md new file mode 100644 index 0000000..c93f1d3 --- /dev/null +++ b/openspec/changes/add-anydoc-input/proposal.md @@ -0,0 +1,29 @@ +## Why + +`espipe` currently treats local file documents as text, so PDFs and office formats cannot be ingested even though the existing glob-based file import pipeline already supports recursive document collections. The Rust `anydoc` library can convert these formats to GitHub-Flavored Markdown, allowing non-text ingestion without changing the downstream document or output model. + +## What Changes + +- Add `anydoc` as a local file input preprocessor for supported non-text formats, including PDF, Word, PowerPoint, Excel, OpenDocument, RTF, and EPUB files. +- Convert each supported file to Markdown before constructing the existing file document, preserving `content.`, Markdown handling, and the conditional `origin` metadata. +- Allow existing concrete file lists and recursive glob patterns such as `**/*.pdf` to ingest converted documents. +- Preserve existing CSV, JSON, NDJSON, Toon, Markdown, YAML, text, stdin, and HTTP/HTTPS behavior while attaching source URI metadata to remote streaming inputs. +- Report conversion failures with the source path and keep remote non-streaming anydoc inputs out of scope. +- Do not add `--extensions` in this change; multiple extension patterns can already be supplied as separate local inputs. A future extension-filter option can build on the existing discovery layer. + +## Capabilities + +### New Capabilities + +- `anydoc-input`: Convert supported local non-text documents to Markdown within the input pipeline. + +### Modified Capabilities + +- `file-document-import`: Supported anydoc formats are imported as converted Markdown documents instead of being rejected as binary files. + +## Impact + +- Affects `src/input.rs` only in the ingestion path and adds the `anydoc` crate dependency. +- Adds fixtures and unit/integration coverage for representative converted formats, mixed globs, metadata preservation, and conversion errors. +- Increases dependency graph, compile time, and binary size because anydoc includes parsers for office containers and PDFs. +- Image-only/scanned PDFs remain unsupported because conversion does not provide OCR. diff --git a/openspec/changes/add-anydoc-input/specs/anydoc-input/spec.md b/openspec/changes/add-anydoc-input/specs/anydoc-input/spec.md new file mode 100644 index 0000000..5705fba --- /dev/null +++ b/openspec/changes/add-anydoc-input/specs/anydoc-input/spec.md @@ -0,0 +1,91 @@ +## ADDED Requirements + +### Requirement: Supported local non-text files are converted through anydoc + +The system SHALL use the Rust `anydoc` processor for local regular files with these extensions: `.doc`, `.docx`, `.docm`, `.odt`, `.pdf`, `.ppt`, `.pps`, `.pot`, `.pptx`, `.pptm`, `.ppsx`, `.ppsm`, `.rtf`, `.epub`, `.xls`, `.xlsx`, `.xlsm`, `.xlsb`, `.ods`, and `.odp`. The processor SHALL convert each file to GitHub-Flavored Markdown before file-document construction. + +#### Scenario: A PDF file is imported + +- **WHEN** the user runs `espipe` with a local text-based `.pdf` input +- **THEN** the system converts the PDF to Markdown through anydoc +- **AND** emits one JSON document containing the converted Markdown in the configured content field + +#### Scenario: An office document is imported + +- **WHEN** the user runs `espipe` with a local supported Word, PowerPoint, Excel, or OpenDocument input +- **THEN** the system converts the file to Markdown through anydoc +- **AND** emits one JSON document for the source file + +#### Scenario: A supported file is imported from a mixed local collection + +- **WHEN** file-document input resolves supported anydoc files together with Markdown or text files +- **THEN** each anydoc file is converted at its position in the deterministic file order +- **AND** existing Markdown and text files continue through their existing readers + +### Requirement: Anydoc conversion preserves the existing file-document shape + +The system SHALL construct converted documents using the existing file-document semantics after Markdown conversion. It SHALL store converted Markdown under `content.`, apply the configured `--content` value, and add one `origin` metadata object for multi-file or glob-resolved local imports. The object SHALL contain `scheme`, `authority`, `path`, `query`, `fragment`, and `filename` fields. It SHALL NOT emit the legacy `file.path` or `file.name` metadata. + +#### Scenario: Default content field is used for converted Markdown + +- **WHEN** a supported anydoc file is imported without `--content` +- **THEN** the converted Markdown is stored in `content.body` +- **AND** the document does not expose the original binary bytes as a field + +#### Scenario: Custom content field is used for converted Markdown + +- **WHEN** a supported anydoc file is imported with `--content markdown` +- **THEN** the converted Markdown is stored in `content.markdown` +- **AND** the system does not add `content.body` solely because the source was converted + +#### Scenario: Converted files participate in multi-file origin metadata + +- **WHEN** anydoc files are imported together with another file-document input +- **THEN** each converted document includes an `origin` object +- **AND** its URI components identify the original local file + +#### Scenario: Glob-resolved converted files include complete origin metadata + +- **WHEN** anydoc files are imported through a local glob pattern +- **THEN** each converted document includes `origin.scheme` equal to `file` +- **AND** `origin.path` identifies the containing directory +- **AND** `origin.filename` identifies the original local file +- **AND** absent `origin.authority`, `origin.query`, and `origin.fragment` fields are omitted + +#### Scenario: Remote inputs preserve URI origin metadata + +- **WHEN** an HTTP or HTTPS CSV, NDJSON, or Toon input is imported +- **THEN** each emitted document includes `origin.scheme`, `origin.authority`, `origin.path`, `origin.query`, `origin.fragment`, and `origin.filename` +- **AND** the values reflect the source URI rather than the temporary download file + +### Requirement: Existing local discovery mechanisms support anydoc files + +The system SHALL process supported anydoc files supplied as direct local paths, shell-expanded file lists, or existing local glob patterns. It SHALL not require a separate subprocess or remote service for local conversion. + +#### Scenario: A quoted recursive PDF glob is imported + +- **WHEN** the user runs `espipe '**/*.pdf' output.ndjson` +- **THEN** the existing glob resolver finds matching regular files +- **AND** each matching PDF is converted and emitted as one file document + +#### Scenario: Multiple extension patterns are supplied + +- **WHEN** the user supplies separate local input patterns such as `**/*.pdf`, `**/*.xls`, and `**/*.doc` +- **THEN** the system combines and de-duplicates the resolved paths using existing file discovery rules +- **AND** converts each supported path according to its extension + +### Requirement: Anydoc conversion failures identify the source file + +The system SHALL report anydoc conversion failures through the existing file-input error path, including the source path and the underlying conversion reason when available. It SHALL not emit a synthetic document for a file that anydoc cannot convert. + +#### Scenario: An unsupported document is encountered + +- **WHEN** anydoc reports that a supported-extension file is encrypted, malformed, unsupported, or exceeds a conversion limit +- **THEN** ingestion fails with a diagnostic identifying the source path +- **AND** the diagnostic is written to stderr + +#### Scenario: An image-only PDF is encountered + +- **WHEN** anydoc cannot extract meaningful text from a scanned or image-only PDF +- **THEN** ingestion fails with a path-specific unsupported-conversion diagnostic +- **AND** the system does not claim to perform OCR diff --git a/openspec/changes/add-anydoc-input/specs/file-document-import/spec.md b/openspec/changes/add-anydoc-input/specs/file-document-import/spec.md new file mode 100644 index 0000000..50b4fda --- /dev/null +++ b/openspec/changes/add-anydoc-input/specs/file-document-import/spec.md @@ -0,0 +1,101 @@ +## MODIFIED Requirements + +### Requirement: Local file inputs import documents by file format + +The system SHALL accept one or more local file inputs and import each regular file according to its file format, including conversion through anydoc for supported local non-text formats. + +#### Scenario: Single Markdown file is imported + +- **WHEN** the user runs `espipe` with a local Markdown file input +- **THEN** the system emits one document for that Markdown file + +#### Scenario: Single supported non-text file is imported + +- **WHEN** the user runs `espipe` with a local PDF or other supported anydoc file input +- **THEN** the system converts the file to Markdown +- **AND** the system emits one document for that file + +#### Scenario: Shell-expanded Markdown files are imported + +- **WHEN** the user's shell expands a file pattern into multiple Markdown file arguments before `espipe` starts +- **THEN** the system treats each file argument as an input +- **AND** it emits one document for each regular file + +#### Scenario: Multiple input positionals are provided + +- **WHEN** the user provides more than two positional arguments +- **THEN** the final positional argument is treated as the output URI +- **AND** every preceding positional argument is treated as an input + +### Requirement: Recursive glob inputs import matching files + +The system SHALL accept local glob input patterns, including recursive `**` patterns, and import each matched regular file according to its file format, including anydoc conversion for supported non-text files. + +#### Scenario: Recursive Markdown glob is imported + +- **WHEN** the user runs `espipe` with a local input pattern of `**/*.md` +- **THEN** the system expands the pattern recursively +- **AND** it emits one document for each matched Markdown file + +#### Scenario: Recursive PDF glob is imported + +- **WHEN** the user runs `espipe` with a local input pattern of `**/*.pdf` +- **THEN** the system expands the pattern recursively +- **AND** it converts and emits one document for each matched PDF file + +#### Scenario: Glob matches no files + +- **WHEN** the user provides a local glob pattern that matches no regular files +- **THEN** startup fails before sending any output +- **AND** the error identifies that the glob matched no files + +#### Scenario: Glob matches directories + +- **WHEN** a local glob pattern matches both regular files and directories +- **THEN** the system imports the matched regular files +- **AND** it does not emit documents for matched directories + +### Requirement: Multi-file imports include origin metadata + +The system SHALL add one `origin` object to emitted documents when file-document input resolves more than one regular file or uses a glob pattern. The object SHALL contain `scheme`, `authority`, `path`, `query`, `fragment`, and `filename` fields, and the system SHALL NOT emit the legacy `file.path` or `file.name` metadata. + +#### Scenario: Multiple files are imported + +- **WHEN** file-document input resolves to more than one regular file +- **THEN** each emitted document includes the source file's `origin` object +- **AND** each emitted document does not include a `file` object + +#### Scenario: Single direct file is imported + +- **WHEN** file-document input resolves to one direct file without glob resolution +- **THEN** the emitted document does not include `origin` +- **AND** it does not include a `file` object + +#### Scenario: A glob resolves one or more files + +- **WHEN** file-document input uses a local glob pattern +- **THEN** each emitted document includes `origin.scheme` equal to `file` +- **AND** `origin.path` identifies the containing directory +- **AND** `origin.filename` identifies the source file +- **AND** absent `origin.authority`, `origin.query`, and `origin.fragment` fields are omitted + +### Requirement: Binary files are rejected + +The system SHALL convert local binary files recognized by anydoc into Markdown documents. It SHALL reject file-document inputs that are neither valid UTF-8 text nor recognized supported anydoc formats. + +#### Scenario: Supported binary file is matched + +- **WHEN** a file-document input resolves to a PDF or supported office/container file recognized by anydoc +- **THEN** the system converts the file to Markdown +- **AND** it emits a document for the converted content + +#### Scenario: Unrecognized binary file is matched + +- **WHEN** a file-document input resolves to a binary file that anydoc does not recognize +- **THEN** importing that file fails +- **AND** the error identifies the file as unsupported binary or invalid UTF-8 input + +#### Scenario: Valid UTF-8 text remains supported + +- **WHEN** a file-document input resolves to an unknown-extension file whose contents are valid UTF-8 +- **THEN** the emitted document contains the full file content in the configured `content.` field diff --git a/openspec/changes/add-anydoc-input/tasks.md b/openspec/changes/add-anydoc-input/tasks.md new file mode 100644 index 0000000..367478f --- /dev/null +++ b/openspec/changes/add-anydoc-input/tasks.md @@ -0,0 +1,26 @@ +## 1. Dependency and input integration + +- [x] 1.1 Add the `anydoc` crate dependency at a Rust 1.88-compatible version. +- [x] 1.2 Add extension-gated anydoc routing to `read_file_documents`, preserving the existing specialized readers and excluding CSV from the new branch. +- [x] 1.3 Refactor Markdown file-document construction to accept converted Markdown text so source Markdown and anydoc output share the same content-field, frontmatter, and origin-metadata behavior. +- [x] 1.4 Add the anydoc conversion wrapper using `anydoc::to_markdown`, including source-path context and underlying conversion errors in diagnostics. +- [x] 1.5 Verify direct paths, shell-expanded file lists, recursive globs, mixed anydoc/Markdown collections, deterministic ordering, and lazy conversion continue to use the existing discovery and output pipeline. + +## 2. AnyDoc format coverage and error behavior + +- [x] 2.1 Add representative routing and conversion coverage for one text-based PDF and one Office or OpenDocument file; rely on anydoc's own test suite for coverage of its remaining formats. +- [x] 2.2 Verify the anydoc routing is driven by the crate's recognized format API and excludes existing CSV handling. +- [x] 2.3 Add representative malformed and image-only conversion failure tests and verify diagnostics identify the source path and are written to stderr. +- [x] 2.4 Add a regression test proving unknown valid UTF-8 files and existing Markdown, YAML, JSON, NDJSON, Toon, CSV, stdin, and HTTPS inputs retain their current behavior. + +## 3. Output-shape and integration verification + +- [x] 3.1 Verify converted documents use `content.body` by default and the configured `--content` field when specified. +- [x] 3.2 Verify converted documents use complete `origin` metadata for multi-file and glob imports and omit it for a single direct file. +- [x] 3.3 Verify mixed recursive file imports emit converted and existing documents in the same deterministic path order without changing output serialization. +- [x] 3.4 Update README input documentation with supported anydoc formats, local-only behavior, glob examples, and the limitation that scanned PDFs require OCR outside espipe. + +## 4. Verification + +- [x] 4.1 Run formatting and the complete Rust test suite. +- [x] 4.2 Run the CLI/integration tests that write NDJSON output and confirm existing output consumers require no changes. diff --git a/src/input.rs b/src/input.rs index 642302d..a73d27e 100644 --- a/src/input.rs +++ b/src/input.rs @@ -8,7 +8,7 @@ use reqwest::{ }; use serde_json::{Map, Value, value::RawValue}; use std::{ - collections::BTreeSet, + collections::BTreeMap, ffi::OsStr, fs::{self, File}, io::{BufRead, BufReader, Read, Seek, SeekFrom, Stdin, Write, stdin}, @@ -22,11 +22,13 @@ pub enum Input { source: String, reader: Box>>, first_record: bool, + origin: Option, _temp_file: Option, }, FileCsv { source: String, reader: Box>>, + origin: Option, _temp_file: Option, }, FileToon { @@ -36,6 +38,7 @@ pub enum Input { document_index: usize, buffered_rows: Vec, eof: bool, + origin: Option, _temp_file: Option, }, Stdin { @@ -44,14 +47,25 @@ pub enum Input { FileDocuments { source: String, paths: Vec, + origins: Vec, path_index: usize, documents: Vec>, document_index: usize, content_field: String, - include_file_metadata: bool, + include_origin: bool, }, } +#[derive(Clone, Debug)] +pub(crate) struct OriginMetadata { + scheme: String, + authority: Option, + path: String, + query: Option, + fragment: Option, + filename: String, +} + type CsvRecord = std::collections::HashMap; const REMOTE_NDJSON_ERROR: &str = "JSON payload does not look like required NDJSON input format."; const JSON_LINE_OPENING_ERROR: &str = "Each record must be a JSON object starting with '{'"; @@ -76,9 +90,11 @@ impl Input { if uris.len() == 1 { let uri = uris.into_iter().next().unwrap(); return match uri.scheme().map(|scheme| scheme.as_str()) { - Some("https") => tokio::task::spawn_blocking(move || fetch_remote_input(uri)) - .await - .map_err(|err| eyre!("Remote input fetch task failed: {err}"))?, + Some("http" | "https") => { + tokio::task::spawn_blocking(move || fetch_remote_input(uri)) + .await + .map_err(|err| eyre!("Remote input fetch task failed: {err}"))? + } _ => open_input_values(vec![uri], &content_field), }; } @@ -90,13 +106,17 @@ impl Input { Input::FileJson { reader, first_record, + origin, .. } => { let raw = read_json_line(reader, line_buffer, *first_record)?; *first_record = false; - Ok(raw) + add_origin_to_raw(raw, origin.as_ref()) + } + Input::FileCsv { reader, origin, .. } => { + let raw = read_csv_line(reader)?; + add_origin_to_raw(raw, origin.as_ref()) } - Input::FileCsv { reader, .. } => read_csv_line(reader), Input::FileToon { source, reader, @@ -104,8 +124,19 @@ impl Input { document_index, buffered_rows, eof, + origin, .. - } => read_toon_document(source, reader, pending, document_index, buffered_rows, eof), + } => { + let raw = read_toon_document( + source, + reader, + pending, + document_index, + buffered_rows, + eof, + )?; + add_origin_to_raw(raw, origin.as_ref()) + } Input::Stdin { reader, .. } => read_json_line(reader, line_buffer, false), Input::FileDocuments { .. } => read_file_document_line(self), } @@ -125,7 +156,7 @@ impl TryFrom> for Input { fn try_from(uri: UriRef) -> Result { match uri.scheme().map(|scheme| scheme.as_str()) { - Some("https") => fetch_remote_input(uri), + Some("http" | "https") => fetch_remote_input(uri), _ => open_input_values(vec![uri], "body"), } } @@ -156,11 +187,10 @@ fn validate_content_field(content_field: &str) -> Result<()> { fn open_input_values(uris: Vec>, content_field: &str) -> Result { for uri in &uris { match uri.scheme().map(|scheme| scheme.as_str()) { - Some("https") if uris.len() == 1 => return fetch_remote_input(uri.clone()), - Some("https") => { + Some("http" | "https") if uris.len() == 1 => return fetch_remote_input(uri.clone()), + Some("http" | "https") => { return Err(eyre!("Remote inputs cannot be combined with file imports")); } - Some("http") => return Err(eyre!("Unsupported input scheme: http")), Some("file") | None => {} Some(scheme) => return Err(eyre!("Unsupported input scheme: {scheme}")), } @@ -191,14 +221,10 @@ fn open_input_values(uris: Vec>, content_field: &str) -> Result( @@ -292,12 +318,14 @@ fn open_local_file(path: PathBuf) -> Result { .has_headers(true) .from_reader(local_file_reader(file, &path)), ), + origin: None, _temp_file: None, }), InputKind::Ndjson | InputKind::Json => Ok(Input::FileJson { source, reader: Box::new(BufReader::new(local_file_reader(file, &path))), first_record: true, + origin: None, _temp_file: None, }), InputKind::Toon => Ok(Input::FileToon { @@ -307,35 +335,41 @@ fn open_local_file(path: PathBuf) -> Result { document_index: 0, buffered_rows: Vec::new(), eof: false, + origin: None, _temp_file: None, }), - InputKind::FileDocument => open_file_documents(vec![source], "body"), + InputKind::FileDocument => open_file_documents( + vec![UriRef::parse(source).map_err(|err| eyre!("Invalid local file URI: {err:?}"))?], + "body", + ), } } -fn open_file_documents(values: Vec, content_field: &str) -> Result { - let paths = resolve_file_document_paths(values)?; - let include_file_metadata = paths.len() > 1; +fn open_file_documents(values: Vec>, content_field: &str) -> Result { + let (paths, origins, resolved_from_glob) = resolve_file_document_paths(values)?; + let include_origin = paths.len() > 1 || resolved_from_glob; let source = format!("{} file document(s)", paths.len()); Ok(Input::FileDocuments { source, paths, + origins, path_index: 0, documents: Vec::new(), document_index: 0, content_field: content_field.to_string(), - include_file_metadata, + include_origin, }) } fn read_file_document_line(input: &mut Input) -> Result> { let Input::FileDocuments { paths, + origins, path_index, documents, document_index, content_field, - include_file_metadata, + include_origin, .. } = input else { @@ -351,31 +385,44 @@ fn read_file_document_line(input: &mut Input) -> Result> { let Some(path) = paths.get(*path_index) else { return Err(eyre!("No file document")); }; + let origin = if *include_origin { + origins.get(*path_index) + } else { + None + }; *path_index += 1; - *documents = read_file_documents(path, content_field, *include_file_metadata)?; + *documents = read_file_documents(path, content_field, origin)?; *document_index = 0; } } -fn resolve_file_document_paths(values: Vec) -> Result> { - let mut paths = BTreeSet::new(); +fn resolve_file_document_paths( + values: Vec>, +) -> Result<(Vec, Vec, bool)> { + let mut paths = BTreeMap::new(); let mut any_glob = false; for value in values { - if has_glob_metachar(&value) { + let value_path = value.path().as_str().to_string(); + if has_glob_metachar(&value_path) { any_glob = true; let mut matched_regular_file = false; - for entry in glob(&value).map_err(|err| eyre!("Invalid glob pattern {value}: {err}"))? { - let path = entry.map_err(|err| eyre!("Error expanding glob {value}: {err}"))?; + for entry in glob(&value_path) + .map_err(|err| eyre!("Invalid glob pattern {value_path}: {err}"))? + { + let path = + entry.map_err(|err| eyre!("Error expanding glob {value_path}: {err}"))?; if path.is_file() { matched_regular_file = true; - paths.insert(path); + paths + .entry(path.clone()) + .or_insert_with(|| origin_from_local_path(&path)); } } if !matched_regular_file { - return Err(eyre!("Glob matched no regular files: {value}")); + return Err(eyre!("Glob matched no regular files: {value_path}")); } } else { - let path = PathBuf::from(value); + let path = PathBuf::from(&value_path); if !path.exists() { return Err(eyre!("File input does not exist: {}", path.display())); } @@ -385,10 +432,10 @@ fn resolve_file_document_paths(values: Vec) -> Result> { path.display() )); } - paths.insert(path); + paths.entry(path).or_insert_with(|| origin_from_uri(&value)); } } - for path in &paths { + for path in paths.keys() { let path_str = path.to_string_lossy(); if is_compressed_input(path_str.as_ref()) { return Err(eyre!("Unsupported compressed input format: {path_str}")); @@ -402,7 +449,8 @@ fn resolve_file_document_paths(values: Vec) -> Result> { }; return Err(eyre!("No regular files resolved from {kind}")); } - Ok(paths.into_iter().collect()) + let (paths, origins): (Vec<_>, Vec<_>) = paths.into_iter().unzip(); + Ok((paths, origins, any_glob)) } fn has_glob_metachar(value: &str) -> bool { @@ -419,17 +467,20 @@ fn should_use_file_document(path: &Path) -> bool { fn read_file_documents( path: &Path, content_field: &str, - include_file_metadata: bool, + origin: Option<&OriginMetadata>, ) -> Result>> { match extension(path).as_deref() { - Some("ndjson" | "jsonl") => read_ndjson_file_documents(path, include_file_metadata), - Some("json") => read_json_file_document(path, include_file_metadata), - Some("toon") => read_toon_file_documents(path, include_file_metadata), - Some("yml" | "yaml") => read_yaml_file_document(path, content_field, include_file_metadata), - Some("md" | "markdown") => { - read_markdown_file_document(path, content_field, include_file_metadata) + Some("ndjson" | "jsonl") => read_ndjson_file_documents(path, origin), + Some("json") => read_json_file_document(path, origin), + Some("toon") => read_toon_file_documents(path, origin), + Some("yml" | "yaml") => read_yaml_file_document(path, content_field, origin), + Some("md" | "markdown") => read_markdown_file_document(path, content_field, origin), + _ if anydoc::Format::from_path(path) + .is_some_and(|format| format != anydoc::Format::Csv) => + { + read_anydoc_file_document(path, content_field, origin) } - _ => read_text_file_document(path, content_field, include_file_metadata), + _ => read_text_file_document(path, content_field, origin), } } @@ -441,10 +492,10 @@ fn read_text_file(path: &Path) -> Result { fn read_text_file_document( path: &Path, content_field: &str, - include_file_metadata: bool, + origin: Option<&OriginMetadata>, ) -> Result>> { let text = read_text_file(path)?; - let mut document = base_file_document(path, include_file_metadata); + let mut document = base_file_document(origin); document.insert( "content".to_string(), Value::Object(Map::from_iter([( @@ -458,10 +509,51 @@ fn read_text_file_document( fn read_markdown_file_document( path: &Path, content_field: &str, - include_file_metadata: bool, + origin: Option<&OriginMetadata>, ) -> Result>> { let text = read_text_file(path)?; - let (frontmatter, body) = split_markdown_frontmatter(&text); + read_markdown_text_document(path, &text, content_field, origin) +} + +#[derive(Debug)] +struct AnyDocConversionError { + path: PathBuf, + source: anydoc::ConvertError, +} + +impl std::fmt::Display for AnyDocConversionError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}: {}", self.path.display(), self.source) + } +} + +impl std::error::Error for AnyDocConversionError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + Some(&self.source) + } +} + +fn read_anydoc_file_document( + path: &Path, + content_field: &str, + origin: Option<&OriginMetadata>, +) -> Result>> { + let markdown = anydoc::to_markdown(path).map_err(|source| { + Report::new(AnyDocConversionError { + path: path.to_path_buf(), + source, + }) + })?; + read_markdown_text_document(path, &markdown, content_field, origin) +} + +fn read_markdown_text_document( + path: &Path, + text: &str, + content_field: &str, + origin: Option<&OriginMetadata>, +) -> Result>> { + let (frontmatter, body) = split_markdown_frontmatter(text); let mut content = Map::new(); if let Some(frontmatter) = frontmatter { content = yaml_mapping_to_json_map(frontmatter) @@ -474,7 +566,7 @@ fn read_markdown_file_document( } } content.insert(content_field.to_string(), Value::String(body.to_string())); - let mut document = base_file_document(path, include_file_metadata); + let mut document = base_file_document(origin); document.insert("content".to_string(), Value::Object(content)); raw_documents(vec![document]) } @@ -512,7 +604,7 @@ fn is_end_of_input(err: &eyre::Report) -> bool { fn read_yaml_file_document( path: &Path, content_field: &str, - include_file_metadata: bool, + origin: Option<&OriginMetadata>, ) -> Result>> { let text = read_text_file(path)?; let content = yaml_mapping_to_json_map(&text) @@ -523,7 +615,7 @@ fn read_yaml_file_document( path.display() )); } - let mut document = base_file_document(path, include_file_metadata); + let mut document = base_file_document(origin); document.insert("content".to_string(), Value::Object(content)); raw_documents(vec![document]) } @@ -536,7 +628,10 @@ fn yaml_mapping_to_json_map(text: &str) -> Result> { Ok(map) } -fn read_json_file_document(path: &Path, include_file_metadata: bool) -> Result>> { +fn read_json_file_document( + path: &Path, + origin: Option<&OriginMetadata>, +) -> Result>> { let text = read_text_file(path)?; let mut document = match serde_json::from_str::(&text) { Ok(Value::Object(map)) => map, @@ -553,13 +648,13 @@ fn read_json_file_document(path: &Path, include_file_metadata: bool) -> Result, ) -> Result>> { let text = read_text_file(path)?; let mut docs = Vec::new(); @@ -576,7 +671,7 @@ fn read_ndjson_file_documents( index + 1 )); }; - add_file_metadata(&mut document, path, include_file_metadata); + add_origin_metadata(&mut document, origin); docs.push(RawValue::from_string(Value::Object(document).to_string())?); } Ok(docs) @@ -584,7 +679,7 @@ fn read_ndjson_file_documents( fn read_toon_file_documents( path: &Path, - include_file_metadata: bool, + origin: Option<&OriginMetadata>, ) -> Result>> { let file = File::open(path).map_err(|err| eyre!("{}: {err}", path.display()))?; let mut reader = BufReader::new(Box::new(file) as Box); @@ -605,9 +700,9 @@ fn read_toon_file_documents( &mut eof, ) { Ok(mut raw) => { - if include_file_metadata { + if origin.is_some() { let mut document: Map = serde_json::from_str(raw.get())?; - add_file_metadata(&mut document, path, include_file_metadata); + add_origin_metadata(&mut document, origin); raw = RawValue::from_string(Value::Object(document).to_string())?; } docs.push(raw); @@ -671,34 +766,107 @@ fn toon_row_value_to_raw(source: &str, document_index: usize, row: Value) -> Res RawValue::from_string(Value::Object(row).to_string()).map_err(Into::into) } -fn base_file_document(path: &Path, include_file_metadata: bool) -> Map { +fn base_file_document(origin: Option<&OriginMetadata>) -> Map { let mut document = Map::new(); - add_file_metadata(&mut document, path, include_file_metadata); + add_origin_metadata(&mut document, origin); document } -fn add_file_metadata(document: &mut Map, path: &Path, include_file_metadata: bool) { - if !include_file_metadata { - return; +fn add_origin_metadata(document: &mut Map, origin: Option<&OriginMetadata>) { + if let Some(origin) = origin { + document.insert("origin".to_string(), origin.clone().into_value()); } - document.insert( - "file".to_string(), - Value::Object(Map::from_iter([ - ( - "path".to_string(), - Value::String(path.display().to_string()), - ), - ( - "name".to_string(), - Value::String( - path.file_name() - .and_then(OsStr::to_str) - .unwrap_or_default() - .to_string(), - ), - ), - ])), - ); +} + +fn origin_from_local_path(path: &Path) -> OriginMetadata { + let filename = path + .file_name() + .and_then(OsStr::to_str) + .unwrap_or_default() + .to_string(); + let directory = path.parent().unwrap_or_else(|| Path::new("")); + OriginMetadata { + scheme: "file".to_string(), + authority: None, + path: origin_directory(directory), + query: None, + fragment: None, + filename, + } +} + +fn origin_from_uri(uri: &UriRef) -> OriginMetadata { + let path = uri.path().as_str(); + let path_ref = Path::new(path); + let is_directory = path.is_empty() || path.ends_with('/'); + OriginMetadata { + scheme: uri + .scheme() + .map(|scheme| scheme.as_str().to_string()) + .unwrap_or_else(|| "file".to_string()), + authority: uri + .authority() + .map(|authority| authority.as_str().to_string()), + path: if is_directory { + if path.is_empty() { + "./".to_string() + } else { + path.to_string() + } + } else { + origin_directory(path_ref.parent().unwrap_or_else(|| Path::new(""))) + }, + query: uri.query().map(|query| query.as_str().to_string()), + fragment: uri.fragment().map(|fragment| fragment.as_str().to_string()), + filename: if is_directory { + String::new() + } else { + path_ref + .file_name() + .and_then(OsStr::to_str) + .unwrap_or_default() + .to_string() + }, + } +} + +impl OriginMetadata { + fn into_value(self) -> Value { + let mut object = Map::from_iter([ + ("scheme".to_string(), Value::String(self.scheme)), + ("path".to_string(), Value::String(self.path)), + ("filename".to_string(), Value::String(self.filename)), + ]); + if let Some(authority) = self.authority { + object.insert("authority".to_string(), Value::String(authority)); + } + if let Some(query) = self.query { + object.insert("query".to_string(), Value::String(query)); + } + if let Some(fragment) = self.fragment { + object.insert("fragment".to_string(), Value::String(fragment)); + } + Value::Object(object) + } +} + +fn origin_directory(path: &Path) -> String { + if path.as_os_str().is_empty() { + "./".to_string() + } else { + path.to_string_lossy().into_owned() + } +} + +fn add_origin_to_raw(raw: Box, origin: Option<&OriginMetadata>) -> Result> { + let Some(origin) = origin else { + return Ok(raw); + }; + let Value::Object(mut document) = serde_json::from_str(raw.get())? else { + return Err(eyre!("Input document must be a JSON object")); + }; + document.insert("origin".to_string(), origin.clone().into_value()); + RawValue::from_string(Value::Object(document).to_string()).map_err(Into::into) } fn raw_documents(documents: Vec>) -> Result>> { @@ -712,7 +880,6 @@ fn raw_documents(documents: Vec>) -> Result fn fetch_remote_input(uri: UriRef) -> Result { let client = Client::builder() - .https_only(true) .connect_timeout(REMOTE_CONNECT_TIMEOUT) .timeout(REMOTE_REQUEST_TIMEOUT) .build()?; @@ -754,6 +921,7 @@ fn fetch_remote_input_with_client(uri: UriRef, client: &Client) -> Resul let reader_file = temp_file.reopen()?; let source = uri.to_string(); + let origin = Some(origin_from_uri(&uri)); match kind { InputKind::Csv => Ok(Input::FileCsv { @@ -763,12 +931,14 @@ fn fetch_remote_input_with_client(uri: UriRef, client: &Client) -> Resul .has_headers(true) .from_reader(Box::new(reader_file) as Box), ), + origin: origin.clone(), _temp_file: Some(temp_file), }), InputKind::Ndjson | InputKind::Json => Ok(Input::FileJson { source, reader: Box::new(BufReader::new(Box::new(reader_file) as Box)), first_record: true, + origin: origin.clone(), _temp_file: Some(temp_file), }), InputKind::Toon => Ok(Input::FileToon { @@ -778,6 +948,7 @@ fn fetch_remote_input_with_client(uri: UriRef, client: &Client) -> Resul document_index: 0, buffered_rows: Vec::new(), eof: false, + origin, _temp_file: Some(temp_file), }), InputKind::FileDocument => Err(eyre!("Unsupported remote input format")), @@ -911,8 +1082,9 @@ mod tests { use super::{ Input, InputKind, JSON_LINE_OPENING_ERROR, REMOTE_NDJSON_ERROR, fetch_remote_input_with_client, input_kind_from_path, local_input_kind, open_input_values, - validate_content_field, validate_ndjson_file, + origin_from_local_path, origin_from_uri, validate_content_field, validate_ndjson_file, }; + use base64::Engine as _; use flate2::{Compression, write::GzEncoder}; use fluent_uri::UriRef; use reqwest::blocking::Client; @@ -923,8 +1095,8 @@ mod tests { use std::{ fs, io::{Read, Write}, - net::TcpListener, - path::PathBuf, + net::{TcpListener, TcpStream}, + path::{Path, PathBuf}, sync::{Arc, mpsc}, thread, time::{SystemTime, UNIX_EPOCH}, @@ -973,6 +1145,14 @@ mod tests { .join(name) } + fn write_base64_fixture(source: &str, path: &Path) { + let encoded = fs::read_to_string(fixture_path(source)).unwrap(); + let bytes = base64::engine::general_purpose::STANDARD + .decode(encoded.trim()) + .unwrap(); + fs::write(path, bytes).unwrap(); + } + fn write_gzip(path: &PathBuf, contents: &str) { let file = fs::File::create(path).unwrap(); let mut encoder = GzEncoder::new(file, Compression::default()); @@ -1126,7 +1306,178 @@ mod tests { } #[test] - fn shell_expanded_files_are_sorted_deduplicated_and_include_metadata() { + fn origin_metadata_uses_root_for_relative_paths_and_omits_null_values() { + let value = origin_from_local_path(Path::new("document.pdf")).into_value(); + + assert_eq!(value["scheme"], "file"); + assert_eq!(value["path"], "./"); + assert_eq!(value["filename"], "document.pdf"); + assert!(value.get("authority").is_none()); + assert!(value.get("query").is_none()); + assert!(value.get("fragment").is_none()); + } + + #[test] + fn origin_metadata_handles_uri_root_and_trailing_slash() { + let root = origin_from_uri(&UriRef::parse("https://example.com/".to_string()).unwrap()) + .into_value(); + assert_eq!(root["path"], "/"); + assert_eq!(root["filename"], ""); + + let directory = + origin_from_uri(&UriRef::parse("https://example.com/docs/".to_string()).unwrap()) + .into_value(); + assert_eq!(directory["path"], "/docs/"); + assert_eq!(directory["filename"], ""); + } + + #[test] + fn anydoc_converts_pdf_to_default_markdown_document() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("sample.pdf"); + write_base64_fixture("anydoc/sample.pdf.base64", &path); + let values = collect_values(Input::try_from(uri(&path)).unwrap()); + + assert_eq!(values.len(), 1); + assert!( + values[0]["content"]["body"] + .as_str() + .unwrap() + .contains("Hello PDF") + ); + assert!(values[0].get("origin").is_none()); + } + + #[test] + fn anydoc_converts_rtf_to_custom_content_field() { + let path = fixture_path("anydoc/sample.rtf"); + let values = collect_values(open_input_values(vec![uri(&path)], "markdown").unwrap()); + + assert!( + values[0]["content"]["markdown"] + .as_str() + .unwrap() + .contains("Hello from RTF") + ); + assert!(values[0]["content"].get("body").is_none()); + } + + #[test] + fn anydoc_converts_office_docx_fixture() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("sample.docx"); + write_base64_fixture("anydoc/sample.docx.base64", &path); + + let values = collect_values(Input::try_from(uri(&path)).unwrap()); + + assert_eq!(values.len(), 1); + assert!( + values[0]["content"]["body"] + .as_str() + .unwrap() + .contains("Hello from DOCX") + ); + } + + #[test] + fn anydoc_mixed_file_import_sorts_paths_and_preserves_origin_metadata() { + let dir = tempfile::tempdir().unwrap(); + let pdf = dir.path().join("sample.pdf"); + write_base64_fixture("anydoc/sample.pdf.base64", &pdf); + let rtf = dir.path().join("sample.rtf"); + fs::copy(fixture_path("anydoc/sample.rtf"), &rtf).unwrap(); + let values = collect_values(open_input_values(vec![uri(&rtf), uri(&pdf)], "body").unwrap()); + + assert_eq!(values.len(), 2); + assert_eq!(values[0]["origin"]["scheme"], "file"); + assert!(values[0]["origin"].get("authority").is_none()); + assert_eq!(values[0]["origin"]["filename"], "sample.pdf"); + assert_eq!( + values[0]["origin"]["path"], + dir.path().display().to_string() + ); + assert!(values[0]["origin"].get("query").is_none()); + assert!(values[0]["origin"].get("fragment").is_none()); + assert_eq!(values[1]["origin"]["filename"], "sample.rtf"); + assert!(values[0]["content"]["body"].is_string()); + assert!(values[1]["content"]["body"].is_string()); + } + + #[test] + fn anydoc_recursive_glob_imports_pdf_files() { + let dir = tempfile::tempdir().unwrap(); + let nested = dir.path().join("nested"); + fs::create_dir(&nested).unwrap(); + write_base64_fixture("anydoc/sample.pdf.base64", &nested.join("sample.pdf")); + let pattern = dir + .path() + .join("**") + .join("*.pdf") + .to_string_lossy() + .into_owned(); + + let values = collect_values( + open_input_values(vec![UriRef::parse(pattern).unwrap()], "body").unwrap(), + ); + + assert_eq!(values.len(), 1); + assert!(values[0]["content"]["body"].is_string()); + assert_eq!(values[0]["origin"]["filename"], "sample.pdf"); + assert_eq!(values[0]["origin"]["path"], nested.display().to_string()); + } + + #[test] + fn anydoc_multiple_extension_globs_combine_and_sort_inputs() { + let dir = tempfile::tempdir().unwrap(); + let pdf = dir.path().join("a.pdf"); + let rtf = dir.path().join("b.rtf"); + write_base64_fixture("anydoc/sample.pdf.base64", &pdf); + fs::copy(fixture_path("anydoc/sample.rtf"), &rtf).unwrap(); + let pdf_pattern = dir.path().join("**/*.pdf").to_string_lossy().into_owned(); + let rtf_pattern = dir.path().join("**/*.rtf").to_string_lossy().into_owned(); + + let values = collect_values( + open_input_values( + vec![ + UriRef::parse(rtf_pattern).unwrap(), + UriRef::parse(pdf_pattern).unwrap(), + ], + "body", + ) + .unwrap(), + ); + + assert_eq!(values.len(), 2); + assert_eq!(values[0]["origin"]["filename"], "a.pdf"); + assert_eq!(values[1]["origin"]["filename"], "b.rtf"); + } + + #[test] + fn anydoc_conversion_error_includes_source_path() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("invalid.pdf"); + fs::write(&path, b"not a PDF").unwrap(); + + let err = read_err(open_input_values(vec![uri(&path)], "body")); + + assert!(err.contains("invalid.pdf")); + assert!(err.len() > "invalid.pdf".len()); + } + + #[test] + fn anydoc_rejects_image_only_pdf_with_ocr_diagnostic() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("image-only.pdf"); + write_base64_fixture("anydoc/image-only.pdf.base64", &path); + + let err = read_err(open_input_values(vec![uri(&path)], "body")); + + assert!(err.contains("image-only.pdf")); + assert!(err.contains("OCR is required")); + } + + #[test] + fn shell_expanded_files_are_sorted_deduplicated_and_include_origin_metadata() { let dir = tempfile::tempdir().unwrap(); let b = dir.path().join("b.txt"); let a = dir.path().join("a.txt"); @@ -1139,8 +1490,8 @@ mod tests { assert_eq!(values.len(), 2); assert_eq!(values[0]["content"]["body"], "alpha"); assert_eq!(values[1]["content"]["body"], "bravo"); - assert_eq!(values[0]["file"]["name"], "a.txt"); - assert_eq!(values[1]["file"]["name"], "b.txt"); + assert_eq!(values[0]["origin"]["filename"], "a.txt"); + assert_eq!(values[1]["origin"]["filename"], "b.txt"); } #[test] @@ -1163,6 +1514,13 @@ mod tests { assert_eq!(values.len(), 2); assert_eq!(values[0]["content"]["body"], "child"); assert_eq!(values[1]["content"]["body"], "root"); + assert_eq!(values[0]["origin"]["filename"], "child.md"); + assert_eq!(values[0]["origin"]["path"], nested.display().to_string()); + assert_eq!(values[1]["origin"]["filename"], "root.md"); + assert_eq!( + values[1]["origin"]["path"], + dir.path().display().to_string() + ); } #[test] @@ -1232,14 +1590,14 @@ mod tests { } #[test] - fn single_direct_file_document_omits_file_metadata() { + fn single_direct_file_document_omits_origin_metadata() { let dir = tempfile::tempdir().unwrap(); let path = dir.path().join("note.txt"); fs::write(&path, "hello").unwrap(); let values = collect_values(open_input_values(vec![uri(&path)], "body").unwrap()); - assert!(values[0].get("file").is_none()); + assert!(values[0].get("origin").is_none()); } #[test] @@ -1448,7 +1806,7 @@ mod tests { } #[test] - fn toon_file_in_multi_input_includes_file_metadata() { + fn toon_file_in_multi_input_includes_origin_metadata() { let dir = tempfile::tempdir().unwrap(); let text = dir.path().join("a.txt"); let toon = dir.path().join("b.toon"); @@ -1461,7 +1819,7 @@ mod tests { assert_eq!(values.len(), 2); assert_eq!(values[0]["content"]["body"], "alpha"); assert_eq!(values[1]["id"], 2); - assert_eq!(values[1]["file"]["name"], "b.toon"); + assert_eq!(values[1]["origin"]["filename"], "b.toon"); } #[test] @@ -1567,14 +1925,35 @@ mod tests { } #[test] - fn http_input_scheme_is_rejected() { - let uri = UriRef::parse("http://example.com/data.ndjson".to_string()).unwrap(); + fn unsupported_input_scheme_is_rejected() { + let uri = UriRef::parse("ftp://example.com/data.ndjson".to_string()).unwrap(); match Input::try_from(uri) { - Ok(_) => panic!("http input should be rejected"), - Err(err) => assert!(err.to_string().contains("Unsupported input scheme: http")), + Ok(_) => panic!("ftp input should be rejected"), + Err(err) => assert!(err.to_string().contains("Unsupported input scheme: ftp")), } } + #[test] + fn remote_http_fetch_preserves_origin_uri_components() { + let (base_url, _requests, handle) = + spawn_http_server("200 OK", "text/csv", "name,count\nalpha,2\n"); + let uri = + UriRef::parse(format!("{base_url}/docs/data.csv?download=1#row").to_string()).unwrap(); + let authority = uri.authority().unwrap().as_str().to_string(); + let values = collect_values( + fetch_remote_input_with_client(uri, &Client::builder().build().unwrap()).unwrap(), + ); + + assert_eq!(values[0]["name"], "alpha"); + assert_eq!(values[0]["origin"]["scheme"], "http"); + assert_eq!(values[0]["origin"]["authority"], authority); + assert_eq!(values[0]["origin"]["path"], "/docs"); + assert_eq!(values[0]["origin"]["filename"], "data.csv"); + assert_eq!(values[0]["origin"]["query"], "download=1"); + assert_eq!(values[0]["origin"]["fragment"], "row"); + handle.join().unwrap(); + } + #[test] fn json_extension_is_accepted_for_local_input_detection() { let path = PathBuf::from("/tmp/example.json"); @@ -1588,12 +1967,20 @@ mod tests { spawn_https_server("200 OK", "text/csv", "name,count\nalpha,2\n"); let client = test_https_client(); let uri = UriRef::parse(format!("{base_url}/download").to_string()).unwrap(); + let authority = uri.authority().unwrap().as_str().to_string(); let mut input = fetch_remote_input_with_client(uri, &client).unwrap(); let mut line = String::new(); let value = input.read_line(&mut line).unwrap(); let actual: serde_json::Value = serde_json::from_str(value.get()).unwrap(); - assert_eq!(actual, serde_json::json!({"name":"alpha","count":"2"})); + assert_eq!(actual["name"], "alpha"); + assert_eq!(actual["count"], "2"); + assert_eq!(actual["origin"]["scheme"], "https"); + assert_eq!(actual["origin"]["authority"], authority); + assert_eq!(actual["origin"]["path"], "/"); + assert_eq!(actual["origin"]["filename"], "download"); + assert!(actual["origin"].get("query").is_none()); + assert!(actual["origin"].get("fragment").is_none()); let request = requests.recv().unwrap(); let accept_header = request @@ -1627,11 +2014,18 @@ mod tests { let (base_url, _requests, handle) = spawn_https_server("200 OK", "application/octet-stream", "id: 1\nname: Alpha\n"); let client = test_https_client(); - let uri = UriRef::parse(format!("{base_url}/events.toon").to_string()).unwrap(); + let uri = + UriRef::parse(format!("{base_url}/events.toon?download=1#page").to_string()).unwrap(); let values = collect_values(fetch_remote_input_with_client(uri, &client).unwrap()); - assert_eq!(values, vec![serde_json::json!({"id":1,"name":"Alpha"})]); + assert_eq!(values[0]["id"], 1); + assert_eq!(values[0]["name"], "Alpha"); + assert_eq!(values[0]["origin"]["scheme"], "https"); + assert_eq!(values[0]["origin"]["path"], "/"); + assert_eq!(values[0]["origin"]["filename"], "events.toon"); + assert_eq!(values[0]["origin"]["query"], "download=1"); + assert_eq!(values[0]["origin"]["fragment"], "page"); handle.join().unwrap(); } @@ -1644,7 +2038,11 @@ mod tests { let values = collect_values(fetch_remote_input_with_client(uri, &client).unwrap()); - assert_eq!(values, vec![serde_json::json!({"id":1,"name":"Alpha"})]); + assert_eq!(values[0]["id"], 1); + assert_eq!(values[0]["name"], "Alpha"); + assert_eq!(values[0]["origin"]["scheme"], "https"); + assert_eq!(values[0]["origin"]["path"], "/"); + assert_eq!(values[0]["origin"]["filename"], "download"); handle.join().unwrap(); } @@ -1712,6 +2110,55 @@ mod tests { .unwrap() } + fn spawn_http_server( + status: &str, + content_type: &str, + body: &str, + ) -> (String, mpsc::Receiver, thread::JoinHandle<()>) { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let port = listener.local_addr().unwrap().port(); + let status = status.to_string(); + let content_type = content_type.to_string(); + let body = body.to_string(); + let (tx, rx) = mpsc::channel(); + + let handle = thread::spawn(move || { + let (stream, _) = listener.accept().unwrap(); + serve_http_request(stream, &tx, &status, &content_type, &body); + }); + + (format!("http://localhost:{port}"), rx, handle) + } + + fn serve_http_request( + mut stream: TcpStream, + tx: &mpsc::Sender, + status: &str, + content_type: &str, + body: &str, + ) { + let mut request = Vec::new(); + let mut buf = [0u8; 1024]; + loop { + let count = stream.read(&mut buf).unwrap(); + if count == 0 { + break; + } + request.extend_from_slice(&buf[..count]); + if request.windows(4).any(|window| window == b"\r\n\r\n") { + break; + } + } + + tx.send(String::from_utf8(request).unwrap()).unwrap(); + let response = format!( + "HTTP/1.1 {status}\r\nContent-Type: {content_type}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len() + ); + stream.write_all(response.as_bytes()).unwrap(); + stream.flush().unwrap(); + } + fn spawn_https_server( status: &str, content_type: &str, diff --git a/tests/file_output.rs b/tests/file_output.rs index 6ebf377..70ba2e3 100644 --- a/tests/file_output.rs +++ b/tests/file_output.rs @@ -1,9 +1,10 @@ +use base64::Engine as _; use flate2::read::GzDecoder; use serde_json::Value; use std::{ fs, io::Read, - path::PathBuf, + path::{Path, PathBuf}, process::Command, time::{SystemTime, UNIX_EPOCH}, }; @@ -25,6 +26,14 @@ fn temp_output_path(filename: &str) -> PathBuf { dir.join(filename) } +fn write_base64_fixture(name: &str, path: &Path) { + let encoded = fs::read_to_string(fixture_path(name)).expect("read base64 fixture"); + let bytes = base64::engine::general_purpose::STANDARD + .decode(encoded.trim()) + .expect("decode base64 fixture"); + fs::write(path, bytes).expect("write decoded fixture"); +} + fn validate_bulk_schema(lines: &[&str]) { assert!( lines.len() % 2 == 0, @@ -77,6 +86,127 @@ fn cli_writes_bulk_output_to_file() { validate_bulk_schema(&lines); } +#[test] +fn cli_converts_anydoc_pdf_to_existing_file_document_output() { + let input_path = temp_output_path("sample.pdf"); + write_base64_fixture("anydoc/sample.pdf.base64", &input_path); + let output_path = temp_output_path("anydoc.ndjson"); + + let status = Command::new(env!("CARGO_BIN_EXE_espipe")) + .arg(input_path) + .arg(&output_path) + .status() + .expect("run espipe"); + + assert!(status.success(), "espipe exited with failure"); + + let contents = fs::read_to_string(&output_path).expect("read output file"); + let document: Value = serde_json::from_str(contents.trim()).expect("document json"); + assert!( + document["content"]["body"] + .as_str() + .expect("body string") + .contains("Hello PDF") + ); + assert!(document.get("origin").is_none()); + assert!(document.get("file").is_none()); +} + +#[test] +fn cli_converts_mixed_anydoc_and_markdown_inputs_without_changing_shape() { + let anydoc_path = temp_output_path("sample.pdf"); + write_base64_fixture("anydoc/sample.pdf.base64", &anydoc_path); + let markdown_path = fixture_path("glob_docs").join("alpha.md"); + let output_path = temp_output_path("mixed-anydoc.ndjson"); + + let status = Command::new(env!("CARGO_BIN_EXE_espipe")) + .arg(anydoc_path) + .arg(markdown_path) + .arg(&output_path) + .arg("--content") + .arg("markdown") + .status() + .expect("run espipe"); + + assert!(status.success(), "espipe exited with failure"); + + let contents = fs::read_to_string(&output_path).expect("read output file"); + let documents: Vec = contents + .lines() + .filter(|line| !line.is_empty()) + .map(|line| serde_json::from_str(line).expect("document json")) + .collect(); + + assert_eq!(documents.len(), 2); + assert!(documents.iter().any(|document| { + document["content"]["markdown"] + .as_str() + .is_some_and(|body| body.contains("Hello PDF")) + })); + assert!(documents.iter().any(|document| { + document["content"]["markdown"] + .as_str() + .is_some_and(|body| body.contains("Alpha")) + })); + assert!(documents.iter().all(|document| { + document.get("file").is_none() + && document["origin"]["scheme"] == "file" + && document["origin"]["path"].is_string() + && document["origin"]["filename"].is_string() + })); +} + +#[test] +fn cli_reports_anydoc_conversion_errors_on_stderr_with_source_path() { + let input_path = temp_output_path("invalid.pdf"); + let output_path = temp_output_path("invalid-anydoc.ndjson"); + fs::write(&input_path, b"not a PDF").expect("write invalid input"); + + let output = Command::new(env!("CARGO_BIN_EXE_espipe")) + .arg(&input_path) + .arg(&output_path) + .output() + .expect("run espipe"); + + assert!( + !output.status.success(), + "espipe should reject invalid input" + ); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains("invalid.pdf"), + "stderr should identify input" + ); + let (_, detail) = stderr + .split_once("invalid.pdf") + .expect("stderr should include error detail after the source path"); + assert!( + !detail.trim().is_empty(), + "stderr should include error detail beyond the source path" + ); +} + +#[test] +fn cli_reports_image_only_pdf_requires_ocr_on_stderr() { + let input_path = temp_output_path("image-only.pdf"); + let output_path = temp_output_path("image-only-anydoc.ndjson"); + write_base64_fixture("anydoc/image-only.pdf.base64", &input_path); + + let output = Command::new(env!("CARGO_BIN_EXE_espipe")) + .arg(&input_path) + .arg(&output_path) + .output() + .expect("run espipe"); + + assert!( + !output.status.success(), + "espipe should reject image-only PDF" + ); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!(stderr.contains("image-only.pdf")); + assert!(stderr.contains("OCR is required")); +} + #[test] fn cli_rejects_multi_file_input_to_non_ndjson_file_output_before_writing() { let first_input = fixture_path("glob_docs").join("alpha.md"); @@ -168,8 +298,9 @@ fn cli_accepts_multi_file_input_to_ndjson_file_output() { assert!(status.success(), "espipe exited with failure"); let contents = fs::read_to_string(&output_path).expect("read output file"); - assert!(contents.contains(r#""name":"alpha.md""#)); - assert!(contents.contains(r#""name":"bravo.md""#)); + assert!(contents.contains(r#""filename":"alpha.md""#)); + assert!(contents.contains(r#""filename":"bravo.md""#)); + assert!(!contents.contains(r#""file":{"#)); } #[test] @@ -193,8 +324,9 @@ fn cli_accepts_multi_file_input_to_gzip_ndjson_file_output() { decoder .read_to_string(&mut contents) .expect("decompress output"); - assert!(contents.contains(r#""name":"alpha.md""#)); - assert!(contents.contains(r#""name":"bravo.md""#)); + assert!(contents.contains(r#""filename":"alpha.md""#)); + assert!(contents.contains(r#""filename":"bravo.md""#)); + assert!(!contents.contains(r#""file":{"#)); } #[test] diff --git a/tests/fixtures/anydoc/image-only.pdf.base64 b/tests/fixtures/anydoc/image-only.pdf.base64 new file mode 100644 index 0000000..1ceac03 --- /dev/null +++ b/tests/fixtures/anydoc/image-only.pdf.base64 @@ -0,0 +1 @@ +JVBERi0xLjQKMSAwIG9iago8PCAvVHlwZSAvQ2F0YWxvZyAvUGFnZXMgMiAwIFIgPj4KZW5kb2JqCjIgMCBvYmoKPDwgL1R5cGUgL1BhZ2VzIC9LaWRzIFszIDAgUl0gL0NvdW50IDEgPj4KZW5kb2JqCjMgMCBvYmoKPDwgL1R5cGUgL1BhZ2UgL1BhcmVudCAyIDAgUiAvTWVkaWFCb3ggWzAgMCAxIDFdIC9SZXNvdXJjZXMgPDwgL1hPYmplY3QgPDwgL0ltMSA1IDAgUiA+PiA+PiAvQ29udGVudHMgNCAwIFIgPj4KZW5kb2JqCjQgMCBvYmoKPDwgL0xlbmd0aCAyNyA+PgpzdHJlYW0KcQoxIDAgMCAxIDAgMCBjbQovSW0xIERvClEKZW5kc3RyZWFtCmVuZG9iago1IDAgb2JqCjw8IC9UeXBlIC9YT2JqZWN0IC9TdWJ0eXBlIC9JbWFnZSAvV2lkdGggMSAvSGVpZ2h0IDEgL0NvbG9yU3BhY2UgL0RldmljZUdyYXkgL0JpdHNQZXJDb21wb25lbnQgOCAvTGVuZ3RoIDEgPj4Kc3RyZWFtCgAKZW5kc3RyZWFtCmVuZG9iagp4cmVmCjAgNgowMDAwMDAwMDAwIDY1NTM1IGYgCjAwMDAwMDAwMDkgMDAwMDAgbiAKMDAwMDAwMDA1OCAwMDAwMCBuIAowMDAwMDAwMTE1IDAwMDAwIG4gCjAwMDAwMDAyNDEgMDAwMDAgbiAKMDAwMDAwMDMxNyAwMDAwMCBuIAp0cmFpbGVyCjw8IC9TaXplIDYgL1Jvb3QgMSAwIFIgPj4Kc3RhcnR4cmVmCjQ2MQolJUVPRgo= diff --git a/tests/fixtures/anydoc/sample.docx.base64 b/tests/fixtures/anydoc/sample.docx.base64 new file mode 100644 index 0000000..2b94a40 --- /dev/null +++ b/tests/fixtures/anydoc/sample.docx.base64 @@ -0,0 +1 @@ +UEsDBBQAAAAIAHJcDF0Vlbo5fQAAAKEAAAARAAAAd29yZC9kb2N1bWVudC54bWxFzr8OAiEMBvBXIfcA9uLgQJBFBzdXVwTuT0IpKTXo23ucg8vvS9t8SU3TgfwLYxb1xpSrbudhESkaoPoloqsHKjFvt4kYnWwjz9CIQ2HysdY1z5jgOI4nQLfmwZqmnxQ+PUuHO2JvMSVSExOq6/3yMNCXXd4tu78i/J+yX1BLAQIUAxQAAAAIAHJcDF0Vlbo5fQAAAKEAAAARAAAAAAAAAAAAAACAAQAAAAB3b3JkL2RvY3VtZW50LnhtbFBLBQYAAAAAAQABAD8AAACsAAAAAAA= diff --git a/tests/fixtures/anydoc/sample.pdf.base64 b/tests/fixtures/anydoc/sample.pdf.base64 new file mode 100644 index 0000000..29752f4 --- /dev/null +++ b/tests/fixtures/anydoc/sample.pdf.base64 @@ -0,0 +1 @@ +JVBERi0xLjQKMSAwIG9iago8PCAvVHlwZSAvQ2F0YWxvZyAvUGFnZXMgMiAwIFIgPj4KZW5kb2JqCjIgMCBvYmoKPDwgL1R5cGUgL1BhZ2VzIC9LaWRzIFszIDAgUl0gL0NvdW50IDEgPj4KZW5kb2JqCjMgMCBvYmoKPDwgL1R5cGUgL1BhZ2UgL1BhcmVudCAyIDAgUiAvTWVkaWFCb3ggWzAgMCA2MTIgNzkyXSAvUmVzb3VyY2VzIDw8IC9Gb250IDw8IC9GMSA0IDAgUiA+PiA+PiAvQ29udGVudHMgNSAwIFIgPj4KZW5kb2JqCjQgMCBvYmoKPDwgL1R5cGUgL0ZvbnQgL1N1YnR5cGUgL1R5cGUxIC9CYXNlRm9udCAvSGVsdmV0aWNhID4+CmVuZG9iago1IDAgb2JqCjw8IC9MZW5ndGggNDEgPj4Kc3RyZWFtCkJUCi9GMSAxOCBUZgo3MiA3MjAgVGQKKEhlbGxvIFBERikgVGoKRVQKZW5kc3RyZWFtCmVuZG9iagp4cmVmCjAgNgowMDAwMDAwMDAwIDY1NTM1IGYgCjAwMDAwMDAwMDkgMDAwMDAgbiAKMDAwMDAwMDA1OCAwMDAwMCBuIAowMDAwMDAwMTE1IDAwMDAwIG4gCjAwMDAwMDAyNDEgMDAwMDAgbiAKMDAwMDAwMDMxMSAwMDAwMCBuIAp0cmFpbGVyCjw8IC9TaXplIDYgL1Jvb3QgMSAwIFIgPj4Kc3RhcnR4cmVmCjQwMQolJUVPRgo= diff --git a/tests/fixtures/anydoc/sample.rtf b/tests/fixtures/anydoc/sample.rtf new file mode 100644 index 0000000..9f1a35c --- /dev/null +++ b/tests/fixtures/anydoc/sample.rtf @@ -0,0 +1,4 @@ +{\rtf1\ansi\deff0 +{\fonttbl{\f0 Helvetica;}} +\f0\fs24 Hello from RTF.\par +} diff --git a/tests/index_template.rs b/tests/index_template.rs index ae54890..d03c4ef 100644 --- a/tests/index_template.rs +++ b/tests/index_template.rs @@ -407,10 +407,10 @@ fn cli_globs_fixture_documents_with_pipeline_and_template() { .map(|request| request.body.as_str()) .collect::>() .join("\n"); - assert!(bulk_body.contains(r#""name":"alpha.md""#)); - assert!(bulk_body.contains(r#""name":"bravo.md""#)); - assert!(bulk_body.contains(r#""name":"charlie.md""#)); - assert!(bulk_body.contains(r#""name":"delta.md""#)); + assert!(bulk_body.contains(r#""filename":"alpha.md""#)); + assert!(bulk_body.contains(r#""filename":"bravo.md""#)); + assert!(bulk_body.contains(r#""filename":"charlie.md""#)); + assert!(bulk_body.contains(r#""filename":"delta.md""#)); assert!(!bulk_body.contains("ignored.tmp")); assert!(bulk_body.contains("\"markdown\":\"# Alpha\\n\\nFirst document.")); assert!(bulk_body.contains("\"markdown\":\"# Bravo\\n\\nSecond document."));