diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d9d3613..42e4b14 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,7 +8,28 @@ on: env: CARGO_TERM_COLOR: always +permissions: + contents: read + +concurrency: + group: ci-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: + msrv: + name: Minimum Rust 1.88 + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Install minimum Rust toolchain + run: rustup toolchain install 1.88.0 --profile minimal + + - name: Check all targets + run: cargo +1.88.0 check --all-targets + build-test: strategy: fail-fast: false @@ -38,10 +59,13 @@ jobs: run: rustup toolchain install stable --target ${{ matrix.target }} --profile minimal - name: Build - run: cargo build --verbose --target ${{ matrix.target }} + run: cargo build --verbose --all-targets --target ${{ matrix.target }} - name: Run tests - run: cargo test --verbose --target ${{ matrix.target }} + run: cargo test --verbose --all-targets --target ${{ matrix.target }} + + - name: Run documentation tests + run: cargo test --verbose --doc --target ${{ matrix.target }} lint: name: Lint & Format @@ -63,4 +87,12 @@ jobs: run: cargo fmt --all -- --check - name: Run clippy - run: cargo clippy -- -D warnings + run: cargo clippy --all-targets -- -D warnings + + - name: Check documentation + env: + RUSTDOCFLAGS: -D warnings + run: cargo doc --no-deps + + - name: Check package contents + run: cargo package --no-verify diff --git a/BENCHMARKS.md b/BENCHMARKS.md new file mode 100644 index 0000000..009fc1b --- /dev/null +++ b/BENCHMARKS.md @@ -0,0 +1,100 @@ +# Benchmarks + +This project benchmarks correctness before speed. Every cross-tool run uses one +fixture and verifies identical sorted path sets before timing begins. + +## Environment + +Results below were collected on 2026-08-08 with: + +- MacBook Pro, Apple M5 Pro (15 cores), 24 GB memory +- Darwin 27.0.0, arm64 +- Rust 1.96.1 +- `fd` 10.4.2 +- ripgrep 15.1.0 +- hyperfine 1.20.0 + +Absolute filesystem timings vary with cache state, storage, directory shape, +antivirus/indexing activity, and operating system. Treat these results as a +reproducible local snapshot, not a universal leaderboard. + +## Cross-tool comparison + +`benchmarks/compare.sh` generates 50,000 files across 250 directories. Ten +extensions are distributed evenly, so each command emits exactly 5,000 `.rs` +paths. The fixture contains no hidden files or ignore rules because POSIX +`find` does not implement the same policy as the other tools. Standard output +is redirected to `/dev/null` during timing. + +| Tool | Mean ± σ | Min | Max | Relative | +|:---|---:|---:|---:|---:| +| `rust_search` | 30.7 ± 0.9 ms | 29.9 ms | 33.2 ms | 1.00 | +| `ripgrep --files` | 31.0 ± 0.7 ms | 30.1 ms | 32.2 ms | 1.01 ± 0.04 | +| `fd` | 35.0 ± 1.4 ms | 33.2 ms | 37.7 ms | 1.14 ± 0.06 | +| `find` | 95.9 ± 3.2 ms | 88.1 ms | 99.0 ms | 3.13 ± 0.14 | + +The variance is high enough that `rust_search` and `ripgrep --files` should be +considered broadly comparable in this sample. The stronger conclusions are +that result parity was achieved and that all four tools can be rerun against +the same generated workload. + +## Before-and-after throughput + +A separate hyperfine run compared the string API in commit `260286c` with this +working tree on the same warm 100,000-file fixture. Both builds emitted 10,000 +paths. Each received five warmups and 15 measured runs. + +| Revision | Mean ± σ | Range | Relative | +|:---|---:|---:|---:| +| Current | 57.1 ± 3.9 ms | 52.3–68.2 ms | 1.00 | +| `260286c` baseline | 78.1 ± 3.8 ms | 71.8–86.2 ms | 1.37× | + +The improvement comes primarily from literal string matching instead of a +regex on every candidate, a burst-tolerant bounded result channel, and reduced +allocation in the path pipeline. + +## Library microbenchmarks + +`cargo bench --bench bench_search -- controlled` uses a 100,000-file fixture, +three warmups, and ten measured runs. This sample used `strsim` 0.11.1. + +| Scenario | Results | Median | +|:---|---:|---:| +| One extension (`rs`) | 10,000 | 44.423 ms | +| Two extensions (`rs`, `txt`) | 20,000 | 47.047 ms | +| Filename substring plus extension | 5,000 | 48.644 ms | +| Extension with limit 100 | 100 | 3.678 ms | +| Similarity sort | 10,000 | 0.912 ms | +| Time to first result | 1 | 1.145 ms | + +## Product comparison + +The tools overlap, but they are not interchangeable: + +| Product | Best fit | Important distinction | +|:---|:---|:---| +| `rust_search` | Embedding search in a Rust application | Typed builder, custom closures, metadata filters, lossless/error-aware iterators | +| `fd` | Interactive shell use | Mature end-user CLI with rich output and execution options | +| `ripgrep --files` | File enumeration alongside content search | Part of a highly optimized text-search CLI | +| POSIX `find` | Portable shell scripts and filesystem predicates | Ubiquitous, but no `.gitignore` policy by default | +| `ignore` | Building a lower-level ignore-aware walker | The traversal engine used by `rust_search` | +| `jwalk` | Parallel streamed directory traversal | Lower-level walking API with optional per-directory sorting | +| `walkdir` | Small, established sequential traversal | Simpler dependency and no parallel traversal | + +`rust_search` should be evaluated as a high-level application library, not as +a replacement for every CLI or directory-walking primitive. + +## Reproducing + +```console +cargo bench --bench bench_search -- controlled +./benchmarks/compare.sh +``` + +The comparison script writes its Markdown table to +`target/search-comparison.md` by default. Set `RUNS` or pass an output path to +change those defaults: + +```console +RUNS=25 ./benchmarks/compare.sh benchmark-results.md +``` diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..d172c6b --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,45 @@ +# Changelog + +## Unreleased + +### Added + +- Streaming, lossless `PathBuf` results through `build_paths()`. +- Error-aware traversal through `build_results()` and `SearchError`. +- Multiple-extension searches with `extensions()`. +- File, directory, and combined result targeting. +- Minimum depth, symlink following, filesystem-boundary, thread-count, and + early maximum-file-size controls. +- `similarity_sort_paths()` for `PathBuf` collections. +- Reproducible controlled and cross-tool benchmark suites. + +### Changed + +- Search construction now returns immediately and traverses in the background. +- Result delivery uses a bounded, burst-tolerant channel, so slow consumers do + not cause memory usage to scale with every match. +- Default searches consistently return files and symbolic links, not a mixture + that sometimes included directories. +- Search inputs are treated as literal text. Regex punctuation no longer has + implicit or panic-prone behavior. +- Similarity scoring uses `strsim` 0.11. +- Home-directory discovery uses `dirs` 6 and the duplicate development + dependency was removed. +- The verified minimum supported Rust version is now declared as 1.88. + +### Fixed + +- Strict matching without an explicit extension now works as documented. +- Case-insensitive extension matching now includes differently cased suffixes. +- Concurrent result limits now reserve exactly the configured number of slots. +- A zero result limit avoids starting filesystem traversal. +- Custom-filter panics, worker panics, and filesystem errors can be observed through + `build_results()`. +- Multiple built-in metadata filters share one metadata lookup per entry. + +### Performance + +- A 15-run, output-equivalent A/B benchmark on 100,000 files measured the new + string API at 57.1 ms versus 78.1 ms for commit `260286c` (1.37× faster). +- A controlled 10,000-item similarity sort measured 0.912 ms after the + `strsim` upgrade. diff --git a/Cargo.toml b/Cargo.toml index cf6c4c5..4e7cfe9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,6 +3,7 @@ name = "rust_search" version = "2.2.0" description = "Blazingly fast file search library built in Rust" edition = "2021" +rust-version = "1.88" authors = ["Parth Jadhav "] license = "MIT" readme = "README.md" @@ -12,27 +13,28 @@ categories = ["filesystem", "algorithms"] include = [ "/.github/**", "/benches/**", + "/benchmarks/**", + "/examples/**", "/src/**", "/tests/**", "/Cargo.toml", + "/BENCHMARKS.md", + "/CHANGELOG.md", "/LICENSE-MIT", "/README.md", + "/ROADMAP.md", "/learnings.md", ] # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -regex = "1" ignore = "0.4" -dirs = "4.0.0" -strsim = "0.10.0" +dirs = "6" +strsim = "0.11" crossbeam-channel = "0.5.15" rayon = "1.11.0" -[dev-dependencies] -dirs = "4.0.0" - [[bench]] name = "bench_search" harness = false diff --git a/README.md b/README.md index 82f25b7..4884936 100644 --- a/README.md +++ b/README.md @@ -13,34 +13,57 @@ Blazingly fast file search crate built in Rust 🔥 +`rust_search` is an embeddable filesystem-search library for applications that +need `fd`-style traversal without launching a subprocess. It streams matches +from a parallel, `.gitignore`-aware walker and provides filename, extension, +depth, size, time, directory-exclusion, and custom-closure filters. + +Highlights: + +- Streaming results with bounded memory and prompt cancellation on drop +- Plain-text filename matching: punctuation is literal and cannot create an invalid regex +- Files, directories, or both, across one or more roots +- Lossless `PathBuf` results and an error-aware iterator when needed +- Multiple extensions, hidden/ignored-file controls, symlink following, and filesystem-boundary controls +- Similarity ranking for both `String` and `PathBuf` results + ## 📦 Usage -Please report any problems you encounter when using rust search here: [Issues](https://github.com/ParthJadhav/rust_search/issues) +Please report problems in [GitHub Issues](https://github.com/ParthJadhav/rust_search/issues). -Add `rust_search = "2.2.0"` in Cargo.toml. +The latest published crates.io release is `2.1.0`: ```toml [dependencies] -rust_search = "2.2.0" +rust_search = "2.1" ``` +This repository is preparing `2.2.0`. To test the unreleased API shown below, +depend on the Git repository until the release is published: + +```toml +[dependencies] +rust_search = { git = "https://github.com/ParthJadhav/Rust_Search" } +``` + +The unreleased version supports Rust 1.88 and newer. + ## Examples -- General use +### General search ```rust use rust_search::SearchBuilder; let search: Vec = SearchBuilder::default() .location("~/path/to/directory") - .search_input("what to search") + .search_input("report") .more_locations(vec!["/anotherPath/to/search", "/keepAddingIfYouWant/"]) - .limit(1000) // results to return - .ext("extension") - .strict() - .depth(1) + .extensions(["md", "txt"]) + .limit(1000) + .depth(4) .ignore_case() - .hidden() + .exclude_dirs(["node_modules", "target"]) .build() .collect(); @@ -49,7 +72,10 @@ for path in search { } ``` -- Sort the output by similarity with the input +`build()` returns immediately; traversal continues in the background while the +iterator is consumed. Dropping the iterator cancels the remaining walk. + +### Sort output by similarity ```rust use rust_search::{SearchBuilder, similarity_sort}; @@ -63,54 +89,63 @@ let mut search: Vec = SearchBuilder::default() .build() .collect(); -similarity_sort(&mut search, &search_input); +similarity_sort(&mut search, search_input); for path in search { println!("{:?}", path); } ``` -> search **without** similarity sort -> `["afly.txt", "bfly.txt", "flyer.txt", "fly.txt"]` +Search without similarity sort: `["afly.txt", "bfly.txt", "flyer.txt", "fly.txt"]` -> search **with** similarity sort -> `["fly.txt", "flyer.txt", "afly.txt", "bfly.txt",]` +Search with similarity sort: `["fly.txt", "flyer.txt", "afly.txt", "bfly.txt"]` -- To get all the files with a specific extension in a directory, use: +### Choose the result representation ```rust use rust_search::SearchBuilder; +use std::path::PathBuf; -let files: Vec = SearchBuilder::default() +let paths: Vec = SearchBuilder::default() .location("/path/to/directory") - .ext("file_extension") - .build() + .ext("rs") + .build_paths() .collect(); ``` -- To get all the files in a directory, use: +Use `build_results()` when traversal failures must not be skipped: ```rust use rust_search::SearchBuilder; -let files: Vec = SearchBuilder::default() - .location("/path/to/directory") - .depth(1) - .build() - .collect(); +for result in SearchBuilder::default().location("/path/to/directory").build_results() { + match result { + Ok(path) => println!("{}", path.display()), + Err(error) => eprintln!("search error: {error}"), + } +} ``` -- To include files ignored by `.gitignore`, use: +### Search directories and control traversal ```rust use rust_search::SearchBuilder; -let files: Vec = SearchBuilder::default() +let directories: Vec = SearchBuilder::default() .location("/path/to/directory") + .directories() + .min_depth(1) + .depth(5) + .follow_links(false) + .same_file_system(true) .git_ignore(false) + .hidden() .build() .collect(); ``` -- To skip expensive directories while walking, use: +Files are the default. Use `.directories()` for directories only or +`.files_and_directories()` for both. + +### Skip expensive directories while walking ```rust use rust_search::SearchBuilder; @@ -122,29 +157,31 @@ let files: Vec = SearchBuilder::default() .collect(); ``` -To filter files by `date_created`, `date_modified`, `file_size` and/or `custom_filter`, use: +### Filter by metadata or a custom closure ```rust use rust_search::{FileSize, FilterExt, SearchBuilder}; use std::time::{Duration, SystemTime}; let search: Vec = SearchBuilder::default() - .location("~/path/to/directory") - .file_size_greater(FileSize::Kilobyte(200.0)) - .file_size_smaller(FileSize::Megabyte(10.0)) - .created_after(SystemTime::now() - Duration::from_secs(3600 * 24 * 10)) - .created_before(SystemTime::now()) - .modified_after(SystemTime::now() - Duration::from_secs(3600 * 24 * 5)) - .custom_filter(|dir| dir.metadata().unwrap().is_file()) - .custom_filter(|dir| !dir.metadata().unwrap().permissions().readonly()) - .build() - .collect(); + .location("~/path/to/directory") + .max_file_size(FileSize::Megabyte(10.0)) + .file_size_greater(FileSize::Kilobyte(200.0)) + .created_after(SystemTime::now() - Duration::from_secs(3600 * 24 * 10)) + .created_before(SystemTime::now()) + .modified_after(SystemTime::now() - Duration::from_secs(3600 * 24 * 5)) + .custom_filter(|dir| { + dir.metadata() + .is_ok_and(|metadata| !metadata.permissions().readonly()) + }) + .build() + .collect(); ``` Custom filters can capture values from their environment: ```rust -use rust_search::{FilterExt, SearchBuilder}; +use rust_search::SearchBuilder; let suffix = ".rs".to_string(); let search: Vec = SearchBuilder::default() @@ -154,45 +191,42 @@ let search: Vec = SearchBuilder::default() .collect(); ``` -👉 For more examples, please refer to the [Documentation](https://docs.rs/rust_search/latest/rust_search/) +For the complete API, see the [documentation](https://docs.rs/rust_search/latest/rust_search/). ## ⚙️ Benchmarks -The difference in sample size is due to the fact that fd and glob are different tools and have different use cases. fd is a command line tool that searches for files and directories. glob is a library that can be used to search for files and directories. The benchmark is done on a `MacBook` Air M2, 16 GB Unified memory. - -Benchmarks are done using [hyperfine](https://github.com/sharkdp/hyperfine), -Benchmarks files are available in the [benchmarks](https://drive.google.com/drive/folders/1ug6ojNixS5jAe6Lh6M0o2d3tku73zQ9w?usp=sharing) drive folder. - -### - Rust Search vs Glob - -The benchmark was done on a directories containing 300K files. - -| Command / Library | Mean \[s] | Min \[s] | Max \[s] | Relative | -|:---|---:|---:|---:|---:| -| `rust_search` | 1.317 ± 0.002 | 1.314 | 1.320 | 1.00 | -| `glob` | 22.728 ± 0.023 | 22.690 | 22.746 | 17.25 ± 0.03 | +The repository includes a reproducible comparison against `fd`, +`ripgrep --files`, and POSIX `find`. The script generates one shared fixture, +verifies that every tool returns the same sorted path set, and only then runs +`hyperfine`. ---- +On an Apple M5 Pro with a warm 50,000-file fixture (5,000 `.rs` matches), one +10-run sample produced: -### - Rust Search vs FD +| Tool | Mean | Range | Relative | +|:---|---:|---:|---:| +| `rust_search` | 30.7 ms | 29.9–33.2 ms | 1.00 | +| `ripgrep --files` | 31.0 ms | 30.1–32.2 ms | 1.01× | +| `fd` | 35.0 ms | 33.2–37.7 ms | 1.14× | +| `find` | 95.9 ms | 88.1–99.0 ms | 3.13× | -The benchmark was done on a directories containing 45K files. +`rust_search` and `ripgrep --files` are effectively tied in this sample, so the +numbers should be read as a local snapshot rather than a universal ranking. A +separate 15-run A/B test on 100,000 files measured the current implementation +at 57.1 ms versus 78.1 ms for commit `260286c`, a 1.37× throughput improvement +with equal result counts. -| Command / Library | Mean \[ms] | Min \[ms] | Max \[ms] | Relative | -|:---|---:|---:|---:|---:| -| `rust_search` | 680.5 ± 2.1 | 678.3 | 683.6 | 1.00 | -| `fd -e .js` | 738.7 ± 10.2 | 720.8 | 746.7 | 1.09 ± 0.02 | +Reproduce the library benchmark and cross-tool comparison with: ---- - -### Results:- - -```diff -+ rust_search is 17.25 times faster than Glob. - -+ rust_search** is 1.09 times faster than FD. +```console +cargo bench --bench bench_search -- controlled +./benchmarks/compare.sh ``` +See [BENCHMARKS.md](BENCHMARKS.md) for methodology, versions, internal timings, +and a product-level comparison. Filesystem benchmarks are sensitive to cache +state, directory shape, storage, security software, and operating system. + ## 👨‍💻 Contributors Any contributions would be greatly valued as this library is still in its early stages. diff --git a/ROADMAP.md b/ROADMAP.md new file mode 100644 index 0000000..8141772 --- /dev/null +++ b/ROADMAP.md @@ -0,0 +1,62 @@ +# Product direction + +## Purpose + +`rust_search` should be the easiest way to embed fast, ignore-aware filesystem +search in a Rust application. Its advantage is not owning a novel directory +walker; it is turning the proven `ignore` traversal engine into a small, +composable product API with matching, filtering, ranking, streaming, and clear +control over correctness tradeoffs. + +The crate should remain an application library. Competing directly with `fd` +as a full CLI or with `walkdir` as a minimal primitive would dilute that focus. + +## Current product principles + +1. Correctness before benchmark claims: compare equal outputs on one fixture. +2. Literal input by default: ordinary filenames must never be interpreted as code. +3. Stream by default: first results and cancellation matter as much as total throughput. +4. Lossless and observable when requested: offer `PathBuf` and error-aware APIs. +5. Explicit policy: hidden files, ignore rules, links, mount boundaries, and entry + types must be caller choices. +6. No invisible stale cache: caching requires an invalidation model, not only a map. + +## Release priorities + +### P0 — stabilize the next release + +- Decide whether the corrected files-only default and asynchronous construction + warrant a major version. They match the documentation but may expose latent + caller assumptions. +- Keep the declared Rust 1.88 minimum covered by CI when dependencies change. +- Run the comparison smoke test on Linux, macOS, and Windows runners and retain + historical JSON results without failing CI on noisy timing thresholds. +- Add property tests for matcher combinations and platform-specific tests for + symbolic links, permissions, non-UTF-8 Unix paths, and filesystem boundaries. +- Document the crate's currently unpublished repository version versus the + latest crates.io release before publishing. + +### P1 — improve query power without weakening safety + +- Add explicit glob and regex query modes behind a fallible builder. Keep + `search_input()` literal. +- Add top-k ranked search so callers do not need to collect and sort every match. +- Expose progress statistics and an explicit cancellation handle for long-running + UI searches. +- Add deterministic ordering as an opt-in mode with clearly benchmarked cost. + +### P2 — broaden integration + +- Evaluate a small C ABI for issue #33 in a separate crate so the core library + remains idiomatic and safe. +- Reproduce issue #34 on musl with syscall profiles before changing traversal + strategy. +- Consider an async adapter only when it can integrate with runtimes without + pretending filesystem traversal itself is non-blocking. + +## Explicitly deferred + +Issue #8 requests caching. A process-local result cache would return stale paths +after creates, deletes, renames, ignore-file edits, or permission changes. It +should not ship until the product has a watcher-backed invalidation strategy, +bounded storage, per-root policy keys, and measurable repeat-query demand. diff --git a/benches/bench_search.rs b/benches/bench_search.rs index b4b8af0..faaa486 100644 --- a/benches/bench_search.rs +++ b/benches/bench_search.rs @@ -4,8 +4,8 @@ use std::io::Write; use std::path::PathBuf; use std::time::{Duration, Instant}; -const WARMUP_ITERS: usize = 1; -const BENCH_ITERS: usize = 3; +const WARMUP_ITERS: usize = 3; +const BENCH_ITERS: usize = 10; fn median(times: &mut [Duration]) -> Duration { times.sort(); @@ -14,9 +14,8 @@ fn median(times: &mut [Duration]) -> Duration { /// Create a controlled test directory with many files for benchmarking. fn create_test_dir(num_dirs: usize, files_per_dir: usize) -> PathBuf { - let dir = std::env::temp_dir().join("rust_search_bench"); - let _ = fs::remove_dir_all(&dir); - fs::create_dir_all(&dir).unwrap(); + let dir = std::env::temp_dir().join(format!("rust_search_bench_{}", std::process::id())); + fs::create_dir(&dir).expect("benchmark directory already exists"); let extensions = [ "rs", "txt", "md", "json", "toml", "yaml", "py", "js", "ts", "css", @@ -88,6 +87,105 @@ fn run_sort_bench( (count, med) } +fn run_first_result_bench Option>( + label: &str, + warmup: usize, + iters: usize, + f: F, +) -> Duration { + for _ in 0..warmup { + assert!(f().is_some()); + } + + let mut times = Vec::with_capacity(iters); + for _ in 0..iters { + let start = Instant::now(); + assert!(f().is_some()); + times.push(start.elapsed()); + } + + let med = median(&mut times); + eprintln!("{label:<28} first result, median {med:>12.3?}"); + med +} + +fn run_controlled_benchmarks() { + eprintln!("=== Controlled (100,000 files) ===\n"); + let dir = create_test_dir(500, 200); + + run_bench("ctrl/ext_only (.rs)", WARMUP_ITERS, BENCH_ITERS, || { + SearchBuilder::default() + .location(&dir) + .ext("rs") + .build() + .collect() + }); + run_bench( + "ctrl/multi_ext (.rs,.txt)", + WARMUP_ITERS, + BENCH_ITERS, + || { + SearchBuilder::default() + .location(&dir) + .extensions(["rs", "txt"]) + .build() + .collect() + }, + ); + run_bench( + "ctrl/name+ext (file_00,rs)", + WARMUP_ITERS, + BENCH_ITERS, + || { + SearchBuilder::default() + .location(&dir) + .search_input("file_00") + .ext("rs") + .build() + .collect() + }, + ); + run_bench( + "ctrl/ext+limit (.rs,100)", + WARMUP_ITERS, + BENCH_ITERS, + || { + SearchBuilder::default() + .location(&dir) + .ext("rs") + .limit(100) + .build() + .collect() + }, + ); + let ctrl_base: Vec = SearchBuilder::default() + .location(&dir) + .ext("rs") + .build() + .collect(); + run_sort_bench( + "ctrl/sort", + &ctrl_base, + "file_0042", + WARMUP_ITERS, + BENCH_ITERS, + ); + run_first_result_bench( + "ctrl/time_to_first (.rs)", + WARMUP_ITERS, + BENCH_ITERS, + || { + SearchBuilder::default() + .location(&dir) + .ext("rs") + .build() + .next() + }, + ); + + fs::remove_dir_all(&dir).expect("failed to clean up benchmark directory"); +} + fn main() { let arg = std::env::args().nth(1).unwrap_or_default(); match arg.as_str() { @@ -121,6 +219,7 @@ fn main() { .collect(); run_sort_bench("sort", &base, "main", WARMUP_ITERS, BENCH_ITERS); } + "controlled" => run_controlled_benchmarks(), "all" => { let home = dirs::home_dir().unwrap(); @@ -154,45 +253,8 @@ fn main() { .collect(); run_sort_bench("home/sort", &base, "main", WARMUP_ITERS, BENCH_ITERS); - // Controlled benchmarks - eprintln!("\n=== Controlled (100,000 files) ===\n"); - let dir = create_test_dir(500, 200); - - run_bench("ctrl/ext_only (.rs)", WARMUP_ITERS, BENCH_ITERS, || { - SearchBuilder::default() - .location(&dir) - .ext("rs") - .build() - .collect() - }); - run_bench( - "ctrl/ext+input (file_00.rs)", - WARMUP_ITERS, - BENCH_ITERS, - || { - SearchBuilder::default() - .location(&dir) - .search_input("file_00") - .ext("rs") - .build() - .collect() - }, - ); - - let ctrl_base: Vec = SearchBuilder::default() - .location(&dir) - .ext("rs") - .build() - .collect(); - run_sort_bench( - "ctrl/sort", - &ctrl_base, - "file_0042", - WARMUP_ITERS, - BENCH_ITERS, - ); - - let _ = fs::remove_dir_all(&dir); + eprintln!(); + run_controlled_benchmarks(); eprintln!("\n=== Done ==="); } @@ -228,9 +290,9 @@ fn main() { .collect() }); - // 4. Search with regex pattern + extension + // 4. Search with a filename query and extension run_bench( - "system/regex+ext (main*.rs)", + "system/name+ext (main,rs)", WARMUP_ITERS, BENCH_ITERS, || { @@ -324,7 +386,7 @@ fn main() { eprintln!("\n=== Done ==="); } _ => { - eprintln!("Usage: bench_search [search|limit|sort|all|system]"); + eprintln!("Usage: bench_search [search|limit|sort|controlled|all|system]"); } } } diff --git a/benchmarks/compare.sh b/benchmarks/compare.sh new file mode 100755 index 0000000..6171ae9 --- /dev/null +++ b/benchmarks/compare.sh @@ -0,0 +1,66 @@ +#!/bin/sh +set -eu + +REPOSITORY=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +OUTPUT=${1:-"$REPOSITORY/target/search-comparison.md"} +RUNS=${RUNS:-10} +FIXTURE=$(mktemp -d "${TMPDIR:-/tmp}/rust-search-bench.XXXXXX") +trap 'rm -rf "$FIXTURE"' EXIT HUP INT TERM + +if command -v fd >/dev/null 2>&1; then + FD=fd +elif command -v fdfind >/dev/null 2>&1; then + FD=fdfind +else + echo "fd (or fdfind) is required" >&2 + exit 1 +fi + +for tool in cargo hyperfine python3 rg find; do + if ! command -v "$tool" >/dev/null 2>&1; then + echo "$tool is required" >&2 + exit 1 + fi +done + +python3 "$REPOSITORY/benchmarks/generate_fixture.py" "$FIXTURE" +cargo build --manifest-path "$REPOSITORY/Cargo.toml" --release --example search +RUST_SEARCH="$REPOSITORY/target/release/examples/search" + +rust_results="$FIXTURE/.rust-search-results" +fd_results="$FIXTURE/.fd-results" +rg_results="$FIXTURE/.ripgrep-results" +find_results="$FIXTURE/.find-results" + +"$RUST_SEARCH" "$FIXTURE" rs | LC_ALL=C sort > "$rust_results" +"$FD" --type f --extension rs . "$FIXTURE" | LC_ALL=C sort > "$fd_results" +rg --files --glob '*.rs' "$FIXTURE" | LC_ALL=C sort > "$rg_results" +find "$FIXTURE" -type f -name '*.rs' | LC_ALL=C sort > "$find_results" + +rust_count=$(wc -l < "$rust_results" | tr -d ' ') +fd_count=$(wc -l < "$fd_results" | tr -d ' ') +rg_count=$(wc -l < "$rg_results" | tr -d ' ') +find_count=$(wc -l < "$find_results" | tr -d ' ') + +if [ "$rust_count" != "$fd_count" ] || [ "$rust_count" != "$rg_count" ] || [ "$rust_count" != "$find_count" ]; then + echo "result counts differ: rust_search=$rust_count fd=$fd_count rg=$rg_count find=$find_count" >&2 + exit 1 +fi + +for candidate in "$fd_results" "$rg_results" "$find_results"; do + if ! cmp -s "$rust_results" "$candidate"; then + echo "result paths differ between rust_search and $candidate" >&2 + diff -u "$rust_results" "$candidate" | sed -n '1,80p' >&2 + exit 1 + fi +done + +mkdir -p "$(dirname -- "$OUTPUT")" +hyperfine --warmup 3 --runs "$RUNS" --export-markdown "$OUTPUT" \ + --command-name rust_search "'$RUST_SEARCH' '$FIXTURE' rs > /dev/null" \ + --command-name fd "'$FD' --type f --extension rs . '$FIXTURE' > /dev/null" \ + --command-name 'ripgrep --files' "rg --files --glob '*.rs' '$FIXTURE' > /dev/null" \ + --command-name find "find '$FIXTURE' -type f -name '*.rs' > /dev/null" + +echo "verified an identical set of $rust_count paths for every tool" +echo "wrote $OUTPUT" diff --git a/benchmarks/generate_fixture.py b/benchmarks/generate_fixture.py new file mode 100755 index 0000000..a2aca81 --- /dev/null +++ b/benchmarks/generate_fixture.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 +"""Generate a deterministic, tool-neutral file-search benchmark fixture.""" + +from __future__ import annotations + +import argparse +from pathlib import Path + + +EXTENSIONS = ("rs", "txt", "md", "json", "toml", "yaml", "py", "js", "ts", "css") + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("root", type=Path) + parser.add_argument("--directories", type=int, default=250) + parser.add_argument("--files-per-directory", type=int, default=200) + return parser.parse_args() + + +def main() -> None: + args = parse_args() + args.root.mkdir(parents=True, exist_ok=True) + if any(args.root.iterdir()): + raise SystemExit(f"refusing to populate non-empty directory: {args.root}") + + for directory_index in range(args.directories): + directory = args.root / f"dir_{directory_index:04d}" + directory.mkdir() + for file_index in range(args.files_per_directory): + extension = EXTENSIONS[file_index % len(EXTENSIONS)] + path = directory / f"file_{file_index:04d}.{extension}" + path.write_bytes(b"benchmark fixture\n") + + total = args.directories * args.files_per_directory + print(f"generated {total} files below {args.root}") + + +if __name__ == "__main__": + main() diff --git a/examples/search.rs b/examples/search.rs new file mode 100644 index 0000000..f7323b4 --- /dev/null +++ b/examples/search.rs @@ -0,0 +1,28 @@ +use std::{env, process}; + +use rust_search::SearchBuilder; + +fn main() { + let mut args = env::args().skip(1); + let Some(root) = args.next() else { + eprintln!("usage: search [limit]"); + process::exit(2); + }; + let Some(extension) = args.next() else { + eprintln!("usage: search [limit]"); + process::exit(2); + }; + + let mut builder = SearchBuilder::default().location(root).ext(extension); + if let Some(limit) = args.next() { + let limit = limit.parse().unwrap_or_else(|error| { + eprintln!("invalid limit {limit:?}: {error}"); + process::exit(2); + }); + builder = builder.limit(limit); + } + + for path in builder.build() { + println!("{path}"); + } +} diff --git a/learnings.md b/learnings.md index e468b4f..24e9d02 100644 --- a/learnings.md +++ b/learnings.md @@ -1,5 +1,10 @@ # Performance Optimization Learnings +> Historical note: this document records the optimization work that landed in +> commit `13fb0f2`. It is preserved as an engineering log, not as the current +> benchmark source of truth. See [`BENCHMARKS.md`](BENCHMARKS.md) for the +> reproducible suite and latest measurements. + ## Summary Achieved significant performance improvements across all benchmarks through 6 iterative diff --git a/src/builder.rs b/src/builder.rs index 6b4b9fd..426d301 100644 --- a/src/builder.rs +++ b/src/builder.rs @@ -4,9 +4,22 @@ use std::{ }; use crate::filter::FilterType; -use crate::{utils::replace_tilde_with_home_dir, Search}; +use crate::{utils::replace_tilde_with_home_dir, Search, SearchPaths, SearchResults}; use ignore::DirEntry; +/// The kinds of filesystem entries returned by a search. +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +#[non_exhaustive] +pub enum SearchTarget { + /// Return files and symbolic links, but not directories. + #[default] + Files, + /// Return directories below the configured search roots. + Directories, + /// Return both files and directories below the configured search roots. + FilesAndDirectories, +} + /// Search behavior toggles collected away from the public builder fields. #[derive(Clone, Copy)] pub struct SearchOptions { @@ -14,11 +27,15 @@ pub struct SearchOptions { case_matching: CaseMatching, hidden: Hidden, git_ignore: GitIgnore, + target: SearchTarget, + follow_links: bool, + same_file_system: bool, + threads: Option, } #[derive(Clone, Copy)] enum MatchMode { - Fuzzy, + Contains, Strict, } @@ -56,6 +73,22 @@ impl SearchOptions { pub const fn respect_git_ignore(self) -> bool { matches!(self.git_ignore, GitIgnore::Respect) } + + pub const fn target(self) -> SearchTarget { + self.target + } + + pub const fn follow_links(self) -> bool { + self.follow_links + } + + pub const fn same_file_system(self) -> bool { + self.same_file_system + } + + pub const fn threads(self) -> Option { + self.threads + } } /// Internal search configuration assembled by [`SearchBuilder`]. @@ -63,9 +96,11 @@ pub struct SearchConfig { pub search_location: PathBuf, pub more_locations: Option>, pub search_input: Option, - pub file_ext: Option, + pub file_extensions: Vec, + pub min_depth: Option, pub depth: Option, pub limit: Option, + pub max_file_size: Option, pub options: SearchOptions, pub excluded_dirs: Vec, pub filters: Vec, @@ -80,11 +115,15 @@ pub struct SearchBuilder { /// The search input, default will get all files from locations. search_input: Option, /// The file extension to search for, defaults to get all extensions. - file_ext: Option, + file_extensions: Vec, + /// The minimum depth at which results are yielded. + min_depth: Option, /// The depth to search to, defaults to no limit. depth: Option, /// The limit of results to return, defaults to no limit. limit: Option, + /// Skip files larger than this many bytes during traversal. + max_file_size: Option, /// Behavior toggles for the search. options: SearchOptions, /// Directory names or paths to exclude from traversal. @@ -97,17 +136,43 @@ impl SearchBuilder { /// Build a new [`Search`] instance. #[allow(deprecated)] pub fn build(&self) -> Search { - Search::new(SearchConfig { + Search::new(self.config()) + } + + /// Build a search that yields lossless [`PathBuf`] values. + /// + /// This is preferable to [`SearchBuilder::build`] when paths may contain + /// non-UTF-8 bytes or when the caller will immediately convert strings + /// back into paths. + /// + /// [`PathBuf`]: std::path::PathBuf + pub fn build_paths(&self) -> SearchPaths { + SearchPaths::new(self.config()) + } + + /// Build a search that yields traversal errors as well as lossless paths. + /// + /// The regular builders skip unreadable entries for convenience. Use this + /// method when callers need to report permission, symbolic-link, or other + /// filesystem errors. + pub fn build_results(&self) -> SearchResults { + SearchResults::new(self.config()) + } + + fn config(&self) -> SearchConfig { + SearchConfig { search_location: self.search_location.clone(), more_locations: self.more_locations.clone(), search_input: self.search_input.clone(), - file_ext: self.file_ext.clone(), + file_extensions: self.file_extensions.clone(), + min_depth: self.min_depth, depth: self.depth, limit: self.limit, + max_file_size: self.max_file_size, options: self.options, excluded_dirs: self.excluded_dirs.clone(), filters: self.filters.clone(), - }) + } } /// Set the search location to search in. @@ -161,10 +226,30 @@ impl SearchBuilder { pub fn ext(mut self, ext: impl Into) -> Self { let ext: String = ext.into(); // Remove the dot if it's there. - self.file_ext = Some( - ext.strip_prefix('.') - .map_or_else(|| ext.clone(), str::to_owned), - ); + self.file_extensions = vec![ext + .strip_prefix('.') + .map_or_else(|| ext.clone(), str::to_owned)]; + self + } + + /// Set one or more file extensions to search for. + /// + /// Leading dots are optional. Calling this method or [`SearchBuilder::ext`] + /// replaces any extensions configured by an earlier call. + pub fn extensions(mut self, extensions: I) -> Self + where + I: IntoIterator, + S: Into, + { + self.file_extensions = extensions + .into_iter() + .map(Into::into) + .map(|extension: String| { + extension + .strip_prefix('.') + .map_or_else(|| extension.clone(), str::to_owned) + }) + .collect(); self } @@ -183,8 +268,11 @@ impl SearchBuilder { /// .created_after(SystemTime::now() - Duration::from_secs(3600 * 24 * 10)) /// .created_before(SystemTime::now()) /// .modified_after(SystemTime::now() - Duration::from_secs(3600 * 24 * 5)) - /// .custom_filter(|dir| dir.metadata().unwrap().is_file()) - /// .custom_filter(|dir| !dir.metadata().unwrap().permissions().readonly()) + /// .custom_filter(|dir| dir.metadata().is_ok_and(|metadata| metadata.is_file())) + /// .custom_filter(|dir| { + /// dir.metadata() + /// .is_ok_and(|metadata| !metadata.permissions().readonly()) + /// }) /// .build() /// .collect(); /// ``` @@ -235,6 +323,15 @@ impl SearchBuilder { self } + /// Set the minimum depth at which matching entries are returned. + /// + /// Traversal still begins at each configured root. A value of `1` omits + /// root entries, while `2` starts yielding entries one level below them. + pub const fn min_depth(mut self, depth: usize) -> Self { + self.min_depth = Some(depth); + self + } + /// Set the limit of results to return. This will limit the amount of results returned. /// ### Arguments /// * `limit` - The limit of results to return. @@ -252,6 +349,15 @@ impl SearchBuilder { self } + /// Skip files larger than `size` while walking. + /// + /// Applying the limit in the walker is cheaper than a metadata-based + /// custom filter because oversized files never reach the result callback. + pub fn max_file_size(mut self, size: impl Into) -> Self { + self.max_file_size = Some(size.into()); + self + } + /// Searches for exact match. /// /// For example, if the search input is "Search", the file "Search.rs" will be found, but not "Searcher.rs". @@ -325,6 +431,53 @@ impl SearchBuilder { self } + /// Return only files and symbolic links, which is the default. + pub const fn files(mut self) -> Self { + self.options.target = SearchTarget::Files; + self + } + + /// Return only directories below the configured search roots. + pub const fn directories(mut self) -> Self { + self.options.target = SearchTarget::Directories; + self + } + + /// Return files, symbolic links, and directories below the search roots. + pub const fn files_and_directories(mut self) -> Self { + self.options.target = SearchTarget::FilesAndDirectories; + self + } + + /// Follow symbolic links while traversing directories. + /// + /// Link following is disabled by default. The walker detects symbolic-link + /// loops and stops descending through them. + pub const fn follow_links(mut self, enabled: bool) -> Self { + self.options.follow_links = enabled; + self + } + + /// Choose whether traversal may cross filesystem boundaries. + /// + /// Pass `true` to keep each search root on its original filesystem. This + /// can prevent unexpectedly expensive walks into network or mounted + /// filesystems. It is disabled by default for compatibility. + pub const fn same_file_system(mut self, enabled: bool) -> Self { + self.options.same_file_system = enabled; + self + } + + /// Set the number of parallel walker threads. + /// + /// A value of zero restores the underlying walker's automatic heuristic. + /// If this is not called, `rust_search` uses twice the available parallelism + /// because directory traversal is generally I/O-bound. + pub const fn threads(mut self, threads: usize) -> Self { + self.options.threads = Some(threads); + self + } + /// Exclude a directory name or path from traversal. /// /// Passing `"node_modules"` excludes every directory with that name. @@ -397,14 +550,20 @@ impl Default for SearchBuilder { search_location: std::env::current_dir().expect("Failed to get current directory"), more_locations: None, search_input: None, - file_ext: None, + file_extensions: Vec::new(), + min_depth: None, depth: None, limit: None, + max_file_size: None, options: SearchOptions { - matching: MatchMode::Fuzzy, + matching: MatchMode::Contains, case_matching: CaseMatching::CaseSensitive, hidden: Hidden::Exclude, git_ignore: GitIgnore::Respect, + target: SearchTarget::Files, + follow_links: false, + same_file_system: false, + threads: None, }, excluded_dirs: Vec::new(), filters: vec![], diff --git a/src/filter.rs b/src/filter.rs index 2a35503..74b9b95 100644 --- a/src/filter.rs +++ b/src/filter.rs @@ -2,6 +2,7 @@ use super::SearchBuilder; use ignore::DirEntry; use std::{ cmp::Ordering, + fs::Metadata, panic::{RefUnwindSafe, UnwindSafe}, sync::Arc, time::SystemTime, @@ -33,29 +34,39 @@ impl FilterType { Self::Custom(Arc::new(f)) } - pub fn apply(&self, dir: &DirEntry) -> bool { - if let Ok(m) = dir.metadata() { - match self { - Self::Created(cmp, time) => { - if let Ok(created) = m.created() { - return created.cmp(time) == *cmp; - } - } - Self::Modified(cmp, time) => { - if let Ok(modified) = m.modified() { - return modified.cmp(time) == *cmp; - } - } - Self::FileSize(cmp, size_in_bytes) => { - return m.len().cmp(size_in_bytes) == *cmp; - } - Self::Custom(f) => return f(dir), + const fn requires_metadata(&self) -> bool { + !matches!(self, Self::Custom(_)) + } + + fn apply(&self, dir: &DirEntry, metadata: Option<&Metadata>) -> bool { + match self { + Self::Created(cmp, time) => metadata + .and_then(|metadata| metadata.created().ok()) + .is_some_and(|created| created.cmp(time) == *cmp), + Self::Modified(cmp, time) => metadata + .and_then(|metadata| metadata.modified().ok()) + .is_some_and(|modified| modified.cmp(time) == *cmp), + Self::FileSize(cmp, size_in_bytes) => { + metadata.is_some_and(|metadata| metadata.len().cmp(size_in_bytes) == *cmp) } + Self::Custom(filter) => filter(dir), } - false } } +/// Apply all result filters while reusing a single metadata lookup. +pub fn matches_all(dir: &DirEntry, filters: &[FilterType]) -> bool { + let metadata = filters + .iter() + .any(FilterType::requires_metadata) + .then(|| dir.metadata().ok()) + .flatten(); + + filters + .iter() + .all(|filter| filter.apply(dir, metadata.as_ref())) +} + /// enum to easily convert between `byte_sizes` #[derive(Debug, Clone)] pub enum FileSize { @@ -104,18 +115,18 @@ pub trait FilterExt { fn modified_at(self, t: SystemTime) -> Self; /// files modified after `t`: [`SystemTime`] fn modified_after(self, t: SystemTime) -> Self; - /// files smaller than `size_in_bytes`: [usize] + /// Files smaller than the supplied [`FileSize`]. fn file_size_smaller(self, size: FileSize) -> Self; - /// files equal to `size_in_bytes`: [usize] + /// Files equal to the supplied [`FileSize`]. fn file_size_equal(self, size: FileSize) -> Self; - /// files greater than `size_in_bytes`: [usize] + /// Files greater than the supplied [`FileSize`]. fn file_size_greater(self, size: FileSize) -> Self; /// Custom filter that exposes the [`DirEntry`] directly. /// ```rust /// use rust_search::{SearchBuilder, FilterExt}; /// /// let search: Vec = SearchBuilder::default() - /// .custom_filter(|dir| dir.metadata().unwrap().is_file()) + /// .custom_filter(|dir| dir.metadata().is_ok_and(|metadata| metadata.is_file())) /// .build() /// .collect(); /// ``` diff --git a/src/lib.rs b/src/lib.rs index 3f29197..af60345 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -17,10 +17,10 @@ mod filter; mod search; mod utils; -pub use builder::SearchBuilder; +pub use builder::{SearchBuilder, SearchTarget}; pub use filter::{FileSize, FilterExt, FilterFn}; // export this in order to use it with custom filter functions pub use ignore::DirEntry; -pub use search::Search; -pub use utils::similarity_sort; +pub use search::{Search, SearchError, SearchPaths, SearchResults}; +pub use utils::{similarity_sort, similarity_sort_paths}; diff --git a/src/search.rs b/src/search.rs index 5315d37..c837a46 100644 --- a/src/search.rs +++ b/src/search.rs @@ -1,239 +1,429 @@ use std::{ + error::Error, ffi::OsStr, - path::PathBuf, + fmt, panic, + path::{Path, PathBuf}, sync::{ atomic::{AtomicUsize, Ordering}, Arc, }, }; -use crate::{builder::SearchConfig, utils, SearchBuilder}; -use crossbeam_channel::Sender; +use crate::{ + builder::{SearchConfig, SearchTarget}, + filter, SearchBuilder, +}; +use crossbeam_channel::{Receiver, Sender}; use ignore::types::TypesBuilder; -use ignore::{WalkBuilder, WalkState}; - -/// Matcher strategy for the walk callback. -enum Matcher { - /// The types pre-filter already handles extension matching; accept all entries. - AcceptAll, - /// Simple extension-only check (fallback when types filter setup failed). - ExtOnly(String), - /// Full regex matching on file names. - Regex(regex::Regex), -} +use ignore::{DirEntry, WalkBuilder, WalkState}; -/// A struct that holds the receiver for the search results -/// -/// Can be iterated on to get the next element in the search results -/// -/// # Examples -/// -/// ## Iterate on the results -/// -/// ``` -/// use rust_search::SearchBuilder; -/// -/// let search = SearchBuilder::default() -/// .location("src") -/// .ext("rs") -/// .depth(1) -/// .build(); -/// -/// for path in search { -/// println!("{:?}", path); -/// } -/// ``` -/// -/// ## Collect results into a vector +/// A streaming iterator over search results represented as UTF-8 strings. /// -/// ``` -/// use rust_search::SearchBuilder; -/// -/// let paths_vec: Vec = SearchBuilder::default() -/// .location("src") -/// .ext("rs") -/// .depth(1) -/// .build() -/// .collect(); -/// ``` +/// Filesystem traversal runs in the background. Results are delivered through +/// a bounded channel, so a slow consumer does not cause memory usage to grow +/// with the number of matches. Dropping the iterator cancels the walk as soon +/// as the active filesystem operations return. pub struct Search { - rx: Box>, + paths: SearchPaths, } impl Iterator for Search { type Item = String; fn next(&mut self) -> Option { - self.rx.next() + self.paths.next().map(|path| { + path.into_os_string() + .into_string() + .unwrap_or_else(|path| path.to_string_lossy().into_owned()) + }) } } impl Search { - /// Search for files in a given arguments - /// ### Arguments - /// * `config` - The search configuration assembled by [`SearchBuilder`] + /// Start searching with the configuration assembled by [`SearchBuilder`]. pub(crate) fn new(config: SearchConfig) -> Self { - let SearchConfig { - search_location, - more_locations, - search_input, - file_ext, - depth, - limit, - options, - excluded_dirs, - filters, - } = config; - let search_input = search_input.as_deref(); - let file_ext = file_ext.as_deref(); - let mut walker = WalkBuilder::new(search_location); - - // Use more threads than CPUs for I/O-bound work: while one thread - // waits for I/O, others can make progress. - let cpus = std::thread::available_parallelism().map_or(8, std::num::NonZero::get); - let thread_count = cpus * 2; - - walker - .hidden(!options.include_hidden()) - .git_ignore(options.respect_git_ignore()) - .max_depth(depth) - .threads(thread_count); - - // Pre-filter by extension using ignore's type system when possible. - // This avoids calling our callback for non-matching files. - let mut types_filter_active = false; - if let Some(ext) = file_ext { - let mut types = TypesBuilder::new(); - if types.add("custom", &format!("*.{ext}")).is_ok() { - types.select("custom"); - if let Ok(built) = types.build() { - walker.types(built); - types_filter_active = true; + Self { + paths: SearchPaths::new(config), + } + } +} + +impl Default for Search { + /// Search for files below the current directory with default settings. + fn default() -> Self { + SearchBuilder::default().build() + } +} + +/// A streaming, lossless iterator over search result paths. +/// +/// Construct this iterator with [`SearchBuilder::build_paths`]. Unlike the +/// string-based [`Search`] iterator, this type preserves non-UTF-8 paths. +pub struct SearchPaths { + results: SearchResults, +} + +impl SearchPaths { + pub(crate) fn new(config: SearchConfig) -> Self { + Self { + results: SearchResults::new(config), + } + } +} + +impl Iterator for SearchPaths { + type Item = PathBuf; + + fn next(&mut self) -> Option { + self.results.by_ref().find_map(Result::ok) + } +} + +impl Default for SearchPaths { + /// Search for files below the current directory with default settings. + fn default() -> Self { + SearchBuilder::default().build_paths() + } +} + +/// A streaming iterator over lossless paths and filesystem traversal errors. +/// +/// Construct this iterator with [`SearchBuilder::build_results`] when silently +/// skipping unreadable entries is not appropriate for the application. +pub struct SearchResults { + receiver: Receiver>, +} + +impl SearchResults { + pub(crate) fn new(config: SearchConfig) -> Self { + let thread_count = config.options.threads().unwrap_or_else(|| { + std::thread::available_parallelism() + .map_or(8, std::num::NonZero::get) + .saturating_mul(2) + }); + // A few hundred slots per producer amortize channel contention during + // bursty directory reads while keeping backpressure bounded. Cap the + // queue so a stalled consumer cannot grow memory with result count. + let channel_capacity = thread_count.max(1).saturating_mul(256).clamp(1_024, 16_384); + let (sender, receiver) = crossbeam_channel::bounded(channel_capacity); + + std::thread::Builder::new() + .name("rust-search-walker".to_owned()) + .spawn(move || { + let worker = panic::catch_unwind(panic::AssertUnwindSafe(|| { + run_search(config, &sender, thread_count); + })); + if worker.is_err() { + let _ = sender.send(Err(SearchError::WorkerPanicked)); } - } + }) + .unwrap_or_else(|error| panic!("failed to start filesystem search: {error}")); + + Self { receiver } + } +} + +impl Iterator for SearchResults { + type Item = Result; + + fn next(&mut self) -> Option { + self.receiver.recv().ok() + } +} + +/// An error observed while producing search results. +#[derive(Debug)] +#[non_exhaustive] +pub enum SearchError { + /// The filesystem walker could not read or traverse an entry. + Walk(ignore::Error), + /// A custom result filter panicked. + FilterPanicked, + /// The background search worker panicked outside a custom filter. + WorkerPanicked, +} + +impl fmt::Display for SearchError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Walk(error) => write!(formatter, "filesystem traversal failed: {error}"), + Self::FilterPanicked => formatter.write_str("a custom search filter panicked"), + Self::WorkerPanicked => formatter.write_str("the background search worker panicked"), } + } +} - // Determine the matcher strategy based on search parameters. - let matcher = build_matcher( - search_input, - file_ext, - options.strict(), - options.ignore_case(), - types_filter_active, - ); - - // Prune excluded directories at the walker level so their children are - // never visited. - if !excluded_dirs.is_empty() { - walker.filter_entry(move |entry| !is_excluded_dir(entry, &excluded_dirs)); +impl Error for SearchError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::Walk(error) => Some(error), + Self::FilterPanicked | Self::WorkerPanicked => None, } + } +} - if let Some(locations) = more_locations { - for location in locations { - walker.add(location); - } +impl From for SearchError { + fn from(error: ignore::Error) -> Self { + Self::Walk(error) + } +} + +impl Default for SearchResults { + /// Search below the current directory and preserve traversal errors. + fn default() -> Self { + SearchBuilder::default().build_results() + } +} + +#[derive(Debug)] +enum NameMatch { + Any, + Contains(String), + Exact(String), +} + +#[derive(Debug)] +struct Matcher { + name: NameMatch, + extensions: Vec, + ignore_case: bool, + extension_prefiltered: bool, + target: SearchTarget, +} + +impl Matcher { + fn is_match(&self, entry: &DirEntry) -> bool { + if !self.target_matches(entry) || !self.extension_matches(entry.path()) { + return false; } - let (tx, rx) = crossbeam_channel::unbounded::(); - let matcher = Arc::new(matcher); - let filters = Arc::new(filters); - let counter = Arc::new(AtomicUsize::new(0)); - - walker.build_parallel().run(|| { - let tx: Sender = tx.clone(); - let matcher = Arc::clone(&matcher); - let filters = Arc::clone(&filters); - let counter = Arc::clone(&counter); - - Box::new(move |path_entry| { - if let Ok(entry) = path_entry { - // Check match using borrowed path first, then convert to owned - // only if matched (avoids allocation for non-matching entries). - let is_match = match matcher.as_ref() { - Matcher::AcceptAll => entry.file_type().is_some_and(|ft| !ft.is_dir()), - Matcher::ExtOnly(ext) => { - entry.path().extension() == Some(OsStr::new(ext.as_str())) - } - Matcher::Regex(reg_exp) => { - entry.path().file_name().is_some_and(|file_name| { - let file_name = file_name.to_string_lossy(); - reg_exp.is_match(&file_name) - }) - } - }; - if is_match { - if !filters.iter().all(|f| f.apply(&entry)) { - return WalkState::Continue; - } - - if limit.is_none_or(|l| counter.fetch_add(1, Ordering::Relaxed) < l) { - // Use into_path() for zero-copy PathBuf, then try zero-copy - // String conversion (succeeds for valid UTF-8 paths). - let path_string = entry - .into_path() - .into_os_string() - .into_string() - .unwrap_or_else(|os| os.to_string_lossy().into_owned()); - if tx.send(path_string).is_ok() { - return WalkState::Continue; - } - } - return WalkState::Quit; - } + match &self.name { + NameMatch::Any => true, + NameMatch::Contains(query) => entry.path().file_name().is_some_and(|name| { + if self.ignore_case { + name.to_string_lossy().to_lowercase().contains(query) + } else { + name.to_string_lossy().contains(query.as_str()) } - WalkState::Continue - }) - }); + }), + NameMatch::Exact(query) => self.exact_name_matches(entry.path(), query), + } + } - // Drop the sender so the receiver knows when all results have been sent - drop(tx); + fn target_matches(&self, entry: &DirEntry) -> bool { + let Some(file_type) = entry.file_type() else { + return false; + }; - if let Some(limit) = limit { - Self { - rx: Box::new(rx.into_iter().take(limit)), + let is_file_or_link = file_type.is_file() || file_type.is_symlink(); + match self.target { + SearchTarget::Files => is_file_or_link, + SearchTarget::Directories => file_type.is_dir() && entry.depth() > 0, + SearchTarget::FilesAndDirectories => { + is_file_or_link || (file_type.is_dir() && entry.depth() > 0) } + } + } + + fn extension_matches(&self, path: &Path) -> bool { + if self.extension_prefiltered { + return true; + } + + self.extensions.is_empty() + || path.extension().is_some_and(|candidate| { + self.extensions + .iter() + .any(|extension| self.text_equals(candidate, extension)) + }) + } + + fn exact_name_matches(&self, path: &Path, query: &str) -> bool { + if !self.extensions.is_empty() { + return path + .file_stem() + .is_some_and(|stem| self.text_equals(stem, query)); + } + + let file_name_matches = path + .file_name() + .is_some_and(|name| self.text_equals(name, query)); + if file_name_matches { + return true; + } + + // With no explicit extension, strict matching also accepts an exact + // stem. This implements the documented `Search` -> `Search.rs` + // behavior while still allowing `Cargo.toml` to match by full name. + path.file_stem() + .is_some_and(|stem| self.text_equals(stem, query)) + } + + fn text_equals(&self, candidate: &OsStr, expected: &str) -> bool { + if self.ignore_case { + candidate.to_string_lossy().to_lowercase() == expected } else { - Self { - rx: Box::new(rx.into_iter()), - } + candidate == OsStr::new(expected) } } } -fn build_matcher( - search_input: Option<&str>, - file_ext: Option<&str>, - strict: bool, - ignore_case: bool, - types_filter_active: bool, -) -> Matcher { - if search_input.is_none() && !strict && !ignore_case { - if file_ext.is_some() && types_filter_active { - // Types pre-filter handles extension matching; no additional check needed. - Matcher::AcceptAll - } else if let Some(ext) = file_ext { - // Fallback: simple extension comparison. - Matcher::ExtOnly(ext.to_owned()) - } else { - Matcher::Regex(utils::build_regex_search_input( - search_input, - file_ext, - strict, - ignore_case, - )) +fn run_search( + config: SearchConfig, + sender: &Sender>, + thread_count: usize, +) { + let SearchConfig { + search_location, + more_locations, + search_input, + file_extensions, + min_depth, + depth, + limit, + max_file_size, + options, + excluded_dirs, + filters, + } = config; + + if limit == Some(0) { + return; + } + + let mut walker = WalkBuilder::new(search_location); + walker + .hidden(!options.include_hidden()) + .git_ignore(options.respect_git_ignore()) + .min_depth(min_depth) + .max_depth(depth) + .max_filesize(max_file_size) + .follow_links(options.follow_links()) + .same_file_system(options.same_file_system()) + .threads(thread_count); + + if let Some(locations) = more_locations { + for location in locations { + walker.add(location); } + } + + // Let ignore's globset discard non-matching files before our callback. + // Its type filter is case-sensitive and file-oriented, so other modes use + // the matcher below to preserve their documented semantics. + let extension_prefiltered = if !options.ignore_case() + && options.target() == SearchTarget::Files + && !file_extensions.is_empty() + { + add_extension_filter(&mut walker, &file_extensions) } else { - Matcher::Regex(utils::build_regex_search_input( - search_input, - file_ext, - strict, - ignore_case, - )) + false + }; + + if !excluded_dirs.is_empty() { + walker.filter_entry(move |entry| !is_excluded_dir(entry, &excluded_dirs)); + } + + let matcher = Arc::new(Matcher { + name: match search_input { + None => NameMatch::Any, + Some(input) if options.ignore_case() && options.strict() => { + NameMatch::Exact(input.to_lowercase()) + } + Some(input) if options.ignore_case() => NameMatch::Contains(input.to_lowercase()), + Some(input) if options.strict() => NameMatch::Exact(input), + Some(input) => NameMatch::Contains(input), + }, + extensions: file_extensions, + ignore_case: options.ignore_case(), + extension_prefiltered, + target: options.target(), + }); + let filters = Arc::new(filters); + let result_count = Arc::new(AtomicUsize::new(0)); + + walker.build_parallel().run(|| { + let sender = sender.clone(); + let matcher = Arc::clone(&matcher); + let filters = Arc::clone(&filters); + let result_count = Arc::clone(&result_count); + + Box::new(move |entry| { + let entry = match entry { + Ok(entry) => entry, + Err(error) => { + return if sender.send(Err(error.into())).is_err() { + WalkState::Quit + } else { + WalkState::Continue + }; + } + }; + + if !matcher.is_match(&entry) { + return WalkState::Continue; + } + + let filters_match = panic::catch_unwind(panic::AssertUnwindSafe(|| { + filter::matches_all(&entry, &filters) + })); + match filters_match { + Ok(true) => {} + Ok(false) => return WalkState::Continue, + Err(_) => { + let _ = sender.send(Err(SearchError::FilterPanicked)); + return WalkState::Quit; + } + } + + if !reserve_result(&result_count, limit) { + return WalkState::Quit; + } + + if sender.send(Ok(entry.into_path())).is_err() { + WalkState::Quit + } else { + WalkState::Continue + } + }) + }); +} + +fn add_extension_filter(walker: &mut WalkBuilder, extensions: &[String]) -> bool { + let mut types = TypesBuilder::new(); + for extension in extensions { + if types + .add("rust-search-extension", &format!("*.{extension}")) + .is_err() + { + return false; + } } + types.select("rust-search-extension"); + + types.build().is_ok_and(|types| { + walker.types(types); + true + }) } -fn is_excluded_dir(entry: &ignore::DirEntry, excluded_dirs: &[PathBuf]) -> bool { - if !entry.file_type().is_some_and(|ft| ft.is_dir()) { +fn reserve_result(counter: &AtomicUsize, limit: Option) -> bool { + let Some(limit) = limit else { + return true; + }; + + counter + .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| { + (current < limit).then_some(current + 1) + }) + .is_ok() +} + +fn is_excluded_dir(entry: &DirEntry, excluded_dirs: &[PathBuf]) -> bool { + if !entry + .file_type() + .is_some_and(|file_type| file_type.is_dir()) + { return false; } @@ -249,10 +439,3 @@ fn is_excluded_dir(entry: &ignore::DirEntry, excluded_dirs: &[PathBuf]) -> bool } }) } - -impl Default for Search { - /// Effectively just creates a [`WalkBuilder`] over the current directory - fn default() -> Self { - SearchBuilder::default().build() - } -} diff --git a/src/utils.rs b/src/utils.rs index 58e31b6..6a56425 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -1,36 +1,9 @@ use rayon::prelude::*; -use regex::Regex; +use std::borrow::Cow; use std::cmp::Ordering; use std::path::{Path, PathBuf}; use strsim::jaro_winkler; -const FUZZY_SEARCH: &str = r".*"; - -pub fn build_regex_search_input( - search_input: Option<&str>, - file_ext: Option<&str>, - strict: bool, - ignore_case: bool, -) -> Regex { - let file_type = file_ext.unwrap_or("*"); - let search_input = search_input.unwrap_or(r"\w+"); - - let mut formatted_search_input = if strict { - format!(r"{search_input}\.{file_type}$") - } else { - format!(r"{search_input}{FUZZY_SEARCH}\.{file_type}$") - }; - - if ignore_case { - formatted_search_input = set_case_insensitive(&formatted_search_input); - } - Regex::new(&formatted_search_input).unwrap() -} - -fn set_case_insensitive(formatted_search_input: &str) -> String { - "(?i)".to_owned() + formatted_search_input -} - /// Replace the tilde with the home directory, if it exists /// ### Arguments /// * `path` - The path to replace the tilde with the home directory @@ -45,11 +18,10 @@ pub fn replace_tilde_with_home_dir(path: impl AsRef) -> PathBuf { path.to_path_buf() } -fn file_name_from_path(path: &str) -> &str { - Path::new(path) - .file_name() - .and_then(|f| f.to_str()) - .unwrap_or(path) +fn file_name_from_path(path: &Path) -> Cow<'_, str> { + path.file_name() + .unwrap_or(path.as_os_str()) + .to_string_lossy() } /// This function can be used to sort the given vector on basis of similarity between the input & the vector @@ -81,6 +53,20 @@ fn file_name_from_path(path: &str) -> &str { /// search **with** similarity sort /// `["fly.txt", "flyer.txt", "afly.txt", "bfly.txt",]` pub fn similarity_sort(vector: &mut [String], input: &str) { + similarity_sort_impl(vector, input); +} + +/// Sort paths by their filename similarity to `input`. +/// +/// This is the lossless [`PathBuf`] counterpart to [`similarity_sort`]. +pub fn similarity_sort_paths(vector: &mut [PathBuf], input: &str) { + similarity_sort_impl(vector, input); +} + +fn similarity_sort_impl(vector: &mut [T], input: &str) +where + T: AsRef + Sync, +{ const PARALLEL_SORT_THRESHOLD: usize = 5000; let input = input.to_lowercase(); // Schwartzian transform: precompute all scores, then sort by score. @@ -90,7 +76,7 @@ pub fn similarity_sort(vector: &mut [String], input: &str) { .par_iter() .enumerate() .map(|(i, path)| { - let name = file_name_from_path(path).to_lowercase(); + let name = file_name_from_path(path.as_ref()).to_lowercase(); (i, jaro_winkler(&name, &input)) }) .collect() @@ -99,7 +85,7 @@ pub fn similarity_sort(vector: &mut [String], input: &str) { .iter() .enumerate() .map(|(i, path)| { - let name = file_name_from_path(path).to_lowercase(); + let name = file_name_from_path(path.as_ref()).to_lowercase(); (i, jaro_winkler(&name, &input)) }) .collect() @@ -125,42 +111,17 @@ fn apply_permutation(v: &mut [T], mut order: Vec) { mod tests { use super::*; - #[test] - fn build_regex_fuzzy_no_ext() { - let re = build_regex_search_input(Some("hello"), None, false, false); - assert!(re.is_match("hello.rs")); - assert!(re.is_match("hello_world.txt")); - } - - #[test] - fn build_regex_strict_with_ext() { - let re = build_regex_search_input(Some("hello"), Some("rs"), true, false); - assert!(re.is_match("hello.rs")); - assert!(!re.is_match("hello_world.rs")); - } - - #[test] - fn build_regex_ignore_case() { - let re = build_regex_search_input(Some("Hello"), None, false, true); - assert!(re.is_match("hello.rs")); - assert!(re.is_match("HELLO.txt")); - } - - #[test] - fn build_regex_defaults() { - let re = build_regex_search_input(None, None, false, false); - // Should match any filename with an extension - assert!(re.is_match("anything.txt")); - } - #[test] fn file_name_from_path_normal() { - assert_eq!(file_name_from_path("/some/path/file.txt"), "file.txt"); + assert_eq!( + file_name_from_path(Path::new("/some/path/file.txt")), + "file.txt" + ); } #[test] fn file_name_from_path_no_extension() { - assert_eq!(file_name_from_path("/some/path/file"), "file"); + assert_eq!(file_name_from_path(Path::new("/some/path/file")), "file"); } #[test] diff --git a/tests/filter_tests.rs b/tests/filter_tests.rs index fac9b9a..24bb18a 100644 --- a/tests/filter_tests.rs +++ b/tests/filter_tests.rs @@ -71,6 +71,20 @@ fn file_size_smaller_filter() { assert!(!results.is_empty(), "All fixture files should be < 10KB"); } +#[test] +fn max_file_size_skips_oversized_files_during_walk() { + let results: Vec = SearchBuilder::default() + .location(fixtures_path()) + .max_file_size(FileSize::Byte(0)) + .build() + .collect(); + + assert!( + results.is_empty(), + "non-empty fixtures should be skipped: {results:?}" + ); +} + #[test] fn custom_filter_works() { // Filter to only include files (not directories) diff --git a/tests/search_tests.rs b/tests/search_tests.rs index ed0eceb..201ffe4 100644 --- a/tests/search_tests.rs +++ b/tests/search_tests.rs @@ -1,5 +1,13 @@ -use rust_search::SearchBuilder; -use std::{fs, path::PathBuf}; +use rust_search::{SearchBuilder, SearchError}; +use std::{ + fs, + path::{Path, PathBuf}, + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, + }, + time::Duration, +}; fn fixtures_dir() -> PathBuf { PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures") @@ -48,6 +56,23 @@ fn search_ext_filters_by_extension() { } } +#[test] +fn search_supports_multiple_extensions() { + let results: Vec = SearchBuilder::default() + .location(fixtures_path()) + .extensions(["rs", ".txt"]) + .build() + .collect(); + + assert!(results.iter().any(|result| result.ends_with("hello.rs"))); + assert!(results.iter().any(|result| result.ends_with("world.txt"))); + assert!(results.iter().all(|result| { + Path::new(result) + .extension() + .is_some_and(|extension| extension == "rs" || extension == "txt") + })); +} + #[test] fn search_input_matches_filename() { let results: Vec = SearchBuilder::default() @@ -96,6 +121,26 @@ fn search_depth_limits_traversal() { } } +#[test] +fn search_min_depth_skips_shallow_results() { + let root = fixtures_dir(); + let results: Vec = SearchBuilder::default() + .location(&root) + .min_depth(2) + .build_paths() + .collect(); + + assert!(!results.is_empty()); + assert!(results.iter().all(|result| { + result + .strip_prefix(&root) + .expect("result should remain under root") + .components() + .count() + >= 2 + })); +} + #[test] fn search_limit_caps_results() { let results: Vec = SearchBuilder::default() @@ -103,11 +148,54 @@ fn search_limit_caps_results() { .limit(2) .build() .collect(); - assert!( - results.len() <= 2, - "Expected at most 2 results, got {}", - results.len() - ); + assert_eq!(results.len(), 2, "Expected exactly 2 results"); +} + +#[test] +fn zero_limit_returns_immediately_without_results() { + let results: Vec = SearchBuilder::default() + .location(fixtures_path()) + .limit(0) + .build() + .collect(); + assert!(results.is_empty()); +} + +#[test] +fn default_search_returns_files_not_directories() { + let results: Vec = SearchBuilder::default() + .location(fixtures_path()) + .build() + .collect(); + + assert!(!results.is_empty()); + assert!(results.iter().all(|result| Path::new(result).is_file())); +} + +#[test] +fn directory_target_returns_only_child_directories() { + let root = fixtures_dir(); + let results: Vec = SearchBuilder::default() + .location(&root) + .directories() + .build() + .collect(); + + assert!(!results.is_empty()); + assert!(results.iter().all(|result| Path::new(result).is_dir())); + assert!(!results.iter().any(|result| Path::new(result) == root)); +} + +#[test] +fn combined_target_returns_files_and_directories() { + let results: Vec = SearchBuilder::default() + .location(fixtures_path()) + .files_and_directories() + .build() + .collect(); + + assert!(results.iter().any(|result| Path::new(result).is_file())); + assert!(results.iter().any(|result| Path::new(result).is_dir())); } #[test] @@ -130,6 +218,74 @@ fn search_strict_matches_exact() { } } +#[test] +fn search_strict_without_extension_matches_exact_stem() { + let results: Vec = SearchBuilder::default() + .location(fixtures_path()) + .search_input("hello") + .strict() + .build() + .collect(); + + assert_eq!(results.len(), 1, "strict stem results: {results:?}"); + assert!(results[0].ends_with("hello.rs")); +} + +#[test] +fn search_input_is_literal_and_never_panics_on_regex_characters() { + let dir = temp_fixture_dir("literal_query"); + fs::write(dir.join("[draft].txt"), "draft").expect("failed to write fixture"); + fs::write(dir.join("ordinary.txt"), "ordinary").expect("failed to write fixture"); + + let results: Vec = SearchBuilder::default() + .location(&dir) + .search_input("[") + .build() + .collect(); + + assert_eq!(results.len(), 1, "literal query results: {results:?}"); + assert!(results[0].ends_with("[draft].txt")); + let _ = fs::remove_dir_all(dir); +} + +#[test] +fn extension_is_literal_when_it_contains_glob_characters() { + let dir = temp_fixture_dir("literal_extension"); + fs::write(dir.join("unusual.["), "fixture").expect("failed to write fixture"); + + let results: Vec = SearchBuilder::default() + .location(&dir) + .ext("[") + .build() + .collect(); + + assert_eq!(results.len(), 1, "literal extension results: {results:?}"); + assert!(results[0].ends_with("unusual.[")); + let _ = fs::remove_dir_all(dir); +} + +#[test] +fn ignore_case_applies_to_extensions() { + let dir = temp_fixture_dir("case_extension"); + fs::write(dir.join("UPPER.RS"), "fixture").expect("failed to write fixture"); + + let sensitive: Vec = SearchBuilder::default() + .location(&dir) + .ext("rs") + .build() + .collect(); + let insensitive: Vec = SearchBuilder::default() + .location(&dir) + .ext("rs") + .ignore_case() + .build() + .collect(); + + assert!(sensitive.is_empty()); + assert_eq!(insensitive.len(), 1); + let _ = fs::remove_dir_all(dir); +} + #[test] fn search_ignore_case() { let results: Vec = SearchBuilder::default() @@ -239,6 +395,39 @@ fn search_more_locations() { ); } +#[cfg(unix)] +#[test] +fn search_can_follow_directory_symlinks() { + use std::os::unix::fs::symlink; + + let root = temp_fixture_dir("symlink_root"); + let target = temp_fixture_dir("symlink_target"); + fs::write(target.join("linked.txt"), "fixture").expect("failed to write linked fixture"); + symlink(&target, root.join("linked-directory")).expect("failed to create directory symlink"); + + let without_following: Vec = SearchBuilder::default() + .location(&root) + .ext("txt") + .build() + .collect(); + let with_following: Vec = SearchBuilder::default() + .location(&root) + .ext("txt") + .follow_links(true) + .build() + .collect(); + + assert!(without_following.is_empty()); + assert_eq!( + with_following.len(), + 1, + "followed results: {with_following:?}" + ); + assert!(with_following[0].ends_with("linked-directory/linked.txt")); + let _ = fs::remove_dir_all(root); + let _ = fs::remove_dir_all(target); +} + #[test] fn search_chained_options() { let results: Vec = SearchBuilder::default() @@ -252,3 +441,98 @@ fn search_chained_options() { assert!(!results.is_empty(), "Chained options should find nested.rs"); assert!(results.iter().any(|r| r.contains("nested.rs"))); } + +#[test] +fn build_paths_returns_pathbuf_results() { + let results: Vec = SearchBuilder::default() + .location(fixtures_path()) + .ext("rs") + .build_paths() + .collect(); + + assert!(!results.is_empty()); + assert!(results + .iter() + .all(|result| result.extension().is_some_and(|ext| ext == "rs"))); +} + +#[test] +fn build_results_exposes_traversal_errors() { + let parent = temp_fixture_dir("missing"); + let missing = parent.join("does-not-exist"); + let results: Vec<_> = SearchBuilder::default() + .location(missing) + .build_results() + .collect(); + + assert_eq!(results.len(), 1); + assert!(results[0].is_err()); + let _ = fs::remove_dir_all(parent); +} + +#[test] +fn build_results_exposes_custom_filter_panics() { + let results: Vec<_> = SearchBuilder::default() + .location(fixtures_path()) + .custom_filter(|_| panic!("intentional test panic")) + .build_results() + .collect(); + + assert!(results + .iter() + .any(|result| matches!(result, Err(SearchError::FilterPanicked)))); +} + +#[cfg(all(unix, not(target_os = "macos")))] +#[test] +fn build_paths_preserves_non_utf8_paths() { + use std::ffi::OsString; + use std::os::unix::ffi::OsStringExt; + + let dir = temp_fixture_dir("non_utf8"); + let filename = OsString::from_vec(b"invalid-\xff.txt".to_vec()); + let expected = dir.join(&filename); + fs::write(&expected, "fixture").expect("failed to write fixture"); + + let results: Vec = SearchBuilder::default() + .location(&dir) + .build_paths() + .collect(); + + assert_eq!(results, vec![expected]); + let _ = fs::remove_dir_all(dir); +} + +#[test] +fn building_a_search_does_not_wait_for_traversal() { + let released = Arc::new(AtomicBool::new(false)); + let filter_released = Arc::clone(&released); + let (built_sender, built_receiver) = std::sync::mpsc::channel(); + let location = fixtures_path(); + + let builder = std::thread::spawn(move || { + let search = SearchBuilder::default() + .location(location) + .custom_filter(move |_| { + while !filter_released.load(Ordering::Acquire) { + std::thread::yield_now(); + } + true + }) + .build(); + built_sender.send(search).expect("test receiver dropped"); + }); + + let search = match built_receiver.recv_timeout(Duration::from_secs(2)) { + Ok(search) => search, + Err(error) => { + released.store(true, Ordering::Release); + builder.join().expect("builder thread panicked"); + panic!("build blocked on filesystem traversal: {error}"); + } + }; + + released.store(true, Ordering::Release); + assert!(!search.collect::>().is_empty()); + builder.join().expect("builder thread panicked"); +} diff --git a/tests/utils_tests.rs b/tests/utils_tests.rs index e29ac91..061b5fd 100644 --- a/tests/utils_tests.rs +++ b/tests/utils_tests.rs @@ -1,4 +1,5 @@ -use rust_search::similarity_sort; +use rust_search::{similarity_sort, similarity_sort_paths}; +use std::path::PathBuf; #[test] fn similarity_sort_basic() { @@ -35,3 +36,10 @@ fn similarity_sort_single_element() { similarity_sort(&mut v, "only"); assert_eq!(v[0], "only.txt"); } + +#[test] +fn similarity_sort_paths_supports_pathbufs() { + let mut values = vec![PathBuf::from("afly.txt"), PathBuf::from("fly.txt")]; + similarity_sort_paths(&mut values, "fly"); + assert_eq!(values[0], PathBuf::from("fly.txt")); +}