diff --git a/.agents/skills/quantmind-dev/references/develop-components.md b/.agents/skills/quantmind-dev/references/develop-components.md index 7783585..5f06ff2 100644 --- a/.agents/skills/quantmind-dev/references/develop-components.md +++ b/.agents/skills/quantmind-dev/references/develop-components.md @@ -86,7 +86,7 @@ apply throughout. A public operation is complete only when all of these agree: -1. A stage and name consistent with `docs/design/en/operations.md`. +1. A stage and name consistent with `contexts/design/operations/naming.md`. 2. Typed input and config models, exported from `quantmind.configs`. 3. One intent-oriented `async def` operation exported from `quantmind.flows`, with its result contract exported from the canonical owning layer. diff --git a/.claude/skills/quantmind-dev/references/develop-components.md b/.claude/skills/quantmind-dev/references/develop-components.md index 7783585..5f06ff2 100644 --- a/.claude/skills/quantmind-dev/references/develop-components.md +++ b/.claude/skills/quantmind-dev/references/develop-components.md @@ -86,7 +86,7 @@ apply throughout. A public operation is complete only when all of these agree: -1. A stage and name consistent with `docs/design/en/operations.md`. +1. A stage and name consistent with `contexts/design/operations/naming.md`. 2. Typed input and config models, exported from `quantmind.configs`. 3. One intent-oriented `async def` operation exported from `quantmind.flows`, with its result contract exported from the canonical owning layer. diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..49dd532 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +tests/fixtures/paper/golden/*.pdf binary diff --git a/.gitignore b/.gitignore index 539aa6c..bcb1663 100644 --- a/.gitignore +++ b/.gitignore @@ -25,7 +25,6 @@ demo_data/ # Local-only development scratch notes (not tracked) new_feature.md -docs/design/zh/next-step-architecture.md docs/superpowers/ # Coverage artifacts (generated by pytest --cov) diff --git a/AGENTS.md b/AGENTS.md index f05b5c3..b2bb4d3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -6,6 +6,21 @@ aligned with `CLAUDE.md` (same core rules); update both in the same change. Use [`contexts/README.md`](contexts/README.md) as the repository information entry point for either development or library-usage work. +## Progressive Context Loading + +Pages under `contexts/` are agent-facing references designed for progressive +disclosure: + +1. Read lines 1-80 first. The preview contains `Quick Summary` and `Contents` + sections that explain the page's purpose, authority, and scope. +2. Use that preview to decide whether the page applies. Do not preload sibling + pages or follow unrelated links. +3. When a page applies, read the entire page before changing code, contracts, + or repository guidance. The preview routes work; it does not replace the + detailed contract. +4. Follow directly linked canonical sources only as the task requires. Avoid + deep reference chains and duplicate guidance in working context. + ## What This Is QuantMind is a knowledge extraction and retrieval library for quantitative @@ -73,8 +88,8 @@ the user explicitly authorizes it — fix the underlying issue instead. 6. **No meaningless wrappers** — a method must add logic, abstraction, or a side effect beyond the call it wraps; otherwise inline it. 7. **Name public operations by intent** — follow - `docs/design/en/operations.md`; use stage verbs, and reserve `pipeline` for - deliberate multi-stage composition. + `contexts/design/operations/naming.md`; use stage verbs, and reserve + `pipeline` for deliberate multi-stage composition. ## Tests and Examples diff --git a/CLAUDE.md b/CLAUDE.md index 37109ba..1533a21 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -7,6 +7,21 @@ update both in the same change. Use [`contexts/README.md`](contexts/README.md) as the repository information entry point for either development or library-usage work. +## Progressive Context Loading + +Pages under `contexts/` are agent-facing references designed for progressive +disclosure: + +1. Read lines 1-80 first. The preview contains `Quick Summary` and `Contents` + sections that explain the page's purpose, authority, and scope. +2. Use that preview to decide whether the page applies. Do not preload sibling + pages or follow unrelated links. +3. When a page applies, read the entire page before changing code, contracts, + or repository guidance. The preview routes work; it does not replace the + detailed contract. +4. Follow directly linked canonical sources only as the task requires. Avoid + deep reference chains and duplicate guidance in working context. + ## What This Is QuantMind is a knowledge extraction and retrieval library for quantitative @@ -74,8 +89,8 @@ the user explicitly authorizes it — fix the underlying issue instead. 6. **No meaningless wrappers** — a method must add logic, abstraction, or a side effect beyond the call it wraps; otherwise inline it. 7. **Name public operations by intent** — follow - `docs/design/en/operations.md`; use stage verbs, and reserve `pipeline` for - deliberate multi-stage composition. + `contexts/design/operations/naming.md`; use stage verbs, and reserve + `pipeline` for deliberate multi-stage composition. ## Tests and Examples diff --git a/contexts/README.md b/contexts/README.md index 8edf745..950c378 100644 --- a/contexts/README.md +++ b/contexts/README.md @@ -1,18 +1,44 @@ # QuantMind Repository Context -This directory is the public repository routing layer for coding agents. Start -with the index that matches the work you are doing: +## Quick Summary + +- **Purpose**: Route agents to the smallest set of context pages needed for a + QuantMind task. +- **Read when**: Starting repository development, library usage, or design + work. +- **Load next**: Choose exactly one primary route below, then follow only links + required by the task. +- **Authority**: Design pages record agreed behavior; code and tests show what + the current version implements. + +## Contents + +- [Routes](#routes) +- [Where Information Lives](#where-information-lives) + +## Routes + +This directory is the public repository context system for coding agents and +maintainers. Start with the index that matches the work you are doing: - [Develop QuantMind](dev/README.md) for architecture, contribution, testing, and verification guidance. - [Use QuantMind](usage/README.md) for public operations, examples, and usage documentation. +- [Design QuantMind](design/README.md) for accepted design decisions and + planned behavior that spans packages. -## Source of Truth and Ownership +## Where Information Lives -- Context pages curate links and route readers; they do not copy full guidance. -- Code, public contracts, design documents, `AGENTS.md` / `CLAUDE.md`, and - workflow skills remain canonical for their stated domains. +- Keep each kind of information in one place so agents do not have to compare + competing versions. +- `contexts/design/` records accepted design decisions and planned behavior. +- `contexts/dev/` and `contexts/usage/` route readers to existing rules and + examples instead of copying them. +- Code and tests show current behavior. Design pages list any planned behavior + that is not implemented yet. +- `docs/` remains the home for user-facing guides, examples, and catalogs. It + may link to a design page but must not maintain a second copy of the design. - Update a component's context index only when its discoverable entry points change, not for every implementation change. - Repository maintainers own this shared structure. Component contributors own diff --git a/contexts/design/README.md b/contexts/design/README.md new file mode 100644 index 0000000..f137b8f --- /dev/null +++ b/contexts/design/README.md @@ -0,0 +1,48 @@ +# QuantMind Design + +## Quick Summary + +- **Purpose**: Find accepted designs for QuantMind packages and public + operations. +- **Read when**: A task changes architecture, package responsibilities, public + behavior, or a required validation rule. +- **Load next**: Open only the domain design that matches the task, then read + that page in full before implementation. +- **Authority**: These pages explain agreed behavior. Each page lists any part + that is not implemented yet. + +## Contents + +- [Design Index](#design-index) +- [Organization Rules](#organization-rules) + +This directory records QuantMind engineering decisions. Use it to understand +which package owns each step, how packages work together, and what behavior an +implementation must preserve. + +## Design Index + +| Domain | Design | +|---|---| +| Flow | [Paper extraction from input to validated result](flow/paper.md) | +| Flow | [News collection](flow/news.md) | +| Library | [Local knowledge storage and meaning-based search](library/local.md) | +| Operations | [Public operation naming](operations/naming.md) | + +## Organization Rules + +- Organize designs directly by package or feature, such as `flow/`, `knowledge/`, + `library/`, `operations/`, or `preprocess/`. Do not add an intermediate + `components/` directory. +- Keep this page as the single global design index. Domain directories do not + need their own index. +- Add a design page only for real design content. Do not create empty + directories, placeholders, or speculative component pages. +- State whether a document describes current behavior, planned behavior, or + both. List current gaps so readers can distinguish a plan from working code. +- Write headings and summaries as an action plus a concrete object. Avoid + unexplained project shorthand and define any necessary domain term when it + first appears. +- Use code and tests to check current behavior. Keep `docs/` focused on + user-facing guides, examples, and catalogs; those pages may link here but + must not maintain a second copy of a design. diff --git a/contexts/design/flow/news.md b/contexts/design/flow/news.md new file mode 100644 index 0000000..98766d5 --- /dev/null +++ b/contexts/design/flow/news.md @@ -0,0 +1,276 @@ +# News Collection Design + +## Quick Summary + +- **Purpose**: Define how QuantMind collects public news and records proof of what each source returned. +- **Read when**: Changing `collect_news`, `NewsWindow`, a news source, the `complete` flag, or news checks. +- **Status**: Current design; the open-source package currently supports PR Newswire. +- **Core rule**: Collection returns documents, failures, and whether the full time window was covered. Other code turns documents into knowledge, stores them, schedules runs, and removes irrelevant news. + +## Contents + +- [Scope and Current Support](#scope-and-current-support) +- [Design Principles](#design-principles) +- [Public API](#public-api) +- [Returned Data](#returned-data) +- [Collection Steps](#collection-steps) +- [Failures and the `complete` Flag](#failures-and-the-complete-flag) +- [Who Owns What](#who-owns-what) +- [How to Verify](#how-to-verify) +- [Out of Scope and When to Add Shared Code](#out-of-scope-and-when-to-add-shared-code) +- [Adding a Public News Source](#adding-a-public-news-source) + +## Scope and Current Support + +This page defines how the open-source package collects public company news. The +first version supports PR Newswire only. It has one collection function, one +time-window input, repeatable HTML cleanup, and visible item failures. + +Its public name follows the +[operation naming rules](../operations/naming.md): collection returns the news +as published plus fetch details. A separate operation turns those documents +into structured knowledge. + +The primary requirement is that any caller can request a complete, one-shot +collection of a past time window. A daily poll is therefore not a separate +operation; it is simply a short window evaluated on a schedule. + +## Design Principles + +1. **Ask for news, not fetch details.** Callers choose a source and time window. + They do not choose RSS, listing pages, how to move between pages, or article + parsing rules. +2. **Return every source row.** QuantMind does not silently remove duplicate + source rows. Repeated rows remain repeated output records, + and may share the same stable ID. +3. **Show partial failures.** The returned batch includes item failures. Invalid + inputs still raise before network work starts. +4. **Hide source implementation details.** PR Newswire may later use a public + mechanism other than listing pages without changing the public function. +5. **List supported sources explicitly.** The first version selects from a + closed source list. It does not expose a provider plugin API or registry. +6. **Keep downstream choices separate.** The calling pipeline owns storage, + duplicate handling, relevance rules, output formats, and scheduling. + +## Public API + +Callers use one entry point: + +```python +from datetime import datetime, timezone + +from quantmind.configs import NewsCollectionCfg, NewsWindow +from quantmind.flows import collect_news + +batch = await collect_news( + NewsWindow( + source="pr-newswire", + start=datetime(2026, 7, 13, tzinfo=timezone.utc), + end=datetime(2026, 7, 14, tzinfo=timezone.utc), + ), + cfg=NewsCollectionCfg(retain_raw_html=False), +) +``` + +`NewsWindow` requires timezone-aware timestamps. Its `[start, end)` interval +includes `start` and excludes `end`. Regular collection and historical runs +use the same call; there are no separate `poll_*`, `backfill_*`, or +`fetch_wire_*` public functions. + +`NewsCollectionCfg` keeps the shared `BaseFlowCfg` fields so it works with the +repository's shared config handling. Its only +collection-specific field is `retain_raw_html`, which controls whether fetched +article HTML bytes remain in the result. It defaults to `False`, which is the +storage-efficient behavior. The article is still fetched, hashed, parsed, and +described by fetch details; only its byte payload is discarded. The collector +does not otherwise use the shared model, tracing, or SDK-run fields. + +### Collection records are not knowledge + +QuantMind keeps collected documents separate from structured knowledge: + +| Operation | Result | Owning package | +|-----------|--------|-----------------| +| `collect_news` | Source documents, fetch details, failures, and whether the full window was scanned | `quantmind.preprocess` | +| future `extract_news_knowledge` | Extracted financial events | `quantmind.knowledge.News` | + +`NewsDocument` is therefore not a `KnowledgeItem`. It carries HTTP details, +raw bytes, parsing output, and collection status that remain useful before any +LLM or business data format is chosen. `knowledge.News` is a structured +financial event with entities, sentiment, financial importance +(`materiality`), source links, and text for embeddings. A pipeline may use both +types, but one cannot replace the other. + +## Returned Data + +`collect_news` returns a `NewsBatch` with four fields. Import these public types +from `quantmind.preprocess`: + +- `documents`: successfully collected `NewsDocument` observations; +- `failures`: lightweight `NewsFailure` records for work that could not be + completed; +- `observed_count`: the number of source rows successfully converted into + in-window observations, before article processing; +- `complete`: whether the listing scan covered the full requested window. + +A `NewsDocument` contains the source name, stable ID, preferred article URL, +title, publisher, publication time, cleaned Markdown, content hash, ticker +hints, and two `NewsArtifact` records: + +- a small record of the public listing row; +- an article record containing fetch details and a content hash. Its + `bytes` field is `None` unless `retain_raw_html=True`. + +`NewsArtifact` records what was fetched. It contains the content hash, +content type, source and resolved URLs, status, headers, and fetch time, with +an optional byte payload. + +Repeated listing rows are not removed. If two rows point to the same release, +the batch contains two observations with the same stable ID. A consumer can +use that ID to update the same database row safely while still recording what +QuantMind observed. + +## Collection Steps + +```text +NewsWindow + -> select the source collector + -> scan public listing pages from newest to oldest + -> keep rows inside [start, end) + -> fetch each linked article + -> convert HTML to Markdown + -> NewsDocument or NewsFailure + -> NewsBatch +``` + +PR Newswire discovery is based on its public news-release listing rather than +the latest-items RSS snapshot. Pages are read newest to oldest until an +observation strictly older than the requested start is seen. The strict +boundary matters because several rows with the same minute-level timestamp may +span two pages. A caller can therefore rerun a past-day request without a +previously saved cursor. + +PR Newswire exposes listing timestamps at minute precision. Scheduled callers +should therefore use minute-aligned window bounds, as the live end-to-end check +does. + +RSS remains an internal parser and a live check. It is not a public +`NewsInput`: choosing a feed is an implementation detail, and a limited feed +snapshot cannot prove that it covered a complete time window. + +The HTTP layer limits retries, waits between attempts, honors `Retry-After`, +and limits requests per host. PR Newswire-specific URL construction and +listing HTML parsing stay in the PR Newswire source module. + +`NewsWindow.source` lists every supported source. `collect_news` has an +explicit branch for each one, so the type checker catches a source name added +without a matching collector. + +## Failures and the `complete` Flag + +Configuration errors raise immediately. Examples include a naive timestamp, +an empty source, an unsupported source, or `end <= start`. + +After collection begins, one item failure does not stop independent items. +Each `NewsFailure` records the source, failed step, URL, optional item ID, error +category, and message. + +`complete=False` when any of the following is true: + +- a listing page could not be fetched or parsed; +- the listing scan stopped before crossing the window start. + +Article failures stay in `failures` but do not change whether all listing rows +were found. A caller can distinguish "the full time window was scanned" from +"every found article was processed." It can store successful records and retry +failed articles separately. An empty batch is complete only when the listing +scan crossed the requested start. + +## Who Owns What + +QuantMind owns: + +- scanning public listings and fetching articles; +- repeatable HTML cleanup; +- stable IDs and content hashes; +- honest batch counts, failure records, and completeness. + +The consuming production pipeline owns: + +- its schedule and when its GitHub Action runs; +- database storage, saved progress, and safe repeated writes; +- rules for removing duplicates; +- rules that remove irrelevant company news; +- later data formats, added fields, and database writes; +- shared batch callbacks, metrics, and monitoring. + +This split lets a separate ingestion job request the last day in one call, +apply its own rules, and write its own data format without adding those choices +to the open-source library. + +## How to Verify + +Use separate offline and live checks. + +### Offline repository checks + +`bash scripts/verify.sh` runs repeatable unit tests, linting, typing, and +coverage. News tests use saved HTML/RSS files and mocked HTTP responses. +They cover window validation, time edges, multi-page listings, duplicate +observations, raw-byte retention, retries, partial failures, and completeness. + +### Live end-to-end news check + +`python scripts/verify_news_e2e.py` is the news live-network check. It performs +three limited checks: + +1. fetch and parse the official PR Newswire RSS feed; +2. scan PR Newswire listing rows for the preceding 24 hours and prove that the + scan crossed the window start; +3. fetch up to 25 unique articles and find the first supported exchange-coded + symbol in every group that has one (100% recall). + +The ticker check uses a separate comparison regex rather than the production +extractor. It accepts plain, Markdown-link, and emphasis-wrapped symbols. It +ignores later symbols in a multi-symbol list and exchanges that the extractor +does not support. The script prints PASS/FAIL plus short listing, article, +failure, and recall summaries. It fails when RSS or listing data is invalid, +no sampled article parses, or supported symbols fall below 100% recall. A +parsed sample with no supported symbols reports `SKIP` and still passes. + +The `news` job in `.github/workflows/e2e.yml` runs this check once daily, when +started manually, and on pull requests that change its direct dependencies. +The required `.github/workflows/ci.yml` workflow remains network-free so local +development and unit tests remain repeatable. + +## Out of Scope and When to Add Shared Code + +The first version does not include GlobeNewswire, Business Wire, authenticated +feeds, continuous cursors, storage, duplicate removal, financial importance +scoring, or a generic batch-operation base class. + +When a second source is implemented, compare its real behavior with PR +Newswire first. Add a shared collector interface only for behavior that both +working collectors actually share. Keep the public +`collect_news(NewsWindow, *, cfg)` call unchanged. + +## Adding a Public News Source + +A coding agent adding a second source follows this checklist: + +1. Add the source name to `NewsWindow.source`. +2. Add one private `quantmind/preprocess/.py` collector. +3. Add one explicit branch to `collect_news`; the type checker must pass. +4. Add saved-input tests for success, time edges, duplicate observations, + partial failures, and the `complete` flag. +5. Add a test proving that only the selected collector is called. +6. Update the supported-source table in `docs/README.md`, this design, and the + focused example if its common path changes. +7. Add or extend a component-specific live verifier and its named job in the + existing `.github/workflows/e2e.yml` when the integration depends on a + public network endpoint. Add its command to `docs/README.md`; do not add the + command to root agent guidance. + +Only after two real collectors share behavior should they use a common Python +`Protocol`. Never add a source only to the input `Literal`; the type checker is +expected to reject that incomplete change. diff --git a/contexts/design/flow/paper.md b/contexts/design/flow/paper.md new file mode 100644 index 0000000..ca26d50 --- /dev/null +++ b/contexts/design/flow/paper.md @@ -0,0 +1,350 @@ +# Paper Extraction: End-to-End Design + +## Quick Summary + +- **Purpose**: Define how a paper input becomes a validated `Paper`. +- **Read when**: Changing paper inputs, parsing, section trees, source tracking, page ranges, or future PageIndex support. +- **Status**: Planned design; [Current Gaps](#current-gaps) lists what is not implemented yet. +- **Core rule**: A model or PageIndex may suggest a section tree. Code creates the final IDs, links, order, page ranges, citations, and source-backed text. +- **Page numbering**: PDF page ranges start at 1 and include both the first and last page. + +## Contents + +- [Overview](#overview) +- [Who Owns Each Step](#who-owns-each-step) +- [Supported Inputs and Fetching](#supported-inputs-and-fetching) +- [Keep PDF Pages Separate](#keep-pdf-pages-separate) +- [Which Source Provides Each Field](#which-source-provides-each-field) +- [Model Output Is a Draft](#model-output-is-a-draft) +- [Build and Validate the Final Paper](#build-and-validate-the-final-paper) +- [Rules for a Valid Paper](#rules-for-a-valid-paper) +- [Get Node Content from Source Pages](#get-node-content-from-source-pages) +- [Retries and Failures by Step](#retries-and-failures-by-step) +- [What Paper Extraction Does Not Do](#what-paper-extraction-does-not-do) +- [How PageIndex Can Fit Later](#how-pageindex-can-fit-later) +- [Fixed Paper Test Data](#fixed-paper-test-data) +- [Current Gaps](#current-gaps) + +## Overview + +Paper extraction has six steps and returns one validated result: + +```mermaid +flowchart LR + input["Paper input"] --> resolve["Resolve and fetch"] + resolve --> source["Keep pages separate
and record source facts"] + source --> draft["Suggest a section tree"] + draft --> assembly["Build the final Paper in code"] + assembly --> validate["Validate the tree and source links"] + validate --> paper["Validated Paper"] +``` + +Code, not the model, records source facts, creates IDs and tree links, copies +text from source pages, creates citations, and validates the result. A model or +future PageIndex integration may suggest sections, titles, summaries, and page +ranges, but that output remains a draft. + +Paper extraction does not require PageIndex. It first preserves ordered source +pages and passes them through a small `PaperSourceDocument -> draft` interface. +PageIndex can later implement that interface without changing the `Paper` type +or creating the final IDs. + +## Who Owns Each Step + +This operation creates one extraction result. It does not store papers, search +across them, or write answers. + +| Work | Owner | +|---|---| +| Resolve identifiers, fetch bytes, parse pages, and hash source content | `quantmind.preprocess` | +| Configure the operation and select the input variant | `quantmind.configs` | +| Suggest a section tree and build the final `Paper` | `quantmind.flows` | +| Define the `Paper`, `TreeKnowledge`, `TreeNode`, source, citation, and extraction models | `quantmind.knowledge` | +| Store knowledge and make it searchable by meaning | `quantmind.library` | +| Navigate a tree and write an answer | A future `quantmind.mind` consumer or agent application | + +The `flow/` directory groups this work because it spans several packages. It +does not require a new `*_flow` public API and does not override the +[public operation naming rules](../operations/naming.md). + +## Supported Inputs and Fetching + +### Supported inputs + +The planned pipeline accepts the existing `PaperInput` types with the following +behavior: + +| Input | What happens | +|---|---| +| `ArxivIdentifier` | Resolve the identifier to an exact arXiv version, provider metadata, and PDF bytes. Preserve the resolved URL and the time that version became available. | +| `HttpUrl` | Follow a limited number of redirects and accept supported PDF, HTML, Markdown, or plain-text content. Record the final URL and exact fetched content. | +| `LocalFilePath` | Read a supported PDF, HTML, Markdown, or plain-text file. The caller owns file retention; extraction records a local source reference and content hash. | +| `RawText` | Treat the supplied text as one source document without pages. It can produce a `Paper`, but it cannot claim PDF page ranges. | + +`DoiIdentifier` remains unsupported until an open-access resolver can fetch the +exact paper content. A DOI landing page alone contains metadata and may not +contain the paper itself. + +### Explicitly unsupported inputs + +V1 does not claim support for: + +- password-protected or corrupt PDFs; +- image-only scans that require OCR; +- authenticated, paywalled, or dynamically rendered sources without a + separately authorized fetcher; +- unsupported binary or media content types; +- a DOI that cannot be resolved to accessible paper content; +- a source whose exact fetched content cannot be identified. + +Unsupported input fails before a model or tree builder runs. The pipeline must +not send an error page, login page, or unresolved metadata page to a model and +call the result a successfully extracted paper. + +### Fetching rules + +Fetching produces one exact source version. Resolve redirects, arXiv revisions, +and the content selected by the server before parsing. The final URI and +SHA-256 hash identify the exact bytes used. A retry may receive changed +content; when the hash changes, treat it as a new source version rather than +silently merging it with the earlier attempt. + +## Keep PDF Pages Separate + +PDF parsing must preserve ordered pages before any tree builder runs. The next +step receives data shaped like this: + +```text +PaperSourceDocument +├── source details and exact content hash +├── page or character range format +└── ordered pages + ├── page number + ├── extracted text, including an empty string when the page has no text + └── optional parser-provided layout or outline hints +``` + +Required properties: + +- PDF page ranges in the fixed test PDF, model draft, and citations start at 1 + and include both the first and last page. +- Every physical page remains represented and in order. Empty pages are not + dropped because doing so would renumber later page references. +- Text cleanup may remove parser noise, but it must not erase the page + boundary or change which page owns an anchor. +- Record the exact source hash, parser name and version, and cleanup version + with the extraction run. +- HTML, Markdown, plain text, and `RawText` have no pages. They retain source + text and character positions but do not invent PDF page numbers. +- A page-based tree builder, including a future PageIndex integration, accepts + only a source document whose range unit is `pdf_page`. + +`PaperSourceDocument` is a temporary value used during extraction, not another +public knowledge model. The caller keeps raw PDF or HTML files; this design +does not embed them inside `Paper`. + +## Which Source Provides Each Field + +Facts from the fetched source take priority over model suggestions. A model may +fill a missing summary or section title, but it must not overwrite a URL, +content hash, publication time, or other known source fact. + +| Field or fact | Who sets it and where it is stored | +|---|---| +| Resolved source URI and source kind | Resolver/fetcher; `SourceRef.kind` and `SourceRef.uri` | +| Exact fetched bytes and content hash | Fetcher; `SourceRef.content_hash`, while the caller keeps the raw file | +| Fetch time | Fetcher clock; `SourceRef.fetched_at` | +| Publication/version time | Provider metadata for the exact version; do not replace it with the first-version date | +| Source availability time | Provider metadata for the exact version; `Paper.available_at`. Use fetch time as the latest possible availability time only when the publication time is unknown. | +| Latest date covered by the paper | A study or data cutoff stated by the source; otherwise use the exact version's publication time for `Paper.as_of`, never fetch time | +| Authors and title | Provider or document metadata; `Paper.authors` and root title unless that metadata is missing or clearly invalid | +| Extraction model, operation, run, and time | Runtime; `ExtractionRef` | +| Parser and cleanup versions | Runtime; keep them with the extraction run until `Paper` has a dedicated field | +| Summaries, section titles, methodology, findings, and limitations | Model or tree builder; validate them before building the final `Paper` | +| Final item/node IDs and tree links | Code only | + +For an arXiv revision, `available_at` refers to the exact revision used, not the +first submission date. `as_of` is the latest date covered by the paper, not the +time it was fetched. Unknown publication or availability data remains unknown +rather than being guessed by a model. + +## Model Output Is a Draft + +A model or PageIndex integration returns a draft with only the information +needed to suggest a section tree. It may contain: + +- temporary node keys used only by that integration; +- candidate titles and summaries; +- an ordered parent/child outline using those temporary keys; +- suggested page ranges using the source document's range format; +- confidence values or errors needed to accept, repair, or reject the draft. + +A draft must not set the final `Paper.id`, `TreeNode.node_id`, +`parent_id`/`children_ids`, copied source text, known source facts, or a +object tied to one provider and exposed through the public result. + +The interface is intentionally small: one `PaperSourceDocument` produces one +draft. The default implementation can use an LLM; a future PageIndex +integration can return the same draft shape. Neither defines the public +`Paper` type. + +## Build and Validate the Final Paper + +Code treats the accepted draft as untrusted input and performs these steps: + +1. Generate the final item and node IDs. Temporary integration IDs never + appear in the returned result. +2. Set `parent_id` and ordered `children_ids` together for each parent-child + link so they cannot disagree. +3. Assign sibling positions from the accepted order. +4. Check page ranges against the preserved source pages. +5. Copy source content from the stated page range and build citations from the + same pages. Model-written paraphrases never become source text or quotes. +6. Apply known source facts and extraction run details. +7. Construct the final `Paper` and run every validation rule below. + +Code owns the final IDs, but separate extraction runs do not need to create the +same IDs. Merging results across runs is a separate decision. + +## Rules for a Valid Paper + +A successful `Paper` satisfies all of the following before it is returned: + +### IDs and root + +- `root_node_id` identifies exactly one entry in `nodes`. +- Every dictionary key equals that node's `node_id`. +- The root has `parent_id=None`; every other node has exactly one parent. +- All item and node IDs are created by code and unique within the result. + +### Parent and child links + +- Every referenced parent and child exists. +- A parent lists each child, and each child points back to that parent. +- A child appears at most once in a parent's `children_ids`. +- The same accepted draft produces the same sibling order, and sibling + positions are unique within one parent. + +### Reachability and safe traversal + +- Every node is reachable from the root. +- The graph has no cycles and no node points to itself. +- No node is shared by multiple parents. +- Depth-first walk and root-to-node path lookup finish for every node. Looking + up an unknown node returns the documented safe result. + +### Source page ranges and citations + +- A PDF page range uses `pdf_page`, starts at 1 or later, has + `start_page <= end_page`, and ends within the preserved PDF page count. +- Citation page ranges also start at 1, include both ends, and stay within the + PDF page count. +- Citation quotes and node content come from the identified source range. +- Inputs without pages do not carry made-up PDF page ranges. +- Sibling page ranges may overlap. A child range is not required to fit + completely inside its parent range unless a future rule adds that + requirement. + +## Get Node Content from Source Pages + +Branch nodes may retain source content. `content=None` remains useful for a +navigation-only node, but being a branch is not itself a reason to discard its +source text. + +When content is present, code copies it from the node's 1-based inclusive page +range. Page-level ranges can include a heading or paragraph also used by an +adjacent node; this is expected when ranges overlap. Do not ask the model to +reproduce source text or create a parent node by joining model summaries. + +Choosing a section and loading its source text remain separate operations. An +agent can read titles and summaries to select a node, then fetch that node's +page range. A future PageIndex integration may help build the outline, but it +does not fetch the source text. + +## Retries and Failures by Step + +| Step | Same input, same result? | Retry and failure behavior | +|---|---|---| +| Input validation and fetching rules | Yes, for one configuration | Invalid or unsupported input fails immediately. | +| Network fetch | No; the remote source may change | Retry only a limited number of temporary failures. Record and hash the exact successful response. Permanent HTTP or content failures stop the operation. | +| PDF or text parsing | Yes, for the same bytes and parser version | Parser failure stops before the model or tree builder runs. Do not skip corrupt pages or renumber later pages. | +| Model or PageIndex draft | No when a model is used | Limited retries may repair network or invalid-draft failures against the same source version. Record every attempt. | +| Build the final `Paper` | Yes for one accepted draft and source version, except for new UUID values | Invalid links, page ranges, or conflicts with source facts reject the draft. | +| Final validation | Yes | Any failure rejects the entire result; never return a partial `Paper` as success. | + +A successful result means one `Paper` passed its source, page-range, and tree +checks. Fetching bytes, receiving a model response, or constructing a Pydantic +object alone is not success. A failure records enough about the failed step and +source to diagnose or retry it, but it does not store a partial `Paper`. + +## What Paper Extraction Does Not Do + +Paper extraction returns one validated `Paper`. Other components handle: + +- storing and safely updating it in `LocalKnowledgeLibrary`; +- generating embeddings and building search records; +- searching across a collection; +- PageIndex-style tree navigation within one selected document; +- writing answers, managing conversation state, and citing a final response; +- deciding whether to keep raw source bytes. + +Keeping this work separate allows Paper extraction to be implemented before +PageIndex. PageIndex can then be added without replacing collection-wide search +or stored knowledge. + +## How PageIndex Can Fit Later + +Future integration must preserve these decisions: + +1. PageIndex receives ordered source pages before they are joined into one text. +2. Its node IDs remain temporary. Code creates the final IDs and tree links. +3. It may propose titles, summaries, ordering, and 1-based inclusive page + ranges through the draft described above. +4. It does not become the `Paper`, `TreeKnowledge`, or `TreeNode` type. +5. Navigation uses titles and summaries; loading source text separately fetches + the selected page range. +6. Sibling page ranges may overlap, and a child range does not need to fit + completely inside its parent range. + +## Fixed Paper Test Data + +The fixed test files live at: + +```text +tests/fixtures/paper/golden/ +├── paper.pdf +└── expected.json +``` + +It is a small four-page test paper with a nested subsection, multi-page +sections, and intentionally overlapping sibling page ranges. `expected.json` +contains only stable facts: page count, titles and paths, 1-based inclusive +page ranges, distinctive text anchors, tree shape, and validation rules. It +does not pin summaries or any other wording produced by a model. + +The offline test parses the fixed PDF, checks every text anchor against its +declared page, validates the tree and all rules, and confirms that overlapping +ranges work. Paper extraction and future PageIndex work must reuse these files +rather than create a competing test paper. + +## Current Gaps + +The repository does not yet guarantee the target pipeline above: + +- `pdf_to_markdown()` concatenates non-empty page text and drops page + boundaries and empty pages. +- `paper_flow()` sends the flattened document to one extraction agent and asks + it to return the final `Paper` directly. +- The model currently controls IDs, edges, citations, source fields, and + content instead of returning the limited draft described above. +- Resolved source URL, content hash, publication/version time, fetch time, and + exact-version availability are not consistently collected from the source. +- `DoiIdentifier` raises `NotImplementedError` because there is no accessible + content resolver. +- `TreeKnowledge` provides traversal helpers but does not yet check every root, + link, reachability, cycle, or page-range rule when it is created. +- The structured-output failure tracked by issue #91 remains separate from + this design issue. + +Those gaps are future implementation work. Adding PageIndex first would not fix +them and is not required to implement the Paper steps above. diff --git a/contexts/design/library/local.md b/contexts/design/library/local.md new file mode 100644 index 0000000..e3aad98 --- /dev/null +++ b/contexts/design/library/local.md @@ -0,0 +1,124 @@ +# Local Knowledge Library Design + +## Quick Summary + +- **Purpose**: Define how the local library stores validated knowledge and searches it by meaning. +- **Read when**: Changing `LocalKnowledgeLibrary`, search records, filters, query results, or file ownership. +- **Status**: Current design owned at runtime by `quantmind.library`. +- **Core rule**: Stored `BaseKnowledge` is the data to keep. Search text and vectors can be rebuilt. The caller stores raw PDF, HTML, and media files. +- **User guide**: [`docs/library.md`](../../../docs/library.md) + +## Contents + +- [Key Decisions](#key-decisions) +- [Who Owns What](#who-owns-what) +- [Stored Knowledge and Rebuildable Search Data](#stored-knowledge-and-rebuildable-search-data) +- [What Can Match a Query](#what-can-match-a-query) +- [When to Rebuild Search Data](#when-to-rebuild-search-data) +- [Time Fields and Look-Ahead](#time-fields-and-look-ahead) +- [Why SQLite and Simple Exact Ranking](#why-sqlite-and-simple-exact-ranking) +- [Out of Scope](#out-of-scope) + +## Key Decisions + +`quantmind.library` stores validated QuantMind knowledge and searches it by +meaning. It replaces an older design for one generic storage layer that would +hide raw files, knowledge JSON, embeddings, and indexes behind one backend API. + +The public API is intentionally small: + +- `LocalKnowledgeLibrary` +- `SemanticQuery` +- `SemanticHit` + +There is no public `Storage`, `VectorStore`, `Retriever`, provider registry, or +backend class hierarchy. Add a shared backend API only after a second working +implementation proves which behavior is truly shared. + +## Who Owns What + +| Package or caller | Responsibility | +|---|---| +| `quantmind.knowledge` | Define immutable knowledge models and the text used for embeddings; perform no I/O | +| `quantmind.library` | Store validated knowledge, maintain rebuildable search records, and return `SemanticHit` results | +| `quantmind.flows` | Produce validated knowledge and optionally pass it to a library | +| `quantmind.mind` or an agent application | Search the library and use matches to write answers | +| Caller or source-specific pipeline | Retain raw PDF, HTML, media, and operational files | + +The V1 library does not store raw source files. `SourceRef` and citations point +back to the source, but they do not contain the source file itself. + +## Stored Knowledge and Rebuildable Search Data + +Validated `BaseKnowledge` is the record that must be preserved. Embeddings, +the text sent to the embedder, filter columns, and vector indexes can all be +rebuilt, even when they share one SQLite database with the knowledge records. + +The local implementation separates three concerns: + +- `knowledge_items` stores one complete validated knowledge item. +- `knowledge_nodes` stores each `TreeNode` in a separate row with its parent, + position, content hash, and owning item. +- `semantic_records` stores searchable text and vector details for whole items, + roots, and nodes. + +Concrete types do not get a table per Pydantic subtype. Storing each complete +item plus separate node rows lets the library rebuild the original typed item +and later navigate a tree one node at a time. + +## What Can Match a Query + +- A `FlattenKnowledge` item produces one searchable record from its exact + `embedding_text()` value. +- A `TreeKnowledge` item produces one record for the whole item with + `node_id=None`, plus one record for every non-root node. +- Each record says whether it represents a whole item or one node. +- `SemanticHit.matched_text` is the exact text used to rank the match. +- Callers use `get(item_id)` to load the full knowledge item and node paths. + +Putting unchanged knowledge with the same item ID is safe to repeat and reuses +its embeddings. The library does not merge separate extraction runs that +created different item IDs. + +## When to Rebuild Search Data + +Every stored vector records the information needed to decide whether it is +reusable: embedding model, dimension, embedding-text hash, source content hash, +knowledge model version, and embedding-text format version. + +When any of those values changes, rebuild only the affected search records. +Deleting an item removes the item, its nodes, and its search records in one +transaction. Invalid vector sizes or bytes, unreadable knowledge data, and +search rows without an item raise an error instead of returning an incomplete +match that looks valid. + +## Time Fields and Look-Ahead + +`as_of` and `available_at` answer different questions: + +- `as_of` is the latest date covered by the knowledge. +- `available_at` is when that source version became observable. + +`available_at_before` excludes records that became available after the cutoff +or have no known availability time. Filtering only by `as_of_before` can still +leak future information. Apply source kind, item type, confidence, tags, tree +ID, and both time cutoffs before ranking results. + +## Why SQLite and Simple Exact Ranking + +SQLite provides transactions, foreign keys, and reliable reconstruction of +typed knowledge. NumPy can compare every embedding with cosine similarity at +the current local data size. This simple exact ranking can later be replaced +without changing user code. + +A future approximate or remote index may replace the private search +implementation. It does not replace stored knowledge and must return the same +`SemanticHit` type regardless of provider. + +## Out of Scope + +- raw source file storage; +- a generic framework for retrieving context and writing answers; +- a public embedder, vector-store, retriever, or backend registry; +- PageIndex tree construction or navigation; +- merging knowledge across runs without a separate stable-ID design. diff --git a/contexts/design/operations/naming.md b/contexts/design/operations/naming.md new file mode 100644 index 0000000..69d4cf8 --- /dev/null +++ b/contexts/design/operations/naming.md @@ -0,0 +1,77 @@ +# Public Operation Names + +## Quick Summary + +- **Purpose**: Define clear names for public QuantMind functions and their input, config, and result types. +- **Read when**: Adding or renaming a public function, operation type, or pipeline. +- **Status**: Use these rules for new names. Keep the existing `paper_flow` name until a separate change explains how callers will migrate. +- **Core rule**: Use a verb that says what the function does. Use `pipeline` only when one function combines several public operations. Do not use `flow` as a verb. + +## Contents + +- [Scope](#scope) +- [Name Patterns](#name-patterns) +- [Type Names](#type-names) +- [Current API](#current-api) +- [Review Checklist](#review-checklist) + +## Scope + +This page defines how to name public QuantMind functions. It does not rename +packages or existing functions. Any existing name change needs a separate plan +for users who depend on the old name. + +The runtime library API is designed for Python callers. Repository guidance +for coding agents belongs in `AGENTS.md`, skills, docs, fixtures, and checks; +it does not belong in the runtime API. + +## Name Patterns + +Start a public function name with a verb that describes its result: + +| Work | Name pattern | What it does | +|-------|--------------|----------------| +| Collection | `collect_` | Fetch source documents and the details needed to verify them | +| Knowledge extraction | `extract__knowledge` | Turn documents into typed `quantmind.knowledge` values | +| Index construction | `build__index` | Turn documents or knowledge into a search index | +| Analysis | `analyze_` | Domain inputs to an analytical result | +| Generic execution | `batch_run` and similar helpers | Apply an operation to a batch without changing what the operation means | +| Pipeline | `__pipeline` | Combine several named operations into one reusable function | + +`flow` is not a verb that explains a result. Do not add a public `*_flow` name +only because a function performs several steps. Name the result precisely, or +use `*_pipeline` when the function combines multiple public operations. + +## Type Names + +- Input types describe what the caller supplies, such as `NewsWindow` or + `PaperInput`. +- Config types name the domain and stage, such as `NewsCollectionCfg` or a + future `PaperExtractionCfg`. +- Result types describe returned data, such as `NewsBatch`, `Paper`, or + `PageIndex`. +- Keep provider names out of public function names unless callers are choosing + provider-specific behavior. + +## Current API + +- `collect_news` is a collection operation and follows these naming rules. +- `batch_run` is a generic batch helper, not a news or paper operation. +- `paper_flow` is an existing extraction API with an old name. Do not copy that + pattern for new operations. Renaming it requires a separate migration plan. + +The current `quantmind.flows` package continues to contain public operations in +this release. Renaming it to `operations` or adding a separate `pipelines` +package is outside this page. + +## Review Checklist + +Before adding a public callable, state: + +1. What work it performs. +2. What it accepts and returns. +3. Whether it is one operation or combines several operations. +4. Why its verb describes the returned result. + +If those answers are unclear, define the behavior before choosing a public +name. diff --git a/contexts/dev/README.md b/contexts/dev/README.md index 30d1e4e..edd5cb9 100644 --- a/contexts/dev/README.md +++ b/contexts/dev/README.md @@ -1,20 +1,39 @@ # Develop QuantMind +## Quick Summary + +- **Purpose**: Route contributors and coding agents to existing development + rules without copying them here. +- **Read when**: Extending, fixing, reviewing, testing, or publishing QuantMind. +- **Load next**: Select the single row that matches the task; load additional + sources only when that workflow explicitly requires them. +- **Required check**: Run `bash scripts/verify.sh` for every repository change. + +## Contents + +- [Rule Index](#rule-index) +- [Verification Rule](#verification-rule) + +## Rule Index + Use this index when extending or maintaining the repository. Follow the linked -canonical source instead of treating this page as a replacement for it. +rule instead of treating this page as a replacement for it. -| Need | Canonical source | +| Need | Read | |---|---| -| Architecture constraints and module ownership | [`AGENTS.md`](../../AGENTS.md) or [`CLAUDE.md`](../../CLAUDE.md) | +| Package responsibilities and coding rules | [`AGENTS.md`](../../AGENTS.md) or [`CLAUDE.md`](../../CLAUDE.md) | | Component-development workflow | [`quantmind-dev` component workflow](../../.agents/skills/quantmind-dev/references/develop-components.md) | | Issue and pull-request labels | [Repository label guidance](labels.md) | | Issue and pull-request body formatting | [GitHub writing style](github-writing.md) | | Public operation and source catalog | [`docs/README.md`](../../docs/README.md) | -| Public operation naming | [Operation naming contract](../../docs/design/en/operations.md) | -| Component designs | [`docs/design/`](../../docs/design/) | +| Public operation naming | [Operation naming rules](../design/operations/naming.md) | +| Component designs | [Design index](../design/README.md) | | Test patterns and coverage | [`tests/`](../../tests/) | | Focused runnable examples | [`examples/`](../../examples/) | -| Deterministic verification | [`scripts/verify.sh`](../../scripts/verify.sh) | +| All local checks | [`scripts/verify.sh`](../../scripts/verify.sh) | + +## Verification Rule Run `bash scripts/verify.sh` for every change. For a public-network component, -also run the bounded live verifier listed in the public component catalog. +also run the limited live-network check listed in the public component +catalog. diff --git a/contexts/dev/github-writing.md b/contexts/dev/github-writing.md index ef1485c..d886e93 100644 --- a/contexts/dev/github-writing.md +++ b/contexts/dev/github-writing.md @@ -1,18 +1,32 @@ # GitHub Writing Style -This guide is the canonical source-format contract for QuantMind Issue, Pull -Request, Discussion, and comment bodies. It applies to text submitted to -GitHub, not to Markdown files stored in the repository. +## Quick Summary -## No Hard-wrapped Prose +- **Purpose**: Define how raw Markdown should be formatted in QuantMind GitHub + issues, pull requests, discussions, and comments. +- **Read when**: Drafting or editing any text submitted to GitHub. +- **Key rule**: Keep each paragraph, list item, checklist item, table row, and + blockquote paragraph on one line in the raw Markdown. +- **Required check**: Read the raw body back after writing and remove + formatter-introduced hard wrapping without changing meaning. -GitHub wraps rendered prose for the viewer. Do not insert source newlines at a -fixed width, including 80 columns. +## Contents -- Keep each logical paragraph on one physical source line. -- Keep each list or checklist item on one physical source line. -- Keep each blockquote paragraph on one physical source line after `>`. -- Keep each Markdown table row on one physical source line. +- [Do Not Wrap Prose at a Fixed Width](#do-not-wrap-prose-at-a-fixed-width) +- [Write Workflow](#write-workflow) + +This guide applies to text submitted to GitHub, not to Markdown files stored in +the repository. + +## Do Not Wrap Prose at a Fixed Width + +GitHub wraps text for the viewer. Do not insert line breaks in the raw body at +a fixed width, including 80 columns. + +- Keep each paragraph on one line in the raw Markdown. +- Keep each list or checklist item on one line. +- Keep each blockquote paragraph on one line after `>`. +- Keep each Markdown table row on one line. - Preserve line structure inside fenced code blocks. - Use blank lines, headings, lists, code fences, tables, or an intentional Markdown hard break only when they express real structure. @@ -24,14 +38,14 @@ to wrap prose. ## Write Workflow -1. Draft the complete body with one logical prose unit per source line. +1. Draft the complete body with one paragraph or list item per source line. 2. Preserve the selected Issue or Pull Request template and its checklist. 3. Submit the body as written, preferably with `--body-file` when using `gh`. 4. Read the raw body back after creation or editing and check that prose was not hard-wrapped before considering the write complete. -When Prettier is already available, use its no-wrap mode as a preflight on the -temporary body file: +When Prettier is already available, use its no-wrap mode to check the temporary +body file before submission: ```bash prettier --write --parser markdown --prose-wrap never /tmp/github-body.md @@ -39,7 +53,7 @@ prettier --check --parser markdown --prose-wrap never /tmp/github-body.md ``` Do not introduce a repository-wide Markdown formatter or change Python's -80-column setting for this purpose. The preflight applies only to the temporary +80-column setting for this purpose. This check applies only to the temporary GitHub body. When updating an existing hard-wrapped body, normalize the complete body while diff --git a/contexts/dev/labels.md b/contexts/dev/labels.md index 0ecb799..0d5764d 100644 --- a/contexts/dev/labels.md +++ b/contexts/dev/labels.md @@ -1,55 +1,74 @@ # Repository Label Guidance -This file is the canonical decision guide for labeling QuantMind issues and -pull requests. Labels describe three independent dimensions: +## Quick Summary + +- **Purpose**: Select consistent labels for QuantMind issues and pull requests. +- **Read when**: Creating, updating, triaging, or reviewing an issue or PR. +- **Label count**: Use exactly one `type:` label, normally one `area:` label + (at most two), and optional `impact:` labels. +- **Selection rule**: Label the work's actual purpose and main area, not every + file it touches. + +## Contents + +- [How Many Labels to Use](#how-many-labels-to-use) +- [Type Labels](#type-labels) +- [Area Labels](#area-labels) +- [Impact Labels](#impact-labels) +- [Resolution and Community Labels](#resolution-and-community-labels) +- [Issues and Pull Requests](#issues-and-pull-requests) +- [Examples](#examples) +- [Rename Existing Labels](#rename-existing-labels) + +Labels describe three separate parts of the work: - `type:` answers what kind of change is proposed or delivered. -- `area:` answers which repository ownership surface is affected. -- `impact:` identifies special compatibility or live-network risk. +- `area:` answers which part of the repository owns the work. +- `impact:` identifies breaking changes or live-network risk. -Choose labels from the work's intent and accepted scope, not only from the -files it happens to touch. +Choose labels from the work's purpose and agreed scope, not only from the files +it happens to touch. -## Cardinality +## How Many Labels to Use Every issue and pull request must have **exactly one** `type:` label. Normally choose **one** `area:` label. Use two only when the work genuinely -spans two ownership surfaces; never use more than two. +belongs to two main areas; never use more than two. -`impact:` labels are optional and additive. Resolution and community labels -do not count toward these limits. +Add `impact:` labels only when needed. Resolution and community labels do not +count toward these limits. ## Type Labels | Label | Use when | |---|---| -| `type: bug` | Existing behavior does not meet its contract or documented expectation. | +| `type: bug` | Existing behavior does not match its documented or expected behavior. | | `type: feature` | The work adds a new capability or observable behavior. | | `type: refactor` | Existing implementation is reorganized without intentionally changing public behavior. | | `type: docs` | The primary deliverable is documentation, guidance, or explanatory example text. | | `type: maintenance` | The work performs specific repository upkeep, such as dependency, configuration, release, or housekeeping changes. | | `type: design` | The primary deliverable is an architecture decision, RFC, or design that precedes implementation. | -`type: maintenance` is not a miscellaneous bucket. If the intended outcome is +`type: maintenance` is not a catch-all label. If the intended outcome is unclear, clarify the work before labeling it. ## Area Labels -| Label | Ownership surface | +| Label | Main area | |---|---| | `area: contexts` | Repository information routing and discovery under `contexts/`. This does not mean all documentation. | | `area: harness` | Contributor and agent controls: `AGENTS.md`, `CLAUDE.md`, skills, CI, verification scripts, hooks, templates, and rulesets. | -| `area: knowledge` | Knowledge models, schemas, serialization, and representation under `quantmind/knowledge/`. | -| `area: configs` | Typed inputs and configuration contracts under `quantmind/configs/`. | -| `area: preprocess` | Deterministic acquisition, parsing, cleaning, formatting, and source handling under `quantmind/preprocess/`. | +| `area: knowledge` | Knowledge models, stored formats, serialization, and representation under `quantmind/knowledge/`. | +| `area: configs` | Typed inputs and configuration models under `quantmind/configs/`. | +| `area: preprocess` | Fetching, parsing, cleaning, formatting, and source handling under `quantmind/preprocess/`. | | `area: flows` | Public operation implementations under `quantmind/flows/`. It is not a generic synonym for pipeline or orchestration. | -| `area: mind` | Memory, tools, MCP integration, and the cognitive layer under `quantmind/mind/`. | +| `area: mind` | Memory, tools, MCP integration, and agent reasoning under `quantmind/mind/`. | | `area: utils` | The narrowly owned utilities surface under `quantmind/utils/`. | | `area: examples` | Examples are the primary deliverable, rather than supporting files for another area. | | `area: packaging` | Installation, builds, dependencies, releases, and package metadata. | -Area labels describe ownership, not every touched path. For example, a news +Area labels describe the main work, not every touched path. For example, a news design-document-only change is `type: docs` + `area: preprocess`, while a skill-only guidance change is `type: docs` + `area: harness`. @@ -57,7 +76,7 @@ skill-only guidance change is `type: docs` + `area: harness`. | Label | Use when | |---|---| -| `impact: breaking` | The accepted change can break a public API, serialized contract, or supported compatibility boundary. | +| `impact: breaking` | The accepted change can break a public API, stored data format, or supported integration. | | `impact: live-network` | The work depends on or changes real public-network behavior, external availability, or a live smoke test. | ## Resolution and Community Labels @@ -76,7 +95,7 @@ including its no-hard-wrap rule. - Label a pull request from its actual diff. - A pull request that closes an issue should normally align with the issue's type and area. If implementation differs, label the pull request accurately - and update the issue when its accepted scope changed. + and update the issue when its agreed scope changed. - Re-evaluate labels when scope changes; do not preserve a stale label merely because it was applied first. @@ -86,12 +105,12 @@ including its no-hard-wrap rule. |---|---|---| | Context framework in [#101](https://github.com/LLMQuant/quant-mind/issues/101) | `type: feature`, `area: contexts` | It added a new routing capability. | | CI/E2E consolidation in [#102](https://github.com/LLMQuant/quant-mind/issues/102) | `type: refactor`, `area: harness`, `impact: live-network` | It restructures repository verification, including live smoke tests. | -| This label-system issue and its PR | `type: docs`, `area: contexts`, `area: harness` | The canonical deliverable is context guidance routed through contributor controls. | -| Fix `paper_flow` contract behavior | `type: bug`, `area: flows` | An existing public operation is broken. | +| This label-system issue and its PR | `type: docs`, `area: contexts`, `area: harness` | The main result is context guidance routed through contributor controls. | +| Fix documented `paper_flow` behavior | `type: bug`, `area: flows` | An existing public operation is broken. | | Update only the news design document | `type: docs`, `area: preprocess` | The document belongs to the news preprocessing surface, not contexts. | | Add a public news source | `type: feature`, `area: preprocess`, `impact: live-network` | It adds acquisition behavior backed by a real external source. | -## Existing Label Migration +## Rename Existing Labels Rename labels instead of deleting and recreating them so GitHub preserves history and current assignments: @@ -105,6 +124,6 @@ history and current assignments: | `example` | `area: examples` | | `harness` | `area: harness` | -After renaming, create the missing taxonomy labels and audit open issues and +After renaming, create the missing labels and audit open issues and pull requests. Remove accidental multiple `type:` labels, choose one or two areas from accepted scope, and leave resolution/community labels intact. diff --git a/contexts/usage/README.md b/contexts/usage/README.md index 93289a3..3f973b1 100644 --- a/contexts/usage/README.md +++ b/contexts/usage/README.md @@ -1,18 +1,39 @@ # Use QuantMind +## Quick Summary + +- **Purpose**: Route library users and agents to current public operations, + inputs, results, examples, and guides. +- **Read when**: Calling QuantMind as a library or selecting a supported public + operation. +- **Load next**: Start with the component row that matches the requested + operation; do not load unrelated component designs. +- **Import rule**: Import inputs and configs from `quantmind.configs`, + operations from `quantmind.flows`, and result types from the package shown in + the component catalog. + +## Contents + +- [Public Usage Sources](#public-usage-sources) +- [Where to Import From](#where-to-import-from) + +## Public Usage Sources + Use this index when calling QuantMind as a library. These links point to the current public API, focused examples, and component-specific guidance. -| Need | Canonical source | +| Need | Read | |---|---| | Current operations, inputs, results, and sources | [Public component catalog](../../docs/README.md) | | Installation and common usage | [Root README usage](../../README.md#-usage-examples) | -| Paper extraction | [Paper guide](../../docs/papers.md) | -| News collection | [News design and behavior](../../docs/design/en/news.md) | -| Local semantic search | [Library guide](../../docs/library.md) and [focused example](../../examples/library/README.md) | +| Paper extraction | [Paper extraction design](../design/flow/paper.md) | +| News collection | [News design and behavior](../design/flow/news.md) | +| Search local knowledge by meaning | [Library guide](../../docs/library.md) and [focused example](../../examples/library/README.md) | | Runnable operation examples | [`examples/flows/`](../../examples/flows/) | | Focused preprocessing examples | [`examples/preprocess/`](../../examples/preprocess/) | +## Where to Import From + Import public inputs and configs from `quantmind.configs`, public operations -from `quantmind.flows`, and result contracts from the owning layer identified -by the component catalog. +from `quantmind.flows`, and result types from the package shown in the +component catalog. diff --git a/docs/README.md b/docs/README.md index 1a05f3e..ea4599d 100644 --- a/docs/README.md +++ b/docs/README.md @@ -6,7 +6,7 @@ acquisition mechanics remain internal unless they are intentionally documented as a public preprocessing primitive. Public callable names follow the -[operation naming contract](design/en/operations.md). The runtime API serves +[operation naming contract](../contexts/design/operations/naming.md). The runtime API serves Python callers; coding-agent guidance lives in the repository development harness. @@ -14,8 +14,8 @@ harness. | Operation | Import | Input and config | Result | Example | Design or guide | |---|---|---|---|---|---| -| Paper extraction | `quantmind.flows.paper_flow` | `PaperInput`, `PaperFlowCfg` | `Paper` | [README usage](../README.md#-usage-examples) | [Papers](papers.md) | -| News collection | `quantmind.flows.collect_news` | `NewsWindow`, `NewsCollectionCfg` | `NewsBatch` from `quantmind.preprocess` | [Collect news](../examples/flows/collect_news.py) | [News collection design](design/en/news.md) | +| Paper extraction | `quantmind.flows.paper_flow` | `PaperInput`, `PaperFlowCfg` | `Paper` | [README usage](../README.md#-usage-examples) | [Paper E2E design](../contexts/design/flow/paper.md) | +| News collection | `quantmind.flows.collect_news` | `NewsWindow`, `NewsCollectionCfg` | `NewsBatch` from `quantmind.preprocess` | [Collect news](../examples/flows/collect_news.py) | [News collection design](../contexts/design/flow/news.md) | | Bounded fan-out | `quantmind.flows.batch_run` | Operation inputs and shared config | `BatchResult` | [README usage](../README.md#-usage-examples) | API docstrings | | Local semantic search | `quantmind.library.LocalKnowledgeLibrary` | `BaseKnowledge`, `SemanticQuery` | `list[SemanticHit]` | [Library example](../examples/library/README.md) | [Library guide](library.md) | diff --git a/docs/design/en/assets/storage_system_design.png b/docs/design/en/assets/storage_system_design.png deleted file mode 100644 index 275b9ed..0000000 Binary files a/docs/design/en/assets/storage_system_design.png and /dev/null differ diff --git a/docs/design/en/news.md b/docs/design/en/news.md deleted file mode 100644 index 4943126..0000000 --- a/docs/design/en/news.md +++ /dev/null @@ -1,260 +0,0 @@ -# News Collection Design - -## Status and Scope - -This document defines the OSS contract for collecting public company news in -QuantMind. The MVP supports PR Newswire only. It is deliberately small: one -intent-oriented collection operation, one time-window input, deterministic -preprocessing, and explicit partial-failure reporting. - -Its public name follows the -[operation naming contract](operations.md): collection returns source-faithful -evidence and remains separate from semantic knowledge extraction. - -The primary requirement is that any caller can request a complete, one-shot -collection of a past time window. A daily poll is therefore not a separate -operation; it is simply a short window evaluated on a schedule. - -## Design Principles - -1. **Express intent, not acquisition mechanics.** Callers ask for news from a - source and time window. They do not select RSS, listing pages, pagination, - or article-body rules. -2. **Return honest observations.** QuantMind does not silently deduplicate - source rows. Repeated source observations remain repeated output records, - and may share the same stable identity. -3. **Make partial work inspectable.** Item failures are values in the returned - batch. Invalid inputs still raise before network work starts. -4. **Keep source policy internal.** PR Newswire discovery can change from - listing pages to another public mechanism without changing the collection - contract. -5. **Make the supported set explicit.** The MVP uses exhaustive source - dispatch. It does not expose a provider protocol or a provider registry. -6. **Separate collection from production policy.** Persistence, deduplication, - rule-based pruning, target schemas, and scheduling belong to the consuming - data pipeline. - -## Public API Contract - -Callers use one entry point: - -```python -from datetime import datetime, timezone - -from quantmind.configs import NewsCollectionCfg, NewsWindow -from quantmind.flows import collect_news - -batch = await collect_news( - NewsWindow( - source="pr-newswire", - start=datetime(2026, 7, 13, tzinfo=timezone.utc), - end=datetime(2026, 7, 14, tzinfo=timezone.utc), - ), - cfg=NewsCollectionCfg(retain_raw_html=False), -) -``` - -`NewsWindow` uses timezone-aware timestamps and the half-open interval -`[start, end)`. Both regular collection and historical backfill use this same -call. There are no separate `poll_*`, `backfill_*`, or `fetch_wire_*` public -entry points. - -`NewsCollectionCfg` retains the repository's shared `BaseFlowCfg` fields so it -works with the common typed-operation and magic-input tooling. Its only -collection-specific field is `retain_raw_html`, which controls whether fetched -article HTML bytes remain in the result. It defaults to `False`, which is the -conservative and storage-efficient behavior. The article is still fetched, -hashed, parsed, and represented by metadata; only its byte payload is -discarded. The deterministic collector does not otherwise consume the shared -model, tracing, or SDK-run fields. - -### Collection records are not knowledge - -QuantMind keeps source evidence and semantic extraction as separate contracts: - -| Operation | Result | Canonical layer | -|-----------|--------|-----------------| -| `collect_news` | Source-faithful documents, artifacts, failures, and coverage | `quantmind.preprocess` | -| future `extract_news_knowledge` | Extracted financial events | `quantmind.knowledge.News` | - -`NewsDocument` is therefore not a `KnowledgeItem`. It carries HTTP evidence, -raw bytes, parsing output, and collection status that remain useful before any -LLM or business schema is chosen. `knowledge.News` is a compact semantic event -with entities, sentiment, materiality, provenance, and an embedding view. The -two may later be composed, but neither substitutes for the other. - -## Returned Data - -`collect_news` returns a `NewsBatch` with four concepts. The collection -contracts are publicly imported from `quantmind.preprocess`: - -- `documents`: successfully collected `NewsDocument` observations; -- `failures`: lightweight `NewsFailure` records for work that could not be - completed; -- `observed_count`: the number of source rows successfully normalized into - in-window observations, before article processing; -- `complete`: whether discovery proved coverage of the requested window. - -A `NewsDocument` contains the source name, stable identity, canonical URL, -title, publisher, publication time, cleaned Markdown, content hash, ticker -hints, and two evidence artifacts: - -- a small discovery artifact representing the public source row; -- an article artifact containing fetch metadata and a content hash. Its - `bytes` field is `None` unless `retain_raw_html=True`. - -`NewsArtifact` is the common evidence shape. It records the content hash, -content type, source and resolved URLs, status, headers, and fetch time, with -an optional byte payload. - -Repeated listing rows are not removed. If two rows point to the same release, -the batch contains two observations with the same stable identity. The -consumer can use that identity for an idempotent upsert without losing what -QuantMind actually observed. - -## Collection Pipeline - -```text -NewsWindow - -> source dispatch - -> newest-to-oldest public discovery pages - -> observations inside [start, end) - -> linked article fetch - -> deterministic HTML-to-Markdown normalization - -> NewsDocument or NewsFailure - -> NewsBatch -``` - -PR Newswire discovery is based on its public news-release listing rather than -the latest-items RSS snapshot. Pages are read newest to oldest until an -observation strictly older than the requested start is seen. The strict -boundary matters because several rows with the same minute-level timestamp may -span two pages. This makes a past-day request independently replayable instead -of dependent on a previously saved cursor. - -PR Newswire exposes listing timestamps at minute precision. Scheduled callers -should therefore use minute-aligned window bounds, as the live E2E does. - -RSS remains a lower-level parser and a live component check. It is not a -high-level `NewsInput`, because feed selection is a source implementation -detail and a bounded feed snapshot cannot prove complete time-window coverage. - -The HTTP layer owns bounded retries, backoff, `Retry-After` handling, and -per-host rate limits. PR Newswire-specific URL construction and listing HTML -parsing stay in the PR Newswire source module. - -Supported source names form a closed set in `NewsWindow.source`. -`collect_news` dispatches them exhaustively, so widening the input schema -without implementing the corresponding collector fails static verification. - -## Failure and Completeness Semantics - -Configuration errors raise immediately. Examples include a naive timestamp, -an empty source, an unsupported source, or `end <= start`. - -After collection begins, recoverable failures are recorded and independent -items continue. Each `NewsFailure` identifies the source, processing stage, -URL, optional item identity, error category, and message. - -`complete=False` when any of the following is true: - -- a discovery page could not be fetched or parsed; -- discovery stopped before crossing the window start; - -Article failures remain explicit in `failures` but do not change discovery -completeness. A caller can therefore distinguish "the source window was fully -enumerated" from "every observed article was processed." It can safely persist -successful records while separately monitoring and replaying the failed -portion. An empty batch is not considered complete unless discovery has -positively crossed the requested start. - -## Responsibility Boundary - -QuantMind owns: - -- public-source discovery and article acquisition; -- deterministic normalization; -- stable identities and evidence hashes; -- honest batch counts, failure records, and completeness. - -The consuming production pipeline owns: - -- its schedule and GitHub Action invocation; -- durable storage, watermarks, and idempotent upserts; -- deduplication policy; -- rule-based company-news pruning; -- downstream schemas, enrichment, and database writes; -- shared batch hooks, metrics, and monitoring infrastructure. - -This boundary lets a separate ingestion job request the last day in one call, -apply its own rules, and write its own schema without coupling those policies -to this OSS library. - -## Verification Contract - -Verification has two intentionally separate layers. - -### Deterministic verification - -`bash scripts/verify.sh` runs deterministic unit tests, linting, typing, and -coverage. News tests use saved HTML/RSS fixtures and mocked HTTP responses. -They cover window validation, time boundaries, pagination, duplicate -observations, raw-byte retention, retries, partial failures, and completeness. - -### Live news E2E - -`python scripts/verify_news_e2e.py` is the component-specific live-network news -smoke test. It performs three bounded checks: - -1. fetch and parse the official PR Newswire RSS feed; -2. discover PR Newswire listing observations for the preceding 24 hours and - prove that discovery crossed the window start; -3. fetch up to 25 unique article observations and require 100% ticker-hint - recall for the first supported exchange-coded symbol in each group. - -The ticker control uses an oracle regex that is independent from the production -extractor and accepts plain, Markdown-link, and emphasis-wrapped symbols. It -intentionally ignores later members of multi-symbol lists and exchanges outside -the production whitelist. The gate prints component-level PASS/FAIL records and -compact discovery, article-sample, failure, and recall summaries. It exits -non-zero if RSS or discovery is invalid, no sampled article parses, or a sample -with supported exchange-coded symbols falls below 100% recall. A parsed sample -with zero supported symbols reports `SKIP` and passes neutrally. - -The `news` job in `.github/workflows/e2e.yml` runs this check once daily, on -manual dispatch, and on pull requests that change its precise dependency paths. -The required `.github/workflows/ci.yml` workflow remains network-free so local -development and unit tests stay deterministic. - -## Non-Goals and Extension Rule - -The MVP does not include GlobeNewswire, Business Wire, authenticated feeds, -continuous cursors, storage, deduplication, business materiality scoring, or a -generic batch-operation base class. - -When a second source is implemented, compare its real behavior with PR -Newswire first. Extract a shared provider interface only for behavior the two -implementations genuinely share. The public -`collect_news(NewsWindow, *, cfg)` contract should remain unchanged. - -## Adding a Public News Source - -A coding agent adding a second source follows this closed checklist: - -1. Add the source name to `NewsWindow.source`. -2. Add one private `quantmind/preprocess/.py` collector. -3. Add one explicit branch to `collect_news`; the exhaustive type check must - remain green. -4. Add fixture-based success, boundary, duplicate-observation, - partial-failure, and completeness tests for the source. -5. Add a routing test proving only the selected collector is called. -6. Update the supported-source table in `docs/README.md`, this design, and the - focused example if its common path changes. -7. Add or extend a component-specific live verifier and its named job in the - existing `.github/workflows/e2e.yml` when the integration depends on a - public network endpoint. Add its command to `docs/README.md`; do not add the - command to root agent guidance. - -Only after two real collectors expose shared behavior should a common -`Protocol` be considered. A new source must never be added only to the input -Literal: static verification is expected to reject that incomplete change. diff --git a/docs/design/en/operations.md b/docs/design/en/operations.md deleted file mode 100644 index c1866b0..0000000 --- a/docs/design/en/operations.md +++ /dev/null @@ -1,62 +0,0 @@ -# Public Operation Naming - -## Scope - -This document defines how public QuantMind callables communicate intent. It is -a naming contract, not a package-layout migration. Existing names may be -changed in focused compatibility work; they are not silently renamed here. - -The runtime library API is designed for Python callers. Repository guidance -for coding agents belongs to the development harness (`AGENTS.md`, skills, -docs, fixtures, and verification), not in the runtime API description. - -## Operation Stages - -Public names use a stage verb plus the domain or result: - -| Stage | Name pattern | Transformation | -|-------|--------------|----------------| -| Collection | `collect_` | External sources to source-faithful documents and evidence | -| Knowledge extraction | `extract__knowledge` | Documents to typed `quantmind.knowledge` values | -| Index construction | `build__index` | Documents or knowledge to a retrieval index | -| Analysis | `analyze_` | Domain inputs to an analytical result | -| Generic execution | `batch_run` and similar combinators | Apply an operation without changing its domain meaning | -| Composed recipe | `__pipeline` | Compose multiple named stages into a reusable pipeline | - -`flow` is not a domain verb. Do not add new public `*_flow` names merely -because a callable performs several steps. Use a precise operation name, or a -`*_pipeline` name only when the callable deliberately composes multiple public -stages as a reusable recipe. - -## Type Names - -- Input types describe caller intent, such as `NewsWindow` or `PaperInput`. -- Config types name the domain and stage, such as `NewsCollectionCfg` or a - future `PaperExtractionCfg`. -- Result types describe returned data, such as `NewsBatch`, `Paper`, or - `PageIndex`. -- Provider names stay out of public operation names unless provider-specific - behavior is itself the public contract. - -## Current API - -- `collect_news` is a collection operation and follows this contract. -- `batch_run` is a generic execution combinator, not a news or paper stage. -- `paper_flow` is an existing semantic-extraction API with legacy naming. It - is not the naming precedent for new operations; any rename belongs in a - separate compatibility change. - -The current `quantmind.flows` package remains the apex implementation namespace -for this release. Whether it should become `operations` or be split from a -future `pipelines` package is deliberately outside this document. - -## Review Checklist - -Before adding a public callable, state: - -1. Its operation stage. -2. Its input and result contracts. -3. Whether it is one operation or a genuine multi-stage pipeline. -4. Why its verb matches the observable result. - -If those answers are unclear, settle the contract before adding a public name. diff --git a/docs/design/en/storage.md b/docs/design/en/storage.md deleted file mode 100644 index 2fa583e..0000000 --- a/docs/design/en/storage.md +++ /dev/null @@ -1,132 +0,0 @@ -# QuantMind Storage Layer: Design Document - -![Storage System Design](./assets/storage_system_design.png) - -## 1. Introduction & Motivation (The "Why") - -The QuantMind framework is designed to process vast amounts of unstructured financial content into a structured, queryable knowledge base. The **Storage Layer** is the heart of this system, serving as the persistent, reliable, and efficient foundation for this knowledge base. - -Its primary motivations are: - -- **Centralized Data Management**: To provide a single source of truth for all processed data, including original source files, structured knowledge, vector embeddings, and operational metadata. -- **Decoupling & Modularity**: To separate the logic of data persistence from the components that produce or consume it (e.g., `Parsers`, `Taggers`, `Retrievers`). This allows other components to operate without worrying about the underlying storage mechanism (e.g., file system, database). -- **Efficiency & Performance**: The knowledge base can grow to millions of items. An efficient storage and retrieval mechanism is critical for the system's overall performance, especially for lookup-intensive operations. -- **Data Integrity & Resilience**: To ensure that data is stored correctly and consistently, and to provide mechanisms for recovery and maintenance in case of inconsistencies. -- **Extensibility**: To design a system that can easily be extended to support different storage backends (e.g., local files, cloud storage, a vector database) without requiring changes to the core application logic. - -## 2. Architecture & Core Concepts (The "How") - -The storage layer is built on a set of core principles: an abstract interface, a concrete file-based implementation, and a high-performance indexing system. - -### 2.1. `BaseStorage`: The Abstract Contract - -The architecture starts with `quantmind.storage.base.BaseStorage`, an abstract base class (ABC) that defines the universal interface for any storage implementation. This ensures that any new storage backend will be a drop-in replacement. - -The contract manages four distinct types of data: - -- **Raw Files**: The original, unprocessed source files (e.g., PDFs from ArXiv, raw HTML). -- **Knowledges**: Structured `KnowledgeItem` objects (including specialized types like `Paper`), typically stored as JSON. These represent the processed, value-added data. -- **Embeddings**: Vector representations of the knowledge items, used for semantic search and retrieval. -- **Extra**: Auxiliary data, primarily used for storing indexes and other metadata that supports the storage layer's operations. - -### 2.2. Specialized Knowledge Processing - -To handle the unique needs of different data types, the `BaseStorage` interface includes a specialized processing method: - -```python -def process_knowledge(self, knowledge: KnowledgeItem) -> str: -``` - -This method acts as a router. While it stores any `KnowledgeItem` by default, it contains special logic for subtypes. For instance, when it receives a `Paper` object: - -1. It first stores the `Paper` metadata as a knowledge item. -2. It then checks if the `Paper` has a `pdf_url`. -3. If a URL exists and the corresponding PDF is not already in the raw file storage, it automatically downloads the content and stores it using `store_raw_file`. - -This design elegantly handles type-specific logic within the storage layer, simplifying the upstream workflow agents. - -### 2.3. `LocalStorage`: A Concrete Filesystem Implementation - -`quantmind.storage.local_storage.LocalStorage` is the default, file-based implementation of `BaseStorage`. - -#### Directory Structure - -It organizes data on the filesystem in a clear, hierarchical structure defined by `LocalStorageConfig`: - -```text -/ -├── 📂 raw_files/ # Raw source files (e.g., paper_id.pdf) -├── 📂 knowledges/ # Structured data (e.g., paper_id.json) -├── 📂 embeddings/ # Vector embeddings (e.g., paper_id.json) -└── 📂 extra/ # Indexes and other metadata - ├── 📄 raw_files_index.json - ├── 📄 knowledges_index.json - └── 📄 embeddings_index.json -``` - -#### High-Performance Indexing System - -A key challenge with a simple filesystem approach is the inefficiency of lookups. Iterating through a directory to find a file (`glob`) is an O(n) operation, which becomes unacceptably slow as the dataset grows. - -To solve this, `LocalStorage` implements a **high-performance, persistent indexing system**. - -**How it Works:** - -1. **Index Files**: For each core data type (`raw_files`, `knowledges`, `embeddings`), a corresponding JSON index file is maintained in the `extra/` directory. This index is a simple key-value map: - - `file_id` -> `{ "path": "relative/path/to/file", "extension": ".pdf" }` - -2. **In-Memory Cache**: On initialization, `LocalStorage` loads these JSON files into in-memory dictionaries (`self._raw_files_index`, etc.) for instantaneous O(1) lookups. - -3. **Real-time Updates**: Every `store_*` or `delete_*` operation automatically updates both the in-memory index and the corresponding JSON file on disk, ensuring data consistency. - - ```python - # Example: store_raw_file - def store_raw_file(...): - # ... write file to disk ... - - # Update and save the index - self._raw_files_index[file_id] = {"path": ..., "extension": ...} - self._save_index("raw_files") - ``` - -4. **Efficient Lookups**: `get_*` methods now first consult the in-memory index for a near-instant lookup. The costly filesystem scan is only used as a last-resort fallback. - - ```python - # Example: get_raw_file - def get_raw_file(self, file_id: str): - # 1. Fast O(1) lookup in the index - if file_id in self._raw_files_index: - return ... - - # 2. Fallback to O(n) directory scan (if not found in index) - # ... glob filesystem ... - # If found, add to index for future fast lookups - ``` - -**Resilience and Self-Healing:** - -The indexing system is designed to be robust: - -- **Automatic Rebuilding**: If an index file is missing or corrupt on startup, `LocalStorage` will automatically scan the corresponding data directory and rebuild the index from scratch. -- **Self-Healing**: If an index entry points to a file that has been deleted from the filesystem externally, the `get_*` method will detect this, remove the stale entry from the index, and save the corrected index. -- **Manual Rebuild**: A public `rebuild_all_indexes()` method is provided for manual maintenance and data recovery. - -## 3. Configuration - -The storage layer is configured via `quantmind.config.storage.LocalStorageConfig`, a Pydantic model that allows for easy setup, primarily defining the `storage_dir`. - -```python -from quantmind.config import LocalStorageConfig - -# Default configuration -config = LocalStorageConfig() # Uses ./data - -# Custom configuration -config = LocalStorageConfig(storage_dir="/path/to/my/storage") -``` - -The config model's `model_post_init` hook automatically handles the creation of the storage directory and all its required subdirectories. - -## 4. Conclusion - -The QuantMind storage layer is a modular, extensible, and high-performance system for managing the project's knowledge base. By combining a clear abstract interface with an efficient, self-healing, file-based implementation, it provides a robust foundation that can scale effectively while remaining easy to manage and extend. The recent addition of the indexing system transforms its lookup performance from a potential bottleneck into a key strength. diff --git a/tests/contexts/__init__.py b/tests/contexts/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/contexts/test_paper_golden.py b/tests/contexts/test_paper_golden.py new file mode 100644 index 0000000..991c0e5 --- /dev/null +++ b/tests/contexts/test_paper_golden.py @@ -0,0 +1,133 @@ +import json +import unittest +from pathlib import Path + +import pymupdf + +_GOLDEN = Path(__file__).resolve().parents[1] / "fixtures" / "paper" / "golden" + + +class PaperGoldenFixtureTests(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + cls.expected = json.loads( + (_GOLDEN / "expected.json").read_text(encoding="utf-8") + ) + with pymupdf.open(_GOLDEN / "paper.pdf") as document: + cls.page_texts = [page.get_text() for page in document] + + def test_document_uses_pdf_golden_span_contract(self) -> None: + document = self.expected["document"] + + self.assertEqual(document["span_unit"], "pdf_page") + self.assertEqual(document["indexing"], "1-based-inclusive") + self.assertEqual(len(self.page_texts), document["page_count"]) + self.assertIn(document["title"], self.page_texts[0]) + + def test_titles_paths_spans_and_anchors_match_pdf(self) -> None: + page_count = self.expected["document"]["page_count"] + nodes = {node["key"]: node for node in self.expected["nodes"]} + + for key, node in nodes.items(): + with self.subTest(node=key): + start = node["span"]["start"] + end = node["span"]["end"] + self.assertGreaterEqual(start, 1) + self.assertLessEqual(start, end) + self.assertLessEqual(end, page_count) + + span_text = "\n".join(self.page_texts[start - 1 : end]) + self.assertIn(node["title"], span_text) + + anchor_pages = set() + for anchor in node["anchors"]: + page = anchor["page"] + anchor_pages.add(page) + self.assertGreaterEqual(page, start) + self.assertLessEqual(page, end) + self.assertIn(anchor["text"], self.page_texts[page - 1]) + + if start != end: + self.assertTrue({start, end}.issubset(anchor_pages)) + + expected_path = [] + cursor = node + path_keys = set() + while cursor is not None: + cursor_key = cursor["key"] + self.assertNotIn(cursor_key, path_keys) + path_keys.add(cursor_key) + expected_path.append(cursor["title"]) + parent = cursor["parent"] + cursor = nodes[parent] if parent is not None else None + self.assertEqual(node["path"], list(reversed(expected_path))) + + def test_topology_satisfies_declared_tree_invariants(self) -> None: + required_invariants = { + "single_root", + "node_keys_unique", + "parents_exist", + "children_exist", + "parent_child_bidirectional", + "all_nodes_reachable", + "acyclic", + "spans_within_document", + } + self.assertEqual(set(self.expected["invariants"]), required_invariants) + + nodes_list = self.expected["nodes"] + nodes = {node["key"]: node for node in nodes_list} + self.assertEqual(len(nodes), len(nodes_list)) + + roots = [node for node in nodes_list if node["parent"] is None] + self.assertEqual(len(roots), 1) + root = roots[0] + self.assertEqual(root["key"], self.expected["document"]["root"]) + + child_owners = {key: 0 for key in nodes} + for key, node in nodes.items(): + with self.subTest(node=key): + self.assertEqual( + len(node["children"]), len(set(node["children"])) + ) + if node["parent"] is not None: + self.assertIn(node["parent"], nodes) + for child_key in node["children"]: + self.assertIn(child_key, nodes) + self.assertEqual(nodes[child_key]["parent"], key) + child_owners[child_key] += 1 + + self.assertEqual(child_owners[root["key"]], 0) + for key, owner_count in child_owners.items(): + if key != root["key"]: + self.assertEqual(owner_count, 1) + + visiting = set() + visited = set() + + def visit(key: str) -> None: + self.assertNotIn(key, visiting, f"cycle detected at {key}") + if key in visited: + return + visiting.add(key) + for child_key in nodes[key]["children"]: + visit(child_key) + visiting.remove(key) + visited.add(key) + + visit(root["key"]) + self.assertEqual(visited, set(nodes)) + + def test_fixture_preserves_overlapping_sibling_spans(self) -> None: + nodes = {node["key"]: node for node in self.expected["nodes"]} + method = nodes["method"] + limitations = nodes["limitations"] + + self.assertEqual(method["parent"], limitations["parent"]) + self.assertLessEqual( + limitations["span"]["start"], method["span"]["end"] + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/fixtures/paper/golden/expected.json b/tests/fixtures/paper/golden/expected.json new file mode 100644 index 0000000..cc31564 --- /dev/null +++ b/tests/fixtures/paper/golden/expected.json @@ -0,0 +1,144 @@ +{ + "schema_version": "1.0", + "document": { + "title": "A Synthetic Cross-Sectional Momentum Study", + "root": "paper", + "page_count": 4, + "span_unit": "pdf_page", + "indexing": "1-based-inclusive" + }, + "nodes": [ + { + "key": "paper", + "title": "A Synthetic Cross-Sectional Momentum Study", + "path": [ + "A Synthetic Cross-Sectional Momentum Study" + ], + "span": { + "start": 1, + "end": 4 + }, + "anchors": [ + { + "page": 1, + "text": "A Synthetic Cross-Sectional Momentum Study" + }, + { + "page": 4, + "text": "This page closes the synthetic paper fixture." + } + ], + "parent": null, + "children": [ + "introduction", + "method", + "limitations" + ] + }, + { + "key": "introduction", + "title": "1 Introduction", + "path": [ + "A Synthetic Cross-Sectional Momentum Study", + "1 Introduction" + ], + "span": { + "start": 1, + "end": 2 + }, + "anchors": [ + { + "page": 1, + "text": "The introduction defines a twelve-month lookback signal." + }, + { + "page": 2, + "text": "The continuation discusses turnover and transaction costs." + } + ], + "parent": "paper", + "children": [] + }, + { + "key": "method", + "title": "2 Method", + "path": [ + "A Synthetic Cross-Sectional Momentum Study", + "2 Method" + ], + "span": { + "start": 2, + "end": 4 + }, + "anchors": [ + { + "page": 2, + "text": "The method ranks securities by trailing excess return." + }, + { + "page": 4, + "text": "Risk scaling completes the method on the final page." + } + ], + "parent": "paper", + "children": [ + "portfolio-construction" + ] + }, + { + "key": "portfolio-construction", + "title": "2.1 Portfolio Construction", + "path": [ + "A Synthetic Cross-Sectional Momentum Study", + "2 Method", + "2.1 Portfolio Construction" + ], + "span": { + "start": 3, + "end": 4 + }, + "anchors": [ + { + "page": 3, + "text": "The portfolio subsection forms equal-weighted quintiles." + }, + { + "page": 4, + "text": "The top and bottom quintiles define the long-short portfolio." + } + ], + "parent": "method", + "children": [] + }, + { + "key": "limitations", + "title": "3 Limitations", + "path": [ + "A Synthetic Cross-Sectional Momentum Study", + "3 Limitations" + ], + "span": { + "start": 4, + "end": 4 + }, + "anchors": [ + { + "page": 4, + "text": "The fixture uses synthetic observations and no investable returns." + } + ], + "parent": "paper", + "children": [] + } + ], + "invariants": [ + "single_root", + "node_keys_unique", + "parents_exist", + "children_exist", + "parent_child_bidirectional", + "all_nodes_reachable", + "acyclic", + "spans_within_document" + ] +} diff --git a/tests/fixtures/paper/golden/paper.pdf b/tests/fixtures/paper/golden/paper.pdf new file mode 100644 index 0000000..4dee59b Binary files /dev/null and b/tests/fixtures/paper/golden/paper.pdf differ diff --git a/tests/test_contexts.py b/tests/test_contexts.py index c84e538..31bf945 100644 --- a/tests/test_contexts.py +++ b/tests/test_contexts.py @@ -4,6 +4,49 @@ class TestContextEntryPoints(unittest.TestCase): + def test_context_pages_support_progressive_disclosure(self) -> None: + repo_root = Path(__file__).resolve().parents[1] + context_root = repo_root / "contexts" + contents_link = re.compile(r"^- \[[^]]+]\(#([^)]+)\)$", re.MULTILINE) + + def github_anchor(heading: str) -> str: + normalized = re.sub(r"[^\w\s-]", "", heading.lower()) + return re.sub(r"[\s-]+", "-", normalized).strip("-") + + for page in sorted(context_root.rglob("*.md")): + lines = page.read_text(encoding="utf-8").splitlines() + preview_lines = lines[:80] + preview = "\n".join(preview_lines) + contents_targets = set(contents_link.findall(preview)) + detail_targets = { + github_anchor(line.removeprefix("## ")) + for line in lines + if line.startswith("## ") + and line not in {"## Quick Summary", "## Contents"} + } + with self.subTest(page=page.relative_to(repo_root)): + self.assertIn("## Quick Summary", preview) + self.assertIn("## Contents", preview) + self.assertLess( + preview_lines.index("## Quick Summary"), + preview_lines.index("## Contents"), + ) + self.assertEqual(contents_targets, detail_targets) + + def test_agent_guides_define_progressive_context_loading(self) -> None: + repo_root = Path(__file__).resolve().parents[1] + + for guide_path in ("AGENTS.md", "CLAUDE.md"): + guide = (repo_root / guide_path).read_text(encoding="utf-8") + with self.subTest(guide=guide_path): + self.assertIn("## Progressive Context Loading", guide) + self.assertIn("Read lines 1-80 first.", guide) + self.assertIn("read the entire page", guide) + self.assertIn( + "The preview routes work; it does not replace the", + guide, + ) + def test_context_index_targets_exist(self) -> None: repo_root = Path(__file__).resolve().parents[1] context_indexes = ( @@ -11,6 +54,11 @@ def test_context_index_targets_exist(self) -> None: "contexts/dev/README.md", "contexts/dev/github-writing.md", "contexts/dev/labels.md", + "contexts/design/README.md", + "contexts/design/flow/news.md", + "contexts/design/flow/paper.md", + "contexts/design/library/local.md", + "contexts/design/operations/naming.md", "contexts/usage/README.md", ) markdown_link = re.compile(r"\[[^]]+]\(([^)]+)\)")