From 646d4fc7419970a6975d01ea8f9f0547cacc2b26 Mon Sep 17 00:00:00 2001 From: pkuwkl Date: Thu, 16 Jul 2026 14:38:14 +0800 Subject: [PATCH 1/5] feat(library): add local semantic knowledge library --- docs/README.md | 1 + docs/library.md | 76 +++ examples/library/semantic_search.py | 42 ++ pyproject.toml | 28 +- quantmind/knowledge/_base.py | 11 +- quantmind/library/__init__.py | 6 + quantmind/library/_embed.py | 40 ++ quantmind/library/_ports.py | 22 + quantmind/library/_projection.py | 153 ++++++ quantmind/library/_types.py | 71 +++ quantmind/library/local.py | 777 ++++++++++++++++++++++++++++ tests/knowledge/test_base.py | 13 + tests/library/__init__.py | 1 + tests/library/test_local.py | 607 ++++++++++++++++++++++ tests/library/test_types.py | 34 ++ 15 files changed, 1875 insertions(+), 7 deletions(-) create mode 100644 docs/library.md create mode 100644 examples/library/semantic_search.py create mode 100644 quantmind/library/__init__.py create mode 100644 quantmind/library/_embed.py create mode 100644 quantmind/library/_ports.py create mode 100644 quantmind/library/_projection.py create mode 100644 quantmind/library/_types.py create mode 100644 quantmind/library/local.py create mode 100644 tests/library/__init__.py create mode 100644 tests/library/test_local.py create mode 100644 tests/library/test_types.py diff --git a/docs/README.md b/docs/README.md index 9b61f89..2b127ed 100644 --- a/docs/README.md +++ b/docs/README.md @@ -17,6 +17,7 @@ harness. | Paper extraction | `quantmind.flows.paper_flow` | `PaperInput`, `PaperFlowCfg` | `Paper` | [README usage](../README.md#-usage-examples) | [Papers](papers.md) | | News collection | `quantmind.flows.collect_news` | `NewsWindow`, `NewsCollectionCfg` | `NewsBatch` from `quantmind.preprocess` | [Collect news](../examples/flows/collect_news.py) | [News collection design](design/en/news.md) | | Bounded fan-out | `quantmind.flows.batch_run` | Operation inputs and shared config | `BatchResult` | [README usage](../README.md#-usage-examples) | API docstrings | +| Local semantic search | `quantmind.library.LocalKnowledgeLibrary` | `BaseKnowledge`, `SemanticQuery` | `list[SemanticHit]` | [Semantic search](../examples/library/semantic_search.py) | [Library guide](library.md) | Import public inputs and configs from `quantmind.configs` and current public operations from `quantmind.flows`. Import result contracts from the canonical diff --git a/docs/library.md b/docs/library.md new file mode 100644 index 0000000..c5463e8 --- /dev/null +++ b/docs/library.md @@ -0,0 +1,76 @@ +# Local Semantic Knowledge Library + +`quantmind.library` persists canonical `BaseKnowledge` in SQLite and ranks +rebuildable NumPy indexes with exact cosine similarity. It is a financial +knowledge API, not a generic RAG framework: provider and storage details remain +private, and search returns typed QuantMind evidence without generating an +answer. + +## Retrieval grain + +- Each `FlattenKnowledge` item produces one target from its exact + `embedding_text()` projection. +- Each `TreeKnowledge` produces one item target for the root with + `node_id=None`, plus one target for every non-root `TreeNode`. +- Canonical JSON is the source of truth. Embeddings and filter columns are + derived data and can be replaced by re-putting the canonical item. + +The durable vector metadata records the embedding model and dimension, exact +projection hash, source content hash, knowledge schema version, and projection +schema version. Re-putting an unchanged item ID reuses its vectors. A changed +node projection replaces only that node vector; model, dimension, source hash, +or schema changes replace the affected item's vectors. + +## Financial-time filters + +`as_of` is the information cutoff represented by the knowledge. `available_at` +is when its source became observable. They intentionally remain separate: + +- `as_of_before` includes records whose information cutoff is at or before the + query cutoff. +- `available_at_before` includes only records with a known availability time at + or before the query cutoff. Unknown availability is excluded. + +Consequently, `as_of_before` alone does not prevent look-ahead. Ingestion flows +should populate `available_at` from publication time. When publication time is +unknown, the caller can copy `SourceRef.fetched_at` to `available_at` as a +conservative upper bound. + +`item_types` and `source_kinds` use any-of matching. Every requested tag must be +present. `confidence`, `tree_id`, and both time cutoffs combine with those +filters before ranking. + +## Usage + +```python +from quantmind.library import LocalKnowledgeLibrary, SemanticQuery + +library = await LocalKnowledgeLibrary.open( + ".quantmind/library.db", + embedding_model="text-embedding-3-small", +) +try: + await library.put(paper) + hits = await library.search( + SemanticQuery( + text="management expects capital expenditure to increase", + available_at_before=research_cutoff, + top_k=10, + ) + ) + evidence = await library.get(hits[0].item_id) +finally: + await library.close() +``` + +`SemanticHit.matched_text` is the exact projection used in ranking. Use +`get(item_id)` to resolve the full canonical item and, for node hits, the node +identified by `node_id`. + +Missing IDs raise `KeyError`. Stored canonical payloads that no longer validate +and orphaned or mismatched derived records raise `RuntimeError` with stale-data +context. Invalid vector bytes and inconsistent stored dimensions raise a clear +corrupt-index `RuntimeError`; provider or query dimension mismatches raise +`ValueError`. `delete()` does not deserialize canonical JSON, so a stale +canonical payload can still be removed together with all derived targets in one +transaction. diff --git a/examples/library/semantic_search.py b/examples/library/semantic_search.py new file mode 100644 index 0000000..607ccfc --- /dev/null +++ b/examples/library/semantic_search.py @@ -0,0 +1,42 @@ +"""Store and search one canonical knowledge item locally.""" + +import asyncio +from datetime import datetime, timezone + +from quantmind.knowledge import News, SourceRef +from quantmind.library import LocalKnowledgeLibrary, SemanticQuery + + +async def main() -> None: + """Persist one news item and print matching semantic evidence.""" + published_at = datetime(2026, 7, 16, 8, tzinfo=timezone.utc) + item = News( + as_of=published_at, + available_at=published_at, + source=SourceRef(kind="rss", uri="https://example.com/fed-rates"), + headline="Federal Reserve holds rates steady", + event_type="monetary_policy", + entities=["Federal Reserve"], + tags=["macro", "rates"], + ) + library = await LocalKnowledgeLibrary.open( + ".quantmind/library.db", + embedding_model="text-embedding-3-small", + ) + try: + await library.put(item) + hits = await library.search( + SemanticQuery( + text="central bank interest-rate decision", + available_at_before=datetime.now(timezone.utc), + top_k=3, + ) + ) + for hit in hits: + print(hit.score, hit.matched_text) + finally: + await library.close() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/pyproject.toml b/pyproject.toml index 50d6fe9..f15b0f3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -119,10 +119,9 @@ reportIncompatibleVariableOverride = "none" # import-linter: architectural boundary contracts # ---------------------------------------------------------------------------- # Encodes the target architecture: utils, knowledge, and preprocess are -# leaves; configs depends only on knowledge; flows + magic is the apex -# layer (PR5). The transitional packages (config/, flow/, llm/, models/) -# were deleted in PR5; they remain in the forbidden lists as a tripwire -# against accidental re-introduction during future refactors. +# leaves; configs depends only on knowledge; library depends only on +# knowledge; flows + magic is the apex layer. The transitional packages +# (config/, flow/, llm/, models/) remain forbidden as a tripwire. [tool.importlinter] root_packages = ["quantmind"] @@ -135,6 +134,7 @@ forbidden_modules = [ "quantmind.configs", "quantmind.flows", "quantmind.knowledge", + "quantmind.library", "quantmind.magic", "quantmind.preprocess", ] @@ -146,6 +146,7 @@ source_modules = ["quantmind.knowledge"] forbidden_modules = [ "quantmind.configs", "quantmind.flows", + "quantmind.library", "quantmind.magic", "quantmind.preprocess", "quantmind.utils", @@ -157,6 +158,7 @@ type = "forbidden" source_modules = ["quantmind.configs"] forbidden_modules = [ "quantmind.flows", + "quantmind.library", "quantmind.magic", "quantmind.preprocess", ] @@ -169,9 +171,27 @@ forbidden_modules = [ "quantmind.configs", "quantmind.flows", "quantmind.knowledge", + "quantmind.library", "quantmind.magic", ] +[[tool.importlinter.contracts]] +name = "library only depends on knowledge" +type = "forbidden" +source_modules = ["quantmind.library"] +forbidden_modules = [ + "quantmind.config", + "quantmind.configs", + "quantmind.flow", + "quantmind.flows", + "quantmind.llm", + "quantmind.magic", + "quantmind.mind", + "quantmind.models", + "quantmind.preprocess", + "quantmind.utils", +] + [[tool.importlinter.contracts]] name = "flows + magic is apex (no transitional package imports)" type = "forbidden" diff --git a/quantmind/knowledge/_base.py b/quantmind/knowledge/_base.py index d2165af..b6ae31c 100644 --- a/quantmind/knowledge/_base.py +++ b/quantmind/knowledge/_base.py @@ -7,9 +7,10 @@ - `GraphKnowledge` — cross-item edges (placeholder, future PR) The `as_of` field is mandatory by design: financial knowledge is only useful -when its time-of-validity is explicit. `source` and `extraction` are typed -references (not bare strings) so dedup, audit, and re-runs all have stable -keys. +when its information cutoff is explicit. Optional `available_at` records when +the source became observable so research can prevent look-ahead independently +from information time. `source` and `extraction` are typed references (not bare +strings) so dedup, audit, and re-runs all have stable keys. Subclasses MUST override `embedding_text()` to declare what string the store layer should embed for them. The contract is enforced at runtime, not at @@ -80,6 +81,10 @@ class BaseKnowledge(BaseModel): # Time (financial mandate) as_of: datetime = Field(..., description="Information cutoff time.") + available_at: datetime | None = Field( + default=None, + description="Time the source became observable to researchers.", + ) created_at: datetime = Field( default_factory=lambda: datetime.now(timezone.utc) ) diff --git a/quantmind/library/__init__.py b/quantmind/library/__init__.py new file mode 100644 index 0000000..edfde4b --- /dev/null +++ b/quantmind/library/__init__.py @@ -0,0 +1,6 @@ +"""Local semantic retrieval for canonical QuantMind knowledge.""" + +from quantmind.library._types import SemanticHit, SemanticQuery +from quantmind.library.local import LocalKnowledgeLibrary + +__all__ = ["LocalKnowledgeLibrary", "SemanticHit", "SemanticQuery"] diff --git a/quantmind/library/_embed.py b/quantmind/library/_embed.py new file mode 100644 index 0000000..d2ed343 --- /dev/null +++ b/quantmind/library/_embed.py @@ -0,0 +1,40 @@ +"""Private OpenAI embedding implementation.""" + +from collections.abc import Sequence +from typing import Any + +from openai import AsyncOpenAI + + +class _OpenAIEmbeddingProvider: + """Generate embeddings without exposing provider response types.""" + + def __init__(self) -> None: + self._client: AsyncOpenAI | None = None + + async def embed( + self, + texts: Sequence[str], + *, + model: str, + dimensions: int | None, + ) -> list[list[float]]: + client = self._client + if client is None: + client = AsyncOpenAI() + self._client = client + kwargs: dict[str, Any] = { + "input": list(texts), + "model": model, + "encoding_format": "float", + } + if dimensions is not None: + kwargs["dimensions"] = dimensions + response = await client.embeddings.create(**kwargs) + ordered = sorted(response.data, key=lambda item: item.index) + return [item.embedding for item in ordered] + + async def close(self) -> None: + if self._client is not None: + await self._client.close() + self._client = None diff --git a/quantmind/library/_ports.py b/quantmind/library/_ports.py new file mode 100644 index 0000000..eda69fe --- /dev/null +++ b/quantmind/library/_ports.py @@ -0,0 +1,22 @@ +"""Private side-effect seams for the local knowledge library.""" + +from collections.abc import Sequence +from typing import Protocol + + +class _EmbeddingProvider(Protocol): + """Private embedding seam used by production code and deterministic tests.""" + + async def embed( + self, + texts: Sequence[str], + *, + model: str, + dimensions: int | None, + ) -> Sequence[Sequence[float]]: + """Embed texts in input order.""" + ... + + async def close(self) -> None: + """Release provider-owned resources.""" + ... diff --git a/quantmind/library/_projection.py b/quantmind/library/_projection.py new file mode 100644 index 0000000..3d578ab --- /dev/null +++ b/quantmind/library/_projection.py @@ -0,0 +1,153 @@ +"""Canonical serialization and searchable projection rules.""" + +import hashlib +import json +from dataclasses import dataclass +from uuid import UUID + +from pydantic import ValidationError + +from quantmind.knowledge import ( + BaseKnowledge, + Earnings, + Factor, + FlattenKnowledge, + News, + Paper, + PaperKnowledgeCard, + Thesis, + TreeKnowledge, +) + +_PROJECTION_SCHEMA_VERSION = "1" + +_KNOWLEDGE_CLASSES: dict[str, type[BaseKnowledge]] = { + f"{knowledge_type.__module__}:{knowledge_type.__qualname__}": knowledge_type + for knowledge_type in ( + Earnings, + Factor, + News, + Paper, + PaperKnowledgeCard, + Thesis, + ) +} + + +@dataclass(frozen=True) +class _Projection: + """One stable item or node target to embed.""" + + target_id: str + node_id: UUID | None + text: str + projection_hash: str + tree_id: UUID | None + + +def _canonical_payload(item: BaseKnowledge) -> tuple[str, str, str]: + """Serialize a supported canonical item and return its stable hash.""" + knowledge_class = f"{type(item).__module__}:{type(item).__qualname__}" + if knowledge_class not in _KNOWLEDGE_CLASSES: + raise TypeError( + f"Unsupported knowledge type '{type(item).__name__}'; " + "only canonical quantmind.knowledge types can be persisted" + ) + payload = json.dumps( + item.model_dump(mode="json"), + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ) + content_hash = hashlib.sha256(payload.encode("utf-8")).hexdigest() + return knowledge_class, payload, content_hash + + +def _load_canonical( + *, + item_id: str, + knowledge_class: str, + item_type: str, + schema_version: str, + payload: str, + canonical_hash: str, +) -> BaseKnowledge: + """Validate stored canonical bytes against identity and schema metadata.""" + actual_hash = hashlib.sha256(payload.encode("utf-8")).hexdigest() + if actual_hash != canonical_hash: + raise RuntimeError( + f"Stale canonical knowledge for item '{item_id}': content hash mismatch" + ) + model = _KNOWLEDGE_CLASSES.get(knowledge_class) + if model is None: + raise RuntimeError( + f"Stale canonical knowledge for item '{item_id}': " + f"unsupported stored type '{knowledge_class}'" + ) + try: + item = model.model_validate_json(payload) + except ValidationError as exc: + raise RuntimeError( + f"Stale canonical knowledge for item '{item_id}': " + "stored payload no longer validates" + ) from exc + if ( + str(item.id) != item_id + or item.item_type != item_type + or item.schema_version != schema_version + ): + raise RuntimeError( + f"Stale canonical knowledge for item '{item_id}': identity or schema " + "metadata does not match the payload" + ) + return item + + +def _project_knowledge(item: BaseKnowledge) -> list[_Projection]: + """Project flat items and trees at the issue-defined retrieval grain.""" + if not isinstance(item, (FlattenKnowledge, TreeKnowledge)): + raise TypeError( + "LocalKnowledgeLibrary supports FlattenKnowledge and TreeKnowledge" + ) + + root_text = item.embedding_text() + if not root_text: + raise ValueError("knowledge embedding_text() must not be empty") + tree_id = item.id if isinstance(item, TreeKnowledge) else None + projections = [ + _Projection( + target_id=f"item:{item.id}", + node_id=None, + text=root_text, + projection_hash=hashlib.sha256( + root_text.encode("utf-8") + ).hexdigest(), + tree_id=tree_id, + ) + ] + + if isinstance(item, TreeKnowledge): + if item.root_node_id not in item.nodes: + raise ValueError("tree root_node_id is not present in nodes") + for node_id, node in sorted( + item.nodes.items(), key=lambda pair: str(pair[0]) + ): + if node_id != node.node_id: + raise ValueError("tree node map key does not match node_id") + if node_id == item.root_node_id: + continue + text = node.embedding_text() + if not text: + raise ValueError("tree node embedding_text() must not be empty") + projections.append( + _Projection( + target_id=f"node:{item.id}:{node_id}", + node_id=node_id, + text=text, + projection_hash=hashlib.sha256( + text.encode("utf-8") + ).hexdigest(), + tree_id=item.id, + ) + ) + return projections diff --git a/quantmind/library/_types.py b/quantmind/library/_types.py new file mode 100644 index 0000000..9fd970e --- /dev/null +++ b/quantmind/library/_types.py @@ -0,0 +1,71 @@ +"""Public domain types for semantic knowledge retrieval.""" + +from datetime import datetime +from typing import Literal +from uuid import UUID + +from pydantic import BaseModel, ConfigDict, Field, field_validator + +from quantmind.knowledge import Citation, SourceRef + + +class SemanticQuery(BaseModel): + """A financial-time-aware semantic query over canonical knowledge.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + text: str = Field(min_length=1) + item_types: list[str] | None = None + source_kinds: ( + list[ + Literal[ + "arxiv", + "http", + "doi", + "local", + "rss", + "transcript", + "manual", + ] + ] + | None + ) = None + confidence: Literal["low", "medium", "high"] | None = None + tags: list[str] | None = None + tree_id: UUID | None = None + as_of_before: datetime | None = None + available_at_before: datetime | None = None + top_k: int = Field(default=10, ge=1) + + @field_validator("text") + @classmethod + def _text_must_not_be_blank(cls, value: str) -> str: + stripped = value.strip() + if not stripped: + raise ValueError("query text must not be blank") + return stripped + + @field_validator("as_of_before", "available_at_before") + @classmethod + def _cutoffs_must_be_timezone_aware( + cls, value: datetime | None + ) -> datetime | None: + if value is not None and value.tzinfo is None: + raise ValueError("financial-time cutoffs must be timezone-aware") + return value + + +class SemanticHit(BaseModel): + """Auditable evidence returned by semantic ranking.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + item_id: UUID + node_id: UUID | None + item_type: str + score: float + matched_text: str + as_of: datetime + available_at: datetime | None + source: SourceRef + citations: list[Citation] diff --git a/quantmind/library/local.py b/quantmind/library/local.py new file mode 100644 index 0000000..e1522cd --- /dev/null +++ b/quantmind/library/local.py @@ -0,0 +1,777 @@ +"""Opinionated SQLite and NumPy semantic knowledge library.""" + +import asyncio +import json +import sqlite3 +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Any +from uuid import UUID + +import numpy as np +from numpy.typing import NDArray +from typing_extensions import Self + +from quantmind.knowledge import BaseKnowledge, TreeKnowledge +from quantmind.library._embed import _OpenAIEmbeddingProvider +from quantmind.library._ports import _EmbeddingProvider +from quantmind.library._projection import ( + _PROJECTION_SCHEMA_VERSION, + _canonical_payload, + _load_canonical, + _project_knowledge, + _Projection, +) +from quantmind.library._types import SemanticHit, SemanticQuery + +_DATABASE_SCHEMA_VERSION = 1 + + +@dataclass(frozen=True) +class _IndexRecord: + """Validated metadata aligned with one NumPy matrix row.""" + + target_id: str + item_id: UUID + node_id: UUID | None + item_type: str + matched_text: str + as_of: float + available_at: float | None + source_kind: str + confidence: str + tags: frozenset[str] + tree_id: UUID | None + + +def _timestamp(value: datetime, field_name: str) -> float: + """Normalize an aware financial timestamp for deterministic filtering.""" + if value.tzinfo is None or value.utcoffset() is None: + raise ValueError(f"{field_name} must be timezone-aware") + return value.astimezone(timezone.utc).timestamp() + + +def _coerce_provider_vectors( + values: Any, + *, + expected_count: int, + expected_dimensions: int | None, +) -> NDArray[np.float32]: + """Validate provider output before it can enter the durable index.""" + try: + vectors = np.asarray(values, dtype=np.float32) + except (TypeError, ValueError) as exc: + raise ValueError( + "Embedding provider returned non-rectangular data" + ) from exc + if vectors.ndim != 2 or vectors.shape[0] != expected_count: + raise ValueError( + "Embedding provider returned an unexpected number or shape of vectors" + ) + dimensions = int(vectors.shape[1]) + if dimensions < 1: + raise ValueError("Embedding provider returned zero-dimensional vectors") + if expected_dimensions is not None and dimensions != expected_dimensions: + raise ValueError( + "Embedding dimension mismatch: provider returned " + f"{dimensions}, expected {expected_dimensions}" + ) + if not np.isfinite(vectors).all(): + raise ValueError("Embedding provider returned non-finite values") + if np.any(np.linalg.norm(vectors, axis=1) == 0): + raise ValueError("Embedding provider returned a zero vector") + return np.ascontiguousarray(vectors, dtype=np.float32) + + +def _decode_stored_vector( + blob: bytes, dimension: int, target_id: str +) -> NDArray[np.float32]: + """Decode and validate a persisted vector as corrupt-index protection.""" + if dimension < 1 or len(blob) != dimension * np.dtype(" None: + """Create the V1 schema or reject an incompatible local database.""" + version_row = db.execute("PRAGMA user_version").fetchone() + version = int(version_row[0]) + if version not in (0, _DATABASE_SCHEMA_VERSION): + raise RuntimeError( + "Stale knowledge library schema: database version " + f"{version}, expected {_DATABASE_SCHEMA_VERSION}" + ) + if version == _DATABASE_SCHEMA_VERSION: + return + db.executescript( + f""" + CREATE TABLE knowledge_items ( + item_id TEXT PRIMARY KEY, + knowledge_class TEXT NOT NULL, + item_type TEXT NOT NULL, + schema_version TEXT NOT NULL, + canonical_json TEXT NOT NULL, + canonical_hash TEXT NOT NULL, + target_count INTEGER NOT NULL CHECK (target_count > 0) + ); + + CREATE TABLE semantic_records ( + target_id TEXT PRIMARY KEY, + item_id TEXT NOT NULL, + node_id TEXT, + item_type TEXT NOT NULL, + matched_text TEXT NOT NULL, + as_of REAL NOT NULL, + available_at REAL, + source_kind TEXT NOT NULL, + confidence TEXT NOT NULL, + tags_json TEXT NOT NULL, + tree_id TEXT, + embedding_model TEXT NOT NULL, + dimension INTEGER NOT NULL, + projection_hash TEXT NOT NULL, + source_content_hash TEXT, + knowledge_schema_version TEXT NOT NULL, + projection_schema_version TEXT NOT NULL, + item_canonical_hash TEXT NOT NULL, + embedding BLOB NOT NULL, + FOREIGN KEY (item_id) REFERENCES knowledge_items(item_id) + ON DELETE CASCADE + ); + + CREATE INDEX semantic_records_item_id + ON semantic_records(item_id); + CREATE INDEX semantic_records_filters + ON semantic_records(item_type, source_kind, confidence, tree_id); + + PRAGMA user_version = {_DATABASE_SCHEMA_VERSION}; + """ + ) + + +class LocalKnowledgeLibrary: + """Persist and semantically search canonical QuantMind knowledge locally.""" + + def __init__( + self, + db: sqlite3.Connection, + *, + embedding_model: str, + embedding_dimensions: int | None, + embedding_provider: _EmbeddingProvider, + ) -> None: + self._db: sqlite3.Connection | None = db + self._embedding_model = embedding_model + self._embedding_dimensions = embedding_dimensions + self._embedding_provider = embedding_provider + self._lock = asyncio.Lock() + self._index_records: list[_IndexRecord] | None = None + self._index_matrix: NDArray[np.float32] | None = None + + @classmethod + async def open( + cls, + path: str | Path, + *, + embedding_model: str, + embedding_dimensions: int | None = None, + _embedding_provider: _EmbeddingProvider | None = None, + ) -> Self: + """Open a local library without performing network I/O. + + Args: + path: SQLite database path or ``":memory:"``. + embedding_model: Provider model identity recorded with every vector. + embedding_dimensions: Optional requested vector dimension. + + Returns: + An open local library. + """ + if not embedding_model.strip(): + raise ValueError("embedding_model must not be blank") + if embedding_dimensions is not None and embedding_dimensions < 1: + raise ValueError("embedding_dimensions must be positive") + supplied_path = str(path) + database_path = ( + supplied_path + if supplied_path == ":memory:" + else str(Path(supplied_path).expanduser()) + ) + if database_path != ":memory:": + Path(database_path).parent.mkdir(parents=True, exist_ok=True) + db: sqlite3.Connection | None = None + try: + db = sqlite3.connect( + database_path, + isolation_level=None, + ) + db.row_factory = sqlite3.Row + db.execute("PRAGMA foreign_keys = ON") + db.execute("PRAGMA busy_timeout = 5000") + _initialize_schema(db) + except sqlite3.DatabaseError as exc: + if db is not None: + db.close() + raise RuntimeError( + f"Corrupt knowledge library database at '{database_path}'" + ) from exc + except Exception: + if db is not None: + db.close() + raise + provider = _embedding_provider or _OpenAIEmbeddingProvider() + return cls( + db, + embedding_model=embedding_model, + embedding_dimensions=embedding_dimensions, + embedding_provider=provider, + ) + + async def put(self, item: BaseKnowledge) -> None: + """Atomically persist canonical knowledge and its affected projections.""" + async with self._lock: + if self._db is None: + raise RuntimeError("LocalKnowledgeLibrary is closed") + knowledge_class, payload, canonical_hash = _canonical_payload(item) + projections = _project_knowledge(item) + as_of = _timestamp(item.as_of, "BaseKnowledge.as_of") + available_at = ( + _timestamp(item.available_at, "BaseKnowledge.available_at") + if item.available_at is not None + else None + ) + tags_json = json.dumps( + item.tags, + ensure_ascii=False, + separators=(",", ":"), + ) + existing_rows = self._db.execute( + "SELECT * FROM semantic_records WHERE item_id = ?", + (str(item.id),), + ).fetchall() + existing = {str(row["target_id"]): row for row in existing_rows} + source_content_hash = item.source.content_hash + + affected: list[_Projection] = [] + retained: dict[str, tuple[bytes, int]] = {} + for projection in projections: + row = existing.get(projection.target_id) + needs_embedding = row is None + if row is not None: + needs_embedding = any( + ( + str(row["embedding_model"]) + != self._embedding_model, + str(row["projection_hash"]) + != projection.projection_hash, + row["source_content_hash"] != source_content_hash, + str(row["knowledge_schema_version"]) + != item.schema_version, + str(row["projection_schema_version"]) + != _PROJECTION_SCHEMA_VERSION, + self._embedding_dimensions is not None + and int(row["dimension"]) + != self._embedding_dimensions, + ) + ) + if needs_embedding: + affected.append(projection) + else: + assert row is not None + blob = bytes(row["embedding"]) + dimension = int(row["dimension"]) + _decode_stored_vector( + blob, + dimension, + projection.target_id, + ) + retained[projection.target_id] = (blob, dimension) + + generated: dict[str, tuple[bytes, int]] = {} + if affected: + provider_values = await self._embedding_provider.embed( + [projection.text for projection in affected], + model=self._embedding_model, + dimensions=self._embedding_dimensions, + ) + vectors = _coerce_provider_vectors( + provider_values, + expected_count=len(affected), + expected_dimensions=self._embedding_dimensions, + ) + for projection, vector in zip(affected, vectors, strict=True): + stored = np.asarray(vector, dtype=" BaseKnowledge: + """Return validated canonical knowledge or report not-found/stale data.""" + async with self._lock: + if self._db is None: + raise RuntimeError("LocalKnowledgeLibrary is closed") + row = self._db.execute( + "SELECT * FROM knowledge_items WHERE item_id = ?", + (str(item_id),), + ).fetchone() + if row is None: + derived_count = int( + self._db.execute( + "SELECT COUNT(*) FROM semantic_records WHERE item_id = ?", + (str(item_id),), + ).fetchone()[0] + ) + if derived_count: + raise RuntimeError( + f"Stale index data for item '{item_id}': " + "derived records exist without canonical knowledge" + ) + raise KeyError(f"Knowledge item '{item_id}' not found") + return _load_canonical( + item_id=str(row["item_id"]), + knowledge_class=str(row["knowledge_class"]), + item_type=str(row["item_type"]), + schema_version=str(row["schema_version"]), + payload=str(row["canonical_json"]), + canonical_hash=str(row["canonical_hash"]), + ) + + async def search(self, query: SemanticQuery) -> list[SemanticHit]: + """Rank filtered projections with deterministic exact cosine similarity.""" + async with self._lock: + if self._db is None: + raise RuntimeError("LocalKnowledgeLibrary is closed") + if self._index_records is None or self._index_matrix is None: + self._rebuild_numpy_index() + assert self._index_records is not None + assert self._index_matrix is not None + index_records = self._index_records + index_matrix = self._index_matrix + + as_of_before = ( + _timestamp(query.as_of_before, "SemanticQuery.as_of_before") + if query.as_of_before is not None + else None + ) + available_at_before = ( + _timestamp( + query.available_at_before, + "SemanticQuery.available_at_before", + ) + if query.available_at_before is not None + else None + ) + item_types = set(query.item_types) if query.item_types else None + source_kinds = ( + set(query.source_kinds) if query.source_kinds else None + ) + required_tags = set(query.tags) if query.tags else None + candidates: list[int] = [] + for index, record in enumerate(index_records): + if ( + item_types is not None + and record.item_type not in item_types + ): + continue + if ( + source_kinds is not None + and record.source_kind not in source_kinds + ): + continue + if ( + query.confidence is not None + and record.confidence != query.confidence + ): + continue + if required_tags is not None and not required_tags.issubset( + record.tags + ): + continue + if ( + query.tree_id is not None + and record.tree_id != query.tree_id + ): + continue + if as_of_before is not None and record.as_of > as_of_before: + continue + if available_at_before is not None and ( + record.available_at is None + or record.available_at > available_at_before + ): + continue + candidates.append(index) + + if not candidates: + return [] + provider_values = await self._embedding_provider.embed( + [query.text], + model=self._embedding_model, + dimensions=self._embedding_dimensions, + ) + query_vectors = _coerce_provider_vectors( + provider_values, + expected_count=1, + expected_dimensions=self._embedding_dimensions, + ) + query_vector = query_vectors[0] + if query_vector.shape[0] != index_matrix.shape[1]: + raise ValueError( + "Embedding dimension mismatch: query vector has " + f"{query_vector.shape[0]} dimensions but the index has " + f"{index_matrix.shape[1]}" + ) + query_vector = query_vector / np.linalg.norm(query_vector) + scores = index_matrix[candidates] @ query_vector + ranked = sorted( + zip(candidates, scores, strict=True), + key=lambda pair: ( + -float(pair[1]), + index_records[pair[0]].target_id, + ), + )[: query.top_k] + + canonical: dict[UUID, BaseKnowledge] = {} + hits: list[SemanticHit] = [] + for index, score in ranked: + record = index_records[index] + item = canonical.get(record.item_id) + if item is None: + row = self._db.execute( + "SELECT * FROM knowledge_items WHERE item_id = ?", + (str(record.item_id),), + ).fetchone() + if row is None: + raise RuntimeError( + f"Stale index data for item '{record.item_id}': " + "canonical knowledge is missing" + ) + item = _load_canonical( + item_id=str(row["item_id"]), + knowledge_class=str(row["knowledge_class"]), + item_type=str(row["item_type"]), + schema_version=str(row["schema_version"]), + payload=str(row["canonical_json"]), + canonical_hash=str(row["canonical_hash"]), + ) + canonical[record.item_id] = item + if isinstance(item, TreeKnowledge): + if record.node_id is None: + citations = item.root().citations + else: + node = item.nodes.get(record.node_id) + if node is None: + raise RuntimeError( + f"Stale index data for target " + f"'{record.target_id}': canonical node is missing" + ) + citations = node.citations + else: + if record.node_id is not None: + raise RuntimeError( + f"Stale index data for target '{record.target_id}': " + "node target belongs to non-tree knowledge" + ) + citations = item.citations + hits.append( + SemanticHit( + item_id=record.item_id, + node_id=record.node_id, + item_type=item.item_type, + score=float(score), + matched_text=record.matched_text, + as_of=item.as_of, + available_at=item.available_at, + source=item.source, + citations=citations, + ) + ) + return hits + + async def delete(self, item_id: UUID) -> None: + """Transactionally remove canonical knowledge and every derived target.""" + async with self._lock: + if self._db is None: + raise RuntimeError("LocalKnowledgeLibrary is closed") + exists = self._db.execute( + "SELECT 1 FROM knowledge_items WHERE item_id = ?", + (str(item_id),), + ).fetchone() + if exists is None: + derived_count = int( + self._db.execute( + "SELECT COUNT(*) FROM semantic_records WHERE item_id = ?", + (str(item_id),), + ).fetchone()[0] + ) + if derived_count: + raise RuntimeError( + f"Stale index data for item '{item_id}': " + "cannot delete derived records without canonical knowledge" + ) + raise KeyError(f"Knowledge item '{item_id}' not found") + try: + self._db.execute("BEGIN IMMEDIATE") + self._db.execute( + "DELETE FROM semantic_records WHERE item_id = ?", + (str(item_id),), + ) + self._db.execute( + "DELETE FROM knowledge_items WHERE item_id = ?", + (str(item_id),), + ) + self._db.execute("COMMIT") + except Exception: + if self._db.in_transaction: + self._db.execute("ROLLBACK") + raise + self._index_records = None + self._index_matrix = None + + async def close(self) -> None: + """Close SQLite and provider-owned resources; repeated calls are safe.""" + async with self._lock: + if self._db is None: + return + db = self._db + self._db = None + self._index_records = None + self._index_matrix = None + try: + await self._embedding_provider.close() + finally: + db.close() + + def _rebuild_numpy_index(self) -> None: + """Validate durable records and rebuild the exact-cosine matrix.""" + assert self._db is not None + orphan = self._db.execute( + """ + SELECT r.item_id + FROM semantic_records AS r + LEFT JOIN knowledge_items AS i ON i.item_id = r.item_id + WHERE i.item_id IS NULL + LIMIT 1 + """ + ).fetchone() + if orphan is not None: + raise RuntimeError( + f"Stale index data for item '{orphan['item_id']}': " + "derived records exist without canonical knowledge" + ) + incomplete = self._db.execute( + """ + SELECT i.item_id, i.target_count, COUNT(r.target_id) AS actual_count + FROM knowledge_items AS i + LEFT JOIN semantic_records AS r ON r.item_id = i.item_id + GROUP BY i.item_id, i.target_count + HAVING COUNT(r.target_id) != i.target_count + LIMIT 1 + """ + ).fetchone() + if incomplete is not None: + raise RuntimeError( + f"Stale index data for item '{incomplete['item_id']}': expected " + f"{incomplete['target_count']} targets, found " + f"{incomplete['actual_count']}" + ) + + rows = self._db.execute( + """ + SELECT r.*, i.canonical_hash AS current_canonical_hash, + i.schema_version AS current_schema_version + FROM semantic_records AS r + JOIN knowledge_items AS i ON i.item_id = r.item_id + ORDER BY r.target_id + """ + ).fetchall() + records: list[_IndexRecord] = [] + vectors: list[NDArray[np.float32]] = [] + index_dimension: int | None = None + for row in rows: + target_id = str(row["target_id"]) + dimension = int(row["dimension"]) + if str(row["embedding_model"]) != self._embedding_model: + raise RuntimeError( + f"Stale index data for target '{target_id}': embedding model " + "changed; re-put the canonical item" + ) + if ( + self._embedding_dimensions is not None + and dimension != self._embedding_dimensions + ): + raise RuntimeError( + f"Stale index data for target '{target_id}': embedding " + "dimension changed; re-put the canonical item" + ) + if ( + str(row["projection_schema_version"]) + != _PROJECTION_SCHEMA_VERSION + or str(row["knowledge_schema_version"]) + != str(row["current_schema_version"]) + or str(row["item_canonical_hash"]) + != str(row["current_canonical_hash"]) + ): + raise RuntimeError( + f"Stale index data for target '{target_id}': projection or " + "canonical metadata changed; re-put the canonical item" + ) + if index_dimension is None: + index_dimension = dimension + elif dimension != index_dimension: + raise RuntimeError( + "Corrupt index data: stored targets have inconsistent " + "embedding dimensions" + ) + vector = _decode_stored_vector( + bytes(row["embedding"]), dimension, target_id + ) + try: + parsed_tags = json.loads(str(row["tags_json"])) + except json.JSONDecodeError as exc: + raise RuntimeError( + f"Corrupt index data for target '{target_id}': invalid tags" + ) from exc + if not isinstance(parsed_tags, list) or not all( + isinstance(tag, str) for tag in parsed_tags + ): + raise RuntimeError( + f"Corrupt index data for target '{target_id}': invalid tags" + ) + node_value = row["node_id"] + tree_value = row["tree_id"] + try: + record = _IndexRecord( + target_id=target_id, + item_id=UUID(str(row["item_id"])), + node_id=UUID(str(node_value)) if node_value else None, + item_type=str(row["item_type"]), + matched_text=str(row["matched_text"]), + as_of=float(row["as_of"]), + available_at=( + float(row["available_at"]) + if row["available_at"] is not None + else None + ), + source_kind=str(row["source_kind"]), + confidence=str(row["confidence"]), + tags=frozenset(parsed_tags), + tree_id=UUID(str(tree_value)) if tree_value else None, + ) + except (TypeError, ValueError) as exc: + raise RuntimeError( + f"Corrupt index data for target '{target_id}': invalid metadata" + ) from exc + records.append(record) + vectors.append( + np.asarray(vector / np.linalg.norm(vector), dtype=np.float32) + ) + + self._index_records = records + if vectors: + self._index_matrix = np.ascontiguousarray( + np.vstack(vectors), dtype=np.float32 + ) + else: + self._index_matrix = np.empty((0, 0), dtype=np.float32) diff --git a/tests/knowledge/test_base.py b/tests/knowledge/test_base.py index eae7380..f1fc840 100644 --- a/tests/knowledge/test_base.py +++ b/tests/knowledge/test_base.py @@ -102,6 +102,19 @@ def test_default_confidence_is_medium(self): item = _ConcreteKnowledge(as_of=_now(), source=_src()) self.assertEqual(item.confidence, "medium") + def test_available_at_is_optional_and_round_trips(self): + item = _ConcreteKnowledge( + as_of=_now(), + available_at=_now() + timedelta(hours=2), + source=_src(), + ) + revived = _ConcreteKnowledge.model_validate_json(item.model_dump_json()) + self.assertEqual(revived.available_at, _now() + timedelta(hours=2)) + + def test_available_at_defaults_to_unknown(self): + item = _ConcreteKnowledge(as_of=_now(), source=_src()) + self.assertIsNone(item.available_at) + def test_default_schema_version(self): item = _ConcreteKnowledge(as_of=_now(), source=_src()) self.assertEqual(item.schema_version, "1.0") diff --git a/tests/library/__init__.py b/tests/library/__init__.py new file mode 100644 index 0000000..56155a9 --- /dev/null +++ b/tests/library/__init__.py @@ -0,0 +1 @@ +"""Tests for the local semantic knowledge library.""" diff --git a/tests/library/test_local.py b/tests/library/test_local.py new file mode 100644 index 0000000..1cdd065 --- /dev/null +++ b/tests/library/test_local.py @@ -0,0 +1,607 @@ +import sqlite3 +import tempfile +import unittest +from collections.abc import Sequence +from datetime import datetime, timezone +from pathlib import Path +from uuid import UUID, uuid4 + +from quantmind.knowledge import Citation, News, Paper, SourceRef, TreeNode +from quantmind.library import LocalKnowledgeLibrary, SemanticQuery + + +class _FakeEmbeddingProvider: + def __init__( + self, + *, + dimension: int = 2, + vectors: dict[str, list[float]] | None = None, + ) -> None: + self.dimension = dimension + self.vectors = vectors or {} + self.calls: list[tuple[str, int | None, tuple[str, ...]]] = [] + self.closed = False + + async def embed( + self, + texts: Sequence[str], + *, + model: str, + dimensions: int | None, + ) -> list[list[float]]: + self.calls.append((model, dimensions, tuple(texts))) + output: list[list[float]] = [] + for text in texts: + configured = self.vectors.get(text) + if configured is not None: + output.append(configured) + continue + size = dimensions or self.dimension + seed = sum( + (index + 1) * ord(char) for index, char in enumerate(text) + ) + output.append( + [ + float(1 + ((seed >> (offset * 3)) % 17)) + for offset in range(size) + ] + ) + return output + + async def close(self) -> None: + self.closed = True + + +def _time(day: int, hour: int = 0) -> datetime: + return datetime(2026, 7, day, hour, tzinfo=timezone.utc) + + +def _news( + headline: str, + *, + item_id: UUID | None = None, + as_of: datetime | None = None, + available_at: datetime | None = None, + source_kind: str = "rss", + source_hash: str | None = None, + confidence: str = "medium", + tags: list[str] | None = None, + citations: list[Citation] | None = None, +) -> News: + return News( + id=item_id or uuid4(), + as_of=as_of or _time(1), + available_at=available_at, + source=SourceRef( + kind=source_kind, # type: ignore[arg-type] + uri=f"https://example.com/{headline}", + content_hash=source_hash, + ), + confidence=confidence, # type: ignore[arg-type] + citations=citations or [], + tags=tags or [], + headline=headline, + event_type="capital_expenditure", + entities=["Alpha Corp"], + timestamp=as_of or _time(1), + ) + + +def _paper() -> tuple[Paper, UUID, UUID]: + root_id = uuid4() + methods_id = uuid4() + results_id = uuid4() + root = TreeNode( + node_id=root_id, + title="Capital Spending Outlook", + summary="Company-wide investment outlook", + citations=[Citation(source_id="paper", page=1, node_id=root_id)], + children_ids=[methods_id, results_id], + ) + methods = TreeNode( + node_id=methods_id, + parent_id=root_id, + title="Methodology", + summary="Survey of management guidance", + citations=[Citation(source_id="paper", page=2, node_id=methods_id)], + ) + results = TreeNode( + node_id=results_id, + parent_id=root_id, + position=1, + title="Results", + summary="Capital expenditure is expected to rise", + ) + paper = Paper( + as_of=_time(1), + available_at=_time(2), + source=SourceRef(kind="arxiv", content_hash="paper-v1"), + confidence="high", + tags=["macro", "rates", "capex"], + citations=[Citation(source_id="paper", page=1)], + root_node_id=root_id, + nodes={root_id: root, methods_id: methods, results_id: results}, + arxiv_id="2607.00111", + ) + return paper, methods_id, results_id + + +class LocalKnowledgeLibraryTests(unittest.IsolatedAsyncioTestCase): + def setUp(self) -> None: + self._temporary_directory = tempfile.TemporaryDirectory() + self.db_path = Path(self._temporary_directory.name) / "library.db" + + def tearDown(self) -> None: + self._temporary_directory.cleanup() + + async def test_flat_put_get_and_deterministic_best_first_search(self): + alpha = _news( + "Alpha expands capacity", + available_at=_time(2), + tags=["capex"], + citations=[Citation(source_id="rss:alpha", quote="capacity rises")], + ) + beta = _news("Beta pauses investment", available_at=_time(2)) + query_text = "management increases capital expenditure" + provider = _FakeEmbeddingProvider( + vectors={ + alpha.embedding_text(): [1.0, 0.0], + beta.embedding_text(): [0.0, 1.0], + query_text: [1.0, 0.0], + } + ) + library = await LocalKnowledgeLibrary.open( + self.db_path, + embedding_model="fake-2d", + embedding_dimensions=2, + _embedding_provider=provider, + ) + try: + await library.put(beta) + await library.put(alpha) + hits = await library.search(SemanticQuery(text=query_text, top_k=2)) + self.assertEqual([hit.item_id for hit in hits], [alpha.id, beta.id]) + self.assertAlmostEqual(hits[0].score, 1.0) + self.assertIsNone(hits[0].node_id) + self.assertEqual(hits[0].matched_text, alpha.embedding_text()) + self.assertEqual(hits[0].source, alpha.source) + self.assertEqual(hits[0].citations, alpha.citations) + self.assertEqual(hits[0].available_at, alpha.available_at) + self.assertEqual(await library.get(alpha.id), alpha) + finally: + await library.close() + + async def test_tree_root_and_non_root_nodes_use_exact_grain_and_filters( + self, + ): + paper, methods_id, results_id = _paper() + unrelated = _news("Unrelated event", tags=["macro", "rates"]) + query_text = "capital expenditure outlook" + provider = _FakeEmbeddingProvider( + vectors={ + paper.embedding_text(): [1.0, 0.0], + paper.nodes[methods_id].embedding_text(): [0.8, 0.2], + paper.nodes[results_id].embedding_text(): [0.9, 0.1], + unrelated.embedding_text(): [0.0, 1.0], + query_text: [1.0, 0.0], + } + ) + library = await LocalKnowledgeLibrary.open( + self.db_path, + embedding_model="fake-2d", + embedding_dimensions=2, + _embedding_provider=provider, + ) + try: + await library.put(unrelated) + await library.put(paper) + hits = await library.search( + SemanticQuery( + text=query_text, + item_types=["paper"], + source_kinds=["arxiv"], + confidence="high", + tags=["macro", "rates"], + tree_id=paper.id, + as_of_before=_time(1), + available_at_before=_time(2), + top_k=10, + ) + ) + self.assertEqual(len(hits), 3) + self.assertEqual(sum(hit.node_id is None for hit in hits), 1) + self.assertEqual( + {hit.node_id for hit in hits if hit.node_id is not None}, + {methods_id, results_id}, + ) + root_hit = next(hit for hit in hits if hit.node_id is None) + self.assertEqual(root_hit.matched_text, paper.embedding_text()) + self.assertEqual(root_hit.citations, paper.root().citations) + methods_hit = next(hit for hit in hits if hit.node_id == methods_id) + self.assertEqual( + methods_hit.matched_text, + paper.nodes[methods_id].embedding_text(), + ) + self.assertEqual( + methods_hit.citations, + paper.nodes[methods_id].citations, + ) + stored = await library.get(methods_hit.item_id) + self.assertIsInstance(stored, Paper) + assert isinstance(stored, Paper) + self.assertEqual( + stored.find_path(methods_id)[-1].node_id, methods_id + ) + finally: + await library.close() + + async def test_as_of_and_availability_cutoffs_are_distinct(self): + safe = _news( + "Known after cutoff", + as_of=_time(1), + available_at=_time(3), + ) + unknown = _news("Unknown availability", as_of=_time(1)) + later_information = _news( + "Later information available early", + as_of=_time(5), + available_at=_time(2), + ) + provider = _FakeEmbeddingProvider() + library = await LocalKnowledgeLibrary.open( + self.db_path, + embedding_model="fake-2d", + embedding_dimensions=2, + _embedding_provider=provider, + ) + try: + for item in (safe, unknown, later_information): + await library.put(item) + as_of_hits = await library.search( + SemanticQuery(text="cutoff", as_of_before=_time(2), top_k=10) + ) + self.assertEqual( + {hit.item_id for hit in as_of_hits}, + {safe.id, unknown.id}, + ) + availability_hits = await library.search( + SemanticQuery( + text="cutoff", + available_at_before=_time(2, 12), + top_k=10, + ) + ) + self.assertEqual( + {hit.item_id for hit in availability_hits}, + {later_information.id}, + ) + combined = await library.search( + SemanticQuery( + text="cutoff", + as_of_before=_time(2), + available_at_before=_time(4), + top_k=10, + ) + ) + self.assertEqual([hit.item_id for hit in combined], [safe.id]) + finally: + await library.close() + + async def test_reput_is_idempotent_and_invalidates_only_changed_metadata( + self, + ): + item = _news( + "Original headline", + source_hash="source-v1", + tags=["old"], + ) + provider = _FakeEmbeddingProvider() + library = await LocalKnowledgeLibrary.open( + self.db_path, + embedding_model="fake-2d", + embedding_dimensions=2, + _embedding_provider=provider, + ) + try: + await library.put(item) + self.assertEqual(len(provider.calls), 1) + await library.put(item) + self.assertEqual(len(provider.calls), 1) + + metadata_only = item.model_copy(update={"tags": ["new"]}) + await library.put(metadata_only) + self.assertEqual(len(provider.calls), 1) + + changed_text = metadata_only.model_copy( + update={"headline": "Changed headline"} + ) + await library.put(changed_text) + self.assertEqual( + provider.calls[-1][2], (changed_text.embedding_text(),) + ) + + changed_source = changed_text.model_copy( + update={ + "source": changed_text.source.model_copy( + update={"content_hash": "source-v2"} + ) + } + ) + await library.put(changed_source) + self.assertEqual(len(provider.calls), 3) + + changed_schema = changed_source.model_copy( + update={"schema_version": "1.1"} + ) + await library.put(changed_schema) + self.assertEqual(len(provider.calls), 4) + finally: + await library.close() + + async def test_tree_projection_change_invalidates_only_one_node(self): + paper, methods_id, _ = _paper() + provider = _FakeEmbeddingProvider() + library = await LocalKnowledgeLibrary.open( + self.db_path, + embedding_model="fake-2d", + embedding_dimensions=2, + _embedding_provider=provider, + ) + try: + await library.put(paper) + self.assertEqual(len(provider.calls[0][2]), 3) + changed_node = paper.nodes[methods_id].model_copy( + update={"summary": "A newly described survey method"} + ) + changed_nodes = dict(paper.nodes) + changed_nodes[methods_id] = changed_node + changed_paper = paper.model_copy(update={"nodes": changed_nodes}) + await library.put(changed_paper) + self.assertEqual( + provider.calls[-1][2], (changed_node.embedding_text(),) + ) + + schema_change = changed_paper.model_copy( + update={"schema_version": "1.1"} + ) + await library.put(schema_change) + self.assertEqual(len(provider.calls[-1][2]), 3) + finally: + await library.close() + + async def test_model_and_dimension_changes_have_explicit_stale_rebuild_path( + self, + ): + item = _news("Model metadata") + first = _FakeEmbeddingProvider(dimension=2) + library = await LocalKnowledgeLibrary.open( + self.db_path, + embedding_model="model-a", + embedding_dimensions=2, + _embedding_provider=first, + ) + await library.put(item) + await library.close() + + second = _FakeEmbeddingProvider(dimension=2) + library = await LocalKnowledgeLibrary.open( + self.db_path, + embedding_model="model-b", + embedding_dimensions=2, + _embedding_provider=second, + ) + with self.assertRaisesRegex(RuntimeError, "Stale index data.*model"): + await library.search(SemanticQuery(text="query")) + self.assertEqual(second.calls, []) + await library.put(item) + self.assertEqual(len(second.calls), 1) + await library.close() + + third = _FakeEmbeddingProvider(dimension=3) + library = await LocalKnowledgeLibrary.open( + self.db_path, + embedding_model="model-b", + embedding_dimensions=3, + _embedding_provider=third, + ) + try: + await library.put(item) + self.assertEqual(third.calls[0][1], 3) + hits = await library.search(SemanticQuery(text="query")) + self.assertEqual([hit.item_id for hit in hits], [item.id]) + finally: + await library.close() + + async def test_index_rebuilds_from_sqlite_without_reembedding_items(self): + item = _news("Persistent item", available_at=_time(2)) + first = _FakeEmbeddingProvider() + library = await LocalKnowledgeLibrary.open( + self.db_path, + embedding_model="fake-2d", + embedding_dimensions=2, + _embedding_provider=first, + ) + await library.put(item) + await library.close() + + second = _FakeEmbeddingProvider() + library = await LocalKnowledgeLibrary.open( + self.db_path, + embedding_model="fake-2d", + embedding_dimensions=2, + _embedding_provider=second, + ) + try: + self.assertEqual(await library.get(item.id), item) + self.assertEqual(second.calls, []) + hits = await library.search(SemanticQuery(text="persistent")) + self.assertEqual([hit.item_id for hit in hits], [item.id]) + self.assertEqual(second.calls[0][2], ("persistent",)) + finally: + await library.close() + + async def test_delete_removes_canonical_root_and_nodes_transactionally( + self, + ): + paper, _, _ = _paper() + provider = _FakeEmbeddingProvider() + library = await LocalKnowledgeLibrary.open( + self.db_path, + embedding_model="fake-2d", + embedding_dimensions=2, + _embedding_provider=provider, + ) + try: + await library.put(paper) + await library.delete(paper.id) + with self.assertRaises(KeyError): + await library.get(paper.id) + with self.assertRaises(KeyError): + await library.delete(paper.id) + calls_before_search = len(provider.calls) + self.assertEqual( + await library.search(SemanticQuery(text="query")), [] + ) + self.assertEqual(len(provider.calls), calls_before_search) + finally: + await library.close() + with sqlite3.connect(self.db_path) as db: + self.assertEqual( + db.execute("SELECT COUNT(*) FROM knowledge_items").fetchone()[ + 0 + ], + 0, + ) + self.assertEqual( + db.execute("SELECT COUNT(*) FROM semantic_records").fetchone()[ + 0 + ], + 0, + ) + + async def test_stale_canonical_get_fails_but_delete_can_recover(self): + item = _news("Stale canonical") + provider = _FakeEmbeddingProvider() + library = await LocalKnowledgeLibrary.open( + self.db_path, + embedding_model="fake-2d", + embedding_dimensions=2, + _embedding_provider=provider, + ) + try: + await library.put(item) + with sqlite3.connect(self.db_path) as db: + db.execute( + "UPDATE knowledge_items SET canonical_json = '{}' " + "WHERE item_id = ?", + (str(item.id),), + ) + with self.assertRaisesRegex( + RuntimeError, "Stale canonical knowledge" + ): + await library.get(item.id) + await library.delete(item.id) + with self.assertRaises(KeyError): + await library.get(item.id) + finally: + await library.close() + + async def test_orphan_and_missing_derived_data_are_reported_as_stale(self): + item = _news("Orphan target") + provider = _FakeEmbeddingProvider() + library = await LocalKnowledgeLibrary.open( + self.db_path, + embedding_model="fake-2d", + embedding_dimensions=2, + _embedding_provider=provider, + ) + await library.put(item) + await library.close() + with sqlite3.connect(self.db_path) as db: + db.execute("PRAGMA foreign_keys = OFF") + db.execute( + "DELETE FROM knowledge_items WHERE item_id = ?", (str(item.id),) + ) + + reopened_provider = _FakeEmbeddingProvider() + library = await LocalKnowledgeLibrary.open( + self.db_path, + embedding_model="fake-2d", + embedding_dimensions=2, + _embedding_provider=reopened_provider, + ) + try: + with self.assertRaisesRegex(RuntimeError, "Stale index data"): + await library.get(item.id) + with self.assertRaisesRegex(RuntimeError, "Stale index data"): + await library.delete(item.id) + with self.assertRaisesRegex(RuntimeError, "Stale index data"): + await library.search(SemanticQuery(text="query")) + finally: + await library.close() + + async def test_corrupt_vector_and_query_dimension_mismatch_fail_clearly( + self, + ): + item = _news("Corrupt vector") + provider = _FakeEmbeddingProvider() + library = await LocalKnowledgeLibrary.open( + self.db_path, + embedding_model="fake-2d", + embedding_dimensions=2, + _embedding_provider=provider, + ) + await library.put(item) + await library.close() + with sqlite3.connect(self.db_path) as db: + db.execute( + "UPDATE semantic_records SET embedding = ? WHERE item_id = ?", + (b"bad", str(item.id)), + ) + + library = await LocalKnowledgeLibrary.open( + self.db_path, + embedding_model="fake-2d", + embedding_dimensions=2, + _embedding_provider=_FakeEmbeddingProvider(), + ) + try: + with self.assertRaisesRegex(RuntimeError, "Corrupt index data"): + await library.search(SemanticQuery(text="query")) + finally: + await library.close() + + mismatch_path = Path(self._temporary_directory.name) / "mismatch.db" + library = await LocalKnowledgeLibrary.open( + mismatch_path, + embedding_model="fake", + embedding_dimensions=2, + _embedding_provider=_FakeEmbeddingProvider(dimension=2), + ) + await library.put(item) + await library.close() + library = await LocalKnowledgeLibrary.open( + mismatch_path, + embedding_model="fake", + _embedding_provider=_FakeEmbeddingProvider(dimension=3), + ) + try: + with self.assertRaisesRegex(ValueError, "dimension mismatch"): + await library.search(SemanticQuery(text="query")) + finally: + await library.close() + + async def test_closed_library_rejects_operations(self): + library = await LocalKnowledgeLibrary.open( + self.db_path, + embedding_model="fake-2d", + _embedding_provider=_FakeEmbeddingProvider(), + ) + await library.close() + await library.close() + with self.assertRaisesRegex(RuntimeError, "closed"): + await library.get(uuid4()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/library/test_types.py b/tests/library/test_types.py new file mode 100644 index 0000000..2476fb0 --- /dev/null +++ b/tests/library/test_types.py @@ -0,0 +1,34 @@ +import unittest +from datetime import datetime + +from pydantic import ValidationError + +import quantmind.library as library +from quantmind.library import SemanticQuery + + +class PublicTypeTests(unittest.TestCase): + def test_package_exports_only_domain_retrieval_types(self): + self.assertEqual( + set(library.__all__), + {"LocalKnowledgeLibrary", "SemanticQuery", "SemanticHit"}, + ) + + def test_query_rejects_blank_text(self): + with self.assertRaises(ValidationError): + SemanticQuery(text=" ") + + def test_query_rejects_naive_financial_cutoff(self): + with self.assertRaisesRegex(ValidationError, "timezone-aware"): + SemanticQuery( + text="rates", + available_at_before=datetime(2026, 7, 16), + ) + + def test_query_rejects_non_positive_top_k(self): + with self.assertRaises(ValidationError): + SemanticQuery(text="rates", top_k=0) + + +if __name__ == "__main__": + unittest.main() From 1661fbb988d1a6b4614eca185e63d8799ffb6f6e Mon Sep 17 00:00:00 2001 From: pkuwkl Date: Thu, 16 Jul 2026 16:08:02 +0800 Subject: [PATCH 2/5] feat(library): add real retrieval scenario --- docs/library.md | 90 ++++++-- examples/library/data/ai_infrastructure.json | 222 +++++++++++++++++++ examples/library/semantic_search.py | 121 ++++++++-- quantmind/library/_projection.py | 143 +++++++++++- quantmind/library/local.py | 197 +++++++++++++--- tests/library/test_example_bundle.py | 162 ++++++++++++++ tests/library/test_local.py | 57 ++++- 7 files changed, 900 insertions(+), 92 deletions(-) create mode 100644 examples/library/data/ai_infrastructure.json create mode 100644 tests/library/test_example_bundle.py diff --git a/docs/library.md b/docs/library.md index c5463e8..4ef86c8 100644 --- a/docs/library.md +++ b/docs/library.md @@ -12,8 +12,36 @@ answer. `embedding_text()` projection. - Each `TreeKnowledge` produces one item target for the root with `node_id=None`, plus one target for every non-root `TreeNode`. -- Canonical JSON is the source of truth. Embeddings and filter columns are - derived data and can be replaced by re-putting the canonical item. +- Canonical typed records are the source of truth. Embeddings and filter + columns are derived data and can be replaced by re-putting the item. + +## Local storage model + +The default local database is SQLite, with separate canonical and derived +concerns: + +- `knowledge_items` stores one aggregate root per `BaseKnowledge`. The type + discriminator and schema version select the concrete Pydantic model. +- `knowledge_nodes` stores every canonical `TreeNode` separately with its item, + parent, position, payload, and content hash. A large tree is not hidden in the + aggregate-root JSON row. +- `semantic_records` stores rebuildable item/root/node projections and vectors. + +Concrete types do not get tables such as `news`, `earnings`, or `papers`. +Creating one table per Pydantic subtype would duplicate common metadata and +require a database migration whenever the knowledge standard adds a type. The +aggregate-root plus normalized-node model preserves typed validation while +giving future tree navigation a stable node-level storage boundary. + +SQLite is the default because canonical knowledge needs transactions, foreign +keys, typed reconstruction, and explicit corrupt/stale/not-found behavior. A +vector database such as Chroma optimizes the derived similarity-search layer; +it does not replace those canonical-storage responsibilities. The current local +scale therefore uses a rebuildable NumPy exact-cosine index backed by +`semantic_records`, without adding another service or dependency. If corpus +size later requires an approximate or remote vector index, that derived layer +can change privately without changing `LocalKnowledgeLibrary`, the canonical +tables, or user code. The durable vector metadata records the embedding model and dimension, exact projection hash, source content hash, knowledge schema version, and projection @@ -42,25 +70,45 @@ filters before ranking. ## Usage -```python -from quantmind.library import LocalKnowledgeLibrary, SemanticQuery - -library = await LocalKnowledgeLibrary.open( - ".quantmind/library.db", - embedding_model="text-embedding-3-small", -) -try: - await library.put(paper) - hits = await library.search( - SemanticQuery( - text="management expects capital expenditure to increase", - available_at_before=research_cutoff, - top_k=10, - ) - ) - evidence = await library.get(hits[0].item_id) -finally: - await library.close() +The bundled AI-infrastructure scenario contains primary-source-backed `News`, +`Earnings`, and a real research `Paper` tree. Set an OpenAI key and run the +complete example: + +```bash +export OPENAI_API_KEY=... +python examples/library/semantic_search.py +``` + +The example: + +1. Validates the bundled JSON as canonical QuantMind models. +2. Persists flat knowledge and normalized tree nodes to SQLite. +3. Closes and reopens the library to demonstrate durable data. +4. Searches a concrete AI-infrastructure question with a no-look-ahead + availability cutoff. +5. Resolves every hit with `get()` and prints source, citation, financial time, + and the root-to-node path and content for tree evidence. + +The bundle's facts and short citations come directly from the +[Compute Trends Across Three Eras of Machine Learning paper](https://arxiv.org/abs/2202.05924), +[Microsoft's FY2025 AI-datacenter investment announcement](https://blogs.microsoft.com/on-the-issues/2025/01/03/the-golden-opportunity-for-american-ai/), +and [NVIDIA's FY2026 Q1 results](https://investor.nvidia.com/news/press-release-details/2025/NVIDIA-Announces-Financial-Results-for-First-Quarter-Fiscal-2026/default.aspx). + +Representative output has this shape; scores depend on the selected embedding +model: + +```text +Scenario: An investor tests whether long-run compute growth... +Persisted 3 knowledge items / 6 targets +Database: .quantmind/ai-infrastructure.db + +Query: What evidence shows demand for AI infrastructure is expanding? + +1. score=0.812 type=paper + path: ... > ... + matched: ... + source: https://... + citation: ... ``` `SemanticHit.matched_text` is the exact projection used in ranking. Use diff --git a/examples/library/data/ai_infrastructure.json b/examples/library/data/ai_infrastructure.json new file mode 100644 index 0000000..f68cad3 --- /dev/null +++ b/examples/library/data/ai_infrastructure.json @@ -0,0 +1,222 @@ +{ + "scenario": "An investor tests whether the long-run growth of machine-learning compute is translating into hyperscaler AI-datacenter investment and accelerator-vendor revenue, while respecting when each source became available.", + "items": [ + { + "id": "b25838d6-c3cc-4bf5-98c6-6f2aa686daf1", + "item_type": "paper", + "schema_version": "1.0", + "as_of": "2022-03-09T00:00:00+00:00", + "available_at": "2022-03-09T00:00:00+00:00", + "created_at": "2026-07-16T15:59:16+08:00", + "source": { + "kind": "arxiv", + "uri": "https://arxiv.org/abs/2202.05924", + "fetched_at": "2026-07-16T15:59:16+08:00" + }, + "confidence": "high", + "citations": [ + { + "source_id": "arxiv:2202.05924v2", + "page": 1, + "quote": "fast-growing compute requirements", + "tree_id": "b25838d6-c3cc-4bf5-98c6-6f2aa686daf1", + "node_id": "4d1ab280-b5a1-46ae-8139-d805038cafe3" + } + ], + "tags": [ + "ai-infrastructure", + "training-compute", + "scaling" + ], + "disclaimers": [ + "The paper measures historical training-compute trends; it does not forecast public-company revenue or investment returns." + ], + "root_node_id": "4d1ab280-b5a1-46ae-8139-d805038cafe3", + "nodes": { + "4d1ab280-b5a1-46ae-8139-d805038cafe3": { + "node_id": "4d1ab280-b5a1-46ae-8139-d805038cafe3", + "parent_id": null, + "position": 0, + "title": "Compute Trends Across Three Eras of Machine Learning", + "summary": "Sevilla and co-authors study the training compute of milestone machine-learning systems and identify pre-deep-learning, deep-learning, and large-scale eras, documenting a sharp acceleration in compute demand after 2010.", + "content": null, + "citations": [ + { + "source_id": "arxiv:2202.05924v2", + "page": 1, + "quote": "three distinct eras", + "tree_id": "b25838d6-c3cc-4bf5-98c6-6f2aa686daf1", + "node_id": "4d1ab280-b5a1-46ae-8139-d805038cafe3" + } + ], + "children_ids": [ + "3e9bc0d2-d270-4f52-8c15-d6b514d6d1ef", + "7a221d20-2a81-4790-9635-4c21a139f727", + "f99e3258-12ab-455c-b717-d89d2ba7cb71" + ] + }, + "3e9bc0d2-d270-4f52-8c15-d6b514d6d1ef": { + "node_id": "3e9bc0d2-d270-4f52-8c15-d6b514d6d1ef", + "parent_id": "4d1ab280-b5a1-46ae-8139-d805038cafe3", + "position": 0, + "title": "Dataset and method", + "summary": "The authors curate training-compute estimates for 123 milestone systems, separate models into historical eras, and fit log-linear trends with bootstrap confidence intervals.", + "content": "The dataset records estimated compute for final training runs. The study compares trends across time and model scale; its reported confidence intervals are based on 1,000 bootstrap samples.", + "citations": [ + { + "source_id": "arxiv:2202.05924v2", + "page": 1, + "quote": "123 milestone Machine Learning systems", + "tree_id": "b25838d6-c3cc-4bf5-98c6-6f2aa686daf1", + "node_id": "3e9bc0d2-d270-4f52-8c15-d6b514d6d1ef" + }, + { + "source_id": "arxiv:2202.05924v2", + "page": 16, + "quote": "bootstrap with B = 1000 samples", + "tree_id": "b25838d6-c3cc-4bf5-98c6-6f2aa686daf1", + "node_id": "3e9bc0d2-d270-4f52-8c15-d6b514d6d1ef" + } + ], + "children_ids": [] + }, + "7a221d20-2a81-4790-9635-4c21a139f727": { + "node_id": "7a221d20-2a81-4790-9635-4c21a139f727", + "parent_id": "4d1ab280-b5a1-46ae-8139-d805038cafe3", + "position": 1, + "title": "Measured compute trends", + "summary": "The paper estimates roughly 20-month doubling before 2010, about six-month doubling for regular-scale deep-learning systems after 2010, and a distinct large-scale trend emerging around 2015.", + "content": "Large-scale systems begin roughly two orders of magnitude above the contemporary regular-scale trend. Their estimated training compute doubles in about ten months, although the authors caution that the category and slope are uncertain.", + "citations": [ + { + "source_id": "arxiv:2202.05924v2", + "page": 3, + "quote": "5.7 months", + "tree_id": "b25838d6-c3cc-4bf5-98c6-6f2aa686daf1", + "node_id": "7a221d20-2a81-4790-9635-4c21a139f727" + }, + { + "source_id": "arxiv:2202.05924v2", + "page": 5, + "quote": "doubling every 9 to 10 months", + "tree_id": "b25838d6-c3cc-4bf5-98c6-6f2aa686daf1", + "node_id": "7a221d20-2a81-4790-9635-4c21a139f727" + } + ], + "children_ids": [] + }, + "f99e3258-12ab-455c-b717-d89d2ba7cb71": { + "node_id": "f99e3258-12ab-455c-b717-d89d2ba7cb71", + "parent_id": "4d1ab280-b5a1-46ae-8139-d805038cafe3", + "position": 2, + "title": "Interpretation limits", + "summary": "The historical association between frontier systems and higher compute budgets supports infrastructure-demand research, but it is not a direct forecast and the paper documents material estimation and selection uncertainty.", + "content": "Only the final training run is counted, while hyperparameter searches are often undisclosed. Corporate labs may also withhold large models, and the boundary between regular-scale and large-scale systems is partly judgmental.", + "citations": [ + { + "source_id": "arxiv:2202.05924v2", + "page": 24, + "quote": "sources of uncertainty and potential weaknesses", + "tree_id": "b25838d6-c3cc-4bf5-98c6-6f2aa686daf1", + "node_id": "f99e3258-12ab-455c-b717-d89d2ba7cb71" + } + ], + "children_ids": [] + } + }, + "arxiv_id": "2202.05924", + "authors": [ + "Jaime Sevilla", + "Lennart Heim", + "Anson Ho", + "Tamay Besiroglu", + "Marius Hobbhahn", + "Pablo Villalobos" + ], + "asset_classes": [ + "equities" + ] + }, + { + "id": "07c75b73-4d11-49f3-a40a-f5b4327e2ec1", + "item_type": "news", + "schema_version": "1.0", + "as_of": "2025-01-03T00:00:00+00:00", + "available_at": "2025-01-03T00:00:00+00:00", + "created_at": "2026-07-16T15:59:16+08:00", + "source": { + "kind": "http", + "uri": "https://blogs.microsoft.com/on-the-issues/2025/01/03/the-golden-opportunity-for-american-ai/", + "fetched_at": "2026-07-16T15:59:16+08:00" + }, + "confidence": "high", + "citations": [ + { + "source_id": "microsoft:on-the-issues:2025-01-03", + "quote": "approximately $80 billion" + } + ], + "tags": [ + "MSFT", + "ai-infrastructure", + "datacenters", + "capex" + ], + "disclaimers": [ + "The source states an investment plan, not completed spending.", + "The source provides a publication date without a time; timestamp and available_at are normalized to midnight UTC." + ], + "headline": "Microsoft plans approximately $80 billion of FY2025 AI-datacenter investment, with more than half in the United States", + "event_type": "capital_investment_plan", + "timestamp": "2025-01-03T00:00:00+00:00", + "entities": [ + "Microsoft", + "AI datacenters", + "United States" + ], + "sentiment": "positive", + "materiality": "high" + }, + { + "id": "9204ab37-62dc-43d6-9e06-290abe2d1ff9", + "item_type": "earnings", + "schema_version": "1.0", + "as_of": "2025-04-27T23:59:59-07:00", + "available_at": "2025-05-28T00:00:00-07:00", + "created_at": "2026-07-16T15:59:16+08:00", + "source": { + "kind": "http", + "uri": "https://investor.nvidia.com/news/press-release-details/2025/NVIDIA-Announces-Financial-Results-for-First-Quarter-Fiscal-2026/default.aspx", + "fetched_at": "2026-07-16T15:59:16+08:00" + }, + "confidence": "high", + "citations": [ + { + "source_id": "nvidia:earnings:fy2026-q1", + "quote": "Data Center revenue of $39.1 billion" + }, + { + "source_id": "nvidia:earnings:fy2026-q1", + "quote": "Revenue is expected to be $45.0 billion" + } + ], + "tags": [ + "NVDA", + "ai-infrastructure", + "accelerators", + "data-center" + ], + "disclaimers": [ + "Revenue is represented in US dollars and EPS is GAAP diluted EPS.", + "The source provides a release date without a time; available_at is normalized to midnight Pacific time.", + "Q2 FY2026 guidance is forward-looking and includes an expected H20 revenue loss from export controls." + ], + "ticker": "NVDA", + "period": "FY2026Q1", + "revenue": 44062000000.0, + "eps": 0.76, + "guidance": "NVIDIA expected Q2 FY2026 revenue of $45.0 billion, plus or minus 2%, including an approximately $8.0 billion H20 revenue loss attributed to export-control limitations.", + "surprise_flags": [] + } + ] +} diff --git a/examples/library/semantic_search.py b/examples/library/semantic_search.py index 607ccfc..e01752d 100644 --- a/examples/library/semantic_search.py +++ b/examples/library/semantic_search.py @@ -1,39 +1,114 @@ -"""Store and search one canonical knowledge item locally.""" +"""Run a persisted semantic search over bundled financial knowledge.""" import asyncio +import json +import os from datetime import datetime, timezone +from pathlib import Path -from quantmind.knowledge import News, SourceRef +from quantmind.knowledge import ( + BaseKnowledge, + Earnings, + News, + Paper, + TreeKnowledge, +) from quantmind.library import LocalKnowledgeLibrary, SemanticQuery +_BUNDLE_PATH = Path(__file__).parent / "data" / "ai_infrastructure.json" +_DEFAULT_LIBRARY_PATH = Path(".quantmind/ai-infrastructure.db") +_KNOWLEDGE_TYPES: dict[str, type[BaseKnowledge]] = { + "earnings": Earnings, + "news": News, + "paper": Paper, +} + + +def _load_bundle() -> tuple[str, list[BaseKnowledge]]: + """Validate bundled JSON as canonical QuantMind knowledge.""" + bundle = json.loads(_BUNDLE_PATH.read_text(encoding="utf-8")) + scenario = str(bundle["scenario"]) + items: list[BaseKnowledge] = [] + for payload in bundle["items"]: + item_type = str(payload["item_type"]) + knowledge_type = _KNOWLEDGE_TYPES.get(item_type) + if knowledge_type is None: + raise ValueError(f"Unsupported bundled item_type: {item_type}") + items.append(knowledge_type.model_validate(payload)) + return scenario, items + + +def _target_count(items: list[BaseKnowledge]) -> int: + """Count the exact item/root/node projections created by the library.""" + return sum( + len(item.nodes) if isinstance(item, TreeKnowledge) else 1 + for item in items + ) + async def main() -> None: - """Persist one news item and print matching semantic evidence.""" - published_at = datetime(2026, 7, 16, 8, tzinfo=timezone.utc) - item = News( - as_of=published_at, - available_at=published_at, - source=SourceRef(kind="rss", uri="https://example.com/fed-rates"), - headline="Federal Reserve holds rates steady", - event_type="monetary_policy", - entities=["Federal Reserve"], - tags=["macro", "rates"], + """Seed, reopen, search, and resolve bundled tree evidence.""" + if not os.getenv("OPENAI_API_KEY"): + raise SystemExit("Set OPENAI_API_KEY before running this example.") + + scenario, items = _load_bundle() + database_path = Path( + os.getenv("QUANTMIND_LIBRARY_PATH", str(_DEFAULT_LIBRARY_PATH)) + ) + embedding_model = os.getenv( + "QUANTMIND_EMBEDDING_MODEL", "text-embedding-3-small" ) + library = await LocalKnowledgeLibrary.open( - ".quantmind/library.db", - embedding_model="text-embedding-3-small", + database_path, + embedding_model=embedding_model, ) try: - await library.put(item) - hits = await library.search( - SemanticQuery( - text="central bank interest-rate decision", - available_at_before=datetime.now(timezone.utc), - top_k=3, - ) + for item in items: + await library.put(item) + finally: + await library.close() + + print(f"Scenario: {scenario}") + print( + f"Persisted {len(items)} knowledge items / {_target_count(items)} targets" + ) + print(f"Database: {database_path}\n") + + library = await LocalKnowledgeLibrary.open( + database_path, + embedding_model=embedding_model, + ) + try: + query = SemanticQuery( + text="What evidence shows demand for AI infrastructure is expanding?", + source_kinds=["http", "arxiv"], + tags=["ai-infrastructure"], + available_at_before=datetime(2026, 1, 1, tzinfo=timezone.utc), + top_k=5, ) - for hit in hits: - print(hit.score, hit.matched_text) + hits = await library.search(query) + print(f"Query: {query.text}\n") + for rank, hit in enumerate(hits, start=1): + item = await library.get(hit.item_id) + print(f"{rank}. score={hit.score:.3f} type={hit.item_type}") + if hit.node_id is not None and isinstance(item, TreeKnowledge): + path = " > ".join( + node.title for node in item.find_path(hit.node_id) + ) + print(f" path: {path}") + node = item.nodes[hit.node_id] + if node.content: + print(f" content: {node.content}") + print(f" matched: {hit.matched_text}") + print(f" source: {hit.source.uri}") + print( + f" as_of={hit.as_of.date()} available_at={hit.available_at}" + ) + for citation in hit.citations: + detail = f" — {citation.quote}" if citation.quote else "" + print(f" citation: {citation.source_id}{detail}") + print() finally: await library.close() diff --git a/quantmind/library/_projection.py b/quantmind/library/_projection.py index 3d578ab..c8c0ed2 100644 --- a/quantmind/library/_projection.py +++ b/quantmind/library/_projection.py @@ -2,6 +2,7 @@ import hashlib import json +from collections.abc import Sequence from dataclasses import dataclass from uuid import UUID @@ -45,22 +46,146 @@ class _Projection: tree_id: UUID | None -def _canonical_payload(item: BaseKnowledge) -> tuple[str, str, str]: - """Serialize a supported canonical item and return its stable hash.""" +@dataclass(frozen=True) +class _CanonicalNode: + """One normalized canonical tree node.""" + + node_id: UUID + parent_id: UUID | None + position: int + payload: str + content_hash: str + + +@dataclass(frozen=True) +class _CanonicalDocument: + """Canonical aggregate root plus separately persisted tree nodes.""" + + knowledge_class: str + item_shape: str + payload: str + canonical_hash: str + nodes: tuple[_CanonicalNode, ...] + + +def _json_payload(value: object) -> str: + """Encode canonical JSON with stable ordering and separators.""" + return json.dumps( + value, + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ) + + +def _canonical_payload(item: BaseKnowledge) -> _CanonicalDocument: + """Split a supported canonical aggregate into root and node records.""" knowledge_class = f"{type(item).__module__}:{type(item).__qualname__}" if knowledge_class not in _KNOWLEDGE_CLASSES: raise TypeError( f"Unsupported knowledge type '{type(item).__name__}'; " "only canonical quantmind.knowledge types can be persisted" ) - payload = json.dumps( - item.model_dump(mode="json"), - ensure_ascii=False, - separators=(",", ":"), - sort_keys=True, + full_payload = _json_payload(item.model_dump(mode="json")) + canonical_hash = hashlib.sha256(full_payload.encode("utf-8")).hexdigest() + if not isinstance(item, TreeKnowledge): + return _CanonicalDocument( + knowledge_class=knowledge_class, + item_shape="flat", + payload=full_payload, + canonical_hash=canonical_hash, + nodes=(), + ) + + nodes: list[_CanonicalNode] = [] + for node_id, node in sorted( + item.nodes.items(), key=lambda pair: str(pair[0]) + ): + node_payload = _json_payload(node.model_dump(mode="json")) + nodes.append( + _CanonicalNode( + node_id=node_id, + parent_id=node.parent_id, + position=node.position, + payload=node_payload, + content_hash=hashlib.sha256( + node_payload.encode("utf-8") + ).hexdigest(), + ) + ) + return _CanonicalDocument( + knowledge_class=knowledge_class, + item_shape="tree", + payload=_json_payload(item.model_dump(mode="json", exclude={"nodes"})), + canonical_hash=canonical_hash, + nodes=tuple(nodes), ) - content_hash = hashlib.sha256(payload.encode("utf-8")).hexdigest() - return knowledge_class, payload, content_hash + + +def _assemble_canonical_payload( + *, + item_id: str, + item_shape: str, + item_payload: str, + expected_node_count: int, + node_records: Sequence[tuple[str, str | None, int, str, str]], +) -> str: + """Rehydrate normalized tree nodes into the canonical Pydantic payload.""" + if item_shape == "flat": + if expected_node_count or node_records: + raise RuntimeError( + f"Stale canonical knowledge for item '{item_id}': " + "flat knowledge unexpectedly has tree nodes" + ) + return item_payload + if item_shape != "tree": + raise RuntimeError( + f"Stale canonical knowledge for item '{item_id}': " + f"unsupported item shape '{item_shape}'" + ) + if len(node_records) != expected_node_count: + raise RuntimeError( + f"Stale canonical knowledge for item '{item_id}': expected " + f"{expected_node_count} canonical nodes, found {len(node_records)}" + ) + try: + root_payload = json.loads(item_payload) + except json.JSONDecodeError as exc: + raise RuntimeError( + f"Stale canonical knowledge for item '{item_id}': invalid root JSON" + ) from exc + if not isinstance(root_payload, dict): + raise RuntimeError( + f"Stale canonical knowledge for item '{item_id}': invalid root payload" + ) + nodes: dict[str, object] = {} + for node_id, parent_id, position, node_payload, node_hash in node_records: + actual_hash = hashlib.sha256(node_payload.encode("utf-8")).hexdigest() + if actual_hash != node_hash: + raise RuntimeError( + f"Stale canonical knowledge for item '{item_id}': " + f"node '{node_id}' content hash mismatch" + ) + try: + parsed_node = json.loads(node_payload) + except json.JSONDecodeError as exc: + raise RuntimeError( + f"Stale canonical knowledge for item '{item_id}': " + f"node '{node_id}' contains invalid JSON" + ) from exc + if ( + not isinstance(parsed_node, dict) + or parsed_node.get("node_id") != node_id + or parsed_node.get("parent_id") != parent_id + or parsed_node.get("position") != position + ): + raise RuntimeError( + f"Stale canonical knowledge for item '{item_id}': " + f"node '{node_id}' metadata does not match its payload" + ) + nodes[node_id] = parsed_node + root_payload["nodes"] = nodes + return _json_payload(root_payload) def _load_canonical( diff --git a/quantmind/library/local.py b/quantmind/library/local.py index e1522cd..39f17e1 100644 --- a/quantmind/library/local.py +++ b/quantmind/library/local.py @@ -18,6 +18,7 @@ from quantmind.library._ports import _EmbeddingProvider from quantmind.library._projection import ( _PROJECTION_SCHEMA_VERSION, + _assemble_canonical_payload, _canonical_payload, _load_canonical, _project_knowledge, @@ -25,7 +26,7 @@ ) from quantmind.library._types import SemanticHit, SemanticQuery -_DATABASE_SCHEMA_VERSION = 1 +_DATABASE_SCHEMA_VERSION = 2 @dataclass(frozen=True) @@ -106,7 +107,7 @@ def _decode_stored_vector( def _initialize_schema(db: sqlite3.Connection) -> None: - """Create the V1 schema or reject an incompatible local database.""" + """Create the current schema or reject an incompatible local database.""" version_row = db.execute("PRAGMA user_version").fetchone() version = int(version_row[0]) if version not in (0, _DATABASE_SCHEMA_VERSION): @@ -122,12 +123,26 @@ def _initialize_schema(db: sqlite3.Connection) -> None: item_id TEXT PRIMARY KEY, knowledge_class TEXT NOT NULL, item_type TEXT NOT NULL, + item_shape TEXT NOT NULL CHECK (item_shape IN ('flat', 'tree')), schema_version TEXT NOT NULL, - canonical_json TEXT NOT NULL, + payload_json TEXT NOT NULL, canonical_hash TEXT NOT NULL, + node_count INTEGER NOT NULL CHECK (node_count >= 0), target_count INTEGER NOT NULL CHECK (target_count > 0) ); + CREATE TABLE knowledge_nodes ( + item_id TEXT NOT NULL, + node_id TEXT NOT NULL, + parent_id TEXT, + position INTEGER NOT NULL, + payload_json TEXT NOT NULL, + content_hash TEXT NOT NULL, + PRIMARY KEY (item_id, node_id), + FOREIGN KEY (item_id) REFERENCES knowledge_items(item_id) + ON DELETE CASCADE + ); + CREATE TABLE semantic_records ( target_id TEXT PRIMARY KEY, item_id TEXT NOT NULL, @@ -156,12 +171,58 @@ def _initialize_schema(db: sqlite3.Connection) -> None: ON semantic_records(item_id); CREATE INDEX semantic_records_filters ON semantic_records(item_type, source_kind, confidence, tree_id); + CREATE INDEX knowledge_nodes_parent + ON knowledge_nodes(item_id, parent_id, position); PRAGMA user_version = {_DATABASE_SCHEMA_VERSION}; """ ) +def _load_stored_canonical( + db: sqlite3.Connection, row: sqlite3.Row +) -> BaseKnowledge: + """Rehydrate and validate one canonical aggregate from normalized rows.""" + item_id = str(row["item_id"]) + node_rows = db.execute( + """ + SELECT node_id, parent_id, position, payload_json, content_hash + FROM knowledge_nodes + WHERE item_id = ? + ORDER BY node_id + """, + (item_id,), + ).fetchall() + payload = _assemble_canonical_payload( + item_id=item_id, + item_shape=str(row["item_shape"]), + item_payload=str(row["payload_json"]), + expected_node_count=int(row["node_count"]), + node_records=[ + ( + str(node_row["node_id"]), + ( + str(node_row["parent_id"]) + if node_row["parent_id"] is not None + else None + ), + int(node_row["position"]), + str(node_row["payload_json"]), + str(node_row["content_hash"]), + ) + for node_row in node_rows + ], + ) + return _load_canonical( + item_id=item_id, + knowledge_class=str(row["knowledge_class"]), + item_type=str(row["item_type"]), + schema_version=str(row["schema_version"]), + payload=payload, + canonical_hash=str(row["canonical_hash"]), + ) + + class LocalKnowledgeLibrary: """Persist and semantically search canonical QuantMind knowledge locally.""" @@ -245,7 +306,7 @@ async def put(self, item: BaseKnowledge) -> None: async with self._lock: if self._db is None: raise RuntimeError("LocalKnowledgeLibrary is closed") - knowledge_class, payload, canonical_hash = _canonical_payload(item) + canonical = _canonical_payload(item) projections = _project_knowledge(item) as_of = _timestamp(item.as_of, "BaseKnowledge.as_of") available_at = ( @@ -324,24 +385,29 @@ async def put(self, item: BaseKnowledge) -> None: self._db.execute( """ INSERT INTO knowledge_items ( - item_id, knowledge_class, item_type, schema_version, - canonical_json, canonical_hash, target_count - ) VALUES (?, ?, ?, ?, ?, ?, ?) + item_id, knowledge_class, item_type, item_shape, + schema_version, payload_json, canonical_hash, + node_count, target_count + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(item_id) DO UPDATE SET knowledge_class = excluded.knowledge_class, item_type = excluded.item_type, + item_shape = excluded.item_shape, schema_version = excluded.schema_version, - canonical_json = excluded.canonical_json, + payload_json = excluded.payload_json, canonical_hash = excluded.canonical_hash, + node_count = excluded.node_count, target_count = excluded.target_count """, ( str(item.id), - knowledge_class, + canonical.knowledge_class, item.item_type, + canonical.item_shape, item.schema_version, - payload, - canonical_hash, + canonical.payload, + canonical.canonical_hash, + len(canonical.nodes), len(projections), ), ) @@ -349,6 +415,27 @@ async def put(self, item: BaseKnowledge) -> None: "DELETE FROM semantic_records WHERE item_id = ?", (str(item.id),), ) + self._db.execute( + "DELETE FROM knowledge_nodes WHERE item_id = ?", + (str(item.id),), + ) + for node in canonical.nodes: + self._db.execute( + """ + INSERT INTO knowledge_nodes ( + item_id, node_id, parent_id, position, + payload_json, content_hash + ) VALUES (?, ?, ?, ?, ?, ?) + """, + ( + str(item.id), + str(node.node_id), + str(node.parent_id) if node.parent_id else None, + node.position, + node.payload, + node.content_hash, + ), + ) for projection in projections: stored_vector = generated.get(projection.target_id) if stored_vector is None: @@ -395,7 +482,7 @@ async def put(self, item: BaseKnowledge) -> None: source_content_hash, item.schema_version, _PROJECTION_SCHEMA_VERSION, - canonical_hash, + canonical.canonical_hash, blob, ), ) @@ -419,24 +506,24 @@ async def get(self, item_id: UUID) -> BaseKnowledge: if row is None: derived_count = int( self._db.execute( - "SELECT COUNT(*) FROM semantic_records WHERE item_id = ?", - (str(item_id),), + """ + SELECT + (SELECT COUNT(*) FROM semantic_records + WHERE item_id = ?) + + + (SELECT COUNT(*) FROM knowledge_nodes + WHERE item_id = ?) + """, + (str(item_id), str(item_id)), ).fetchone()[0] ) if derived_count: raise RuntimeError( - f"Stale index data for item '{item_id}': " - "derived records exist without canonical knowledge" + f"Stale data for item '{item_id}': child records exist " + "without canonical knowledge" ) raise KeyError(f"Knowledge item '{item_id}' not found") - return _load_canonical( - item_id=str(row["item_id"]), - knowledge_class=str(row["knowledge_class"]), - item_type=str(row["item_type"]), - schema_version=str(row["schema_version"]), - payload=str(row["canonical_json"]), - canonical_hash=str(row["canonical_hash"]), - ) + return _load_stored_canonical(self._db, row) async def search(self, query: SemanticQuery) -> list[SemanticHit]: """Rank filtered projections with deterministic exact cosine similarity.""" @@ -547,14 +634,7 @@ async def search(self, query: SemanticQuery) -> list[SemanticHit]: f"Stale index data for item '{record.item_id}': " "canonical knowledge is missing" ) - item = _load_canonical( - item_id=str(row["item_id"]), - knowledge_class=str(row["knowledge_class"]), - item_type=str(row["item_type"]), - schema_version=str(row["schema_version"]), - payload=str(row["canonical_json"]), - canonical_hash=str(row["canonical_hash"]), - ) + item = _load_stored_canonical(self._db, row) canonical[record.item_id] = item if isinstance(item, TreeKnowledge): if record.node_id is None: @@ -601,14 +681,21 @@ async def delete(self, item_id: UUID) -> None: if exists is None: derived_count = int( self._db.execute( - "SELECT COUNT(*) FROM semantic_records WHERE item_id = ?", - (str(item_id),), + """ + SELECT + (SELECT COUNT(*) FROM semantic_records + WHERE item_id = ?) + + + (SELECT COUNT(*) FROM knowledge_nodes + WHERE item_id = ?) + """, + (str(item_id), str(item_id)), ).fetchone()[0] ) if derived_count: raise RuntimeError( - f"Stale index data for item '{item_id}': " - "cannot delete derived records without canonical knowledge" + f"Stale data for item '{item_id}': cannot delete child " + "records without canonical knowledge" ) raise KeyError(f"Knowledge item '{item_id}' not found") try: @@ -617,6 +704,10 @@ async def delete(self, item_id: UUID) -> None: "DELETE FROM semantic_records WHERE item_id = ?", (str(item_id),), ) + self._db.execute( + "DELETE FROM knowledge_nodes WHERE item_id = ?", + (str(item_id),), + ) self._db.execute( "DELETE FROM knowledge_items WHERE item_id = ?", (str(item_id),), @@ -660,6 +751,40 @@ def _rebuild_numpy_index(self) -> None: f"Stale index data for item '{orphan['item_id']}': " "derived records exist without canonical knowledge" ) + orphan_node = self._db.execute( + """ + SELECT n.item_id + FROM knowledge_nodes AS n + LEFT JOIN knowledge_items AS i ON i.item_id = n.item_id + WHERE i.item_id IS NULL + LIMIT 1 + """ + ).fetchone() + if orphan_node is not None: + raise RuntimeError( + f"Stale canonical knowledge for item '{orphan_node['item_id']}': " + "tree nodes exist without an aggregate root" + ) + incomplete_tree = self._db.execute( + """ + SELECT i.item_id, i.item_shape, i.node_count, + COUNT(n.node_id) AS actual_count + FROM knowledge_items AS i + LEFT JOIN knowledge_nodes AS n ON n.item_id = i.item_id + GROUP BY i.item_id, i.item_shape, i.node_count + HAVING COUNT(n.node_id) != i.node_count + OR (i.item_shape = 'flat' AND i.node_count != 0) + OR (i.item_shape = 'tree' AND i.node_count = 0) + LIMIT 1 + """ + ).fetchone() + if incomplete_tree is not None: + raise RuntimeError( + f"Stale canonical knowledge for item " + f"'{incomplete_tree['item_id']}': expected " + f"{incomplete_tree['node_count']} tree nodes, found " + f"{incomplete_tree['actual_count']}" + ) incomplete = self._db.execute( """ SELECT i.item_id, i.target_count, COUNT(r.target_id) AS actual_count diff --git a/tests/library/test_example_bundle.py b/tests/library/test_example_bundle.py new file mode 100644 index 0000000..d62023b --- /dev/null +++ b/tests/library/test_example_bundle.py @@ -0,0 +1,162 @@ +import json +import tempfile +import unittest +from collections.abc import Sequence +from datetime import datetime, timezone +from pathlib import Path +from uuid import UUID + +from quantmind.knowledge import ( + BaseKnowledge, + Earnings, + News, + Paper, + TreeKnowledge, +) +from quantmind.library import LocalKnowledgeLibrary, SemanticQuery + +_BUNDLE_PATH = ( + Path(__file__).parents[2] + / "examples" + / "library" + / "data" + / "ai_infrastructure.json" +) +_KNOWLEDGE_TYPES: dict[str, type[BaseKnowledge]] = { + "earnings": Earnings, + "news": News, + "paper": Paper, +} + + +class _DeterministicEmbeddingProvider: + """Provide stable local vectors so the example scenario stays offline.""" + + async def embed( + self, + texts: Sequence[str], + *, + model: str, + dimensions: int | None, + ) -> list[list[float]]: + del model + if dimensions != 2: + raise ValueError("This test provider requires two dimensions") + return [[1.0, float(1 + sum(map(ord, text)) % 97)] for text in texts] + + async def close(self) -> None: + """Release no resources.""" + + +class ExampleBundleTests(unittest.TestCase): + def setUp(self) -> None: + bundle = json.loads(_BUNDLE_PATH.read_text(encoding="utf-8")) + self.scenario = str(bundle["scenario"]) + self.items = [ + _KNOWLEDGE_TYPES[str(payload["item_type"])].model_validate(payload) + for payload in bundle["items"] + ] + + def test_bundle_is_a_concrete_cross_shape_scenario(self): + self.assertIn("AI", self.scenario) + self.assertEqual( + {type(item) for item in self.items}, + {News, Earnings, Paper}, + ) + self.assertTrue( + all("ai-infrastructure" in item.tags for item in self.items) + ) + + def test_sources_and_financial_times_are_auditable(self): + for item in self.items: + self.assertIsNotNone(item.source.uri) + assert item.source.uri is not None + self.assertTrue(item.source.uri.startswith("https://")) + self.assertIsNotNone(item.as_of.utcoffset()) + self.assertIsNotNone(item.available_at) + assert item.available_at is not None + self.assertIsNotNone(item.available_at.utcoffset()) + self.assertTrue(item.citations) + + def test_paper_tree_has_stable_valid_navigation(self): + paper = next(item for item in self.items if isinstance(item, Paper)) + self.assertEqual(len(paper.nodes), 4) + self.assertEqual( + [node.position for node in paper.children_of(paper.root_node_id)], + [0, 1, 2], + ) + self.assertEqual(len(list(paper.walk_dfs())), 4) + for node_id, node in paper.nodes.items(): + self.assertIsInstance(node_id, UUID) + self.assertEqual(node_id, node.node_id) + path = paper.find_path(node_id) + self.assertEqual(path[0].node_id, paper.root_node_id) + self.assertEqual(path[-1].node_id, node_id) + + def test_bundle_creates_six_documented_semantic_targets(self): + target_count = sum( + len(item.nodes) if isinstance(item, TreeKnowledge) else 1 + for item in self.items + ) + self.assertEqual(target_count, 6) + + +class ExampleBundleSearchTests(unittest.IsolatedAsyncioTestCase): + def setUp(self) -> None: + bundle = json.loads(_BUNDLE_PATH.read_text(encoding="utf-8")) + self.items = [ + _KNOWLEDGE_TYPES[str(payload["item_type"])].model_validate(payload) + for payload in bundle["items"] + ] + self._temporary_directory = tempfile.TemporaryDirectory() + self.db_path = Path(self._temporary_directory.name) / "library.db" + + def tearDown(self) -> None: + self._temporary_directory.cleanup() + + async def test_bundle_put_reopen_search_and_get(self): + library = await LocalKnowledgeLibrary.open( + self.db_path, + embedding_model="deterministic-2d", + embedding_dimensions=2, + _embedding_provider=_DeterministicEmbeddingProvider(), + ) + for item in self.items: + await library.put(item) + await library.close() + + library = await LocalKnowledgeLibrary.open( + self.db_path, + embedding_model="deterministic-2d", + embedding_dimensions=2, + _embedding_provider=_DeterministicEmbeddingProvider(), + ) + try: + hits = await library.search( + SemanticQuery( + text=( + "What evidence shows demand for AI infrastructure " + "is expanding?" + ), + source_kinds=["http", "arxiv"], + tags=["ai-infrastructure"], + available_at_before=datetime( + 2026, 1, 1, tzinfo=timezone.utc + ), + top_k=10, + ) + ) + self.assertEqual(len(hits), 6) + self.assertEqual( + {hit.item_type for hit in hits}, + {"news", "earnings", "paper"}, + ) + self.assertEqual(sum(hit.node_id is not None for hit in hits), 3) + for item in self.items: + self.assertEqual(await library.get(item.id), item) + finally: + await library.close() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/library/test_local.py b/tests/library/test_local.py index 1cdd065..03e8a28 100644 --- a/tests/library/test_local.py +++ b/tests/library/test_local.py @@ -1,3 +1,4 @@ +import json import sqlite3 import tempfile import unittest @@ -195,6 +196,25 @@ async def test_tree_root_and_non_root_nodes_use_exact_grain_and_filters( try: await library.put(unrelated) await library.put(paper) + with sqlite3.connect(self.db_path) as db: + root_row = db.execute( + """ + SELECT item_shape, payload_json, node_count + FROM knowledge_items WHERE item_id = ? + """, + (str(paper.id),), + ).fetchone() + assert root_row is not None + self.assertEqual(root_row[0], "tree") + self.assertNotIn("nodes", json.loads(root_row[1])) + self.assertEqual(root_row[2], 3) + self.assertEqual( + db.execute( + "SELECT COUNT(*) FROM knowledge_nodes WHERE item_id = ?", + (str(paper.id),), + ).fetchone()[0], + 3, + ) hits = await library.search( SemanticQuery( text=query_text, @@ -478,6 +498,12 @@ async def test_delete_removes_canonical_root_and_nodes_transactionally( ], 0, ) + self.assertEqual( + db.execute("SELECT COUNT(*) FROM knowledge_nodes").fetchone()[ + 0 + ], + 0, + ) async def test_stale_canonical_get_fails_but_delete_can_recover(self): item = _news("Stale canonical") @@ -492,7 +518,7 @@ async def test_stale_canonical_get_fails_but_delete_can_recover(self): await library.put(item) with sqlite3.connect(self.db_path) as db: db.execute( - "UPDATE knowledge_items SET canonical_json = '{}' " + "UPDATE knowledge_items SET payload_json = '{}' " "WHERE item_id = ?", (str(item.id),), ) @@ -531,15 +557,40 @@ async def test_orphan_and_missing_derived_data_are_reported_as_stale(self): _embedding_provider=reopened_provider, ) try: - with self.assertRaisesRegex(RuntimeError, "Stale index data"): + with self.assertRaisesRegex(RuntimeError, "Stale data"): await library.get(item.id) - with self.assertRaisesRegex(RuntimeError, "Stale index data"): + with self.assertRaisesRegex(RuntimeError, "Stale data"): await library.delete(item.id) with self.assertRaisesRegex(RuntimeError, "Stale index data"): await library.search(SemanticQuery(text="query")) finally: await library.close() + async def test_corrupt_canonical_tree_node_fails_rehydration(self): + paper, methods_id, _ = _paper() + library = await LocalKnowledgeLibrary.open( + self.db_path, + embedding_model="fake-2d", + embedding_dimensions=2, + _embedding_provider=_FakeEmbeddingProvider(), + ) + try: + await library.put(paper) + with sqlite3.connect(self.db_path) as db: + db.execute( + """ + UPDATE knowledge_nodes SET payload_json = '{}' + WHERE item_id = ? AND node_id = ? + """, + (str(paper.id), str(methods_id)), + ) + with self.assertRaisesRegex( + RuntimeError, "node.*content hash mismatch" + ): + await library.get(paper.id) + finally: + await library.close() + async def test_corrupt_vector_and_query_dimension_mismatch_fail_clearly( self, ): From aa470f032f5f90b8589737c0429e4324f790238f Mon Sep 17 00:00:00 2001 From: pkuwkl Date: Thu, 16 Jul 2026 16:24:25 +0800 Subject: [PATCH 3/5] feat(library): ship prebuilt example index --- docs/library.md | 28 ++++---- examples/library/data/ai_infrastructure.db | Bin 0 -> 98304 bytes examples/library/semantic_search.py | 80 ++++----------------- scripts/build_library_example_bundle.py | 70 ++++++++++++++++++ tests/library/test_example_bundle.py | 73 ++++++++++++------- 5 files changed, 147 insertions(+), 104 deletions(-) create mode 100644 examples/library/data/ai_infrastructure.db create mode 100644 scripts/build_library_example_bundle.py diff --git a/docs/library.md b/docs/library.md index 4ef86c8..e3a52a5 100644 --- a/docs/library.md +++ b/docs/library.md @@ -71,24 +71,31 @@ filters before ranking. ## Usage The bundled AI-infrastructure scenario contains primary-source-backed `News`, -`Earnings`, and a real research `Paper` tree. Set an OpenAI key and run the -complete example: +`Earnings`, and a real research `Paper` tree. Its canonical source JSON is +precompiled into a SQLite database with six `text-embedding-3-small` targets, +so the user-facing example does not perform ingestion or document embedding. +Put `OPENAI_API_KEY` in `.env` and run: ```bash -export OPENAI_API_KEY=... python examples/library/semantic_search.py ``` The example: -1. Validates the bundled JSON as canonical QuantMind models. -2. Persists flat knowledge and normalized tree nodes to SQLite. -3. Closes and reopens the library to demonstrate durable data. -4. Searches a concrete AI-infrastructure question with a no-look-ahead +1. Opens the ready-to-search SQLite bundle through `LocalKnowledgeLibrary`. +2. Embeds only the user's query with `text-embedding-3-small`. +3. Searches a concrete AI-infrastructure question with a no-look-ahead availability cutoff. -5. Resolves every hit with `get()` and prints source, citation, financial time, +4. Resolves every hit with `get()` and prints source, citation, financial time, and the root-to-node path and content for tree evidence. +Maintainers can regenerate the model-specific database from the auditable JSON +after changing the source data or storage schema: + +```bash +python scripts/build_library_example_bundle.py +``` + The bundle's facts and short citations come directly from the [Compute Trends Across Three Eras of Machine Learning paper](https://arxiv.org/abs/2202.05924), [Microsoft's FY2025 AI-datacenter investment announcement](https://blogs.microsoft.com/on-the-issues/2025/01/03/the-golden-opportunity-for-american-ai/), @@ -98,10 +105,7 @@ Representative output has this shape; scores depend on the selected embedding model: ```text -Scenario: An investor tests whether long-run compute growth... -Persisted 3 knowledge items / 6 targets -Database: .quantmind/ai-infrastructure.db - +Bundle: .../examples/library/data/ai_infrastructure.db Query: What evidence shows demand for AI infrastructure is expanding? 1. score=0.812 type=paper diff --git a/examples/library/data/ai_infrastructure.db b/examples/library/data/ai_infrastructure.db new file mode 100644 index 0000000000000000000000000000000000000000..34bc6008ecdbe4c13c9b2ab50bc59a250ef630c5 GIT binary patch literal 98304 zcmeF42Y?hs+V^Y8c}0>K+chA_z=X{^t4$P?pacWzZTHOdu%o*>%gii-iq|Mf1rbHe z+D6O)6@}f^Hj6puFnVV?_4G_<_WM1(OFZ@V?)Bd9eZOy3+U~A;>Iwhnsp_ig>Zz%R zOh{xbEuKu(ni(ymwo`d|O4l@{R5y+tI5z%u;-bUG8_szg?Y6l8rOR$=*hxq9%>PDp z?f8xA>Ey5Leo0TWb948jJC}4?MpEuChd>U290EB6{+kh)zkS!v1IqTzn?Es87qjLZ zTbDep#){3fs_K$4D_vD@rmVWmHmTi4O&K#{+8Ay8q|svz*S5)QEz(Wf;&1Y#ZK?L* zVN+X`hz-?RB(z*6(}_$XSvPq8pe~&URQmGf&+?Q?TeW6gCK0VlS*9%+RTm{Ald>#y9z4H& z=N3%||FWj7W^OB^%WrGiTB>cDqQ&A}I(8o5^W`n8&6trIYg=jC-gj!1K5fLv31hTv zbG3c8lAAG8Gp!7^YSYFXK21A#%J_*RrW~OiH0Fq*ThX=Z+(I>J@-%JI^a&HTCXtZZ z$Y|>vudqP=TgB;^M60b>RmPf=`7Js#U6qV$gy;nAm-R4bn~53$s*0|T&~2^E7Eb>y zF`L8Tx2Q7a%yiW;>15q+617B=N3yjpR&B(J#S(Qht7-wS=C_!{617%cn(9NnoHpiw zF;jlU`ubGz7%M9Etg6~fSN|5vRwS@ALo2YEd}|#yHvnn;Q)^burf!NJUcmga3${?A z?S;H3iJEoExBY%Zx?Bpq9#vd?A;#u3+C=Dhk#*UdXX40rJ zQ@3ciM0|P^%~ruLZ5@r==rI$<&@PS|F?H04(PK7I4Bo#}rvX!ad8(z2m{kL(DzhPz z^zN&+t$EEi_ zUP%43w%j91SDV|m_}Zv(V@4g+>amvm@sqTD2E}X4%%Gv#AbIW%8oZ7F+TMcHw>8Sz z+{kzl6-_qOW&R<+H`=lV$$piZe;egnpRd2oVk@Hk;IX-FschG_xbl3zWn6g)?dfiI zkuGOXj`pgz>V86}I47l+PHEp``eD2D>$55`zh{@OWkG+T83~p6ii5GJuP_uV_LW)w zVqdV#Z$_+OED(>E&1#^VR!gHcY*V-lv&>W-kFRw3nAVGp3F6VfF#ib!-s8HUyddZg z7Ww>zzF^6;V5q#PjDJ3VNx9!&FtlLHC-x@NB44m{n%@rv|2C0E6B$z;Zs~&ZBTp!Z zR+}k$1M#$#DJZXNsHquR(AoxUysbB9S{tutH6%0K6pS`ArZ$R4o|V#4)@-Y;!P4kb z>=`N>7SJMzni}|{RqNw15u-@m>_jYKmT$DKJU$nDihZ*J(6x3JTcpez`tNErrB#ze zTC?WV^9Z+MS|+JQEN#!iqGA63vSLRME!fBqi7vY@u$-s!!1?1<#n4sMDrT)ndt-8Z(vF%(|F1=HRIs zHXSfx#KBrDQPUt)qzyy%v}&uh53bQ;0AhAvuHS3T?|! z5w>+Qqg9hQz6Gq>M63?8GTOmrG!aiky+R`Ckf1j92yvMEugcro)aiQ!!sn zGRYWVrl)SAE?U#zDN?uD595OVO_5TQq=Ag5lC_eZOl4%?kVz$L(!&aX*s2E||FDu` zF6U%Yrq>hO7=RmPgY=|B$B!OAVzY$J3T_#ZTRNVxR82!#OQtmHAQ!>ChJv;z+oHN= z)&t_41pR2MX0Enp>E@Wx{y~2;lSw5ao-c^87DIfkhG;d3+KshLu=6TFTN((Vm2sau z9WZfYWg)rVN+ps~k6N5?h*XxSTUE56Go_{xR5HqtIydRi(Ie2>O!yLY z@syd)q#B}`hLj~~(J0j>WuktXTk4MKi+W9*L?xr^W33dXwsIga>(bFwqCQjA+9*on zIWHCqn9*QKS#dNTDk=(=1Ow4vwAc)gTv%3GT38Y+D=Cfp3yVubg@vVo!jgErq&N~U zDlLi?#-nERwC&sVn0`RNetpi$lZT4GBwA7wDGB)sV}XFLuq+<(nT3AS7cYtwhJqzl z(253r>7gQ+mrvqcdcRj?e!ORKLgLW7~T6YfW=)BAQC3 zlkrS>vd)*OwtR_ny1`19Z&nSpw|;ows^7DX-lBa?^VpQuwT?DJqMX5Gh6cr~lU}p5 zVYT%&mNd=jdaF)alwWV3>l-39jL_sM{L5{$wA9F}t@j$#EghpV-}Gxu_Vj6^T3S%% zf-(uq3;2l}Aq1%r<3*a;64Vmo)9Yw$V%k*CGOz9BQJ&e@`b5oo8nK$H&E{5#xs?0I zcDdRzV>O97X|y&;^uPhIZ5q{0&Xyj7*P0b+BgXqSYqkZxL%pY$R-44=Of}F~n>BHH zLJ|zx*4nqWvTdm`Sg>U)D|LhS1{5D&fkpG4j zHd2$EnI5*OVt6g=R+XcrQS}Fg`$NN}%FImGFeT?p*2|N&0f*1^np2Zr2q9SKGZVw5 z(cIWTPMkV+8kYQ01G(8l0vFL?&1q>FTVwUt+KhQ+AzHdXEEFgTn31x0aV%sN7R6## zBod5At!SxPW|fu}7n$*(6$%Cl%}`NEpsd&|viyPK!m`4`Q`_bBn10ZJetni7Hos?H zUL;sl8Y-oK9Sue4Y14Z8%1WcfzT$Y$G>c1%V`lsp8^wAwPL&@l|MxDX;b`e#r{*MP zGw&t_(6kKm7nKDI1q7KVYq2UwW9jpkZQWinznZD?hI=h_&6?HFQtxPln7yc$2MN!d z1EtHSO|@pzFE(L2!K$m{DW;9H61CipsFTUBaY-(xnYHFzZKRb>q>?jh8c3OFrV{j2 z#w8<>YO`AE-oebvCAC9^S4}dKOp6P)Y#@GJ-?q2h1w3xdNU+rJixinWcZyBRR~iVF z`C_I1B7dkfYR0V)y0i=p0z(Tnjtk;EQGGL0$>XF$B`qTTBf5RylCq?_NZ@QaC@=;2 z^39;n4nnZ0K`&^Frwiu&2bz2@KD1go(ew?{T6ID?H8Q8hytbK;@fuyd7VFTqE|#v) z5*hmJ(8{xb{)!a^3azD~_h`+!xtlt`^guT^s-)5bscY^0{zH$~3(GbEv30BFnE;xp zWHRHut7r-EHhT8g+C6W&|M%)bdfrFHt^1H(>ab*L=5RAY7jbii+@h8GrRHf{&q%1@ zF?>~U^t|FwFzS!fY*?W{Jm3%dX)8)YW$|FtG_9CFSY`!+p)%77g#smkXmKFQQBq_^ zON&i^tSlPu&_Q1Ex6fas@^8*xl>ag}xxX9&IRtVDmR zA&^7hKZL+O?WgCpC>loT|Rey&$iUX5;GVK#DabwlXy%|mz4Q^WyPT)Utu&DFqx;0mjp}NGSMju zO%gRRW^kDtZGBVxFU*p+MGa;lTV}~an`g<3hL!M2t1eUhADIsSC(CCF{w1x8HftSA z%3Dk3^;u<;nyZz`(wcz(KWIqHyuK`~<(gGYlUE_X4c3`o0-*~u+|qqT9gZU&&%se zs6`9jGtNw+fh_Gw*CbitC+~D+?L&%X5q0v~c+(QBwtGuHL-?1b_&2UQEhy&&PP(DC z7E}Nl2>9fHcVwGjNN<9e8t26=Z#r6IA!5n94lU>p__5zZB%PSEEfqA|;2pE9dLN6F zytQwe9BRg8L0Ky^&@ox=gfHr%84Xu$^1tUSjTcHO{Lh*rA^>8F6$y5{`>yfcL$|!W ziEZ*-%bS{ld1kzj<#|zms4x->76ytU!B8*~3R%%uVNr;s8pXk2Nr@FL4DuGx4EcE{ zSz?<0xLFzv@tPnKSaeJ~S}WD%^!?k`TK!usQ9;Nmi$wjgpf46I@%swnML}O_G*IM= z6-SB!g|XsTz=~sS8*6jCtPDGfN_~N#8Sxbs6-9lKl0XSB^vYtvh*=WlPTnN`3$0&U z%-YoYZSjsUQ_XwV&5mWo&3~jB+_Lr9W?f_3UvF-zR4p7fR~cDlwW-RqzS{LFbTsMP zSo6~~KQv3!r+;Whq=8EC9ZDuyXVFyg6sog+p*j%N@;yx^5ueNQK8tlzThyAN+BUb+ zo@`l71X+lZ(FS?->eUpD6~ieOm26zo)vr7O{A9~9 zTehtG-UCgRuCQ2xhmiMu+@?p{=7)nkqT(z(-28@LOEvIPUb+NYOHGzq!jn2J-?s6} zpIYSA09gp+RsEI^ki0I6EC=ARX~~)!7Fcawkw$If5jWecp*YXY2wKH7Dq&!#K(ZOC zR&ciN3$)nFX%Nph_(D@}iw`{6Lx4K&x)>?GR zVg>W8;RV5B%+60umqQd(AA6etcvSfCl@X&5&ng@r|tSSTKd`ir6=9dXcu!c<;Rbop)T_{vUq*7a8DS17AzzwbB&q47EPo_!MdO8LAX*ZNN8?f6zz2)U;(Rb14friH7%GmGT2`<) zRLD2OrT##uFla@J{e|&JAm}fP@pW(Dzxw5$a!zSf*=L)Sdtjxq_ia{oZKcxp4y$bM zu+k2#RNBa9rQHx#?rUMC_Y5n?*Q~N@nv~u#tn8gDm0HrI)U-yWe^Q~cDy-CsCZ$bp zQrUt^Wh|&rdhZJ5tfPECjw9e#1D`of$|;7&9?->+`EjFisw-7?QK@qFZ&t=SA?n2c3(dYllupH!Ein=_d3qG%8gIud~-H z<8;zs3m#nz7K^$yqjU|R^9y_=Q3 zv|PE1(S1kgJ5(zBbo4X9aWOOn$Pv4qK$q1`%D#rY_Q=XZum0sq>niDB@I17)gUwWQ zFKbrryIkK4KHH(&BkNT5YiI@`e{q>|%aHq0qp}O3yQV^^W$+xu^`r259Nk}QQpQ{B zlu-kZ4e0*>@~=a79J*b>vtPNg?}D}i$H;PJ+z!ohaCs8G?ZGhu{hi=-8uu5X|1RL7 zRVd?l?q7v>1G#ODP&!Mzlc$75xU?9=X7|Ajq8yqeyGHjy^u#* z_EOSDaJ?MauR%ks=|_|PVOVKra*uDFZk5U%jgPK^C;rW1qqCYg!+*vgbSo#{M2^sO zMu%HDW}w&El${3eFQ^lfx&BJ{bIyZz0W=ZLcf)fC`rr>6J@hNEkNV*358sd0Df?Pz z(#Y#Z+Oz2OEpq?TsI)_{(;)48(rn}|fyY4X{Tw?g;q?f1-bA?_(ZeR~A$)i_yogEF z4;!wb4(v)Deg(fsZ0w4jrTAt5wonHg;@nu-sGND^fr)=RKjF4c*!%kH3=*DHM;bhOLHVDP&;tkij6gC6={q3MHtYvFy}dT>DBPUwWb?v2PDh7Qv> ze@4FeX@-wy+8@ex>N zJHulK^owKLv+%f*as%OAPg#8B&ftEr=!k6bcOlorvhg+k>jIA_DW420gL3+6_>*rG zfY(asR+TGt0z8*Nb1-&KryO?Mdy?M;j7pK;rAcMMSnGyg&%mz3pdF9?4~6`rmf))BX8*^8UY5uMfgA!k1ab)E5Xd3$dm-R!UzO**>uAfD z@0~VLx8?Kqj#E3#ADkzj8@B01N_m^s1pJFU+q5#^U*y@AH~Q^{wy)~i=66Wm_&>k9 z$}i0Cp8vAfgA!k1ab)E5Xd2rLm-Dh4uKp3{{syPkt zYU_{ibriz3EEMd}RzmOh|JuT_&AY-1MDbTZY~p{_T?O|Lv0PNfgA!k1ab)E5cr>mz~qknJYbt%#<%G=_&Q92U)SG_ zU;7~)*?Ff;?AlF4a+fy!{$E=x>4~e_w?9gCKdwtRPPxAv0yzYJCj@f)|F!Jg=1rfL z2ZR3LFnXe_Qb{_m@K;hd>U290EB6atP!Q$RUtJAcw&Jb_AB}+@b4q z-hD1RVt&uI%(S=Vr_kCmN7n9yf@pQ3CdQ`iO!KD;%8!)cYa(N263IHbm3%W*m5j&P z!oHxquA!!8Xo37>TgzL4w&<~Gsv%f7v|v_4GGi5#7o?L7sVH;VOhz@-MXglEWLx&R z-d6VYP}F4-W{q~7Wgc5+rPEd#9xWw)$9EN54b&&s)y&m0Y&joK)FFA}rZQTpp-xMr znSyVq%UH3Y+Hs&|Y1MPnnTCItpHR+@#JIhxmYhqfBnVPwAfBK;w2%<^^p)oTxSBqGg<1Cn_teFjv z`ec6_%Omp&w%)gHcrWUrS_XTLX^73N1xvF=HmLtMyhd1LS`cqbFo~m+>4bz%L3t35 zHPqIkKHi>&HPwl9CYeg0x@M-+$!Nlhu}!wcQ%RgIV9rgmwGvp?#V9kgnxu^{KqA;< zWzt%@p&mTBNYuqsW;&B=2N~x69;*ggv_?On|k{Qj^Vu_R$&1mr? zC(3O?+FOyY#gfqmflEAuTO|UmbSA+K7J0O3tA_G&w?)tjwwXkxM#6eLQIe`pSuGVv zt4Y)*GUFgiyJ$qz4#9jk;u=mAd>{qZ8`wZ+#`b72=IG6ngPGY~9 zaqK1VI(sCHWq*Y;*wf`9?%CHQ%RVF8k?c1RV=o5TYvD@vTwwnebr$6&u4gX;_GCDS zd-gq1>Kj$y;!$;_iJlRJ{v3ExHTI^em ztb<4|sXkg;vQP6xebgI>N`k0bO1~#xD4d z{WxqN`mnc*x&T|?W$c3QMuJ5>_jh3XH?8&)Zw_&2r;zhC<;$QU=8O&K@;>^pe}?-! zahN6ly0Ew3lRKH~PUyTC zyouZFIqQ_7Uh0RTFLB~Xy?L8D_W(x&{HKSNQ->ee-^M86dJnMu44p-vc3`oRv}*Qh zDdYSB@+hBOiX3dP7s7Wt^f(XtJCU`5{0@}g0op*NG6oVW^GUx0y2sJ;2I#=rmAzP+ zn%K_=8KvdQxfi@wpx+^!U*#Y^#J8>k1`C?lgNSoaXp@xdg#NRMiPgv-)WkkM*uI24 zQtI(LHfqSxFG1!3#M4gjs^sWQ4AwQNEc*>*|4Kdl5j`(PpF82b6rJXx-|pyqCUSa{ zw+h*dkjH*tdKdIs18p3>*U(mcj*O0!`xm(DbJryAr*= zZem|G((#3M8Fa*qwg}8`gl;`_$8t!Uh+n<^zpld`+9Op=ydp0PR`v?8E5cU%=X^&V zKC$nlJ~hMN5uMSeE4bVM9e$C$gS@tXC3deP{|jVZO&al;y^r$9G*(dlY;3|l?S&@x z@BwEq$gV~X{&G%%7j3$pq}&F|%`H>T1(c)iyVNi3O0ExuW+^le!fPP)Z|Pe01mgN5 z$`cp*H~4^f(?ZyXZuSSHKY_20B{rth-u9!+eds1_={uZVcwIuA?*`q==tkYrN8yX- z(D8hHRfxO-+6B>X8T8cM>;!Dzm-`})iP-QScpe3xN6L9-Rw#{qq`bYeE(a_8Yg~tY zPjP+~y8Xf85qP}Lb!XBg-uI;596{Wi!$Cbzcc6DZbYq~wR&@e86`&73GN^m{=U^;; z?M!}G?yoK*j-a~}|DH>`Fcn(-;H<-*Gr$-A`Y`g@??~T?eB#xKaUDet&j9^;?3>SZ zg!0#t_X@nJSI#?JcLBHB68aVJ)S-O<-3!1w3%}vef7aynSERg-UCui2Ahxo-iH*L< z25TLD>Ub~#Z|7F{9M@S>}#j);+p;JTw+FFKsoHoVw;xRs~A>$PA0(YkeK3Yq9Blq&W!Dek2WV0Wb7hmfriy!QRC<6vr>YzFW-M>T5 zWXj2&TB6g3P0C&dZ)uNKQvN8g?f{Qko&yJ<8_zxcOR!&zjNK_qz0eDZnH#a6cEp{{ zaT;Xub>O zCGIsarw*x4xn7S9eD55Aemii!0~?QqHm{le6RA7Ir*y&mE`cuE`*GNAenOC9Lqud_^jfR{%z-m1*!AXsP&qR3NA7;z|FJjjv4xP(5 zE`(+Ub}zu5_F%#OXtI}{20v{D`5)5mPM|z`$umvrxw;biq41zCxJ!^fmOSh?{zg0e z9eNdTpCT^zYDN!q>V$9E&(5BYo@Z0$Hu8uKgR*Wr%1d1$X7!F>P*$PzpYi8Vlti~bAB9t!;pOzx~>T;?E=d9sr!2(cNl5(;WhEc zHCz*ChQwY^%D#x+9npz+bJ@R6or~Nv@P9t{_`~fFeKF^%CNJ)l1>bXs#b=49FYw(! z=*}SRGLA23FS-*iw}3+j?D+uxv_C3Kd_0Ihh)?@qbo(4TH0Ym)?@j1(B=u-tY`75p z>CONTB+QH(61*Bmt)J99Qp7lSV#L0 z&a~U^3&;RJg)P|=o4oOlx&u9?!1EG#&n&~gTvLbKd&0`Sh%@)v6X<}S^&PoKR`wmR z?}zO3;rCP{bpn1h@E;CFXK@cN<4JJQNxz5Kq5fzmk(S|pPkebH_A^$|?gkI>$Fs;9 zMfq;c^goclD`m=v&x2^Yhrs72aK4IXFR_}v86K|__wPX``%oT^OzMmgMmG~2h&SV4 zVy6)v50gf$OMB}?k?|t-72c&>+9ov%S$op{5TjY@qc^7D`Kqu_?+Wih=v#x$#Dla` z>N;p%Cmt7Z9*@qi(N=u0p0PT-XeXrK<^CDl`O}FPtr&Yw$494727P4jPq(Y!0{(r- zUyFX1ke)z3v8y_g_bGDvpzGxo$~_+3Y14FkEaL}_eoZz&+FRi81iTzDd>kIcs5=;6 z&<<&LA@6+BZ-Ae~wbVN>&h{maXAvJ0X{UCF_7X5$jvU%PcQ@Ll4dB?9I{6o955k7s z!E_B+(DrB((2sj}6*e-)bDQY59*wTsahwM>--7oW@Y@%E%_2^N^rv|qXg`P9FSnVt zjQFFDxNFHpST+=s`Wo z%KU(dP2=#-kH|Rciv`sE;sk@4HWGr-dtZx~QzJXtc5fghL zdl7t>qR&2HMSJF)i@X53a;^V`vQ1zy3jM$`+lW3_asL|SeBfG!u7_~2FRL~M9Eg`} z5Zs@GUw3GwPq-s?2^Q*Im@A;erkQNsg7&*TGi&kb0qv}lkCpW(+=8m z;Lr0b`x)mY(BF=I8>l0@a80{rH#hRkCw8mhOI*9JlMf!Yv}4Z`8|Q$_cG$Y4iMcV- zir_Vu^JD1L34Kn0@8|IPD|C#L#b4Qd!|W9d7E<5f>%=+oDSsOD_$=E2drQ%I3V1Jr zW*yj6lYSuOzJL#HkIb*By~(4EG0;_CL42G`9iY9`zJk})lsy^xdy#!}xw5I3Y6x;J zCas(p>Vq!5p(`R*JF?&O&hWh%TSK%hU6FSaW9}4=}X(iy(Tsv1J5s@eGEH@d3$@(??dOy;V~Y&sf)%= z#K-h7&s%u+MlO5hYJ&5p&@{l~er#AoxdghO2?p=Mk9+CUs}qTx&hRDv^h)rf-B3@! zzc>8;&hZ#=-%S6anKCC(M~NLJu{03At4TW-8u~BxkHl#&F!-9XZ_{qkKa@U^A??i- z$iEc-oCaRfo-O14IrMp*`wP%lo_QgT8{kLXG^mpz!|gyGxMn3jH^93)vNM!rY+}F9 zvlY3{WbUc+Mt8709Gx`EV}tP$^m|fQUCJ*hqi=5jTB69)0!ai@jS&AP@NIQ-40b~$QT3&^+_eTG<;B*_j=aW|qzXyr;%dw+3 zv3xAHtU%U5l+UA_A367<^Jm0WgzMW$OOn4EHoQ;SbC5~D!)4CH1#5%(+-ww_4nfEJ zu&EtByS|Be2IL*fJu#s#h2DnGA2|ku83z+b9l0J?sf-Xhy$cOveXSq-za{?~%AU$x z!4KrYCkqc_ciO(Slo8CTu(ODIjV;bK9B*^KhCFoFZZ1{oGH`no-z?%;B)+|ceA*uU zblRal&|AnkoS0e9waf+e18=Z17$+FJfJr5FaRk1SHe@f#UWd&*xAZsHDTlUFn})t0 zi9C1?Z1l!t&R}#kp}UweXNJ9ba3>9&51h;JcL(fR4et0ro;f;LxlQ13BUs-LpC8b- zKfc6Psi*dNV0#03=&wphe~)qTig-dx%Y`)T5qxN*Q!|2r}_V8bcUjs~x%C`ZgnUr##+o;(8- z{A{psz)#WEW^V+$A86y+lP7(=s~XvF8e8crI>d|7AAYnydKEV9h3rD`d9fV4(BUd% z-iFR+!LJ?IC$MKPV)0F6NW0Jt`AeYZc_T5TSzy+lShxbaUWfJ_Y!YhYq)*+vn)I4}3ZjhhVP0hX-|3#zE5FI%h+7C3-vq z&o_`WkvKqF4#Md-oJ{$#zfOGrDPG8MF! z^qK4)P0AI&f`{xW?@~^?w^6x6!H76F&_n8zj3?FIv^O8aXaBI*u1MROeGr<*NTtEm5z(eHg^NjzoX$v9bE0l(GYa3wrubEbXM7$azB5tEGbq#r76ywpkQ->J{Z zp9Kb#HFzdC9mC$dz1@>Mbg(NqK0_vF7u~bPVegr1_rRynYcHeM5w!L7oV%dsTeKzD z<7Z+>#&3!?UpbV$8a@xhdkATVVC$}2mqK$SdeBy=tMSK3>^qn|e4>My)|u;t$RqZ2 z=(D@P|4r<|ui8>DxRQ2#6?}R2Xg5Q9EisMl+Fr!PZO~py8J?ledhBh7ZOzCazBJk_ zo%TdOld_$Or775Q9(g=dwC>Qml;z%*I=L(N&xPraVaHnRI0}8wE%n|Nx$na3CCZ(H zoM$=WVCdrjSLYafcP{*E;eR$|&Zq3&&G-PFcO##Eo=zOOD=9aG_WgTgf~~w?k$F2; zj<4abq1(^ISTS*aFmx9-d3|^puNoiHcF@*2LMMHWPw~Mc=zSkN^YF>tTuVPXi2n2) zjJLr-{M(KE>$o0I`i=0ReRt+J@w`Cy57#OC1@cctH)*dsfNdB)?{gf84EnaVMmg%0 zD?ToP_IZ5q1Uk@v%3ehJGL9%TZ{m~Tl}f9`uNPAjvE)$lsZ=#Ik0i!%oy+ z>Y3CnSCRfW@_6>^M-#ITVL#>cBk{pXbk(t8FY@QXX9Rh9$iPNavc_lV&$D~Kn2Y|&=#2Tu8v`w~4rMjm=A{G~7FdI>T`-c|HR zW^jEB<<7>|^<2N&_m@K;hd>U290LCh2;4p|Z-=VsYQO?thYkb!&6-}; zZhlYICx3p=Y>zhY-P)GroZkAgQOVl+hK!|6W7SCg@*4+ZPKA z^9PDU!Kgo88Y;Cyfq1|l^v8o{X{an7jGCqu^9Rc;zPT+ktxzaX5{MQDq8uefX0)`} z^vBAgan?4_KYEt_#DVnL=r4GE(Y@(g^`_r(D18$8+cG!fz82=4BmD;-{j)XnFX$uM z^1g|Fg{;BQKdDez`ebSaeVFO=QJ5Pr7SM<8OMc&=)I)j((DT zG=1Gkq?^#cK>xK8UT4$iKOH*yj;fCSF=GQ+bD)r|9ZY|AGU>>3zQk_I>5LWZi|NmG z#a71D*>||!2iro>&_CCg($}Kjm)#M1=4xf`Q8U4DF*F6p5xbs1m(@+mzJ@#*=hN4A z`qSs^D(PVGJhY5;ovG-~J0tg9u5Sh(nM;0z`Pi?a8HD`Bta+rr>AXZ=tq{6v=<_Xu z=P0fph1cWg{u=$xx9D5ez+(gYKY)D326h~}UBR;-{o}i!?Z7dT@uiHF=o30m!nZv* zMxehFyiVi(LiFDSTr~QJ$8-NGyy-LRW6)_u= z`L`&`7|M7XIVtiBDaV|YGYr|Cv3opv-UIzocoGM8KJxz#HZO#E&lgtNabWroAaH%7UqkaYq6B1W}E=z<>{ znL863>Gx*E56nAgdm)ds?4_iQ;CeZ-UxS8N(~l~6OChHcQZk5U%jgPK^C;rW1 zqa*WS_|F)GZp^K_CUS&^@s@Kd2lHAk<2z>>ym`keYZ9EVgg|ji8 zKZ2b%QEo@{ka5&Q`0#Rg5tFJPHe5p;U<|9jf?vFGGc4KP$6VLRhLeLQ*+ zmsK6!71uHU@*=-C^EAfDL-+e}$$G z_N|3C?=&3zs&_&s^mT7U?l5$i#`!by#V1!ndpU9{NR#pCV)#A`ExuH5gYN|R;a~d{ z-kIvF;ZMGdzx9>Stzyi50z8*Nb1-&KryO?MynokO?~^S>ewQZZ z9`SEC{CWm<9R}@q^nU=^r!f|;xJzlDz@K`eM&Y9q(f>#IzKZN>=5%_Y-*B+nuhfe> zbt(LqhgUq$WFE%;7MU+X^D>7``VeGA;qf`LXOeG_{sLtl0QalmsZ;J@@`9XSLcX;5 zW1&9yFTg zF9OIt3K?POgU}IIT7>#2b?iH2%;x?OmRA<&Az?Z>qnIDhc;=>tyb`47_o@=;xRTZWnC zA*(DB^~Zv~Sg^$JD~uNfeWlSrkuO#pDGC(EiemvQ-WEn};p5FGj5aeSA8mS51AM1g zos4yp@9;KG5kwojFA}#MK5zWwjjtK`OitVSH=$a;%^S)WkMi{+lNGhtUn3vlWs+Nd z;b$>l(efcC{BOo1#YId4Sa=Hel$4lJtJExvNBw3X;4dxty;HGB#V!nu9&)mzB{MKOF%@`~D zDg6if-)bg(zOU(vFb>pNW2e!Vu-}5$J@hZ>GfJP?mOj_*^o>5C596o*(}{lCmt4zw zrKjK>pxj9MCNkF$K!=@3yO@5=P4r7H6JF>yxS93!9P{bRHBrv^+gFsoAKJlOze68s7&b{?dpq(OqZq4r*Y*v0pJDgY$fVEiOrgx> z9P3G64*!1iqo$KjAI%v>zbivOmcCS$H3Ql_?Cb?LV$)^ViVpfm$e}+e>!Xab(Ca2> zk)gi9{y*}L`(0#9-%Hlr-QC1DA=n}P#g5>07kr*0{|xGeIPRo7y>Ic1{`l|)c(7)|-V>b{b4`C+JyOhiBK-Gf_|hND7C;B4 zstz6LlV=COryU2nDf-tg*X|?OOP;)wRgAUWOGR(;z)9~6Z^kChHR${a_#8ueBlhXY z15Q_S_Ce#EYwA?+);hx*$GYOWyJd-pdmc;HIs_?l3m>!AEzYlOLMT zkbNUqe+XZEZBR#zp2R1yqpwG9C3$18qY8Z{axFT6wSGHwY=3MR4(&VeDuc$M{$By^ z_+8o``3}o{xXJtO#PuUv>d3X&AmggN;DsM0CcO9Z=XpJnJueb1$Aa_3L7r+m_ zvb-mCb_5&na5pd}2S=S4GlIy+AGX4-fzUsOj6vW@%<7MFy*+v-D_O4r{W|Wi3iBNu zI^hcgy|ehwc_HkrVU+bJ+Tq+6;n!P`|03!0Nb3*o+rxh>y8VdFk8}SN?P7c6wMU=t z&|w_@{1E?MjSUBoej++`B<`j{yAQbU4X>loe=#_8CKll7Fy_`~G0!M-FlTdr0KCpe zKgy~5uz3abYdgwyCyuv6uM;ZpF)`m89v_hoewqWXQ?cVZ^dvsBJ7G6-M#dNDCpy!{ z*;UY@v(zgkF_BOCTcCRgnO899_b2c?5gji_f7(*DJGM~g+}Gf7EH>Cpd?N;CZ!^ES zkZZ7aUqs&L_z`_&PEWpx(N|!rJZGpk>Rs%|Co)#o!th*6-gNNa6Rd`FJp)}`cr+s; z3cv2ip$*6mf!;^?5VjJxS>(vtEN2>eUxPhgkS}vJjIm`7+n&t*1<0npW!E%$Yw&I2 zNXfkMYW&!<*;}8hpTYHHuoz7_zI$-CM|K~|OZ$#r9R>f3kV*R>>w&d1z%UGt_Q?N` z`wg@MtI&hC-klB2EOZ-)?jOU4wnsaXqZxjA=+YNFs+zoS+Vp0qC3rKZ zC*Oi-tYNj6qT3Ms_Xl{=r_kOjXRaLjGl|K+qR%35c!a#Ul))!5FK*w-{cg}+ip-wS zN_5mxKd!8LmU(PS1Gchf5FK$0*9PB@xvg?p_G`x<1$Nk(F zf;DmFN=)z3$h;x8%;7j68tQqr1RPG{d^vumol?{dml)Ab$2Qsny&T`*d#zmB6Z`{D zf>haEkW+*%(heL2Zl9o^V2AJ3cF-=z_Ail%Z*;J69)%yzb(zaoz3}B=bYF;F2NP>& zl2!noGq7t2{1zc&7vhZgv45b>oru1C6RIBzZflTD{m!0>jsu|G8~ta(cLaR!y*dLN zcBCA#v_Er?jV`fie?t86Jhd-|*ImewcK-+Rx?y7>bfSA5`WB+clVE=Xn9#1c#HW5d zGRJ}QLTvd7StjS{l>Y|V_(Rri8E=8d>)8D|wC&)B53=7wvp1M-Xj1wPP5d?l{mfGG zZ${2V#84x8Y=@rM@2v;^ppm)@T|01HRpI%;zJcFw$ggDWKe8sEFLhA9V^`J095LzM zLwVxWxfh*?VdH*q*c*AQ3zF|a^fBa7cd|TxWj$o}6=a@A9@rXL=)02NiFD#o<`_jM zUE-!Y$Mam%F4zrhXc{ zyE8h$M`DilPS)}{%hC5XXeEwn(CKclufazz!lM`E{s0|uB;O<15$t=3_T+Dr8A+P7 zsYhT#7wY|8@I4jUmE3oP*WLK;a_HuA3`XuB(Q^#=o(u2A*myrNe+&A7y-xdL^gz}d z9A~4$v@qY9a-2$h%!Xeda4+P%9DS#P&E4pG0(IwO(vBi+5#`SkdhRa*x95;An2trq zM)Y`%^gFrVi|=!KlQs!FuP2VXYd@efMdrdukYvFMc#SPpB-kcJGMOu zCTWiS8Hard9`qC4?U4;1dnNX*fku2p{OfKb-_BCLE4pHfJRj}zx!#X{c$oXW;4_J_ z!d}Gb2hd%J9nbQ7r$4Pdfo?pP?4793m%{gI__n9MS#+&Fj8T($PO*jU+%dq82(q=(7AHSiqI|-R*(GMO;+tW?F&5nlku!<*4xeXvPbcfhwLb7JCVf7% zU9kgx**f@K!5DK8y1c_Pl)5Tycy>JWPs8^H>>t#u4CZ#m6@7 ziz+8xh$Zb{e0mKuU~5bw{$AlZemMU66S6~mmqcox?rrLFs`xreRBwWqQ9Nqln%^fl;A`=I4>re7fQ4(=yh zOBF4LkHC(Pv6VQKF}O?I+3z+f zZ3pC2=cF#^3VguVXhhDFV0a$<&VYa3`j+1h8WiUD5cqE9AoSXoqZ9rp!&kH$#%|co z^U-*mn550ro~9i5De2Qp;=CPsA5nfKbokvH>oQj&{V8KKv_GJu*l+|oHG|83Gn4u`in{+EwD>`%jms{@wmqP~n1jBk z)Fb^)@*f3@%hBz9bh@w!JH#KHzeTqNq>TfQpWw;(U)D!x%Q&A1cC-ube9B#qO&Rn& z2HKb4`5tA7d!sAqYvHwkv=H=fqXW9iI7u4iFL1^O?xln zS8pvg@#_8={-<#qM;Y`IEFAo!cBkBE(#`=drR?3HrGtudDY{B35V$NWcrT!Uv!TVA8-A(!3v1u^Zv;hwO%XXyf(cGWb z=)IrTdO-U;Ja0nAaCG)_Pu-FAx6T99mA~VIEWCH8?mPe;{xfPx-;V3m=q?z5y*v-x z(a4&Kf9}JF)6l7wbjDQfv7`^Aj=vSAuZ~?0alaFEf-Ch*`k)&91^GQCjc1uQ2whKt z_d;}JJSD%+By&?XeIe=hW=}`|hdEP^j0*G)5>w|we>6O=CNBPf9n5)YH^X}=e&m@a zeGPX7ys3}IVWiE4hr|H*=ljeW7}5NNK2|LwFpH*%hcZm(h!zHl!`um0qp56>}_XDvYX5@dde%;ETR3^a@3 zH-s_brIcCC{deSjO_@Q|UvQOrA@eoa+u^}?__C%$U4?D0p=(#{eTKYeh?`qTUqbrd z(6fIj*mMc{--GQ3!Sg8O^neb`oLgw?UWNzXh-8<*FO6KDrS>Q2 ze=oE*guUM{GoHgYpMm$0*iHNy)PJ2eBKs3^SD`!DY3~v<9g)|GGR5%eOTNVR8uaZC zzwet^3y-e*a?qyA*w=X;oser&2h}c=r`|Zrc$P88n_UiGE(gyotv5U_MW4MH1E>nx zILaPHInFMz?Tl?!?qOW-gj}BI2G}{^V(bGZA<}4r-2LHyF*Z=f_#^o{p~rskw=!a9>59Ay^AV<*FGEvP!B7ig@?>*$a7A{0?rwv^&u|)0^R}iuS3>) z^83;b@Z6QUne}1YyXd=^{8yS->x2G_Ij+BxzBZT~3jJgF2AiBB(s!hc!~KI{zC{BA z={r8a{cYfOJo(7fPKVETl%wQx1U~0yzY72>h-H+|jdr>zn?v|H7O8 zwrq0PmN)%v;p4sOpJOqVSMgO$gD1{Jl;e-`Rn_%sG@b>5oW%<9y2c zxb8&1iSdWc7)IvYvuneAD@$K!FL=>M(R*=Ezg^wmv@>$z=rJ8$k0F~h<88qN{ph0`qv-3s!+8N~oDam_ z(M|l;8a(K4s%x;d9du8_gZV4FJ@UFB>m79A_h^j{v54n*ghu=_pqc>o_2(NF#n zeHbIE_KfT1a&5!oLekM$ev`*RCxaN&Zw1GfD8C!H%&AlcV`u#}^03)Ch#2_>zO%t< zZ)D9!SHXw=uj5C@chTWuaH~N_#%)FbAD=?`27LP^GM`5m=`){3`i`WXjy?`$j)B+1 z;CeP=MEcQ==uEy2Pj5cyOl%?^vOUO~2i;)e4L=!`oNva4yRqSGY~2~%CWpQ6|J_if z_f54v1o@{^b{V>FhratFTgH)NDE}6`KIJ%oGJ1*kjwpLSGWz19M@icqzETHgh!5a@ z60y#hNa~?h3Ra9aRHBjJfI?6F=;77@UB{7M3?93H-%r?!-uju~*$G_sAsrfhhG2-D z&%isy`2gbJPH4)|XDPBT;k+xd&%wqar02u)YUEu5uHE6YGwEH>r3RYgxCa-Bm+UNb zzMk>qBa{J49h+T^aqwEw^O+03ypnIV!5|91`HY9^NZ%Pfhc@&63*CrkjWL#UCi>~< zW^vycj6Vh!=oDjYgSuy52S)IAvc%Eb@Slv`y~t-SM_m|ZOoxBFanIOK%|Vv}(mFE+ zWWLz@h5;YAeVh2rP<+3ZGN*t6ZHm?h+J5M`5FI)r6Fl6r;Q^+q8oBR7hY#gFolX7M z-$MVJIDY2-uN+|OJV%)#cp*!EFFhMZHtnfS%-XMm(WT&mTo)W<&R!c#+G_3@OJ<)! zRtOo#LUST##;h9SZ5un}eXo3rr5{B6ZbEIY`=}PFu&cC1|R$>vF)~_ z{L9$zI%%{u*+V#Y!p_CyBTwxNFNrCMUD`%_2hPZpZ@G-y@xfm>e+z!hqd2!?PbvE0 zca3_HrG2-Lxk^*DSJhQ_xo1rr*dz-iQ&hNX6&boP90HGI4*?t40OJlYt`H9 zpLnU>Z z_SN12n~p%{Fv?IL+>z+G8e6FkMt9~R{)Row6X`3%-nWPDbEMr!yz%=p+GE5Y&mfn$ za~t9H9`Y{6W@wEgp~oNc4Whd{JbNKG!u8kCJxlo|=)iN%c@SGhBb&PA#q$F4{wVkG zdZn@D_lU2jENzNI&}h@UBLsNIX6>wKKj?e z_X_Ct0E6d=&E?$FmgqB}qhDbwu8A*4;(&UpPeBLjf%`SS??V1C_+E-_<7oF*a~;Cw zUAXQDj$_DAlJ_+Fvi8VXLEdoqo`RkQ#0c1^e(=SQ+Wzo98M`0lz09|~OJiOs+ZWl( z&`sLA1)P7x7UsH)HQ09(vGxdj-$I|eD)`MHc+KJd8_wOJIUJkwITpf~wHMC$=u8`v z1vC3X#c7Trkjr#L@-G-JR<&$MxhR(;*)8 zK9&4-2?NwmafSyu6XHj-2yr}0gAFTBv?M|?`j5;9YCXl}vKKs$`vu+`~ z41Ly8b^vKlA|pUs-5$P|V*_?)FMXOV2XE$)a9S3>(rO5c2az9bue&qUT z-u3p!hI7#2#xUzK;58eai8pzNDZh`DEgz(O~Wgmjq2=qGxpAh>l^;r66(w@t^dYOlJR+BdvT|^$fRMX+n z2|p8yS)Qlzy=a#ASx%g1^Iqh22CwbW;To~OiQhYf=X!WcKk7t$Oe`71imZp$X{VjB z*sZPQcdxOH`tMwbJs%*C_E_Hm8B4%o4~}oR7Yy-}R*u{N*Mp#!K0pKj+_)kFgpSj)zko_gS-h~cvvI9T}FNf z(pSObajuV`9u(tq^wxW$-+S03{f$MGe+HW*E@z_?vF%QVZVzatQT{5{jm`(x7vPN? zS+}6_npoeD-3K%?<^bb8!IJ(@7N4nCDDyn@S3@sxJP@AqNmKB%IQK-Kj_^Mj-3zI^ z)H!DfX>s)DS>+rAjjRQl!acF)XyCCJnybL^Ir=z{VasB8cSXiZ`n9K!NB>ajSM~&C zNPF`G`aMbdqhT;Y{*CAwAb&CG25CG89BG@8k3(4tfiAltj~FnPav)dU6Kl2TbpmAv zQFbu4T*&om=s)LRY?Mtxa~b6hCk7?vUL%f9M_!P$d(h<@u)Y9{U*-Nxc;1V>Pmw+n zc>`(h@Pm2+dBmn#gI?3Pz7*T8;JAr$tSgXprOF0})1X1V^9aXI*tihgAEZnV_?||5 zDAH%3=bhZ6yE_0sZii1V0&Cg^S?{C~3*NWlymvM7X`>wK}BxrW8e2pJS$0ifqdFT<3!Tp-0z3|hme*gE*VD~?eM`<&@Q6veHGrboF7_a9U_=x zvt5QwYluta**tIMxvtJZCl?>CMPIIsZp2dq=@rBQ&q{4~@Hi5k$6%Y(@yWCkPjKHE znRDR1khCx17s8h(f(P;M9*#Yyz-J-#_jU59&)#oel{L{{hTkrf>Cg54@SM*3QtH3- z!Hs9}^^fqrF6^y;a(93a&oo#3eFpkm32*UBA9U^ro#2L#4g4YV3Th|L1Hl75WIjiG z3LUTIIsn}(5@+1k6O+i+uI3=l-KS`yXQAVM6}%f|u4ggvaS_)a!*>-p--1ptbf&Il z&w}T%__K_dI0~#!px*8cpONq{An#=QCf`E09ro@+`3orf8TX^%xeysIW0#D1Pla|I zea5lS{DrdX(4A+sT`p}H{4Dr?g{=eWYrKH|L&1mm(wD=Rm~jq3?^W=8o3vTblV{V% zl4qRu1wJWkqMhX$TMenV_*~Za8<#`Bp12`aveFMOg63O-Y?e;hs&JWfHciz(9p{hRd1uMIQ4$N#T@;}rTQ;G+Er zS!crkBl28$df3n_aW~K%I!-&bxeMPRJ$ImBHaIg?n(6Mg8W^< z_Iv2hgBDw5?o-yNWpCk{n2@!L8qazeH#sZOk2>SMt7fdEAAqb&@O3jf(T><)Xn%>0 zhhh)yqIw1%9f;@QT=RT!UI)KJ!184DB#vZFsCIrM<1hU6DYWw_uTgda=cA!L6uPfS z9{{$WpzEvH+mSSMc6JZ54yD5Tej_`cn3Na`k&Z0}KFGca|JSHD0phY5x_!xC1mA<8 zrH<(DL$eor-vG;<$UB3WBhIu^bmciBV<|P5V?K13U|+BB|7q_|fUc;neStTliX$pc zQEX>HFyH*~!2#PbYQzbHQ;hxb4T50`AR!vH5k)kL3W}l;j~d4~AfkXAV#heo6B_4s zigAonoTAD7?L*$pd+**`m8y5EUe%3Nr>nd7UVE+oT6^u?YY(Tp`oDnUUe{-8YnQv< zj&94~84Ld>uq~GN3jSU2I6(Rp?9+>W(_R0NKF`9uIeCl8yBd8j`D~Y}tMJ@SuXmme z)0@hy?OGoj#*fij`j5fR{3XN|#UgdU-o>rt4kPn5dDp-^74HO19h&*_@eOf*_9Jpm zcGSS^$eWz`PJC#|{1uu5$mw)mDE%Y*!;JltQ|Z2&d8)lCtG+$eU@rGv*GH1qsN8*U zKkTd?;#qjslk*iGhvBhT$zB9>`m(Thr+h8CrSi^}_DaQLihgIydxzf*!bka7OqzbV z9A6B7|NpP=u-;!Y zs=cPUuC}(duCAl8v7x1{t)a$?0$Une8ZG&Ec->!9N8_kL?QLx}H66_j^$ndhP4&(7 zN(}mM`Tu{phnVNjVuM4(T9d>cbHoG2F^M^pY^FT%Ls}O5)r-4^$uk~_))7O5*zO|t ztBFZ=6sPQgZl!W_oQ;=@2h9rlx#9G~qp_Yl2jAKKLlwT-L zxmBEEE+pL^1~FWI37)HoajwGe&*GecczMTLvYq>{rHg06fAZ3m${AC{Z&#vtAMum< zo%{fN2g>hBuX(W2ClUh{hbY(V{DT$70Qk4iWjI>6ql=aMwfOc4vEL|qY$*K!Jg=eS zE$%OL7Q<(~>2m=-m!lO=#bUWgOj+z8?H1S$#jC4B{2k^)%d_Zr5t{wbh)D}Ed{)Sx zA*K%VyLaPv{}A(7c&m&25$6VT8h=4M5e9PzStDJ(gO&A z`7_w*-WA5{X=~!M zp5r01*)RR8{KuDjMFbh<{j=N9uuIxcnFY$+t{(bUjH`$G-k>3BN1PfJ@DO@A1T zugU}9e@OaLF)^IU`S2V9iYlgL)EeSbxYNEUPjOHIrWk4sO+0~ z(IbpMvfmrmK0@E)@m@-wqtvG{K^W`C-!-OuQkmZJ*r`}dUa-k!@@_@T&XIOe&^wHK zqxHx#w^SZV_n_MfW#2Zg;y+X~aLW4tO?|{|27Z=yR<4+e~{gt_kl}h=c!c zqHRAWttVdmBA%$79ma1DVw*Ym9*5r#>9>p?ZRi)`d!#&Ly!=f0UHHkZ()uX-B6-Gh zWxf<&#|GwI@;b5x!@!sG43DEK`T*DWvxBtcZTzox+~Jr5i#8v9s;seQ@ff@d-1F7) zS@iEW)SS0@vb)HEEocAmAFArOd?6h+QXduRkF&$?=*$+OZld{cUyRSra8GAT(AiR=A$?0&VET4^Y(&ATZu;U zU;Sr%G(D^vr{@}5Ux>dt&sJMuo|cb>`rsq6`PTFZdBb2DrR@Fi^5^iMitH=(ei2*K zu~>=EZEW|2d-lm*XXgjuoS}})NyPgoe~asNhZ-x={}nWY(3qb|%%g|@(M312&2Hp{ z_Rm+-AESXIexH4`i}XJE*PtV(T3_&{$Ko*t?{$WnTVsm@j9GsT!>-cLckGSF;qdXP zxHsODO4g60>C=+6q(8_P`r~7syVyhdr=_oh{|W4Mlk#we_sgbRy5EZZQuZv^Puq%i zllPh9Y4Z4e^aJ$zmV{5{ui?`l7WzaRD#KTEeSKKF&7W`_Jk+=6)c|4vlqwK6c%GzyqCtL5e!h8IbSO4X0 z;W}6OsdQdU=B8}DfGs^cV*Pj6PZHPQosoHVMW3s$@C?Y*b6A&5dgxE})$y-NQGU06 zV**=kp-zV4`(3!zU347GUsbCA`^ou1K3w{4p!c(Hy!3* zD|ev!8Ux2{G<+weQ}sJJ2|W&W-OfIH;Qbz3MB3DE)&H9C9LTRiU&TkOdy!5ha~sE> z)cFPGa~q|u6_S2+LesM@$R~!`o4ttJo<(;n@Q4ODw^o~Dp_me9qLja zUA&B@H+^qmt4H|a1Twzy|D|itjA0XfR`x5jN73sCuxcym>*#|2_oes!`e@I_=oU2l z%74q+&|-RAw0zb3lRreWfij=tKTthJY_gc$*r(W+-}P2k>NmX}O=#C8av~T`bN(35 zH}w;tzolaqZ1N>LmKSU_4c-2(Z&+>*8k<(+e?+!6 z7H^K{zT|FCKi`Kg?t`T?6m6-hUwVi#yDGbaZH}Oy=XzM{ieKU@>>v8xY#Du*xHi{S z=y%I}F4lI-?3L^T=lRN-^9$>z<#m+Z%Ax&~lc7;LtW7L-&v(PxSNTV<=`*9>N$c+Z zMrGfsRNHhO>d}0BnSbYeIPJ%VcbE79n#<{Ug?|1z<(@#l0X=U(!=}Y7yw^nYy8GYS zyK0Rp=}xaV)X4{YPv4z{KJXkgH^a3d{rj)57mh6trBkT$KUvp(fsc`qKZhUR3Evn= zGV&OMGy0Z zchJetj*-@rZFYm>TGxxzV>dXLmaIjhyOUpu)x-ZAW6z6hGc=FEavy#43(*?t`g9J?U-$Yj(*wR z%BX|To};7r)44EjPsZM4vv-=~p-zg|;8N$Itp>TF4U~JMxtNU~aK9YyL-FI=VT~^l zOQ*+bhsG+^xb`Z%y2+a>Z)bK^f7vK@VbkngJj!^@!`pK!e+I2~m3+Zo`;l`Q+I5|+ z@kbvivl<<&mE^a>-d){3Mkbx6t1f9hK_(Qh84{yp&GYIu4iC17cpoo4#8MBz z)Wla_mKOR{V~XevGUUe#VDXHt#y~xkyPEu4@D08n`lj>vARWX17t>Z}whH}nd1E@g zN1rd1mlk|3ty52P(A-V#X)yEC&>zG*%Igk;KCQePT6LE`jPHkNhtu;?w(XCvcqi3{ z%fFZ2Mi;ipwUI>p5_x82+I4!Ra?|O)9=)DX?f~*%!Ly(8>=4E~`D|&|z><(XUiwAS z_QMa}>?`+YtDpP%$>WaewDXrM)&Bs7v3b}}5uT-?56G``O;-9+#rLBe_jx9~P3E@x zx9{@H`_Ru;K34udd@sRoaFEZ|-&L+V+68Rh*LhQTJqNQeHZbNZo>Z?rlxrvBHnb-FU!}+K>bXT(&))Lx z6|tzipbtOE`moU={M23kiS%A8#4TtaQ+6KR>d>w3&^P6Wz;cfCec{_5&V!}zO{RDt z{zm#&p1rS&pO(m_^E8Cy%jYPL@#eO39@6#v6=MMPSAgimirTCvDf1efhfVzjR zc)_uba_Tv}Zz|mhrl9i?Jgh zz%qu6Wn^i`(fRDU0?mV^>e_3rJrzsIJ`b<)&c<2!R5Hm;&QmYpIlIJp4}7lFKJ@X4 z=V@^>49`lN9gd)@x~*6Du!QGgae@3tVD842*TZ&+du7UZqTQ06yWsl~nfQfwSQPNZ zr?AN*%KsegA=0#&akuY~sb8b2H(zh2p1{6b!IIV^l5 z3GGRJM&j^%GMz8yi+`2Z4d0A>Eqhw;jSmrA@miG@R2j|-3CuS61~MX zd^XHAm6yYI6g}RPHfKl_OUvieTU|v@%U@j|bv&H!OMj7Gz36wivig|#cHiafp}u!j zZ=0#NXN-6D<|}K~r2iQ2(3gG2uAY=}+%;fwDWQ1OAp@;`&4Fz&I)EjeaNR8SLRb z3E491$vw3hZLRzQd+>{7sybBn*?4@_aROs@#gOVgonjZ)JHjJ|D>kC1J~Nq%M;-a6 zI_BxSJIOEM)lZozZGf%GNm)6w&z z@-r~Rc-=sjwpg9Z`MtESrQJpjpG=j@4|4qpA6W)ZPrj%>DrUOwi+3$q`=S4UUHNu) zKAAh{!{*^L7hiEyq2CO9i{kU-hraAnG>!6pJJ|oWoVUm01hyEA&ppNrAub5^{T__h zldZnPTu5;``Wolq_}!0}=W8m~4`ZI7U+~rB0oT*$=UJ4##W&z6H}JX8M}4I1&hCf7 zt>1`Xi@(p$9z*x6a>vO35xU#y)>IK!kT)0oz53%TjTxU<&R^MNV5PbbDmxO6N9oeT zaTZ^&2R>a3*2!qP3^gYNd+6Kr583wY_5%GnDj6l=C zNA8yvo`ZDAH!OK413G<9`3(G)|9$=c^Qgz?|3XW3->Ukp1-`Yww-)%^$4FI z>~qQq-Mjbc^ONZR{`_D~b6a!csOI{C4ehnH0~=a8>j$Y zPtc6-`03=~W5V|aCv;AUCys3$KRIrlIC0X1X=BED$-lP+Y&)nX9_2%WV)+uFASM2X6kiqiKNuYI(;c&LjGi)O;^;BQ zj6QPf;Uo4s#5Vx{vrh&7Z){Ky-7#(JqyDS!|J8PO)HXCW*Vg%-U%P+x*Vney4(e#F zYj1Ax4S+%QZM98x9i1JW^|ke_-uvG^sHvg0(I$Y7`u3*U2A>A-{lAv_LA4FF?e(>d zz7o*V+0wISP&OMEWZ+&ury(a(pSanj%hl8Lt*G2R`W>{0g-Agt1MS?+Wi6$POaA zZn^ngG3d@@Jtsc;9NlT?$BAF+(3i!?Z;&rGD((^AErP!hU-LrwIe44@jQcBd3!din zW8*1W>TXO!Ryc@-yTf#!0+_*X$NB`06u!DQ{Ex(DMxsbd~-e1Mq z>)_qT^^Fz(ja%W}*YF($dysK3J{!AUn=OtO8)k%&yKEUi3=$8yXd7 zR*e7X-Jgw&3G&ZeFT+0;%g$C8##rU<%F!w2FPVA&{AltjaPXH zivTBpFjSQJ=i(fkDjmM;s21Kjx+V2Z34IPLwQE2y53!0z~;l*M?9aY(`Y~Exn#Am z2~08lv%$tAV(0RD;_36~k|_IE`Qq&`4z8}#b?3viY&jGDJ@I^2*^6PMbJ>_B>dP*@ z@qS*}QbYhe}Fv&7fT<& z*3Ysj49RcI+3cb{yseM8UmKqz{T1aNc05g2Iz$KYhiA2yCtw-NemzRoBXKlrk?o{uGaDt&@20@hjbjpZ`^ zMOu-z!3uP6|5Tc}kL+4wDD{+WJJkPR*(c_2d#fWdqa)bt20FA$AIiSkdGNjDW&UV9 z6#i$Gj6k=Cw9sCr(`gC3#;NJe(vp&TKvxgXOg=ds&RfwRs-Jq)HM^w?E9PvZwYjH% znEy{VR(Ic#ey=0Q4DIWoQ1|H7X*7dh?OstI>V+@It;+URQm-^8u-`U~Isnd@87 zy{L^0g6SOP|4{PI@1eeJ3eyf`(IA(}?%+wlqW;OW}>J9QIX1y3)uANjxhi!j3zUqz4e*y42QpTY=B zJ^+uA?8ZOB`dPLqd3W+#KA2sI{t#p9A@H2$`YG65mp9^%_B@4tD&ZT&PWS|$xI@`1 zrL6;7n3Ly2={EdI8%ly-U&=>5;uFUBao@q#)ZiUas?LL@VNQG^+V}D9NzS$8rRwPm z@0al3qwwFb_$BS_6ZzNjZS|8aLZfeqrUdNDbd&Z+IDVr}?{M<$+mC!A$=JYPH2&nKZjh}rE}(%X>n3dPF;Fub#DG-ZA_qGB!V+Eyc%vt74&g z>p;tfVI4eYhp;E3s;jx;Vh;RgsB`s~{a)GHiggS1cLaWS$p3XX@Dd9o52MjXXYZ5w z28_o$?=IgnB_0LSmayI9y1%lQlWYAY?DZ(dvDNNu?7f*`FJinzdYCKI4vR9(GfTdE zp^P>ip3UjoY@|Mlt<~Q<==PQWmb{h(6Jyx6f17A@m8Q&{s_)O<(JG0k(bBvoQdkWfzgU-%((}O`uhMMK2_AY5A#VE;sIN#|4P|Cc>=bZ?HRcic6Cu`&*fPq|3Be- ze3t)a^VIPPe;0?+U=D)hAs8~vHmz}R`yP1>g@~pnflPjmlwJ3N!Bps_k-h2 zI9^~2_``a6d4oWw9>i$TZ1o`42{C1M7yRhsPn56aA6vPf1N$|wJcr+l`no;nTwk%C z$sSirBRhSv#I|TWvqD?Sx($sI{cygZVt*~0{T$AX+~bpdAnpp!wt4jZ8Elcd?Js>7 zXF8V`>m%Dv6_eAt@tyg>O6@%-EJYJH}+I9~-r zKeE5ZXMj3=UfRj{T~MmV_=WZw_AsQg@qVoomA`_wD_VU}_&;^9XFO}L7=mr~W*7Aw zZwGtmx8B#zZ-k?nE#BZy)*ZrnalAS|qHn5S4SjpOKEC7?dz1a5^bj*`JJk0*;M#&- z3!J;i=cl=6T(T|tp6Vm`Pbd00&Z8>g0e-SCJO7yen+&PGJ2w&2gtdZ*UxfExWJ~4! z()CCBjgIVz1<*aP6zOrl;WaV98UrF{XN9Z5AIWA_$kL5js&$dHWtv|QX ze!xXq5z&q&aSrLtbQC+3Kfq8vt zv&q>R&PU-3efneYZKCXylJSY_lcbT6vUTw?T5)WWxjqce9oX7MrJg;q`3Ew;0I3*3qcn z`WyLttJsG8;q=-ZpN-gcA)afp)u!(CrD;8zo#cKwT~?#(w83IwW!G|ybbn8!8gnP~ zD}ROGDSYf>Sk+7X8-6)L9fbO=#rt~seaQYo-SVB}c39{i!y3kb)qZz#IKNR=91$&1 z#&auEXYm|%D8ulXvwFw|IWJXyva&a$Z*$$`&~B3OJl%mEw3XyWJZ@dSYRueSnKh-2 zQ07$WU6p%NzaQf8Gtoc8SB{}ah)?@gJlkNrgl#I0uU&`!ek7bz)xksP52AapXPI8R zNR#V8U=&-27%Xn%zq=`iR}5pk8@`@F1%D{S6dFp-va$T?c3T zs&Q>~?JTs3+2p)csjkJR_px*E(_8Il2+#6?c=p7PAB3@1@guy%4e>Ls|G-}QpY%=V z@6+K3GMCHW$q~jHbJ^o<*aoXF@k@FMy)JZ5$7DVnC#YvJWU6fWQ+%vVMyta)Qn}6P zx)QB%X^3TV&(nmxb9!X=RjR(8uTAFTj`!W`Q;Q*ZEzyq#JALV%UCZm^@q1-Xgm-^> zou+*prLEnl%)|WW#F9NH+U;fZJVlwO(7xb!3yyg(j$wQ8NH#*7ycEBICC})9h3(lb zKF;wax_)#zfDVra-1HiO?mD(yC|_HSf9?KgG%srVz41Q~uao)ZUtG^s?(jRo$=ixuL!IC0{3SfM zN}o=@&FS;8`z<{qdb>~Xx}86%|ESg3^EG^HAmIzeTo~al8tBC)=|nof zp7N{Y75UnKuvZu_6yqz^_YZ|Vmrla1iIKGZom&!efK&m#Rn zUphp}{{{ahxcN$Xxa%Xyc!r*QE?VIJ6|uy0{0?@1g=1Z`&FrIZ&+NSlHpsic%on3D zW?_%e*2^1}?CZ8C^BVW9?)OnP;Q9bhwuzn?f}i%A^7U)sdYF#ui{ECs{=~s{>B}%* zB|X?_Z9KI7=nDLn%GVZcyoR4mJ#&LkB~R0xEz{TNHXHBWbUmNU#=pP+|Ba}}8~?hC zVc%;0tp&cdz_%9opK5`{y?chm{w-##85a9b{tt`&Ep;^wt)uFj2R7BUw+(Et$~>^8 zqo!$KT}w^tsE)e!+D>m1_$O%o{-XZL`;XjX*HsJoO$XJ+|0}iE{v-F8qHenZ*81DV9@k!d zqyPB-u$AB+MdQ8eYW|(?*a?$uJD4(M(wI@l1<6w;1QZh{O&QqcLj{v2jE%>R8E3P= z6q^tx?>u_l9VU!!T0HR|3;#i(|6<|4wXLmVY{#V5DHA45MiTY`3|zHq;lE!0Z)~q` zAJx=U+tSwI-~9gO-_&l!zjjn@YfD{yhl_@q8f*E3+L{{LYMPrzH8%Ry*Vr1++0xkA zP#f0&+uLhf+v=KIn%X++8yoAqNuaH+t*NzEazo3YK@H7yEzN^EYZ{sc)i*TQD$v~7 z+1xa$vvE*kdqZbiYg;vDOzsw|h<`)8lkP5lSxv0i=D1L7VGb|sDG&c;2r+pS=JM`F zv;B}&YpQ#RO~qT;mg1Ck9R0;qhoY^BG42s_h|QwQO8&Df_IT0#T=B+5u8%P$yh7~s zJMq*dEPqGXf8lxrjK{!y z5IdNUjg6?gk8+;vz(yAi!{K{}o7e$VzB z;5!Q6W$d*kd%f>CP?@)eRQEDuL-0M54rk$KzM*`r`;6SH#i3_N-^cYBHXf(U_VBMP zt(p&AORW8>7`Tu0ABlrQJi0er?c`nrFKo#`#{hhTt$M=72a zb9nV}{{;Jwg;|}(#tiwcWM4v$0b*tIFkzh~Tf_K<@8uQr`{8{yOdq2?4)$yCzX`4O zkWZ68$Xp9wh#o*QlU>zi*pppsj&A=|GQt{BcAd0nsQ(S4`{oY+nNj|GFfF9(R`lVw zVLqU|y7I3{?=Sym_j60D)@m=q?-$YwbcaiWC)xxi?Kqd7z0P)jX7_c`ogVPfWwNws zeD+!AtI%&<@f|n%KFg<7DAk99T#F#aIF0_fE)iv{A)8rWK`!9}OWWW=~ z$mP@d`VQon-w5B8D88ZRhG;$`V-y)<$ZgS<4@G|>`rzBsAF_T9 zb_nxAab~`5|DZqGO_hVMynt_fga13|yR0zB1M_cIABiIIyA+pWb^>^9)Gd!Pme;S{oe;C)t%@zJ7|6F+9 zC1)iYw&V3K8P+Ew{d&GFdrn5T#`5ZZ{o=^Li_YvB{#TRrfb%nbpl41pSK2hR%2xN+ z??u0R^tV5!(^=%7Mpp30`_*|3omZ$ka~jzox*ZD>TZI2&B|Yc{cVa#xD>>gvp831t zGBUn`^DaE~)%lO${m}h_QuRBI+4t4QMd%j>J)L)hhdmM);~{wd86Q5B?4yi6J#B*H meEzH+GIg2Ug^&5NWMl1h1Gan~-e2Ld6Kv tuple[str, list[BaseKnowledge]]: - """Validate bundled JSON as canonical QuantMind knowledge.""" - bundle = json.loads(_BUNDLE_PATH.read_text(encoding="utf-8")) - scenario = str(bundle["scenario"]) - items: list[BaseKnowledge] = [] - for payload in bundle["items"]: - item_type = str(payload["item_type"]) - knowledge_type = _KNOWLEDGE_TYPES.get(item_type) - if knowledge_type is None: - raise ValueError(f"Unsupported bundled item_type: {item_type}") - items.append(knowledge_type.model_validate(payload)) - return scenario, items +from quantmind.knowledge import TreeKnowledge +from quantmind.library import LocalKnowledgeLibrary, SemanticQuery - -def _target_count(items: list[BaseKnowledge]) -> int: - """Count the exact item/root/node projections created by the library.""" - return sum( - len(item.nodes) if isinstance(item, TreeKnowledge) else 1 - for item in items - ) +_BUNDLE_PATH = Path(__file__).parent / "data" / "ai_infrastructure.db" +_EMBEDDING_MODEL = "text-embedding-3-small" +_EMBEDDING_DIMENSIONS = 1536 async def main() -> None: - """Seed, reopen, search, and resolve bundled tree evidence.""" + """Search and resolve evidence from the prebuilt local bundle.""" + load_dotenv() if not os.getenv("OPENAI_API_KEY"): raise SystemExit("Set OPENAI_API_KEY before running this example.") - scenario, items = _load_bundle() - database_path = Path( - os.getenv("QUANTMIND_LIBRARY_PATH", str(_DEFAULT_LIBRARY_PATH)) - ) - embedding_model = os.getenv( - "QUANTMIND_EMBEDDING_MODEL", "text-embedding-3-small" - ) - - library = await LocalKnowledgeLibrary.open( - database_path, - embedding_model=embedding_model, - ) - try: - for item in items: - await library.put(item) - finally: - await library.close() - - print(f"Scenario: {scenario}") - print( - f"Persisted {len(items)} knowledge items / {_target_count(items)} targets" - ) - print(f"Database: {database_path}\n") - library = await LocalKnowledgeLibrary.open( - database_path, - embedding_model=embedding_model, + _BUNDLE_PATH, + embedding_model=_EMBEDDING_MODEL, + embedding_dimensions=_EMBEDDING_DIMENSIONS, ) try: query = SemanticQuery( @@ -85,9 +32,10 @@ async def main() -> None: source_kinds=["http", "arxiv"], tags=["ai-infrastructure"], available_at_before=datetime(2026, 1, 1, tzinfo=timezone.utc), - top_k=5, + top_k=6, ) hits = await library.search(query) + print(f"Bundle: {_BUNDLE_PATH}") print(f"Query: {query.text}\n") for rank, hit in enumerate(hits, start=1): item = await library.get(hit.item_id) diff --git a/scripts/build_library_example_bundle.py b/scripts/build_library_example_bundle.py new file mode 100644 index 0000000..0028c15 --- /dev/null +++ b/scripts/build_library_example_bundle.py @@ -0,0 +1,70 @@ +"""Compile the library example source into a ready-to-search SQLite bundle.""" + +import asyncio +import json +import os +from pathlib import Path + +from dotenv import load_dotenv + +from quantmind.knowledge import BaseKnowledge, Earnings, News, Paper +from quantmind.library import LocalKnowledgeLibrary + +_ROOT = Path(__file__).parents[1] +_SOURCE_PATH = ( + _ROOT / "examples" / "library" / "data" / "ai_infrastructure.json" +) +_OUTPUT_PATH = _ROOT / "examples" / "library" / "data" / "ai_infrastructure.db" +_EMBEDDING_MODEL = "text-embedding-3-small" +_EMBEDDING_DIMENSIONS = 1536 +_KNOWLEDGE_TYPES: dict[str, type[BaseKnowledge]] = { + "earnings": Earnings, + "news": News, + "paper": Paper, +} + + +def _load_source_items() -> list[BaseKnowledge]: + """Validate the auditable source JSON as canonical knowledge.""" + bundle = json.loads(_SOURCE_PATH.read_text(encoding="utf-8")) + items: list[BaseKnowledge] = [] + for payload in bundle["items"]: + item_type = str(payload["item_type"]) + knowledge_type = _KNOWLEDGE_TYPES.get(item_type) + if knowledge_type is None: + raise ValueError(f"Unsupported source item_type: {item_type}") + items.append(knowledge_type.model_validate(payload)) + return items + + +async def main() -> None: + """Build a fresh model-specific database from the source bundle.""" + load_dotenv() + if not os.getenv("OPENAI_API_KEY"): + raise SystemExit("Set OPENAI_API_KEY before building the bundle.") + + items = _load_source_items() + _OUTPUT_PATH.unlink(missing_ok=True) + succeeded = False + library = await LocalKnowledgeLibrary.open( + _OUTPUT_PATH, + embedding_model=_EMBEDDING_MODEL, + embedding_dimensions=_EMBEDDING_DIMENSIONS, + ) + try: + for item in items: + await library.put(item) + succeeded = True + finally: + await library.close() + if not succeeded: + _OUTPUT_PATH.unlink(missing_ok=True) + + print( + f"Built {_OUTPUT_PATH.relative_to(_ROOT)} with {len(items)} items " + f"using {_EMBEDDING_MODEL}." + ) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/tests/library/test_example_bundle.py b/tests/library/test_example_bundle.py index d62023b..a6d0b8c 100644 --- a/tests/library/test_example_bundle.py +++ b/tests/library/test_example_bundle.py @@ -1,5 +1,5 @@ import json -import tempfile +import sqlite3 import unittest from collections.abc import Sequence from datetime import datetime, timezone @@ -22,6 +22,7 @@ / "data" / "ai_infrastructure.json" ) +_DATABASE_PATH = _BUNDLE_PATH.with_suffix(".db") _KNOWLEDGE_TYPES: dict[str, type[BaseKnowledge]] = { "earnings": Earnings, "news": News, @@ -29,8 +30,11 @@ } -class _DeterministicEmbeddingProvider: - """Provide stable local vectors so the example scenario stays offline.""" +class _QueryEmbeddingProvider: + """Provide one local query vector for the prebuilt OpenAI index.""" + + def __init__(self) -> None: + self.calls: list[tuple[str, int | None, tuple[str, ...]]] = [] async def embed( self, @@ -39,10 +43,10 @@ async def embed( model: str, dimensions: int | None, ) -> list[list[float]]: - del model - if dimensions != 2: - raise ValueError("This test provider requires two dimensions") - return [[1.0, float(1 + sum(map(ord, text)) % 97)] for text in texts] + self.calls.append((model, dimensions, tuple(texts))) + if model != "text-embedding-3-small" or dimensions != 1536: + raise ValueError("Unexpected prebuilt embedding metadata") + return [[1.0, *([0.0] * 1535)] for _ in texts] async def close(self) -> None: """Release no resources.""" @@ -108,28 +112,44 @@ def setUp(self) -> None: _KNOWLEDGE_TYPES[str(payload["item_type"])].model_validate(payload) for payload in bundle["items"] ] - self._temporary_directory = tempfile.TemporaryDirectory() - self.db_path = Path(self._temporary_directory.name) / "library.db" - - def tearDown(self) -> None: - self._temporary_directory.cleanup() - async def test_bundle_put_reopen_search_and_get(self): - library = await LocalKnowledgeLibrary.open( - self.db_path, - embedding_model="deterministic-2d", - embedding_dimensions=2, - _embedding_provider=_DeterministicEmbeddingProvider(), - ) - for item in self.items: - await library.put(item) - await library.close() + async def test_prebuilt_bundle_searches_and_matches_source_json(self): + self.assertTrue(_DATABASE_PATH.is_file()) + with sqlite3.connect(_DATABASE_PATH) as db: + self.assertEqual( + db.execute("SELECT COUNT(*) FROM knowledge_items").fetchone()[ + 0 + ], + 3, + ) + self.assertEqual( + db.execute("SELECT COUNT(*) FROM knowledge_nodes").fetchone()[ + 0 + ], + 4, + ) + self.assertEqual( + db.execute("SELECT COUNT(*) FROM semantic_records").fetchone()[ + 0 + ], + 6, + ) + self.assertEqual( + set( + db.execute( + "SELECT DISTINCT embedding_model, dimension " + "FROM semantic_records" + ).fetchall() + ), + {("text-embedding-3-small", 1536)}, + ) + provider = _QueryEmbeddingProvider() library = await LocalKnowledgeLibrary.open( - self.db_path, - embedding_model="deterministic-2d", - embedding_dimensions=2, - _embedding_provider=_DeterministicEmbeddingProvider(), + _DATABASE_PATH, + embedding_model="text-embedding-3-small", + embedding_dimensions=1536, + _embedding_provider=provider, ) try: hits = await library.search( @@ -154,6 +174,7 @@ async def test_bundle_put_reopen_search_and_get(self): self.assertEqual(sum(hit.node_id is not None for hit in hits), 3) for item in self.items: self.assertEqual(await library.get(item.id), item) + self.assertEqual(len(provider.calls), 1) finally: await library.close() From c05ae7cd2b8e0ded055a5d3a50b31b6868e4e0c0 Mon Sep 17 00:00:00 2001 From: pkuwkl Date: Thu, 16 Jul 2026 18:02:54 +0800 Subject: [PATCH 4/5] refactor(library): separate local implementation --- docs/README.md | 2 +- docs/library.md | 2 +- examples/library/README.md | 39 + examples/library/data/ai_infrastructure.db | Bin 98304 -> 98304 bytes quantmind/library/_internal/__init__.py | 1 + quantmind/library/_internal/exact_cosine.py | 236 +++++ .../index_embeddings.py} | 24 +- .../library/_internal/retrieval_targets.py | 70 ++ quantmind/library/_internal/sqlite_store.py | 789 ++++++++++++++++ quantmind/library/_ports.py | 22 - quantmind/library/_projection.py | 278 ------ quantmind/library/local.py | 844 ++---------------- .../build_ai_infrastructure_bundle.py} | 2 +- tests/library/test_local.py | 37 +- 14 files changed, 1290 insertions(+), 1056 deletions(-) create mode 100644 examples/library/README.md create mode 100644 quantmind/library/_internal/__init__.py create mode 100644 quantmind/library/_internal/exact_cosine.py rename quantmind/library/{_embed.py => _internal/index_embeddings.py} (62%) create mode 100644 quantmind/library/_internal/retrieval_targets.py create mode 100644 quantmind/library/_internal/sqlite_store.py delete mode 100644 quantmind/library/_ports.py delete mode 100644 quantmind/library/_projection.py rename scripts/{build_library_example_bundle.py => examples/build_ai_infrastructure_bundle.py} (98%) diff --git a/docs/README.md b/docs/README.md index 2b127ed..1a05f3e 100644 --- a/docs/README.md +++ b/docs/README.md @@ -17,7 +17,7 @@ harness. | Paper extraction | `quantmind.flows.paper_flow` | `PaperInput`, `PaperFlowCfg` | `Paper` | [README usage](../README.md#-usage-examples) | [Papers](papers.md) | | News collection | `quantmind.flows.collect_news` | `NewsWindow`, `NewsCollectionCfg` | `NewsBatch` from `quantmind.preprocess` | [Collect news](../examples/flows/collect_news.py) | [News collection design](design/en/news.md) | | Bounded fan-out | `quantmind.flows.batch_run` | Operation inputs and shared config | `BatchResult` | [README usage](../README.md#-usage-examples) | API docstrings | -| Local semantic search | `quantmind.library.LocalKnowledgeLibrary` | `BaseKnowledge`, `SemanticQuery` | `list[SemanticHit]` | [Semantic search](../examples/library/semantic_search.py) | [Library guide](library.md) | +| Local semantic search | `quantmind.library.LocalKnowledgeLibrary` | `BaseKnowledge`, `SemanticQuery` | `list[SemanticHit]` | [Library example](../examples/library/README.md) | [Library guide](library.md) | Import public inputs and configs from `quantmind.configs` and current public operations from `quantmind.flows`. Import result contracts from the canonical diff --git a/docs/library.md b/docs/library.md index e3a52a5..703a017 100644 --- a/docs/library.md +++ b/docs/library.md @@ -93,7 +93,7 @@ Maintainers can regenerate the model-specific database from the auditable JSON after changing the source data or storage schema: ```bash -python scripts/build_library_example_bundle.py +python scripts/examples/build_ai_infrastructure_bundle.py ``` The bundle's facts and short citations come directly from the diff --git a/examples/library/README.md b/examples/library/README.md new file mode 100644 index 0000000..99c9a7c --- /dev/null +++ b/examples/library/README.md @@ -0,0 +1,39 @@ +# Local Knowledge Library Example + +This example searches an auditable AI-infrastructure knowledge bundle containing +`News`, `Earnings`, and a research `Paper` tree. + +## Run + +Put `OPENAI_API_KEY` in the repository `.env`, then run from the repository root: + +```bash +python examples/library/semantic_search.py +``` + +The bundled SQLite database already contains six 1536-dimensional +`text-embedding-3-small` document targets. Running the example embeds only the +query, then uses `LocalKnowledgeLibrary.open()`, `search()`, and `get()` to show +ranked evidence and tree paths. + +## Files + +- `semantic_search.py` is the user-facing query example. +- `data/ai_infrastructure.json` is the canonical, reviewable source bundle. +- `data/ai_infrastructure.db` is the generated SQLite library with canonical + records and model-specific embeddings. Do not edit it directly. + +## Rebuild the bundle + +Regenerate the database after changing the source JSON, library schema, +retrieval-target rules, embedding model, or dimension: + +```bash +python scripts/examples/build_ai_infrastructure_bundle.py +``` + +The build validates every source item as canonical QuantMind knowledge and +removes an incomplete database if embedding generation fails. + +See [the library guide](../../docs/library.md) for storage, retrieval-grain, and +financial-time semantics. diff --git a/examples/library/data/ai_infrastructure.db b/examples/library/data/ai_infrastructure.db index 34bc6008ecdbe4c13c9b2ab50bc59a250ef630c5..10f4eedc1727baf2eb29e2226b3f921cca2e7450 100644 GIT binary patch delta 6208 zcmXZg36vGpwFck=h$twI2sl6?3W$a{0e1IoEG4MubHa!QpQb>3KIb{c0dF}XC@3O> z6P6Is5$k-~ZlcxSgh(r~aO5 z(>|%@)#i|lu7!9)CFEl(AxqyB3P}jUnvF@mHR|HH2iU{s~Av1aOVMeQP27nxj$XG&h9ov4)U825ewU zh+p9BCjqQC_7SWry@xi0u+f|`Y@+5+EJvqXzy_W>g7WNZck8>#k|H|fJG-t(G5 zJXa9Xkv+s-y(HTr`Lpo@TSB^7_?MIc{`BtMeMsHb`|Ge8b21m zn`CX$o(b?`YP{DJ@{53egYatP=ODi=(w~~wy)k5+8$)q=Q;6I{mRQ`Xu zlI^5#bVD^1qR)<~gfKx35{DpM*AlW!;hsmGm*MVW{^9J_)Q4^6Rq!gtgUPwn z`_v^Nxk5z!7%t#&b0miV_z%&2gveTQiy*q0tzydhn%7GJldYPay#)}Sg2e6}}nZ*XtTH?dm(-$M}tW^5z#uS%PXNc)0i~oWKZB`7aNBW z{%-Xs+PEeNH5RQ`PaRsSA)Q1{;L%f9t!c7GMKcNTkdl-n5g>tc#*Q*J_cmQetl2INc9 z2k#v!c1KY>Vr(xHH&J3AAZxu-W79-g?gN%=dYf99jLe4~OEzuCsf{RFPC<(16QpQ? zK#qVd)nQKo^iuun^;PF$GFUDZRl2th+iSdqG>ABvOTrgLw7rNpN)ETw-r?{lj`mOw zFCn&DSr@+BYWMa0gZ&;HZGn^;ipq4{A&4>B8|p)Jl;F-WCOw%nR}0%>{J&T!j{I!K zPh)*AfbXQ@R^SU7LUe5{B+h}b42a2zrBjq@ahA8SoZdumLM0T(iL4#lyNzvNyBb~} z^A%iix;YmK_GDZB(Q2ooVD_w4Lo%DQ^BMe$gR(+Zd0D~bNtk}DI4{*+iCh`!9AIO} zFJ1Xkq_?NsWX=Wvd|4p&_WUI_-4VJJ*1d}E8aRAJ7M!K`deOnXUgMbE>MfXjf^?Wjs&&GDP+l!DbBi0a8Z!YRw@+otxCi8f33_ zpiJRJ)yI>JtC#%t<`605cmT!Th0(&{*A)3ko?Dy++akKVV9Sk4Z52nrAKu^)lB4(^ z2In9#(31E`eic|yVA;{Y9lQCCz~|}jq_3NZ6kc{9;A_a<#Ayc@m)E!M1I1Pdj`aN; zg6~kN4Dd(Kwa9ERw;Nwe5j+9b=t?!j`-yC>&MusPuSnVgTtu>R*TXinC)WamcJ*xH z_%|FZGKWINY)ajM=% zAT~3xlg<-Nx&hO++Fv*W4#IMx32*BAZ$Ps+?;)~nYazZzG|irk+V=kJMEU$fv+Y>a zBiK+h&pd}1Kgm3Wmw#i=pCT|fqjG76>ZfvweN2JkdhhE@EE$ujT>r&g-aE_h*2YwF zF~aw&o{rkji*&A{)ear&oSd9(63E%ckc@T^?N8cc_)5P)Rxu@P5!(ByvEqSE#Lc*yB2LUlv3g8W}-U` z*@Hz^_8;!t_=?TC+Alv=%!}{P-;X-dlh~@$9r-*E{$%SvTIP71RS8A8Te;<=&oi=3 zW`7~&8=ODKa7&V@@_2zLFB8!xHk+TDTCeTdKv7*~taHUB#e8=8ZS4L4XC0ePNRB~( zO}Y$TFH3I=HqWS|AF7NN8n{-3HTn80mffgfvq^42V3=pTr`#;siTEy#PDgg2@h<^g zV~nGGo}wFz=yDGCN1~r}b%oo6h!kh9BRNIXR#P#Zx`VaPU|$^y@O;(lhz)zxLR{9v z;fkrN1L9_56MdJr7WcR4agJAEzLoko2H<$Wj_c&_`br#?Uwo~v1FR384q;_;C>9z! zUXbHF=UPE7t;GlY9H3tzR*SntFvuDQ8ut*<2CRCBVmnS^pifK7G|`S$WaV*tANGYl z*Q@9PiVgBjl|s9WG*MjnI*#oFWsM{|!@B^X6FE&e6?Htn7K#D-?kMyBq}*DD=}wr< z6~R;TyS5h6ef2F_67rs;Z-c!h77g-##x3R;jz=N18j!-v?o)t;!n=-x5tR@gaAZ(tMj zALd)d#yzpyS|0lAJBjz2N-vj`Tt)X|xft%d7*EA~ruk>8gNK{sh4XMP!~deCyH&E4 zx2-PgW5SmvF9$fAB)cN?q{;T_sKS2Fme!v=#bV>-eeF0luL$IRW2Yc6LmuQQyB_Jy zRPV-GNPqCG)ptCygVex2Rg;F%=LL>a?AkWk061tbsxvs`tQZJ@zTsq!g2HdQ>}v^6kvWzUcN_0pkq0;@2;@wJj`#MvEq2sqkH}4xtl=}D7b=P& z9AC?*qscebh5T*~=MT12fQ!70IC0{{m+HT<1Yy6Q_9>+9VktRP;wCspdn^0wYV*$V zJkC)gA{_*i`5x+eysNeis2V?Eg54syRzzz6?a=6A0N8@hI*Gt3lpaSi$71Wx;sz>B zMEF*Hz0}Sej#3fMmPYju4~KJ>AkO2kkG#L({WBk(5PY99T?BF(+uf;tgnM~B7TcMQ zIP9WdalVg;$Q5snlsX8*O&%lHd)L&0_A?^56&VXYJ{F0ED*Lz8DEH?z2U#xVrU8#) zi5)7I>+F3BJm?4=&t|GQGX%QM*ngJm@?DGea%p&r(=MWUg)!O6yteKnhluDN-~Bmt zPfJFb(`K>z9N-$;qdb$$m8SGE;IldYv(;f$Jw4^5FE=axy4gx z|E>}WghDrCmUFH@`;hIo)Y*p3_>y3wefbiAf8=O2rjMA=+hnVvyoqLuYpuUNV>?W2 zFGzYw!N^7zI?L%cQ7^3v(Fzqm3-F()val&ci}WL!%p~!GN=RR>gjmYL8qvRiR2fVv z)n}PA-@cq-7OT`!*61)f|Euq(OE4h45wU$Ga{=uDz|sttvu$rY7TKX}rXr`9f&xx< zQ3r1WK9~ciS^k#V=xP2HFsT}k75zrV*YOuZSwE3x<4x|%k?T+5XHl_9c9LF zc9w~Q8$v#k!*M`Wc-oB$FP44U&mw+>^!ObYPBgxfBD?9|0OLYRb%NtNpG4uM4iiUX z^f`5oM7DY|=PO7iH%V_D*Z8-G?K2i(T!+vm(UhrKh4BJy3KXxAkd(>3d#KVA3 zgQNJ0bJ%p(o)5F!gmyuGgZU2`vw=nZDrTvDsS= MMI8IJ5klAh0}TCglmGw# delta 6208 zcmXBY3%FKe+6VBbQc+GtCz6Fkia{!Szwc%djgV8HoMy}_rO)wWieadC)rgQ#NQ`1X~zm@CS&$XVzefZ!1`+g4ZPSZB2zsFm&PAb(2 zO(7dm3(@uEkc}#bkd#9_q8y?LwGa<6w?lKt-!p!;6yk2BkbYGQ>2tMEJ*+tdI?0Q* z5LR0Iu0G^Dll_jpcjyka_6q$y*xYw%NavM9-mf84FJ!A7J4@JES6>PFf^w*iCj3rQ zNaof;*192toy#0iey$Yq=}jCph4g-N>nI;o3(>EdLvkjAjn?NCjj{gCA%20(Pudrp zehB0reFs_pfzC*vb_ebm{r%|P%kSJuDP*TKg#4mf$Okoq{2h%U86l!7 z)Yq0nRHd}5wH39HtjAJsI(s&U=u>jXTK~k_rEIj|__b1qE(YW?b}*DYrk#SNfr1$? z$ZN<8GE~kK?Ke#!ZrdEv;Sh3^jawQ*D~7MF3)Mq7t*Z;sDH!^^7Q!Y5&ub3ZHh`VU z(BG}!VC?`7_5<`=Oo=*f$glyXRRYzW!U035>hRt+=LkuA2Vr1ui;DdHke=#AQ9Jh_pvKosdU91 z)Xa!dB~(QphG^GA_f%8JJBfNer+3KlRUF@^h_;3L3RFfSY~*OXG~CQsdw>=;hI}!( zPk}hB6smQYs zu9S{_v_-~kI6I7i;{o{ykhicQf(WhEoeDW=*aLIZq+u8N6+pU70OX^u=^vy%_As6f z{9It(VDIhv5Lb>8b-!Auu0zR#COJ=DEu)vv^Y*IzrL7u5+>YGrT1WtmuLiWUF+=H> z^&usf-yz~FfeUiGpZUXmMO}ToF+|Ha?ImjV^GgNW4Dd=jXHZ?JWYgvGW6lOy=&TI@ z1S|Oj5gsKS9XMSeiaJIwAUC)!q#FV3-4OCWk{wtM*$uW=y=u<@`)}=k)x>U?U^o4} z^^WvD6pW&cYwW49>~o;oHdpL>P733XmqWT1+VQ0jZX$Fz5QD5;FN)U~X)VuV^v?n2 zEs+iZ0t3~RnyM}stbR^!k|;j_>~?lafE>;4wdlXZyc0FM4v<^e`aMRrD-NOCpc}~O z?F?*S^yj86IZYP&J^(~-&VOX6ue`o#uEOB$<&YHTfGwNcBGPG$zh4g39{_Gi)NYDfJ;@D?SD@QXTCLd~tYvVXIf>LoyH@A1bOo6dkY8){05Q$vlQ%q|lLJ4)bF; zf7|?VK=y11(My2sCbHSSEvzfn>|Y}9NxqGg6g6}%`RTqRYa!X7R*uE=1sLdB5=hM< zbSPv$LZmpd^F_AA;-B@ua)3+~@gj9F5eriQT;@wT??@S2g|n;$@T)nWq7~;vEA1{! zZnU1?ts;lzU(W0Q6Fm@#Dv$eUBlO<~VjScv87Ly1 z%h*UDr7L+7=#J*5adZHnF9BGb?Jon^mF_q!-K{^t93Kg~^LNSa3qY!WI-vVh(D<-u z7|xsPLniX%Q$lZYD8H%bs(W#G5J1+VwVZZU#U*lbAAe@lbpk4WLVXU0l`E%vDrt){1~!CJQ*d9Nh;4W6x)O&51I4wWgvK%VZ2W zYSTa1vbR!J_4!om>LtIWDI^LxI?xV!57o^;eC$}$A+5g?>}LQJSiTnkLq*n|q0XpXz-W7Hbu)IXCjGEB!MI5@ZAE)UeMoMk zyRA0d_?em$&R2ldAK06u&Nc!lHPMrbbC@)YDBJ%BG5!dFPMrV0{3@XM31Bb~K^wJ3 z=_2FqfNrMjQ4to?*$S1?7EQ-Q6^L(rr6N7lcM18ibRQ*uj{X46?um<=bstRFG4Xsh z9;-Bk>fRh}MX9~XVU(`1(30Xu?tnwFa07)k3TBK*?%}XU%}od7-PoyVKk64}|8PJr zmxkxbpDg-<(W=0XFn=07g_nD(#kY$5C}}yDkIH&^y_bQ4fj?22#JCDi(3B|5aF_2c zKy6P(CC9^zU&TZ_PM#+_uRi4Z)!jW6<`k&Sh9#shqU9W&W3`K>KCpUw+vXnZ+D8*Se?0-Z#s@*3f{v`+xJ1IqKz?}d^M zv(N8gaR0J9y8wfJ)J2zR-;?VtsuM-=QUP3K!^l0Xa2Hs&rK5Sqza#T@Q^-27e-c}Y zJ=vr`gWegnEf-!T;v}44Te!uxTs(!~=>%UP;0!2kx-9450l@yvV99v4=vp?1@G#I@ z0d};M9IkIZI>wRx6`6NAdCvGRBAxE5mQv|Si=(;|{eI@A+5h=d4vyJSIjxFUPO7To z;+Xuq@9J`>K10ympQ-Y&2!L@ST95XhI6`^cpWanvJH&b&rpQ%qHolybcgeJ8d!#;@ zY?-w^FuOIdPx*ddf#@*{6Ghl0U(br(`Cc40@r|{R9c}$O-@l2zl|0Te??B1=Gx`Z3Yg`dRR=4;ec9+jrm-CISGCs^Rb@GvUb|RIZf34pZp!G{a`VTUK z3#V{4zP9Dw&ppxe_xb8){>Mh|zgZ$4z(HZOyNEU#@9x6rU>q~{grZUdVT7;iHogbl z>LH&2B;ffWBEw2@i6DSa6j%J7b>~3ASn)d|zql5{sQ{$r&b7V}paaa`TIAm^mCQD* zc82VcBJUi%N$5ZkEpjgO5Jg~M69)qzS1;9}`e$*nMx+Wa%r<_{<6N{K=HI8pHr z0rr9@J9^Vi(f?Y*Uh*!`5122Q+KvrpK)8>CCDv`p;DQR9u<8mf>MDL+^w>-~yf3@1i~AboLrv7`cDHfm*MWsTkhGab2=-N-y0o68ConNH&}F1bBI^jUUN7(OUN^U zoCM6DfOe52n-s?P=H4`SvY-YuguHKs;iDmOW){Dh4q!UMmH^)zd{= zG4<8})|)IR@yHG2wSx`GP{~Tk;BJD|6kHH0Dzu# zA@&0zo&f~L5+_tF*VTX0eb9d1vSxm!K;O0Yn_^$SZ9`voNnhK?JFWPiOuCI@PXT5%^2alN6Hpt;UIzSJ4*o{guE?&ieyF#YdI&4YT#H%h zOw~o+om}OFUz`;GQA!^+85%edXrZ+<`Pj~dS? zhwA;-<}i4p`CR`Da{z^|K-tcj`Sb(qeS`WP$X!e>0U4WIQ!JcEXQ6XBOcnK2<&dE@8Z76}0OZNJ3PM)rXaHVhYay?aBNp?5m4V+&}rhU!N zXXBg5J*h}L(Em&U9t&*c^oEc<%aPn9d+A^8eeL*+TN~GLve}x{q^lstNP>a%bq-LO z99Z_dOnQ!H{C2v|h5Q0C-RLdkv?#h=MZFP|_mfu>X>Sb^#lta5>*fE1f&s$aC_-4B ztw1(&@F|5B%^rbJ?}&lRI8yC#jlmBKb{M%#E^nvflq?j=Lu3BO1%^8~!<*u28AAVLVO>9;7@#zoqXq5w+svTsvU}C-!k(hq`{& zR%!Ff-t>&WqyHquBb45C72&qvb#AiPEtI zw`ubJM)z5TF;Sy5%wnuHq~AIXT|n^+b61e1oY|_;U=cp!=)GE0QvsOFSy7b#koN;c z-OQdBPkJcu=L3a}cp5*g1qXOs@!UzvI*)Pak)O}u^F0{=$Zd*RDpKi;m$BnX9e1&9cWMq@(l;Ug Q3nP7?yG=xXZG_O}KcC@#_5c6? diff --git a/quantmind/library/_internal/__init__.py b/quantmind/library/_internal/__init__.py new file mode 100644 index 0000000..9c39f0c --- /dev/null +++ b/quantmind/library/_internal/__init__.py @@ -0,0 +1 @@ +"""Private implementation details for ``quantmind.library``.""" diff --git a/quantmind/library/_internal/exact_cosine.py b/quantmind/library/_internal/exact_cosine.py new file mode 100644 index 0000000..0a6f0c1 --- /dev/null +++ b/quantmind/library/_internal/exact_cosine.py @@ -0,0 +1,236 @@ +"""Deterministic NumPy exact-cosine filtering and ranking.""" + +from collections.abc import Sequence +from dataclasses import dataclass +from datetime import datetime, timezone +from typing import Any +from uuid import UUID + +import numpy as np +from numpy.typing import NDArray + +from quantmind.library._types import SemanticQuery + + +@dataclass(frozen=True) +class _IndexRecord: + """Durable metadata and vector bytes for one searchable target.""" + + target_id: str + item_id: UUID + node_id: UUID | None + item_type: str + matched_text: str + as_of: float + available_at: float | None + source_kind: str + confidence: str + tags: frozenset[str] + tree_id: UUID | None + dimension: int + embedding: bytes + + +@dataclass(frozen=True) +class _RankedRecord: + """One exact-cosine result before canonical hit resolution.""" + + record: _IndexRecord + score: float + + +def _coerce_provider_vectors( + values: Any, + *, + expected_count: int, + expected_dimensions: int | None, +) -> NDArray[np.float32]: + """Validate provider output before it can enter the durable index.""" + try: + vectors = np.asarray(values, dtype=np.float32) + except (TypeError, ValueError) as exc: + raise ValueError( + "Embedding provider returned non-rectangular data" + ) from exc + if vectors.ndim != 2 or vectors.shape[0] != expected_count: + raise ValueError( + "Embedding provider returned an unexpected number or shape of vectors" + ) + dimensions = int(vectors.shape[1]) + if dimensions < 1: + raise ValueError("Embedding provider returned zero-dimensional vectors") + if expected_dimensions is not None and dimensions != expected_dimensions: + raise ValueError( + "Embedding dimension mismatch: provider returned " + f"{dimensions}, expected {expected_dimensions}" + ) + if not np.isfinite(vectors).all(): + raise ValueError("Embedding provider returned non-finite values") + if np.any(np.linalg.norm(vectors, axis=1) == 0): + raise ValueError("Embedding provider returned a zero vector") + return np.ascontiguousarray(vectors, dtype=np.float32) + + +def _decode_stored_vector( + blob: bytes, dimension: int, target_id: str +) -> NDArray[np.float32]: + """Decode and validate a persisted vector as corrupt-index protection.""" + if dimension < 1 or len(blob) != dimension * np.dtype(" tuple[bytes, int]: + """Encode one validated provider vector for SQLite persistence.""" + return ( + np.asarray(vector, dtype=" float: + """Normalize an aware financial timestamp for deterministic filtering.""" + if value.tzinfo is None or value.utcoffset() is None: + raise ValueError(f"{field_name} must be timezone-aware") + return value.astimezone(timezone.utc).timestamp() + + +class _CosineCandidates: + """Filtered records and normalized vectors ready for one query.""" + + def __init__( + self, + records: tuple[_IndexRecord, ...], + matrix: NDArray[np.float32], + ) -> None: + self._records = records + self._matrix = matrix + + def rank( + self, + query_vector: NDArray[np.float32], + *, + top_k: int, + ) -> list[_RankedRecord]: + """Rank candidates best-first with stable target-ID tie breaking.""" + if query_vector.shape[0] != self._matrix.shape[1]: + raise ValueError( + "Embedding dimension mismatch: query vector has " + f"{query_vector.shape[0]} dimensions but the index has " + f"{self._matrix.shape[1]}" + ) + normalized = query_vector / np.linalg.norm(query_vector) + scores = self._matrix @ normalized + ranked = sorted( + zip(self._records, scores, strict=True), + key=lambda pair: (-float(pair[1]), pair[0].target_id), + )[:top_k] + return [ + _RankedRecord(record=record, score=float(score)) + for record, score in ranked + ] + + +class _ExactCosineIndex: + """Immutable in-memory exact-cosine index rebuilt from SQLite rows.""" + + def __init__( + self, + records: tuple[_IndexRecord, ...], + matrix: NDArray[np.float32], + ) -> None: + self._records = records + self._matrix = matrix + + @classmethod + def build(cls, records: Sequence[_IndexRecord]) -> "_ExactCosineIndex": + """Validate durable vectors and build a normalized matrix.""" + vectors: list[NDArray[np.float32]] = [] + index_dimension: int | None = None + for record in records: + if index_dimension is None: + index_dimension = record.dimension + elif record.dimension != index_dimension: + raise RuntimeError( + "Corrupt index data: stored targets have inconsistent " + "embedding dimensions" + ) + vector = _decode_stored_vector( + record.embedding, + record.dimension, + record.target_id, + ) + vectors.append( + np.asarray(vector / np.linalg.norm(vector), dtype=np.float32) + ) + matrix = ( + np.ascontiguousarray(np.vstack(vectors), dtype=np.float32) + if vectors + else np.empty((0, 0), dtype=np.float32) + ) + return cls(tuple(records), matrix) + + def filter(self, query: SemanticQuery) -> _CosineCandidates | None: + """Apply metadata and financial-time filters before query embedding.""" + as_of_before = ( + _timestamp(query.as_of_before, "SemanticQuery.as_of_before") + if query.as_of_before is not None + else None + ) + available_at_before = ( + _timestamp( + query.available_at_before, + "SemanticQuery.available_at_before", + ) + if query.available_at_before is not None + else None + ) + item_types = set(query.item_types) if query.item_types else None + source_kinds = set(query.source_kinds) if query.source_kinds else None + required_tags = set(query.tags) if query.tags else None + indices: list[int] = [] + for index, record in enumerate(self._records): + if item_types is not None and record.item_type not in item_types: + continue + if ( + source_kinds is not None + and record.source_kind not in source_kinds + ): + continue + if ( + query.confidence is not None + and record.confidence != query.confidence + ): + continue + if required_tags is not None and not required_tags.issubset( + record.tags + ): + continue + if query.tree_id is not None and record.tree_id != query.tree_id: + continue + if as_of_before is not None and record.as_of > as_of_before: + continue + if available_at_before is not None and ( + record.available_at is None + or record.available_at > available_at_before + ): + continue + indices.append(index) + if not indices: + return None + return _CosineCandidates( + tuple(self._records[index] for index in indices), + np.ascontiguousarray(self._matrix[indices], dtype=np.float32), + ) diff --git a/quantmind/library/_embed.py b/quantmind/library/_internal/index_embeddings.py similarity index 62% rename from quantmind/library/_embed.py rename to quantmind/library/_internal/index_embeddings.py index d2ed343..9f45ef8 100644 --- a/quantmind/library/_embed.py +++ b/quantmind/library/_internal/index_embeddings.py @@ -1,13 +1,31 @@ -"""Private OpenAI embedding implementation.""" +"""Private embedding client used to build and query the local index.""" from collections.abc import Sequence -from typing import Any +from typing import Any, Protocol from openai import AsyncOpenAI +class _EmbeddingProvider(Protocol): + """Embedding seam used by production code and deterministic tests.""" + + async def embed( + self, + texts: Sequence[str], + *, + model: str, + dimensions: int | None, + ) -> Sequence[Sequence[float]]: + """Embed texts in input order.""" + ... + + async def close(self) -> None: + """Release provider-owned resources.""" + ... + + class _OpenAIEmbeddingProvider: - """Generate embeddings without exposing provider response types.""" + """Generate index embeddings without exposing provider response types.""" def __init__(self) -> None: self._client: AsyncOpenAI | None = None diff --git a/quantmind/library/_internal/retrieval_targets.py b/quantmind/library/_internal/retrieval_targets.py new file mode 100644 index 0000000..5cc1726 --- /dev/null +++ b/quantmind/library/_internal/retrieval_targets.py @@ -0,0 +1,70 @@ +"""Map canonical knowledge to stable searchable text targets.""" + +import hashlib +from dataclasses import dataclass +from uuid import UUID + +from quantmind.knowledge import BaseKnowledge, FlattenKnowledge, TreeKnowledge + +_PROJECTION_SCHEMA_VERSION = "1" + + +@dataclass(frozen=True) +class _RetrievalTarget: + """One stable item or non-root node target to embed.""" + + target_id: str + node_id: UUID | None + text: str + projection_hash: str + tree_id: UUID | None + + +def _project_knowledge(item: BaseKnowledge) -> list[_RetrievalTarget]: + """Project flat items and trees at the issue-defined retrieval grain.""" + if not isinstance(item, (FlattenKnowledge, TreeKnowledge)): + raise TypeError( + "LocalKnowledgeLibrary supports FlattenKnowledge and TreeKnowledge" + ) + + root_text = item.embedding_text() + if not root_text: + raise ValueError("knowledge embedding_text() must not be empty") + tree_id = item.id if isinstance(item, TreeKnowledge) else None + targets = [ + _RetrievalTarget( + target_id=f"item:{item.id}", + node_id=None, + text=root_text, + projection_hash=hashlib.sha256( + root_text.encode("utf-8") + ).hexdigest(), + tree_id=tree_id, + ) + ] + + if isinstance(item, TreeKnowledge): + if item.root_node_id not in item.nodes: + raise ValueError("tree root_node_id is not present in nodes") + for node_id, node in sorted( + item.nodes.items(), key=lambda pair: str(pair[0]) + ): + if node_id != node.node_id: + raise ValueError("tree node map key does not match node_id") + if node_id == item.root_node_id: + continue + text = node.embedding_text() + if not text: + raise ValueError("tree node embedding_text() must not be empty") + targets.append( + _RetrievalTarget( + target_id=f"node:{item.id}:{node_id}", + node_id=node_id, + text=text, + projection_hash=hashlib.sha256( + text.encode("utf-8") + ).hexdigest(), + tree_id=item.id, + ) + ) + return targets diff --git a/quantmind/library/_internal/sqlite_store.py b/quantmind/library/_internal/sqlite_store.py new file mode 100644 index 0000000..3a7eb12 --- /dev/null +++ b/quantmind/library/_internal/sqlite_store.py @@ -0,0 +1,789 @@ +"""Concrete SQLite persistence for canonical knowledge and index records.""" + +import hashlib +import json +import sqlite3 +from collections.abc import Sequence +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from uuid import UUID + +from pydantic import ValidationError + +from quantmind.knowledge import ( + BaseKnowledge, + Earnings, + Factor, + News, + Paper, + PaperKnowledgeCard, + Thesis, + TreeKnowledge, +) +from quantmind.library._internal.exact_cosine import _IndexRecord +from quantmind.library._internal.retrieval_targets import ( + _PROJECTION_SCHEMA_VERSION, + _RetrievalTarget, +) + +_DATABASE_SCHEMA_VERSION = 2 + +_KNOWLEDGE_CLASSES: dict[str, type[BaseKnowledge]] = { + f"{knowledge_type.__module__}:{knowledge_type.__qualname__}": knowledge_type + for knowledge_type in ( + Earnings, + Factor, + News, + Paper, + PaperKnowledgeCard, + Thesis, + ) +} + + +@dataclass(frozen=True) +class _CanonicalNode: + """One normalized canonical tree node.""" + + node_id: UUID + parent_id: UUID | None + position: int + payload: str + content_hash: str + + +@dataclass(frozen=True) +class _CanonicalDocument: + """Canonical aggregate root plus separately persisted tree nodes.""" + + knowledge_class: str + item_shape: str + payload: str + canonical_hash: str + nodes: tuple[_CanonicalNode, ...] + + +@dataclass(frozen=True) +class _StoredEmbedding: + """Existing vector and invalidation metadata for one target.""" + + target_id: str + embedding_model: str + dimension: int + projection_hash: str + source_content_hash: str | None + knowledge_schema_version: str + projection_schema_version: str + embedding: bytes + + +@dataclass(frozen=True) +class _PreparedPut: + """Validated canonical write plus the vectors it may retain.""" + + item: BaseKnowledge + canonical: _CanonicalDocument + as_of: float + available_at: float | None + tags_json: str + existing_embeddings: dict[str, _StoredEmbedding] + + +def _json_payload(value: object) -> str: + """Encode canonical JSON with stable ordering and separators.""" + return json.dumps( + value, + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ) + + +def _canonical_payload(item: BaseKnowledge) -> _CanonicalDocument: + """Split a supported canonical aggregate into root and node records.""" + knowledge_class = f"{type(item).__module__}:{type(item).__qualname__}" + if knowledge_class not in _KNOWLEDGE_CLASSES: + raise TypeError( + f"Unsupported knowledge type '{type(item).__name__}'; " + "only canonical quantmind.knowledge types can be persisted" + ) + full_payload = _json_payload(item.model_dump(mode="json")) + canonical_hash = hashlib.sha256(full_payload.encode("utf-8")).hexdigest() + if not isinstance(item, TreeKnowledge): + return _CanonicalDocument( + knowledge_class=knowledge_class, + item_shape="flat", + payload=full_payload, + canonical_hash=canonical_hash, + nodes=(), + ) + + nodes: list[_CanonicalNode] = [] + for node_id, node in sorted( + item.nodes.items(), key=lambda pair: str(pair[0]) + ): + node_payload = _json_payload(node.model_dump(mode="json")) + nodes.append( + _CanonicalNode( + node_id=node_id, + parent_id=node.parent_id, + position=node.position, + payload=node_payload, + content_hash=hashlib.sha256( + node_payload.encode("utf-8") + ).hexdigest(), + ) + ) + return _CanonicalDocument( + knowledge_class=knowledge_class, + item_shape="tree", + payload=_json_payload(item.model_dump(mode="json", exclude={"nodes"})), + canonical_hash=canonical_hash, + nodes=tuple(nodes), + ) + + +def _assemble_canonical_payload( + *, + item_id: str, + item_shape: str, + item_payload: str, + expected_node_count: int, + node_records: Sequence[tuple[str, str | None, int, str, str]], +) -> str: + """Rehydrate normalized tree nodes into the canonical Pydantic payload.""" + if item_shape == "flat": + if expected_node_count or node_records: + raise RuntimeError( + f"Stale canonical knowledge for item '{item_id}': " + "flat knowledge unexpectedly has tree nodes" + ) + return item_payload + if item_shape != "tree": + raise RuntimeError( + f"Stale canonical knowledge for item '{item_id}': " + f"unsupported item shape '{item_shape}'" + ) + if len(node_records) != expected_node_count: + raise RuntimeError( + f"Stale canonical knowledge for item '{item_id}': expected " + f"{expected_node_count} canonical nodes, found {len(node_records)}" + ) + try: + root_payload = json.loads(item_payload) + except json.JSONDecodeError as exc: + raise RuntimeError( + f"Stale canonical knowledge for item '{item_id}': invalid root JSON" + ) from exc + if not isinstance(root_payload, dict): + raise RuntimeError( + f"Stale canonical knowledge for item '{item_id}': invalid root payload" + ) + nodes: dict[str, object] = {} + for node_id, parent_id, position, node_payload, node_hash in node_records: + actual_hash = hashlib.sha256(node_payload.encode("utf-8")).hexdigest() + if actual_hash != node_hash: + raise RuntimeError( + f"Stale canonical knowledge for item '{item_id}': " + f"node '{node_id}' content hash mismatch" + ) + try: + parsed_node = json.loads(node_payload) + except json.JSONDecodeError as exc: + raise RuntimeError( + f"Stale canonical knowledge for item '{item_id}': " + f"node '{node_id}' contains invalid JSON" + ) from exc + if ( + not isinstance(parsed_node, dict) + or parsed_node.get("node_id") != node_id + or parsed_node.get("parent_id") != parent_id + or parsed_node.get("position") != position + ): + raise RuntimeError( + f"Stale canonical knowledge for item '{item_id}': " + f"node '{node_id}' metadata does not match its payload" + ) + nodes[node_id] = parsed_node + root_payload["nodes"] = nodes + return _json_payload(root_payload) + + +def _load_canonical( + *, + item_id: str, + knowledge_class: str, + item_type: str, + schema_version: str, + payload: str, + canonical_hash: str, +) -> BaseKnowledge: + """Validate stored canonical bytes against identity and schema metadata.""" + actual_hash = hashlib.sha256(payload.encode("utf-8")).hexdigest() + if actual_hash != canonical_hash: + raise RuntimeError( + f"Stale canonical knowledge for item '{item_id}': content hash mismatch" + ) + model = _KNOWLEDGE_CLASSES.get(knowledge_class) + if model is None: + raise RuntimeError( + f"Stale canonical knowledge for item '{item_id}': " + f"unsupported stored type '{knowledge_class}'" + ) + try: + item = model.model_validate_json(payload) + except ValidationError as exc: + raise RuntimeError( + f"Stale canonical knowledge for item '{item_id}': " + "stored payload no longer validates" + ) from exc + if ( + str(item.id) != item_id + or item.item_type != item_type + or item.schema_version != schema_version + ): + raise RuntimeError( + f"Stale canonical knowledge for item '{item_id}': identity or schema " + "metadata does not match the payload" + ) + return item + + +def _timestamp(value: datetime, field_name: str) -> float: + """Normalize an aware canonical timestamp for SQLite filtering.""" + if value.tzinfo is None or value.utcoffset() is None: + raise ValueError(f"{field_name} must be timezone-aware") + return value.astimezone(timezone.utc).timestamp() + + +def _initialize_schema(db: sqlite3.Connection) -> None: + """Create the current schema or reject an incompatible local database.""" + version_row = db.execute("PRAGMA user_version").fetchone() + version = int(version_row[0]) + if version not in (0, _DATABASE_SCHEMA_VERSION): + raise RuntimeError( + "Stale knowledge library schema: database version " + f"{version}, expected {_DATABASE_SCHEMA_VERSION}" + ) + if version == _DATABASE_SCHEMA_VERSION: + return + db.executescript( + f""" + CREATE TABLE knowledge_items ( + item_id TEXT PRIMARY KEY, + knowledge_class TEXT NOT NULL, + item_type TEXT NOT NULL, + item_shape TEXT NOT NULL CHECK (item_shape IN ('flat', 'tree')), + schema_version TEXT NOT NULL, + payload_json TEXT NOT NULL, + canonical_hash TEXT NOT NULL, + node_count INTEGER NOT NULL CHECK (node_count >= 0), + target_count INTEGER NOT NULL CHECK (target_count > 0) + ); + + CREATE TABLE knowledge_nodes ( + item_id TEXT NOT NULL, + node_id TEXT NOT NULL, + parent_id TEXT, + position INTEGER NOT NULL, + payload_json TEXT NOT NULL, + content_hash TEXT NOT NULL, + PRIMARY KEY (item_id, node_id), + FOREIGN KEY (item_id) REFERENCES knowledge_items(item_id) + ON DELETE CASCADE + ); + + CREATE TABLE semantic_records ( + target_id TEXT PRIMARY KEY, + item_id TEXT NOT NULL, + node_id TEXT, + item_type TEXT NOT NULL, + matched_text TEXT NOT NULL, + as_of REAL NOT NULL, + available_at REAL, + source_kind TEXT NOT NULL, + confidence TEXT NOT NULL, + tags_json TEXT NOT NULL, + tree_id TEXT, + embedding_model TEXT NOT NULL, + dimension INTEGER NOT NULL, + projection_hash TEXT NOT NULL, + source_content_hash TEXT, + knowledge_schema_version TEXT NOT NULL, + projection_schema_version TEXT NOT NULL, + item_canonical_hash TEXT NOT NULL, + embedding BLOB NOT NULL, + FOREIGN KEY (item_id) REFERENCES knowledge_items(item_id) + ON DELETE CASCADE + ); + + CREATE INDEX semantic_records_item_id + ON semantic_records(item_id); + CREATE INDEX semantic_records_filters + ON semantic_records(item_type, source_kind, confidence, tree_id); + CREATE INDEX knowledge_nodes_parent + ON knowledge_nodes(item_id, parent_id, position); + + PRAGMA user_version = {_DATABASE_SCHEMA_VERSION}; + """ + ) + + +class _SQLiteStore: + """Own the concrete SQLite schema, transactions, and record validation.""" + + def __init__(self, db: sqlite3.Connection) -> None: + self._db = db + + @classmethod + def open(cls, path: str | Path) -> "_SQLiteStore": + """Open and initialize a concrete SQLite knowledge store.""" + supplied_path = str(path) + database_path = ( + supplied_path + if supplied_path == ":memory:" + else str(Path(supplied_path).expanduser()) + ) + if database_path != ":memory:": + Path(database_path).parent.mkdir(parents=True, exist_ok=True) + db: sqlite3.Connection | None = None + try: + db = sqlite3.connect(database_path, isolation_level=None) + db.row_factory = sqlite3.Row + db.execute("PRAGMA foreign_keys = ON") + db.execute("PRAGMA busy_timeout = 5000") + _initialize_schema(db) + except sqlite3.DatabaseError as exc: + if db is not None: + db.close() + raise RuntimeError( + f"Corrupt knowledge library database at '{database_path}'" + ) from exc + except Exception: + if db is not None: + db.close() + raise + return cls(db) + + def prepare_put(self, item: BaseKnowledge) -> _PreparedPut: + """Validate a canonical write and load vectors it may retain.""" + rows = self._db.execute( + "SELECT * FROM semantic_records WHERE item_id = ?", + (str(item.id),), + ).fetchall() + existing = { + str(row["target_id"]): _StoredEmbedding( + target_id=str(row["target_id"]), + embedding_model=str(row["embedding_model"]), + dimension=int(row["dimension"]), + projection_hash=str(row["projection_hash"]), + source_content_hash=( + str(row["source_content_hash"]) + if row["source_content_hash"] is not None + else None + ), + knowledge_schema_version=str(row["knowledge_schema_version"]), + projection_schema_version=str(row["projection_schema_version"]), + embedding=bytes(row["embedding"]), + ) + for row in rows + } + return _PreparedPut( + item=item, + canonical=_canonical_payload(item), + as_of=_timestamp(item.as_of, "BaseKnowledge.as_of"), + available_at=( + _timestamp(item.available_at, "BaseKnowledge.available_at") + if item.available_at is not None + else None + ), + tags_json=json.dumps( + item.tags, + ensure_ascii=False, + separators=(",", ":"), + ), + existing_embeddings=existing, + ) + + def put( + self, + prepared: _PreparedPut, + targets: Sequence[_RetrievalTarget], + vectors: dict[str, tuple[bytes, int]], + *, + embedding_model: str, + ) -> None: + """Atomically replace canonical and derived records for one item.""" + item = prepared.item + canonical = prepared.canonical + try: + self._db.execute("BEGIN IMMEDIATE") + self._db.execute( + """ + INSERT INTO knowledge_items ( + item_id, knowledge_class, item_type, item_shape, + schema_version, payload_json, canonical_hash, + node_count, target_count + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(item_id) DO UPDATE SET + knowledge_class = excluded.knowledge_class, + item_type = excluded.item_type, + item_shape = excluded.item_shape, + schema_version = excluded.schema_version, + payload_json = excluded.payload_json, + canonical_hash = excluded.canonical_hash, + node_count = excluded.node_count, + target_count = excluded.target_count + """, + ( + str(item.id), + canonical.knowledge_class, + item.item_type, + canonical.item_shape, + item.schema_version, + canonical.payload, + canonical.canonical_hash, + len(canonical.nodes), + len(targets), + ), + ) + self._db.execute( + "DELETE FROM semantic_records WHERE item_id = ?", + (str(item.id),), + ) + self._db.execute( + "DELETE FROM knowledge_nodes WHERE item_id = ?", + (str(item.id),), + ) + for node in canonical.nodes: + self._db.execute( + """ + INSERT INTO knowledge_nodes ( + item_id, node_id, parent_id, position, + payload_json, content_hash + ) VALUES (?, ?, ?, ?, ?, ?) + """, + ( + str(item.id), + str(node.node_id), + str(node.parent_id) if node.parent_id else None, + node.position, + node.payload, + node.content_hash, + ), + ) + for target in targets: + blob, dimension = vectors[target.target_id] + self._db.execute( + """ + INSERT INTO semantic_records ( + target_id, item_id, node_id, item_type, + matched_text, as_of, available_at, source_kind, + confidence, tags_json, tree_id, embedding_model, + dimension, projection_hash, source_content_hash, + knowledge_schema_version, + projection_schema_version, item_canonical_hash, + embedding + ) VALUES ( + ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, + ?, ?, ? + ) + """, + ( + target.target_id, + str(item.id), + str(target.node_id) + if target.node_id is not None + else None, + item.item_type, + target.text, + prepared.as_of, + prepared.available_at, + item.source.kind, + item.confidence, + prepared.tags_json, + str(target.tree_id) + if target.tree_id is not None + else None, + embedding_model, + dimension, + target.projection_hash, + item.source.content_hash, + item.schema_version, + _PROJECTION_SCHEMA_VERSION, + canonical.canonical_hash, + blob, + ), + ) + self._db.execute("COMMIT") + except Exception: + if self._db.in_transaction: + self._db.execute("ROLLBACK") + raise + + def get(self, item_id: UUID) -> BaseKnowledge: + """Return validated canonical knowledge or report not-found/stale data.""" + row = self._db.execute( + "SELECT * FROM knowledge_items WHERE item_id = ?", + (str(item_id),), + ).fetchone() + if row is None: + derived_count = int( + self._db.execute( + """ + SELECT + (SELECT COUNT(*) FROM semantic_records + WHERE item_id = ?) + + + (SELECT COUNT(*) FROM knowledge_nodes + WHERE item_id = ?) + """, + (str(item_id), str(item_id)), + ).fetchone()[0] + ) + if derived_count: + raise RuntimeError( + f"Stale data for item '{item_id}': child records exist " + "without canonical knowledge" + ) + raise KeyError(f"Knowledge item '{item_id}' not found") + item_key = str(row["item_id"]) + node_rows = self._db.execute( + """ + SELECT node_id, parent_id, position, payload_json, content_hash + FROM knowledge_nodes + WHERE item_id = ? + ORDER BY node_id + """, + (item_key,), + ).fetchall() + payload = _assemble_canonical_payload( + item_id=item_key, + item_shape=str(row["item_shape"]), + item_payload=str(row["payload_json"]), + expected_node_count=int(row["node_count"]), + node_records=[ + ( + str(node_row["node_id"]), + ( + str(node_row["parent_id"]) + if node_row["parent_id"] is not None + else None + ), + int(node_row["position"]), + str(node_row["payload_json"]), + str(node_row["content_hash"]), + ) + for node_row in node_rows + ], + ) + return _load_canonical( + item_id=item_key, + knowledge_class=str(row["knowledge_class"]), + item_type=str(row["item_type"]), + schema_version=str(row["schema_version"]), + payload=payload, + canonical_hash=str(row["canonical_hash"]), + ) + + def delete(self, item_id: UUID) -> None: + """Transactionally remove canonical knowledge and every child record.""" + exists = self._db.execute( + "SELECT 1 FROM knowledge_items WHERE item_id = ?", + (str(item_id),), + ).fetchone() + if exists is None: + derived_count = int( + self._db.execute( + """ + SELECT + (SELECT COUNT(*) FROM semantic_records + WHERE item_id = ?) + + + (SELECT COUNT(*) FROM knowledge_nodes + WHERE item_id = ?) + """, + (str(item_id), str(item_id)), + ).fetchone()[0] + ) + if derived_count: + raise RuntimeError( + f"Stale data for item '{item_id}': cannot delete child " + "records without canonical knowledge" + ) + raise KeyError(f"Knowledge item '{item_id}' not found") + try: + self._db.execute("BEGIN IMMEDIATE") + self._db.execute( + "DELETE FROM semantic_records WHERE item_id = ?", + (str(item_id),), + ) + self._db.execute( + "DELETE FROM knowledge_nodes WHERE item_id = ?", + (str(item_id),), + ) + self._db.execute( + "DELETE FROM knowledge_items WHERE item_id = ?", + (str(item_id),), + ) + self._db.execute("COMMIT") + except Exception: + if self._db.in_transaction: + self._db.execute("ROLLBACK") + raise + + def load_index_records( + self, + *, + embedding_model: str, + embedding_dimensions: int | None, + ) -> list[_IndexRecord]: + """Validate SQLite relationships and load typed exact-index records.""" + orphan = self._db.execute( + """ + SELECT r.item_id + FROM semantic_records AS r + LEFT JOIN knowledge_items AS i ON i.item_id = r.item_id + WHERE i.item_id IS NULL + LIMIT 1 + """ + ).fetchone() + if orphan is not None: + raise RuntimeError( + f"Stale index data for item '{orphan['item_id']}': " + "derived records exist without canonical knowledge" + ) + orphan_node = self._db.execute( + """ + SELECT n.item_id + FROM knowledge_nodes AS n + LEFT JOIN knowledge_items AS i ON i.item_id = n.item_id + WHERE i.item_id IS NULL + LIMIT 1 + """ + ).fetchone() + if orphan_node is not None: + raise RuntimeError( + f"Stale canonical knowledge for item '{orphan_node['item_id']}': " + "tree nodes exist without an aggregate root" + ) + incomplete_tree = self._db.execute( + """ + SELECT i.item_id, i.item_shape, i.node_count, + COUNT(n.node_id) AS actual_count + FROM knowledge_items AS i + LEFT JOIN knowledge_nodes AS n ON n.item_id = i.item_id + GROUP BY i.item_id, i.item_shape, i.node_count + HAVING COUNT(n.node_id) != i.node_count + OR (i.item_shape = 'flat' AND i.node_count != 0) + OR (i.item_shape = 'tree' AND i.node_count = 0) + LIMIT 1 + """ + ).fetchone() + if incomplete_tree is not None: + raise RuntimeError( + f"Stale canonical knowledge for item " + f"'{incomplete_tree['item_id']}': expected " + f"{incomplete_tree['node_count']} tree nodes, found " + f"{incomplete_tree['actual_count']}" + ) + incomplete = self._db.execute( + """ + SELECT i.item_id, i.target_count, COUNT(r.target_id) AS actual_count + FROM knowledge_items AS i + LEFT JOIN semantic_records AS r ON r.item_id = i.item_id + GROUP BY i.item_id, i.target_count + HAVING COUNT(r.target_id) != i.target_count + LIMIT 1 + """ + ).fetchone() + if incomplete is not None: + raise RuntimeError( + f"Stale index data for item '{incomplete['item_id']}': expected " + f"{incomplete['target_count']} targets, found " + f"{incomplete['actual_count']}" + ) + + rows = self._db.execute( + """ + SELECT r.*, i.canonical_hash AS current_canonical_hash, + i.schema_version AS current_schema_version + FROM semantic_records AS r + JOIN knowledge_items AS i ON i.item_id = r.item_id + ORDER BY r.target_id + """ + ).fetchall() + records: list[_IndexRecord] = [] + for row in rows: + target_id = str(row["target_id"]) + dimension = int(row["dimension"]) + if str(row["embedding_model"]) != embedding_model: + raise RuntimeError( + f"Stale index data for target '{target_id}': embedding model " + "changed; re-put the canonical item" + ) + if ( + embedding_dimensions is not None + and dimension != embedding_dimensions + ): + raise RuntimeError( + f"Stale index data for target '{target_id}': embedding " + "dimension changed; re-put the canonical item" + ) + if ( + str(row["projection_schema_version"]) + != _PROJECTION_SCHEMA_VERSION + or str(row["knowledge_schema_version"]) + != str(row["current_schema_version"]) + or str(row["item_canonical_hash"]) + != str(row["current_canonical_hash"]) + ): + raise RuntimeError( + f"Stale index data for target '{target_id}': projection or " + "canonical metadata changed; re-put the canonical item" + ) + try: + parsed_tags = json.loads(str(row["tags_json"])) + except json.JSONDecodeError as exc: + raise RuntimeError( + f"Corrupt index data for target '{target_id}': invalid tags" + ) from exc + if not isinstance(parsed_tags, list) or not all( + isinstance(tag, str) for tag in parsed_tags + ): + raise RuntimeError( + f"Corrupt index data for target '{target_id}': invalid tags" + ) + node_value = row["node_id"] + tree_value = row["tree_id"] + try: + record = _IndexRecord( + target_id=target_id, + item_id=UUID(str(row["item_id"])), + node_id=UUID(str(node_value)) if node_value else None, + item_type=str(row["item_type"]), + matched_text=str(row["matched_text"]), + as_of=float(row["as_of"]), + available_at=( + float(row["available_at"]) + if row["available_at"] is not None + else None + ), + source_kind=str(row["source_kind"]), + confidence=str(row["confidence"]), + tags=frozenset(parsed_tags), + tree_id=UUID(str(tree_value)) if tree_value else None, + dimension=dimension, + embedding=bytes(row["embedding"]), + ) + except (TypeError, ValueError) as exc: + raise RuntimeError( + f"Corrupt index data for target '{target_id}': invalid metadata" + ) from exc + records.append(record) + return records + + def close(self) -> None: + """Close the owned SQLite connection.""" + self._db.close() diff --git a/quantmind/library/_ports.py b/quantmind/library/_ports.py deleted file mode 100644 index eda69fe..0000000 --- a/quantmind/library/_ports.py +++ /dev/null @@ -1,22 +0,0 @@ -"""Private side-effect seams for the local knowledge library.""" - -from collections.abc import Sequence -from typing import Protocol - - -class _EmbeddingProvider(Protocol): - """Private embedding seam used by production code and deterministic tests.""" - - async def embed( - self, - texts: Sequence[str], - *, - model: str, - dimensions: int | None, - ) -> Sequence[Sequence[float]]: - """Embed texts in input order.""" - ... - - async def close(self) -> None: - """Release provider-owned resources.""" - ... diff --git a/quantmind/library/_projection.py b/quantmind/library/_projection.py deleted file mode 100644 index c8c0ed2..0000000 --- a/quantmind/library/_projection.py +++ /dev/null @@ -1,278 +0,0 @@ -"""Canonical serialization and searchable projection rules.""" - -import hashlib -import json -from collections.abc import Sequence -from dataclasses import dataclass -from uuid import UUID - -from pydantic import ValidationError - -from quantmind.knowledge import ( - BaseKnowledge, - Earnings, - Factor, - FlattenKnowledge, - News, - Paper, - PaperKnowledgeCard, - Thesis, - TreeKnowledge, -) - -_PROJECTION_SCHEMA_VERSION = "1" - -_KNOWLEDGE_CLASSES: dict[str, type[BaseKnowledge]] = { - f"{knowledge_type.__module__}:{knowledge_type.__qualname__}": knowledge_type - for knowledge_type in ( - Earnings, - Factor, - News, - Paper, - PaperKnowledgeCard, - Thesis, - ) -} - - -@dataclass(frozen=True) -class _Projection: - """One stable item or node target to embed.""" - - target_id: str - node_id: UUID | None - text: str - projection_hash: str - tree_id: UUID | None - - -@dataclass(frozen=True) -class _CanonicalNode: - """One normalized canonical tree node.""" - - node_id: UUID - parent_id: UUID | None - position: int - payload: str - content_hash: str - - -@dataclass(frozen=True) -class _CanonicalDocument: - """Canonical aggregate root plus separately persisted tree nodes.""" - - knowledge_class: str - item_shape: str - payload: str - canonical_hash: str - nodes: tuple[_CanonicalNode, ...] - - -def _json_payload(value: object) -> str: - """Encode canonical JSON with stable ordering and separators.""" - return json.dumps( - value, - ensure_ascii=False, - separators=(",", ":"), - sort_keys=True, - ) - - -def _canonical_payload(item: BaseKnowledge) -> _CanonicalDocument: - """Split a supported canonical aggregate into root and node records.""" - knowledge_class = f"{type(item).__module__}:{type(item).__qualname__}" - if knowledge_class not in _KNOWLEDGE_CLASSES: - raise TypeError( - f"Unsupported knowledge type '{type(item).__name__}'; " - "only canonical quantmind.knowledge types can be persisted" - ) - full_payload = _json_payload(item.model_dump(mode="json")) - canonical_hash = hashlib.sha256(full_payload.encode("utf-8")).hexdigest() - if not isinstance(item, TreeKnowledge): - return _CanonicalDocument( - knowledge_class=knowledge_class, - item_shape="flat", - payload=full_payload, - canonical_hash=canonical_hash, - nodes=(), - ) - - nodes: list[_CanonicalNode] = [] - for node_id, node in sorted( - item.nodes.items(), key=lambda pair: str(pair[0]) - ): - node_payload = _json_payload(node.model_dump(mode="json")) - nodes.append( - _CanonicalNode( - node_id=node_id, - parent_id=node.parent_id, - position=node.position, - payload=node_payload, - content_hash=hashlib.sha256( - node_payload.encode("utf-8") - ).hexdigest(), - ) - ) - return _CanonicalDocument( - knowledge_class=knowledge_class, - item_shape="tree", - payload=_json_payload(item.model_dump(mode="json", exclude={"nodes"})), - canonical_hash=canonical_hash, - nodes=tuple(nodes), - ) - - -def _assemble_canonical_payload( - *, - item_id: str, - item_shape: str, - item_payload: str, - expected_node_count: int, - node_records: Sequence[tuple[str, str | None, int, str, str]], -) -> str: - """Rehydrate normalized tree nodes into the canonical Pydantic payload.""" - if item_shape == "flat": - if expected_node_count or node_records: - raise RuntimeError( - f"Stale canonical knowledge for item '{item_id}': " - "flat knowledge unexpectedly has tree nodes" - ) - return item_payload - if item_shape != "tree": - raise RuntimeError( - f"Stale canonical knowledge for item '{item_id}': " - f"unsupported item shape '{item_shape}'" - ) - if len(node_records) != expected_node_count: - raise RuntimeError( - f"Stale canonical knowledge for item '{item_id}': expected " - f"{expected_node_count} canonical nodes, found {len(node_records)}" - ) - try: - root_payload = json.loads(item_payload) - except json.JSONDecodeError as exc: - raise RuntimeError( - f"Stale canonical knowledge for item '{item_id}': invalid root JSON" - ) from exc - if not isinstance(root_payload, dict): - raise RuntimeError( - f"Stale canonical knowledge for item '{item_id}': invalid root payload" - ) - nodes: dict[str, object] = {} - for node_id, parent_id, position, node_payload, node_hash in node_records: - actual_hash = hashlib.sha256(node_payload.encode("utf-8")).hexdigest() - if actual_hash != node_hash: - raise RuntimeError( - f"Stale canonical knowledge for item '{item_id}': " - f"node '{node_id}' content hash mismatch" - ) - try: - parsed_node = json.loads(node_payload) - except json.JSONDecodeError as exc: - raise RuntimeError( - f"Stale canonical knowledge for item '{item_id}': " - f"node '{node_id}' contains invalid JSON" - ) from exc - if ( - not isinstance(parsed_node, dict) - or parsed_node.get("node_id") != node_id - or parsed_node.get("parent_id") != parent_id - or parsed_node.get("position") != position - ): - raise RuntimeError( - f"Stale canonical knowledge for item '{item_id}': " - f"node '{node_id}' metadata does not match its payload" - ) - nodes[node_id] = parsed_node - root_payload["nodes"] = nodes - return _json_payload(root_payload) - - -def _load_canonical( - *, - item_id: str, - knowledge_class: str, - item_type: str, - schema_version: str, - payload: str, - canonical_hash: str, -) -> BaseKnowledge: - """Validate stored canonical bytes against identity and schema metadata.""" - actual_hash = hashlib.sha256(payload.encode("utf-8")).hexdigest() - if actual_hash != canonical_hash: - raise RuntimeError( - f"Stale canonical knowledge for item '{item_id}': content hash mismatch" - ) - model = _KNOWLEDGE_CLASSES.get(knowledge_class) - if model is None: - raise RuntimeError( - f"Stale canonical knowledge for item '{item_id}': " - f"unsupported stored type '{knowledge_class}'" - ) - try: - item = model.model_validate_json(payload) - except ValidationError as exc: - raise RuntimeError( - f"Stale canonical knowledge for item '{item_id}': " - "stored payload no longer validates" - ) from exc - if ( - str(item.id) != item_id - or item.item_type != item_type - or item.schema_version != schema_version - ): - raise RuntimeError( - f"Stale canonical knowledge for item '{item_id}': identity or schema " - "metadata does not match the payload" - ) - return item - - -def _project_knowledge(item: BaseKnowledge) -> list[_Projection]: - """Project flat items and trees at the issue-defined retrieval grain.""" - if not isinstance(item, (FlattenKnowledge, TreeKnowledge)): - raise TypeError( - "LocalKnowledgeLibrary supports FlattenKnowledge and TreeKnowledge" - ) - - root_text = item.embedding_text() - if not root_text: - raise ValueError("knowledge embedding_text() must not be empty") - tree_id = item.id if isinstance(item, TreeKnowledge) else None - projections = [ - _Projection( - target_id=f"item:{item.id}", - node_id=None, - text=root_text, - projection_hash=hashlib.sha256( - root_text.encode("utf-8") - ).hexdigest(), - tree_id=tree_id, - ) - ] - - if isinstance(item, TreeKnowledge): - if item.root_node_id not in item.nodes: - raise ValueError("tree root_node_id is not present in nodes") - for node_id, node in sorted( - item.nodes.items(), key=lambda pair: str(pair[0]) - ): - if node_id != node.node_id: - raise ValueError("tree node map key does not match node_id") - if node_id == item.root_node_id: - continue - text = node.embedding_text() - if not text: - raise ValueError("tree node embedding_text() must not be empty") - projections.append( - _Projection( - target_id=f"node:{item.id}:{node_id}", - node_id=node_id, - text=text, - projection_hash=hashlib.sha256( - text.encode("utf-8") - ).hexdigest(), - tree_id=item.id, - ) - ) - return projections diff --git a/quantmind/library/local.py b/quantmind/library/local.py index 39f17e1..6055d05 100644 --- a/quantmind/library/local.py +++ b/quantmind/library/local.py @@ -1,246 +1,48 @@ """Opinionated SQLite and NumPy semantic knowledge library.""" import asyncio -import json -import sqlite3 -from dataclasses import dataclass -from datetime import datetime, timezone from pathlib import Path -from typing import Any from uuid import UUID -import numpy as np -from numpy.typing import NDArray from typing_extensions import Self from quantmind.knowledge import BaseKnowledge, TreeKnowledge -from quantmind.library._embed import _OpenAIEmbeddingProvider -from quantmind.library._ports import _EmbeddingProvider -from quantmind.library._projection import ( +from quantmind.library._internal.exact_cosine import ( + _coerce_provider_vectors, + _decode_stored_vector, + _encode_vector, + _ExactCosineIndex, +) +from quantmind.library._internal.index_embeddings import ( + _EmbeddingProvider, + _OpenAIEmbeddingProvider, +) +from quantmind.library._internal.retrieval_targets import ( _PROJECTION_SCHEMA_VERSION, - _assemble_canonical_payload, - _canonical_payload, - _load_canonical, _project_knowledge, - _Projection, + _RetrievalTarget, ) +from quantmind.library._internal.sqlite_store import _SQLiteStore from quantmind.library._types import SemanticHit, SemanticQuery -_DATABASE_SCHEMA_VERSION = 2 - - -@dataclass(frozen=True) -class _IndexRecord: - """Validated metadata aligned with one NumPy matrix row.""" - - target_id: str - item_id: UUID - node_id: UUID | None - item_type: str - matched_text: str - as_of: float - available_at: float | None - source_kind: str - confidence: str - tags: frozenset[str] - tree_id: UUID | None - - -def _timestamp(value: datetime, field_name: str) -> float: - """Normalize an aware financial timestamp for deterministic filtering.""" - if value.tzinfo is None or value.utcoffset() is None: - raise ValueError(f"{field_name} must be timezone-aware") - return value.astimezone(timezone.utc).timestamp() - - -def _coerce_provider_vectors( - values: Any, - *, - expected_count: int, - expected_dimensions: int | None, -) -> NDArray[np.float32]: - """Validate provider output before it can enter the durable index.""" - try: - vectors = np.asarray(values, dtype=np.float32) - except (TypeError, ValueError) as exc: - raise ValueError( - "Embedding provider returned non-rectangular data" - ) from exc - if vectors.ndim != 2 or vectors.shape[0] != expected_count: - raise ValueError( - "Embedding provider returned an unexpected number or shape of vectors" - ) - dimensions = int(vectors.shape[1]) - if dimensions < 1: - raise ValueError("Embedding provider returned zero-dimensional vectors") - if expected_dimensions is not None and dimensions != expected_dimensions: - raise ValueError( - "Embedding dimension mismatch: provider returned " - f"{dimensions}, expected {expected_dimensions}" - ) - if not np.isfinite(vectors).all(): - raise ValueError("Embedding provider returned non-finite values") - if np.any(np.linalg.norm(vectors, axis=1) == 0): - raise ValueError("Embedding provider returned a zero vector") - return np.ascontiguousarray(vectors, dtype=np.float32) - - -def _decode_stored_vector( - blob: bytes, dimension: int, target_id: str -) -> NDArray[np.float32]: - """Decode and validate a persisted vector as corrupt-index protection.""" - if dimension < 1 or len(blob) != dimension * np.dtype(" None: - """Create the current schema or reject an incompatible local database.""" - version_row = db.execute("PRAGMA user_version").fetchone() - version = int(version_row[0]) - if version not in (0, _DATABASE_SCHEMA_VERSION): - raise RuntimeError( - "Stale knowledge library schema: database version " - f"{version}, expected {_DATABASE_SCHEMA_VERSION}" - ) - if version == _DATABASE_SCHEMA_VERSION: - return - db.executescript( - f""" - CREATE TABLE knowledge_items ( - item_id TEXT PRIMARY KEY, - knowledge_class TEXT NOT NULL, - item_type TEXT NOT NULL, - item_shape TEXT NOT NULL CHECK (item_shape IN ('flat', 'tree')), - schema_version TEXT NOT NULL, - payload_json TEXT NOT NULL, - canonical_hash TEXT NOT NULL, - node_count INTEGER NOT NULL CHECK (node_count >= 0), - target_count INTEGER NOT NULL CHECK (target_count > 0) - ); - - CREATE TABLE knowledge_nodes ( - item_id TEXT NOT NULL, - node_id TEXT NOT NULL, - parent_id TEXT, - position INTEGER NOT NULL, - payload_json TEXT NOT NULL, - content_hash TEXT NOT NULL, - PRIMARY KEY (item_id, node_id), - FOREIGN KEY (item_id) REFERENCES knowledge_items(item_id) - ON DELETE CASCADE - ); - - CREATE TABLE semantic_records ( - target_id TEXT PRIMARY KEY, - item_id TEXT NOT NULL, - node_id TEXT, - item_type TEXT NOT NULL, - matched_text TEXT NOT NULL, - as_of REAL NOT NULL, - available_at REAL, - source_kind TEXT NOT NULL, - confidence TEXT NOT NULL, - tags_json TEXT NOT NULL, - tree_id TEXT, - embedding_model TEXT NOT NULL, - dimension INTEGER NOT NULL, - projection_hash TEXT NOT NULL, - source_content_hash TEXT, - knowledge_schema_version TEXT NOT NULL, - projection_schema_version TEXT NOT NULL, - item_canonical_hash TEXT NOT NULL, - embedding BLOB NOT NULL, - FOREIGN KEY (item_id) REFERENCES knowledge_items(item_id) - ON DELETE CASCADE - ); - - CREATE INDEX semantic_records_item_id - ON semantic_records(item_id); - CREATE INDEX semantic_records_filters - ON semantic_records(item_type, source_kind, confidence, tree_id); - CREATE INDEX knowledge_nodes_parent - ON knowledge_nodes(item_id, parent_id, position); - - PRAGMA user_version = {_DATABASE_SCHEMA_VERSION}; - """ - ) - - -def _load_stored_canonical( - db: sqlite3.Connection, row: sqlite3.Row -) -> BaseKnowledge: - """Rehydrate and validate one canonical aggregate from normalized rows.""" - item_id = str(row["item_id"]) - node_rows = db.execute( - """ - SELECT node_id, parent_id, position, payload_json, content_hash - FROM knowledge_nodes - WHERE item_id = ? - ORDER BY node_id - """, - (item_id,), - ).fetchall() - payload = _assemble_canonical_payload( - item_id=item_id, - item_shape=str(row["item_shape"]), - item_payload=str(row["payload_json"]), - expected_node_count=int(row["node_count"]), - node_records=[ - ( - str(node_row["node_id"]), - ( - str(node_row["parent_id"]) - if node_row["parent_id"] is not None - else None - ), - int(node_row["position"]), - str(node_row["payload_json"]), - str(node_row["content_hash"]), - ) - for node_row in node_rows - ], - ) - return _load_canonical( - item_id=item_id, - knowledge_class=str(row["knowledge_class"]), - item_type=str(row["item_type"]), - schema_version=str(row["schema_version"]), - payload=payload, - canonical_hash=str(row["canonical_hash"]), - ) - class LocalKnowledgeLibrary: """Persist and semantically search canonical QuantMind knowledge locally.""" def __init__( self, - db: sqlite3.Connection, + store: _SQLiteStore, *, embedding_model: str, embedding_dimensions: int | None, embedding_provider: _EmbeddingProvider, ) -> None: - self._db: sqlite3.Connection | None = db + self._store: _SQLiteStore | None = store self._embedding_model = embedding_model self._embedding_dimensions = embedding_dimensions self._embedding_provider = embedding_provider self._lock = asyncio.Lock() - self._index_records: list[_IndexRecord] | None = None - self._index_matrix: NDArray[np.float32] | None = None + self._index: _ExactCosineIndex | None = None @classmethod async def open( @@ -265,333 +67,105 @@ async def open( raise ValueError("embedding_model must not be blank") if embedding_dimensions is not None and embedding_dimensions < 1: raise ValueError("embedding_dimensions must be positive") - supplied_path = str(path) - database_path = ( - supplied_path - if supplied_path == ":memory:" - else str(Path(supplied_path).expanduser()) - ) - if database_path != ":memory:": - Path(database_path).parent.mkdir(parents=True, exist_ok=True) - db: sqlite3.Connection | None = None - try: - db = sqlite3.connect( - database_path, - isolation_level=None, - ) - db.row_factory = sqlite3.Row - db.execute("PRAGMA foreign_keys = ON") - db.execute("PRAGMA busy_timeout = 5000") - _initialize_schema(db) - except sqlite3.DatabaseError as exc: - if db is not None: - db.close() - raise RuntimeError( - f"Corrupt knowledge library database at '{database_path}'" - ) from exc - except Exception: - if db is not None: - db.close() - raise - provider = _embedding_provider or _OpenAIEmbeddingProvider() return cls( - db, + _SQLiteStore.open(path), embedding_model=embedding_model, embedding_dimensions=embedding_dimensions, - embedding_provider=provider, + embedding_provider=( + _embedding_provider or _OpenAIEmbeddingProvider() + ), ) async def put(self, item: BaseKnowledge) -> None: - """Atomically persist canonical knowledge and its affected projections.""" + """Atomically persist canonical knowledge and its affected targets.""" async with self._lock: - if self._db is None: + store = self._store + if store is None: raise RuntimeError("LocalKnowledgeLibrary is closed") - canonical = _canonical_payload(item) - projections = _project_knowledge(item) - as_of = _timestamp(item.as_of, "BaseKnowledge.as_of") - available_at = ( - _timestamp(item.available_at, "BaseKnowledge.available_at") - if item.available_at is not None - else None - ) - tags_json = json.dumps( - item.tags, - ensure_ascii=False, - separators=(",", ":"), - ) - existing_rows = self._db.execute( - "SELECT * FROM semantic_records WHERE item_id = ?", - (str(item.id),), - ).fetchall() - existing = {str(row["target_id"]): row for row in existing_rows} - source_content_hash = item.source.content_hash - - affected: list[_Projection] = [] - retained: dict[str, tuple[bytes, int]] = {} - for projection in projections: - row = existing.get(projection.target_id) - needs_embedding = row is None - if row is not None: + prepared = store.prepare_put(item) + targets = _project_knowledge(item) + existing = prepared.existing_embeddings + affected: list[_RetrievalTarget] = [] + vectors: dict[str, tuple[bytes, int]] = {} + for target in targets: + stored = existing.get(target.target_id) + needs_embedding = stored is None + if stored is not None: needs_embedding = any( ( - str(row["embedding_model"]) - != self._embedding_model, - str(row["projection_hash"]) - != projection.projection_hash, - row["source_content_hash"] != source_content_hash, - str(row["knowledge_schema_version"]) + stored.embedding_model != self._embedding_model, + stored.projection_hash != target.projection_hash, + stored.source_content_hash + != item.source.content_hash, + stored.knowledge_schema_version != item.schema_version, - str(row["projection_schema_version"]) + stored.projection_schema_version != _PROJECTION_SCHEMA_VERSION, self._embedding_dimensions is not None - and int(row["dimension"]) - != self._embedding_dimensions, + and stored.dimension != self._embedding_dimensions, ) ) if needs_embedding: - affected.append(projection) - else: - assert row is not None - blob = bytes(row["embedding"]) - dimension = int(row["dimension"]) - _decode_stored_vector( - blob, - dimension, - projection.target_id, - ) - retained[projection.target_id] = (blob, dimension) + affected.append(target) + continue + assert stored is not None + _decode_stored_vector( + stored.embedding, + stored.dimension, + target.target_id, + ) + vectors[target.target_id] = ( + stored.embedding, + stored.dimension, + ) - generated: dict[str, tuple[bytes, int]] = {} if affected: provider_values = await self._embedding_provider.embed( - [projection.text for projection in affected], + [target.text for target in affected], model=self._embedding_model, dimensions=self._embedding_dimensions, ) - vectors = _coerce_provider_vectors( + generated = _coerce_provider_vectors( provider_values, expected_count=len(affected), expected_dimensions=self._embedding_dimensions, ) - for projection, vector in zip(affected, vectors, strict=True): - stored = np.asarray(vector, dtype=" BaseKnowledge: """Return validated canonical knowledge or report not-found/stale data.""" async with self._lock: - if self._db is None: + store = self._store + if store is None: raise RuntimeError("LocalKnowledgeLibrary is closed") - row = self._db.execute( - "SELECT * FROM knowledge_items WHERE item_id = ?", - (str(item_id),), - ).fetchone() - if row is None: - derived_count = int( - self._db.execute( - """ - SELECT - (SELECT COUNT(*) FROM semantic_records - WHERE item_id = ?) - + - (SELECT COUNT(*) FROM knowledge_nodes - WHERE item_id = ?) - """, - (str(item_id), str(item_id)), - ).fetchone()[0] - ) - if derived_count: - raise RuntimeError( - f"Stale data for item '{item_id}': child records exist " - "without canonical knowledge" - ) - raise KeyError(f"Knowledge item '{item_id}' not found") - return _load_stored_canonical(self._db, row) + return store.get(item_id) async def search(self, query: SemanticQuery) -> list[SemanticHit]: - """Rank filtered projections with deterministic exact cosine similarity.""" + """Rank filtered targets with deterministic exact cosine similarity.""" async with self._lock: - if self._db is None: + store = self._store + if store is None: raise RuntimeError("LocalKnowledgeLibrary is closed") - if self._index_records is None or self._index_matrix is None: - self._rebuild_numpy_index() - assert self._index_records is not None - assert self._index_matrix is not None - index_records = self._index_records - index_matrix = self._index_matrix - - as_of_before = ( - _timestamp(query.as_of_before, "SemanticQuery.as_of_before") - if query.as_of_before is not None - else None - ) - available_at_before = ( - _timestamp( - query.available_at_before, - "SemanticQuery.available_at_before", + if self._index is None: + self._index = _ExactCosineIndex.build( + store.load_index_records( + embedding_model=self._embedding_model, + embedding_dimensions=self._embedding_dimensions, + ) ) - if query.available_at_before is not None - else None - ) - item_types = set(query.item_types) if query.item_types else None - source_kinds = ( - set(query.source_kinds) if query.source_kinds else None - ) - required_tags = set(query.tags) if query.tags else None - candidates: list[int] = [] - for index, record in enumerate(index_records): - if ( - item_types is not None - and record.item_type not in item_types - ): - continue - if ( - source_kinds is not None - and record.source_kind not in source_kinds - ): - continue - if ( - query.confidence is not None - and record.confidence != query.confidence - ): - continue - if required_tags is not None and not required_tags.issubset( - record.tags - ): - continue - if ( - query.tree_id is not None - and record.tree_id != query.tree_id - ): - continue - if as_of_before is not None and record.as_of > as_of_before: - continue - if available_at_before is not None and ( - record.available_at is None - or record.available_at > available_at_before - ): - continue - candidates.append(index) - - if not candidates: + candidates = self._index.filter(query) + if candidates is None: return [] + provider_values = await self._embedding_provider.embed( [query.text], model=self._embedding_model, @@ -602,39 +176,21 @@ async def search(self, query: SemanticQuery) -> list[SemanticHit]: expected_count=1, expected_dimensions=self._embedding_dimensions, ) - query_vector = query_vectors[0] - if query_vector.shape[0] != index_matrix.shape[1]: - raise ValueError( - "Embedding dimension mismatch: query vector has " - f"{query_vector.shape[0]} dimensions but the index has " - f"{index_matrix.shape[1]}" - ) - query_vector = query_vector / np.linalg.norm(query_vector) - scores = index_matrix[candidates] @ query_vector - ranked = sorted( - zip(candidates, scores, strict=True), - key=lambda pair: ( - -float(pair[1]), - index_records[pair[0]].target_id, - ), - )[: query.top_k] + ranked = candidates.rank(query_vectors[0], top_k=query.top_k) canonical: dict[UUID, BaseKnowledge] = {} hits: list[SemanticHit] = [] - for index, score in ranked: - record = index_records[index] + for result in ranked: + record = result.record item = canonical.get(record.item_id) if item is None: - row = self._db.execute( - "SELECT * FROM knowledge_items WHERE item_id = ?", - (str(record.item_id),), - ).fetchone() - if row is None: + try: + item = store.get(record.item_id) + except KeyError as exc: raise RuntimeError( f"Stale index data for item '{record.item_id}': " "canonical knowledge is missing" - ) - item = _load_stored_canonical(self._db, row) + ) from exc canonical[record.item_id] = item if isinstance(item, TreeKnowledge): if record.node_id is None: @@ -659,7 +215,7 @@ async def search(self, query: SemanticQuery) -> list[SemanticHit]: item_id=record.item_id, node_id=record.node_id, item_type=item.item_type, - score=float(score), + score=result.score, matched_text=record.matched_text, as_of=item.as_of, available_at=item.available_at, @@ -670,233 +226,23 @@ async def search(self, query: SemanticQuery) -> list[SemanticHit]: return hits async def delete(self, item_id: UUID) -> None: - """Transactionally remove canonical knowledge and every derived target.""" + """Transactionally remove canonical knowledge and every child record.""" async with self._lock: - if self._db is None: + store = self._store + if store is None: raise RuntimeError("LocalKnowledgeLibrary is closed") - exists = self._db.execute( - "SELECT 1 FROM knowledge_items WHERE item_id = ?", - (str(item_id),), - ).fetchone() - if exists is None: - derived_count = int( - self._db.execute( - """ - SELECT - (SELECT COUNT(*) FROM semantic_records - WHERE item_id = ?) - + - (SELECT COUNT(*) FROM knowledge_nodes - WHERE item_id = ?) - """, - (str(item_id), str(item_id)), - ).fetchone()[0] - ) - if derived_count: - raise RuntimeError( - f"Stale data for item '{item_id}': cannot delete child " - "records without canonical knowledge" - ) - raise KeyError(f"Knowledge item '{item_id}' not found") - try: - self._db.execute("BEGIN IMMEDIATE") - self._db.execute( - "DELETE FROM semantic_records WHERE item_id = ?", - (str(item_id),), - ) - self._db.execute( - "DELETE FROM knowledge_nodes WHERE item_id = ?", - (str(item_id),), - ) - self._db.execute( - "DELETE FROM knowledge_items WHERE item_id = ?", - (str(item_id),), - ) - self._db.execute("COMMIT") - except Exception: - if self._db.in_transaction: - self._db.execute("ROLLBACK") - raise - self._index_records = None - self._index_matrix = None + store.delete(item_id) + self._index = None async def close(self) -> None: """Close SQLite and provider-owned resources; repeated calls are safe.""" async with self._lock: - if self._db is None: + store = self._store + if store is None: return - db = self._db - self._db = None - self._index_records = None - self._index_matrix = None + self._store = None + self._index = None try: await self._embedding_provider.close() finally: - db.close() - - def _rebuild_numpy_index(self) -> None: - """Validate durable records and rebuild the exact-cosine matrix.""" - assert self._db is not None - orphan = self._db.execute( - """ - SELECT r.item_id - FROM semantic_records AS r - LEFT JOIN knowledge_items AS i ON i.item_id = r.item_id - WHERE i.item_id IS NULL - LIMIT 1 - """ - ).fetchone() - if orphan is not None: - raise RuntimeError( - f"Stale index data for item '{orphan['item_id']}': " - "derived records exist without canonical knowledge" - ) - orphan_node = self._db.execute( - """ - SELECT n.item_id - FROM knowledge_nodes AS n - LEFT JOIN knowledge_items AS i ON i.item_id = n.item_id - WHERE i.item_id IS NULL - LIMIT 1 - """ - ).fetchone() - if orphan_node is not None: - raise RuntimeError( - f"Stale canonical knowledge for item '{orphan_node['item_id']}': " - "tree nodes exist without an aggregate root" - ) - incomplete_tree = self._db.execute( - """ - SELECT i.item_id, i.item_shape, i.node_count, - COUNT(n.node_id) AS actual_count - FROM knowledge_items AS i - LEFT JOIN knowledge_nodes AS n ON n.item_id = i.item_id - GROUP BY i.item_id, i.item_shape, i.node_count - HAVING COUNT(n.node_id) != i.node_count - OR (i.item_shape = 'flat' AND i.node_count != 0) - OR (i.item_shape = 'tree' AND i.node_count = 0) - LIMIT 1 - """ - ).fetchone() - if incomplete_tree is not None: - raise RuntimeError( - f"Stale canonical knowledge for item " - f"'{incomplete_tree['item_id']}': expected " - f"{incomplete_tree['node_count']} tree nodes, found " - f"{incomplete_tree['actual_count']}" - ) - incomplete = self._db.execute( - """ - SELECT i.item_id, i.target_count, COUNT(r.target_id) AS actual_count - FROM knowledge_items AS i - LEFT JOIN semantic_records AS r ON r.item_id = i.item_id - GROUP BY i.item_id, i.target_count - HAVING COUNT(r.target_id) != i.target_count - LIMIT 1 - """ - ).fetchone() - if incomplete is not None: - raise RuntimeError( - f"Stale index data for item '{incomplete['item_id']}': expected " - f"{incomplete['target_count']} targets, found " - f"{incomplete['actual_count']}" - ) - - rows = self._db.execute( - """ - SELECT r.*, i.canonical_hash AS current_canonical_hash, - i.schema_version AS current_schema_version - FROM semantic_records AS r - JOIN knowledge_items AS i ON i.item_id = r.item_id - ORDER BY r.target_id - """ - ).fetchall() - records: list[_IndexRecord] = [] - vectors: list[NDArray[np.float32]] = [] - index_dimension: int | None = None - for row in rows: - target_id = str(row["target_id"]) - dimension = int(row["dimension"]) - if str(row["embedding_model"]) != self._embedding_model: - raise RuntimeError( - f"Stale index data for target '{target_id}': embedding model " - "changed; re-put the canonical item" - ) - if ( - self._embedding_dimensions is not None - and dimension != self._embedding_dimensions - ): - raise RuntimeError( - f"Stale index data for target '{target_id}': embedding " - "dimension changed; re-put the canonical item" - ) - if ( - str(row["projection_schema_version"]) - != _PROJECTION_SCHEMA_VERSION - or str(row["knowledge_schema_version"]) - != str(row["current_schema_version"]) - or str(row["item_canonical_hash"]) - != str(row["current_canonical_hash"]) - ): - raise RuntimeError( - f"Stale index data for target '{target_id}': projection or " - "canonical metadata changed; re-put the canonical item" - ) - if index_dimension is None: - index_dimension = dimension - elif dimension != index_dimension: - raise RuntimeError( - "Corrupt index data: stored targets have inconsistent " - "embedding dimensions" - ) - vector = _decode_stored_vector( - bytes(row["embedding"]), dimension, target_id - ) - try: - parsed_tags = json.loads(str(row["tags_json"])) - except json.JSONDecodeError as exc: - raise RuntimeError( - f"Corrupt index data for target '{target_id}': invalid tags" - ) from exc - if not isinstance(parsed_tags, list) or not all( - isinstance(tag, str) for tag in parsed_tags - ): - raise RuntimeError( - f"Corrupt index data for target '{target_id}': invalid tags" - ) - node_value = row["node_id"] - tree_value = row["tree_id"] - try: - record = _IndexRecord( - target_id=target_id, - item_id=UUID(str(row["item_id"])), - node_id=UUID(str(node_value)) if node_value else None, - item_type=str(row["item_type"]), - matched_text=str(row["matched_text"]), - as_of=float(row["as_of"]), - available_at=( - float(row["available_at"]) - if row["available_at"] is not None - else None - ), - source_kind=str(row["source_kind"]), - confidence=str(row["confidence"]), - tags=frozenset(parsed_tags), - tree_id=UUID(str(tree_value)) if tree_value else None, - ) - except (TypeError, ValueError) as exc: - raise RuntimeError( - f"Corrupt index data for target '{target_id}': invalid metadata" - ) from exc - records.append(record) - vectors.append( - np.asarray(vector / np.linalg.norm(vector), dtype=np.float32) - ) - - self._index_records = records - if vectors: - self._index_matrix = np.ascontiguousarray( - np.vstack(vectors), dtype=np.float32 - ) - else: - self._index_matrix = np.empty((0, 0), dtype=np.float32) + store.close() diff --git a/scripts/build_library_example_bundle.py b/scripts/examples/build_ai_infrastructure_bundle.py similarity index 98% rename from scripts/build_library_example_bundle.py rename to scripts/examples/build_ai_infrastructure_bundle.py index 0028c15..b68e0f3 100644 --- a/scripts/build_library_example_bundle.py +++ b/scripts/examples/build_ai_infrastructure_bundle.py @@ -10,7 +10,7 @@ from quantmind.knowledge import BaseKnowledge, Earnings, News, Paper from quantmind.library import LocalKnowledgeLibrary -_ROOT = Path(__file__).parents[1] +_ROOT = Path(__file__).parents[2] _SOURCE_PATH = ( _ROOT / "examples" / "library" / "data" / "ai_infrastructure.json" ) diff --git a/tests/library/test_local.py b/tests/library/test_local.py index 03e8a28..6e1785b 100644 --- a/tests/library/test_local.py +++ b/tests/library/test_local.py @@ -7,7 +7,14 @@ from pathlib import Path from uuid import UUID, uuid4 -from quantmind.knowledge import Citation, News, Paper, SourceRef, TreeNode +from quantmind.knowledge import ( + Citation, + FlattenKnowledge, + News, + Paper, + SourceRef, + TreeNode, +) from quantmind.library import LocalKnowledgeLibrary, SemanticQuery @@ -53,6 +60,13 @@ async def close(self) -> None: self.closed = True +class _UnsupportedKnowledge(FlattenKnowledge): + item_type: str = "unsupported" + + def embedding_text(self) -> str: + return "unsupported canonical knowledge" + + def _time(day: int, hour: int = 0) -> datetime: return datetime(2026, 7, day, hour, tzinfo=timezone.utc) @@ -172,6 +186,27 @@ async def test_flat_put_get_and_deterministic_best_first_search(self): finally: await library.close() + async def test_unsupported_knowledge_fails_before_embedding(self): + provider = _FakeEmbeddingProvider() + library = await LocalKnowledgeLibrary.open( + self.db_path, + embedding_model="fake-2d", + embedding_dimensions=2, + _embedding_provider=provider, + ) + item = _UnsupportedKnowledge( + as_of=_time(1), + source=SourceRef(kind="manual"), + ) + try: + with self.assertRaisesRegex( + TypeError, "Unsupported knowledge type" + ): + await library.put(item) + self.assertEqual(provider.calls, []) + finally: + await library.close() + async def test_tree_root_and_non_root_nodes_use_exact_grain_and_filters( self, ): From 585b37786fca0c0cfb5461a1d6a6d9a7767bcaee Mon Sep 17 00:00:00 2001 From: pkuwkl Date: Thu, 16 Jul 2026 22:04:22 +0800 Subject: [PATCH 5/5] docs(contexts): route library guidance --- AGENTS.md | 1 + CLAUDE.md | 1 + contexts/usage/README.md | 1 + 3 files changed, 3 insertions(+) diff --git a/AGENTS.md b/AGENTS.md index 5a2f200..f05b5c3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -18,6 +18,7 @@ handoff all come from `openai-agents`. | Module | Role | |--------|------| | `quantmind/knowledge/` | Pydantic data standard (`FlattenKnowledge` / `TreeKnowledge` / `GraphKnowledge`) — dependency leaf | +| `quantmind/library/` | Local persistence and semantic retrieval for canonical knowledge — depends only on `knowledge` | | `quantmind/configs/` | Operation cfg + typed input models or unions (`BaseFlowCfg`, `NewsWindow`, `PaperInput`) — depends only on `knowledge` | | `quantmind/preprocess/` | Deterministic fetch / format / clean / time utilities — depends only on `utils` | | `quantmind/flows/` | Apex layer: public library operations (`paper_flow`, `collect_news`, `batch_run`) | diff --git a/CLAUDE.md b/CLAUDE.md index a599942..37109ba 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -19,6 +19,7 @@ handoff all come from `openai-agents`. | Module | Role | |--------|------| | `quantmind/knowledge/` | Pydantic data standard (`FlattenKnowledge` / `TreeKnowledge` / `GraphKnowledge`) — dependency leaf | +| `quantmind/library/` | Local persistence and semantic retrieval for canonical knowledge — depends only on `knowledge` | | `quantmind/configs/` | Operation cfg + typed input models or unions (`BaseFlowCfg`, `NewsWindow`, `PaperInput`) — depends only on `knowledge` | | `quantmind/preprocess/` | Deterministic fetch / format / clean / time utilities — depends only on `utils` | | `quantmind/flows/` | Apex layer: public library operations (`paper_flow`, `collect_news`, `batch_run`) | diff --git a/contexts/usage/README.md b/contexts/usage/README.md index eaea201..93289a3 100644 --- a/contexts/usage/README.md +++ b/contexts/usage/README.md @@ -9,6 +9,7 @@ current public API, focused examples, and component-specific guidance. | Installation and common usage | [Root README usage](../../README.md#-usage-examples) | | Paper extraction | [Paper guide](../../docs/papers.md) | | News collection | [News design and behavior](../../docs/design/en/news.md) | +| Local semantic search | [Library guide](../../docs/library.md) and [focused example](../../examples/library/README.md) | | Runnable operation examples | [`examples/flows/`](../../examples/flows/) | | Focused preprocessing examples | [`examples/preprocess/`](../../examples/preprocess/) |