diff --git a/README.md b/README.md index 9250d80..4975b3a 100644 --- a/README.md +++ b/README.md @@ -172,6 +172,8 @@ rag/ - [Setup Guide](SETUP.md) - **Complete setup instructions with troubleshooting** - [Quick Start Guide](backend/docs/QUICK_START.md) - Condensed setup steps - [Deployment Guide](backend/docs/DEPLOYMENT_GUIDE.md) - Production deployment +- [Evidence Highlighting V2](backend/docs/EVIDENCE_HIGHLIGHTING_V2.md) - Fail-closed source provenance and viewer contract +- [Query-Aware Evidence Chains](backend/docs/HYCE_EVIDENCE_CHAINS_ADOPTION.md) - HyCE-inspired adoption, testing, rollout, and rollback plan - [System Specification](lighthouse.md) - Full architecture spec - [Prompting Guide](PROMPTING_GUIDE.md) - Prompt engineering practices diff --git a/backend/.env.example b/backend/.env.example index a5cc2dd..b066e93 100644 --- a/backend/.env.example +++ b/backend/.env.example @@ -93,6 +93,23 @@ OPENAI_API_KEY=your-openai-api-key-here # REQUIRED - get from https://platform. # LLM_REWRITE_MODEL=gpt-4o-mini # LLM_REWRITE_TIMEOUT=5 +# ============================================================================= +# QUERY-AWARE EVIDENCE CHAINS (CONTROLLED ROLLOUT) +# ============================================================================= +# HyCE-inspired online graph propagation for multi-hop evidence assembly. +# Structural scores rank evidence only; exact citation/highlight verification +# remains independent and fail-closed. Start in shadow/evaluation environments. + +# EVIDENCE_CHAIN_ENABLED=false +# EVIDENCE_CHAIN_MODE=auto # off | auto | on +# EVIDENCE_CHAIN_ROUTE_THRESHOLD=0.55 +# EVIDENCE_CHAIN_MAX_HOPS=3 +# EVIDENCE_CHAIN_MAX_NODES=30 +# EVIDENCE_CHAIN_MAX_SELECTED_NODES=15 +# EVIDENCE_CHAIN_MAX_CHAINS=6 +# EVIDENCE_CHAIN_PROPAGATION_STEPS=10 +# EVIDENCE_CHAIN_RESTART_PROBABILITY=0.35 + # ============================================================================= # OCR SETTINGS (for scanned PDFs and image-based documents) # ============================================================================= diff --git a/backend/alembic/versions/014_evidence_record_v2.py b/backend/alembic/versions/014_evidence_record_v2.py new file mode 100644 index 0000000..92b08a8 --- /dev/null +++ b/backend/alembic/versions/014_evidence_record_v2.py @@ -0,0 +1,28 @@ +"""Persist claim-level evidence records on citation snapshots. + +Revision ID: 014 +Revises: 013 +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + + +revision: str = "014" +down_revision: Union[str, None] = "013" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column( + "citation_snapshots", + sa.Column("evidence_record", postgresql.JSONB(astext_type=sa.Text()), nullable=True), + ) + + +def downgrade() -> None: + op.drop_column("citation_snapshots", "evidence_record") diff --git a/backend/app/config.py b/backend/app/config.py index 4d85aba..cb9311d 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -115,6 +115,19 @@ class Settings(BaseSettings): # Cross-format selector highlighting enable_cross_format_highlighting: bool = True + # Query-aware evidence chains (HyCE-inspired online retrieval layer). + # Feature-gated for controlled A/B rollout; scores rank evidence only and + # never set citation/highlight verification status. + evidence_chain_enabled: bool = False + evidence_chain_mode: str = "auto" # off | auto | on + evidence_chain_route_threshold: float = 0.55 + evidence_chain_max_hops: int = 3 + evidence_chain_max_nodes: int = 30 + evidence_chain_max_selected_nodes: int = 15 + evidence_chain_max_chains: int = 6 + evidence_chain_propagation_steps: int = 10 + evidence_chain_restart_probability: float = 0.35 + # Access Control Layer (ACL) # When enabled, enforces tenant-scoped ABAC/RBAC at every pipeline stage. # Identity resolved from headers: X-Tenant-Id, X-User-Id, X-Roles, X-Groups @@ -149,6 +162,34 @@ def validate_acl_disclosure_mode(cls, v: str) -> str: raise ValueError(f"acl_disclosure_mode must be one of {valid}, got '{v}'") return v + @field_validator("evidence_chain_mode") + @classmethod + def validate_evidence_chain_mode(cls, v: str) -> str: + valid = {"off", "auto", "on"} + if v not in valid: + raise ValueError(f"evidence_chain_mode must be one of {valid}, got '{v}'") + return v + + @field_validator("evidence_chain_route_threshold", "evidence_chain_restart_probability") + @classmethod + def validate_evidence_chain_probability(cls, v: float) -> float: + if not 0.0 < v <= 1.0: + raise ValueError("evidence-chain probabilities must be in (0, 1]") + return v + + @field_validator( + "evidence_chain_max_hops", + "evidence_chain_max_nodes", + "evidence_chain_max_selected_nodes", + "evidence_chain_max_chains", + "evidence_chain_propagation_steps", + ) + @classmethod + def validate_evidence_chain_positive_ints(cls, v: int) -> int: + if v <= 0: + raise ValueError("evidence-chain limits must be positive") + return v + @field_validator("jwt_algorithms") @classmethod def validate_jwt_algorithms(cls, v: list[str]) -> list[str]: @@ -261,6 +302,10 @@ def log_configuration_summary(self) -> None: logger.info(f" OpenAI API Key: {'configured' if self.openai_api_key else 'NOT SET (required for embeddings)'}") logger.info(f" Docling: enabled={self.docling_enabled_default}, mode={self.docling_mode}") logger.info(f" Cross-format highlighting: enabled={self.enable_cross_format_highlighting}") + logger.info( + f" Evidence chains: enabled={self.evidence_chain_enabled}, " + f"mode={self.evidence_chain_mode}" + ) logger.info(f" ACL: enabled={self.acl_enabled}, strict={self.acl_strict_mode}, disclosure={self.acl_disclosure_mode}") if not self.acl_enabled: logger.warning( diff --git a/backend/app/db/models.py b/backend/app/db/models.py index 4eccdb8..9a96a0a 100644 --- a/backend/app/db/models.py +++ b/backend/app/db/models.py @@ -177,6 +177,7 @@ class CitationSnapshot(Base): node_id = Column(String(64), nullable=False) selector_bundle = Column(JSONB, nullable=False) exact_text = Column(Text, nullable=True) + evidence_record = Column(JSONB, nullable=True) answer_hash = Column(String(128), nullable=False) content_hash = Column(String(128), nullable=False) created_at = Column(DateTime(timezone=True), server_default=func.now()) diff --git a/backend/app/graph/chunker.py b/backend/app/graph/chunker.py index 3df1ae2..aa8fcb4 100644 --- a/backend/app/graph/chunker.py +++ b/backend/app/graph/chunker.py @@ -11,6 +11,7 @@ import re import logging +import unicodedata from dataclasses import dataclass, field from typing import List, Optional, Tuple, Any, TYPE_CHECKING @@ -520,6 +521,8 @@ def _create_chunk( "width": page_data.width, "height": page_data.height } + meta["page_rotation"] = int(getattr(page_data, "rotation", 0) or 0) + meta["coordinate_system"] = "normalized_top_left" # Preserve docling structural provenance where available. if getattr(page_data, "meta", None): @@ -527,8 +530,18 @@ def _create_chunk( if page_data.meta.get(key) is not None: meta[key] = page_data.meta.get(key) - # Compute merged bbox from matching text spans - bbox = self._compute_chunk_bbox(text_plain, page_data) + # Retain ordered word provenance. The legacy merged bbox remains + # available for approximate navigation, but V2 evidence is resolved + # from source_spans into precise line rectangles. + source_spans, provenance_status = self._compute_chunk_source_spans( + text_plain, + page_data, + ) + meta["source_spans"] = source_spans + meta["source_span_resolution"] = provenance_status + meta["evidence_schema_version"] = "2.0" + + bbox = self._compute_chunk_bbox(text_plain, page_data, source_spans=source_spans) if bbox: meta["bbox"] = bbox @@ -559,7 +572,9 @@ def _count_tokens(self, text: str) -> int: def _compute_chunk_bbox( self, chunk_text: str, - page_data: Any + page_data: Any, + *, + source_spans: Optional[List[dict]] = None, ) -> Optional[dict]: """Compute merged bounding box for chunk text from page spans. @@ -576,7 +591,22 @@ def _compute_chunk_bbox( if not page_data or not hasattr(page_data, 'text_spans') or not page_data.text_spans: return None - # Normalize chunk text for matching + if source_spans: + point_boxes = [ + span.get("bbox") + for span in source_spans + if isinstance(span.get("bbox"), dict) + ] + if point_boxes: + return { + "x0": round(min(box["x0"] for box in point_boxes), 2), + "y0": round(min(box["y0"] for box in point_boxes), 2), + "x1": round(max(box["x1"] for box in point_boxes), 2), + "y1": round(max(box["y1"] for box in point_boxes), 2), + } + + # Legacy approximate fallback for documents ingested without V2 word + # provenance. chunk_text_normalized = ' '.join(chunk_text.split()).lower() # Find spans that appear in the chunk text @@ -613,3 +643,89 @@ def _compute_chunk_bbox( "x1": round(x1, 2), "y1": round(y1, 2) } + + @staticmethod + def _normalize_provenance_text(value: str) -> str: + return " ".join(unicodedata.normalize("NFKC", value or "").split()).casefold() + + @classmethod + def _compute_chunk_source_spans( + cls, + chunk_text: str, + page_data: Any, + ) -> Tuple[List[dict], str]: + """Resolve a chunk to one unique contiguous sequence of source words. + + Ambiguous and inexact matches deliberately return no verifiable spans. + That makes downstream evidence fail closed instead of highlighting a + plausible-looking but potentially incorrect region. + """ + spans = list(getattr(page_data, "text_spans", None) or []) + if not spans: + return [], "missing_source_words" + + ordered = sorted(spans, key=lambda item: int(getattr(item, "order", 0) or 0)) + stream_parts: List[str] = [] + ranges: List[Tuple[int, int, Any]] = [] + cursor = 0 + for span in ordered: + normalized = cls._normalize_provenance_text(getattr(span, "text", "")) + if not normalized: + continue + if stream_parts: + stream_parts.append(" ") + cursor += 1 + start = cursor + stream_parts.append(normalized) + cursor += len(normalized) + ranges.append((start, cursor, span)) + + page_stream = "".join(stream_parts) + needle = cls._normalize_provenance_text(chunk_text) + if not page_stream or not needle: + return [], "empty_normalized_text" + + occurrences: List[int] = [] + search_from = 0 + while True: + idx = page_stream.find(needle, search_from) + if idx < 0: + break + occurrences.append(idx) + search_from = idx + max(1, len(needle)) + if len(occurrences) > 1: + break + + if not occurrences: + return [], "chunk_not_exact_in_source_words" + if len(occurrences) > 1: + return [], "ambiguous_chunk_in_source_words" + + match_start = occurrences[0] + match_end = match_start + len(needle) + matched: List[dict] = [] + for start, end, span in ranges: + if start >= match_end or end <= match_start: + continue + payload = span.to_dict() if hasattr(span, "to_dict") else { + "text": getattr(span, "text", ""), + "bbox": { + "x0": span.bbox[0], + "y0": span.bbox[1], + "x1": span.bbox[2], + "y1": span.bbox[3], + }, + } + payload["page_no"] = int(page_data.page_no) + matched.append(payload) + + if not matched: + return [], "chunk_match_has_no_source_words" + if not all( + span.get("verifiable") is True + and isinstance(span.get("normalized_bbox"), dict) + and span.get("coordinate_system") == "pdf_points_top_left" + for span in matched + ): + return matched, "approximate_source_provenance" + return matched, "exact_source_words" diff --git a/backend/app/graph/context_packer.py b/backend/app/graph/context_packer.py index a678b3a..061a44d 100644 --- a/backend/app/graph/context_packer.py +++ b/backend/app/graph/context_packer.py @@ -6,7 +6,7 @@ import logging from dataclasses import dataclass, field -from typing import List, Dict, Optional, Any +from typing import List, Dict, Optional, Any, Set from app.db.graph_models import Node, NodeType from .expander import ExpandedContext @@ -78,7 +78,11 @@ def to_text(self, include_citations: bool = True) -> str: marker = f"[{citation.node_id}:{page_no}]" # Keep auxiliary metadata outside [] so citation regexes stay simple. - meta_parts = [f"source={citation.source_type}"] + meta_parts = [ + f"node_id={citation.node_id}", + f"page_no={page_no}", + f"source={citation.source_type}", + ] if citation.label: meta_parts.append(f"label={citation.label}") @@ -121,7 +125,9 @@ def __init__( def pack( self, expanded: ExpandedContext, - query: Optional[str] = None + query: Optional[str] = None, + node_order: Optional[List[str]] = None, + allowed_node_ids: Optional[Set[str]] = None, ) -> PackedContext: """Pack expanded context into ordered blocks. @@ -143,6 +149,36 @@ def pack( total_chars = 0 max_chars = self.max_tokens * self.CHARS_PER_TOKEN + # Query-aware evidence chains provide an explicit, bounded evidence + # order. This branch preserves canonical citations while avoiding + # accidental inclusion of unselected candidate nodes. + if node_order is not None: + nodes_by_id = {node.node_id: node for node in expanded.all_nodes} + for node_id in node_order: + if allowed_node_ids is not None and node_id not in allowed_node_ids: + continue + node = nodes_by_id.get(node_id) + if node is None: + continue + source_type = expanded.node_sources.get(node_id, "chain") + block = self._create_block(node, source_type, order) + if block and (self.include_empty or block.text.strip()): + if total_chars + len(block.text) <= max_chars: + blocks.append(block) + total_chars += len(block.text) + order += 1 + + tokens_estimate = total_chars // self.CHARS_PER_TOKEN + logger.info( + f"Packed {len(blocks)} chain-ordered blocks, ~{tokens_estimate} tokens " + f"(max={self.max_tokens})" + ) + return PackedContext( + blocks=blocks, + total_chars=total_chars, + total_tokens_estimate=tokens_estimate, + ) + # 1. Seed chunks for node in self._sort_chunks(expanded.seed_nodes): block = self._create_block(node, 'seed', order) diff --git a/backend/app/graph/docling_adapter.py b/backend/app/graph/docling_adapter.py index db07c2e..0f7a721 100644 --- a/backend/app/graph/docling_adapter.py +++ b/backend/app/graph/docling_adapter.py @@ -283,7 +283,16 @@ def _get_text_span(item: Any) -> Optional[TextSpan]: t = getattr(bbox, "t", None) or getattr(bbox, "y0", None) or 0 r = getattr(bbox, "r", None) or getattr(bbox, "x1", None) or 0 b = getattr(bbox, "b", None) or getattr(bbox, "y1", None) or 0 - return TextSpan(text=text, bbox=(l, t, r, b)) + # Docling coordinate origins vary by source adapter. Preserve this + # provenance for approximate navigation, but do not mark it as + # legal-grade renderable evidence until the origin is normalized. + return TextSpan( + text=text, + bbox=(l, t, r, b), + coordinate_system="docling_points_unspecified", + extraction_source="docling", + verifiable=False, + ) except Exception: return None diff --git a/backend/app/graph/expander.py b/backend/app/graph/expander.py index bb15e80..8620808 100644 --- a/backend/app/graph/expander.py +++ b/backend/app/graph/expander.py @@ -29,6 +29,9 @@ class ExpandedContext: # For citation tracking node_sources: Dict[str, str] = field(default_factory=dict) # node_id -> source type + # Additional authorized nodes selected by query-aware evidence chaining. + # Empty for the baseline path, preserving existing behavior. + chain_nodes: List[Node] = field(default_factory=list) @property def all_nodes(self) -> List[Node]: @@ -55,6 +58,11 @@ def all_nodes(self) -> List[Node]: if node.node_id not in seen: seen.add(node.node_id) result.append(node) + + for node in self.chain_nodes: + if node.node_id not in seen: + seen.add(node.node_id) + result.append(node) return result diff --git a/backend/app/graph/page_extractor.py b/backend/app/graph/page_extractor.py index daca1e8..07ec0c0 100644 --- a/backend/app/graph/page_extractor.py +++ b/backend/app/graph/page_extractor.py @@ -7,6 +7,7 @@ import io import base64 import logging +import unicodedata from dataclasses import dataclass, field from typing import List, Optional, Tuple from pathlib import Path @@ -20,14 +21,37 @@ @dataclass class TextSpan: - """A text span with bounding box information.""" + """An ordered source word with display-oriented provenance. + + ``bbox`` uses page points with a top-left origin after page rotation. + ``normalized_bbox`` uses the same orientation with values in 0..1 so the + browser never has to mix PyMuPDF coordinates with PDF.js user space. + """ text: str bbox: tuple # (x0, y0, x1, y1) in PDF points + span_id: str = "" + order: int = 0 + block_no: Optional[int] = None + line_no: Optional[int] = None + word_no: Optional[int] = None + normalized_bbox: Optional[dict] = None + coordinate_system: str = "pdf_points_top_left" + extraction_source: str = "native_pdf" + verifiable: bool = True def to_dict(self) -> dict: return { + 'span_id': self.span_id, + 'order': self.order, 'text': self.text, - 'bbox': {'x0': self.bbox[0], 'y0': self.bbox[1], 'x1': self.bbox[2], 'y1': self.bbox[3]} + 'bbox': {'x0': self.bbox[0], 'y0': self.bbox[1], 'x1': self.bbox[2], 'y1': self.bbox[3]}, + 'normalized_bbox': self.normalized_bbox, + 'block_no': self.block_no, + 'line_no': self.line_no, + 'word_no': self.word_no, + 'coordinate_system': self.coordinate_system, + 'extraction_source': self.extraction_source, + 'verifiable': self.verifiable, } @@ -42,6 +66,7 @@ class PageData: used_ocr: bool width: float height: float + rotation: int = 0 text_spans: List['TextSpan'] = field(default_factory=list) # Text with bbox info meta: dict = field(default_factory=dict) @@ -241,11 +266,19 @@ def _extract_page( used_ocr=used_ocr, width=width, height=height, + rotation=int(page.rotation or 0), text_spans=text_spans, meta={ "native_char_count": len(native_text), "final_char_count": len(text_plain), "text_span_count": len(text_spans), + "coordinate_system": "normalized_top_left", + "crop_box": { + "x0": float(page.cropbox.x0), + "y0": float(page.cropbox.y0), + "x1": float(page.cropbox.x1), + "y1": float(page.cropbox.y1), + }, } ) @@ -282,10 +315,11 @@ def _should_use_ocr( ) def _extract_text_spans(self, page: fitz.Page) -> List[TextSpan]: - """Extract text spans with bounding box information. - - Uses PyMuPDF's get_text('dict') to get structured text with coordinates. - Each span represents a continuous text segment with the same formatting. + """Extract ordered words with normalized display rectangles. + + Word-level provenance lets the verifier return only the lines supporting + a claim. Font spans and merged chunk boxes are too coarse for legal or + clinical review. Args: page: PyMuPDF page object @@ -293,29 +327,50 @@ def _extract_text_spans(self, page: fitz.Page) -> List[TextSpan]: Returns: List of TextSpan objects with text and bbox """ - spans = [] + spans: List[TextSpan] = [] try: - # Get structured text data with bounding boxes - text_dict = page.get_text('dict', flags=fitz.TEXT_PRESERVE_WHITESPACE) - - for block in text_dict.get('blocks', []): - # Skip image blocks - if block.get('type') != 0: # type 0 = text block + words = page.get_text("words", sort=True) + display_width = max(float(page.rect.width), 1.0) + display_height = max(float(page.rect.height), 1.0) + rotation_matrix = page.rotation_matrix + + for order, word in enumerate(words): + if len(word) < 8: continue - - for line in block.get('lines', []): - for span in line.get('spans', []): - text = span.get('text', '').strip() - bbox = span.get('bbox') - - if text and bbox: - spans.append(TextSpan( - text=text, - bbox=tuple(bbox) # (x0, y0, x1, y1) - )) + x0, y0, x1, y1, text, block_no, line_no, word_no = word[:8] + text = unicodedata.normalize("NFKC", str(text)).strip() + if not text: + continue + + source_rect = fitz.Rect(float(x0), float(y0), float(x1), float(y1)) + display_rect = source_rect * rotation_matrix if page.rotation else source_rect + display_rect.normalize() + normalized_bbox = { + "x0": max(0.0, min(1.0, display_rect.x0 / display_width)), + "y0": max(0.0, min(1.0, display_rect.y0 / display_height)), + "x1": max(0.0, min(1.0, display_rect.x1 / display_width)), + "y1": max(0.0, min(1.0, display_rect.y1 / display_height)), + } + spans.append( + TextSpan( + text=text, + bbox=( + float(display_rect.x0), + float(display_rect.y0), + float(display_rect.x1), + float(display_rect.y1), + ), + span_id=f"p{page.number + 1}:w{order}", + order=order, + block_no=int(block_no), + line_no=int(line_no), + word_no=int(word_no), + normalized_bbox=normalized_bbox, + ) + ) except Exception as e: - logger.warning(f"Failed to extract text spans with bbox: {e}") + logger.warning(f"Failed to extract word provenance with bbox: {e}") return spans diff --git a/backend/app/llm/openai_client.py b/backend/app/llm/openai_client.py index 625d47f..6ef2616 100644 --- a/backend/app/llm/openai_client.py +++ b/backend/app/llm/openai_client.py @@ -6,6 +6,8 @@ import logging import time +import json +import re from dataclasses import dataclass, field from typing import List, Optional, Dict, Any @@ -31,6 +33,39 @@ logger = logging.getLogger(__name__) +GROUNDED_ANSWER_SCHEMA: Dict[str, Any] = { + "type": "object", + "additionalProperties": False, + "required": ["answer", "citations"], + "properties": { + "answer": {"type": "string"}, + "citations": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": False, + "required": [ + "citation_id", + "node_id", + "page_no", + "exact_quote", + "label", + ], + "properties": { + "citation_id": { + "type": "string", + "pattern": "^C[1-9][0-9]*$", + }, + "node_id": {"type": "string"}, + "page_no": {"type": "integer", "minimum": 1}, + "exact_quote": {"type": "string"}, + "label": {"type": ["string", "null"]}, + }, + }, + }, + }, +} + @dataclass class Citation: @@ -39,9 +74,11 @@ class Citation: Extended for provenance anchoring - enables "click citation → open doc → highlight". """ node_id: str + citation_id: Optional[str] = None page_no: Optional[int] = None label: Optional[str] = None # e.g., "Figure 1", "Table 2" text_snippet: Optional[str] = None + exact_quote: Optional[str] = None # Extended fields for provenance anchoring (populated during citation hydration) doc_id: Optional[str] = None @@ -83,15 +120,13 @@ def get_qa_system_prompt() -> str: IMPORTANT RULES: 1. ONLY use information from the provided context to answer. Do NOT make up information. 2. If the context doesn't contain enough information to answer, say "I cannot find sufficient information in the provided context." -3. ALWAYS cite your sources using the format [node_id:PAGE_NUMBER] or [LABEL:PAGE_NUMBER] for figures/tables. +3. Cite sources in the answer using stable IDs [C1], [C2], and so on. 4. Every factual claim or paragraph MUST have at least one citation. 5. When referencing a figure or table, use its label (e.g., "Figure 1", "Table 2") in your answer. 6. Be concise but thorough. Include specific details from the context. - -CITATION FORMAT EXAMPLES: -- "The study found that X [abc123:3]" -- "As shown in Figure 1 [Figure 1:5], the data indicates..." -- "Table 2 [Table 2:7] presents the following results..." +7. Return the structured object required by the API schema. +8. For every citation copy the smallest complete supporting passage verbatim + into exact_quote. Never paraphrase exact_quote. The context will include: - Chunk text with node_id and page number @@ -149,6 +184,7 @@ class OpenAIClient: RETRY_ATTEMPTS = 3 RETRY_BASE_DELAY_S = 1.0 RETRY_MAX_DELAY_S = 10.0 + CITATION_REPAIR_ATTEMPTS = 2 def __init__( self, @@ -237,7 +273,10 @@ def generate_answer( QUESTION: {question} -Please answer the question based ONLY on the provided context. Remember to cite your sources using [node_id:page] or [Label:page] format.""" +Return a grounded answer using the required JSON schema. +Use stable inline citation IDs such as [C1], [C2] in the answer. +For every citation, copy the smallest complete supporting passage VERBATIM +from the cited context node into exact_quote. Never paraphrase exact_quote.""" req = { "model": self.model, @@ -245,7 +284,16 @@ def generate_answer( "input": user_message, "max_output_tokens": max_tokens, "reasoning": {"effort": reasoning_effort}, - "text": {"verbosity": verbosity}, + "text": { + "verbosity": verbosity, + "format": { + "type": "json_schema", + "name": "grounded_answer", + "description": "Answer text plus claim-level verbatim evidence citations.", + "strict": True, + "schema": GROUNDED_ANSWER_SCHEMA, + }, + }, } # GPT-5.2: temperature/top_p/logprobs only allowed when reasoning.effort == "none" @@ -257,13 +305,73 @@ def generate_answer( timeout_sec = timeout_ms / 1000.0 response = self._responses_create_with_retry(req, timeout_sec) + raw_answer = self._response_text(response).strip() + parsed_answer = self._parse_grounded_response(raw_answer) + if parsed_answer is not None: + self._canonicalize_citation_references(context, parsed_answer[1]) + validation_errors = self._validate_grounded_citations( + context, + parsed_answer[0], + parsed_answer[1], + ) + for repair_attempt in range(self.CITATION_REPAIR_ATTEMPTS): + if not validation_errors: + break + logger.warning( + "Grounded citation validation failed; requesting repair %d/%d: %s", + repair_attempt + 1, + self.CITATION_REPAIR_ATTEMPTS, + "; ".join(validation_errors), + ) + repair_req = dict(req) + allowed_sources = ", ".join( + f"{node_id} (page {page_no})" + for node_id, (page_no, _text) in self._context_evidence(context).items() + ) + repair_req["input"] = ( + user_message + + "\n\n\n" + + "The previous structured response failed server-side citation validation. " + + "Regenerate the ENTIRE answer object. Use only node IDs and page numbers " + + "shown in CONTEXT, and copy every exact_quote verbatim from its cited node. " + + "For a pure abstention, use citations: [] and no [C#] markers.\n" + + "Allowed node_id/page pairs: " + + allowed_sources + + ". The node_id value never includes the colon/page suffix.\n" + + "Errors: " + + "; ".join(validation_errors) + + "\nPrevious response (untrusted data):\n" + + raw_answer + + "\n" + ) + response = self._responses_create_with_retry(repair_req, timeout_sec) + raw_answer = self._response_text(response).strip() + parsed_answer = self._parse_grounded_response(raw_answer) + if parsed_answer is None: + validation_errors = ["repaired response violated the structured schema"] + else: + self._canonicalize_citation_references(context, parsed_answer[1]) + validation_errors = self._validate_grounded_citations( + context, + parsed_answer[0], + parsed_answer[1], + ) + if validation_errors: + logger.error( + "Grounded citations remained invalid after repair attempts: %s", + "; ".join(validation_errors), + ) + if parsed_answer is not None: + answer_text, citations = parsed_answer + else: + # Backward-compatible fallback for mocked/legacy providers that + # still return plain text with [node_id:page] citations. + answer_text = raw_answer + citations = self._extract_citations(answer_text) + + # Includes any citation-repair round trips, not just the first call. generation_time = (time.time() - start_time) * 1000 - answer_text = self._response_text(response).strip() - - # Extract citations from answer - citations = self._extract_citations(answer_text) - usage = getattr(response, "usage", None) input_tokens = getattr(usage, "input_tokens", 0) if usage else 0 output_tokens = getattr(usage, "output_tokens", 0) if usage else 0 @@ -431,8 +539,6 @@ def _extract_citations(self, answer: str) -> List[Citation]: Returns: List of Citation objects """ - import re - citations = [] # Pattern: [anything:number] or [anything] @@ -466,3 +572,146 @@ def _extract_citations(self, answer: str) -> List[Citation]: )) return citations + + @staticmethod + def _parse_grounded_response(text: str) -> Optional[tuple[str, List[Citation]]]: + """Parse and defensively validate the structured grounded answer.""" + try: + payload = json.loads(text) + except (json.JSONDecodeError, TypeError): + return None + if not isinstance(payload, dict) or not isinstance(payload.get("answer"), str): + return None + raw_citations = payload.get("citations") + if not isinstance(raw_citations, list): + return None + + citations: List[Citation] = [] + seen_ids: set[str] = set() + for item in raw_citations: + if not isinstance(item, dict): + return None + citation_id = str(item.get("citation_id") or "").strip() + node_id = str(item.get("node_id") or "").strip() + exact_quote = str(item.get("exact_quote") or "").strip() + page_no = item.get("page_no") + if ( + not re.fullmatch(r"C[1-9][0-9]*", citation_id) + or citation_id in seen_ids + or not node_id + or not exact_quote + or not isinstance(page_no, int) + or page_no < 1 + ): + return None + seen_ids.add(citation_id) + citations.append( + Citation( + citation_id=citation_id, + node_id=node_id, + page_no=page_no, + label=item.get("label"), + exact_quote=exact_quote, + text_snippet=exact_quote, + ) + ) + + answer = payload["answer"].strip() + referenced_ids = set(re.findall(r"\[(C[1-9][0-9]*)\]", answer)) + if referenced_ids != seen_ids: + logger.warning( + "Structured answer citation IDs differ from citation payload: answer=%s payload=%s", + sorted(referenced_ids), + sorted(seen_ids), + ) + return None + return answer, citations + + @staticmethod + def _context_evidence(context: str) -> Dict[str, tuple[int, str]]: + """Parse ContextPacker blocks into the node/page/text trust boundary.""" + evidence: Dict[str, tuple[int, str]] = {} + for raw_block in re.split(r"\n\n---\n\n", context or ""): + block = raw_block.strip() + if not block: + continue + first_line, separator, remainder = block.partition("\n") + marker = re.match(r"^\[([^:\]\n]+):(\d+)\](.*)$", first_line.strip()) + if not marker: + continue + node_id = marker.group(1).strip() + page_no = int(marker.group(2)) + if separator: + node_text = remainder.strip() + else: + # Backward-compatible test/legacy form: [node:page] text + node_text = marker.group(3).strip() + if node_text.startswith("source="): + node_text = "" + evidence[node_id] = (page_no, node_text) + return evidence + + @staticmethod + def _quote_key(text: str) -> str: + """Match the same normalized word stream used by source-span verification.""" + return " ".join(re.findall(r"[A-Za-z0-9]+", text or "")).casefold() + + @classmethod + def _canonicalize_citation_references( + cls, + context: str, + citations: List[Citation], + ) -> None: + """Repair only the unambiguous [NODE_ID:PAGE] copy-format mistake. + + No fuzzy node matching is allowed. A suffix is removed only when both + the base node ID and page exactly match a trusted packed-context block. + """ + evidence = cls._context_evidence(context) + for citation in citations: + if citation.node_id in evidence: + continue + match = re.fullmatch(r"(.+):(\d+)", citation.node_id or "") + if not match: + continue + base_node_id = match.group(1) + suffix_page = int(match.group(2)) + source = evidence.get(base_node_id) + if ( + source is not None + and suffix_page == source[0] + and citation.page_no == source[0] + ): + citation.node_id = base_node_id + + @classmethod + def _validate_grounded_citations( + cls, + context: str, + answer: str, + citations: List[Citation], + ) -> List[str]: + """Reject citations that cannot resolve to exact packed source evidence.""" + evidence = cls._context_evidence(context) + errors: List[str] = [] + inline_ids = set(re.findall(r"\[(C[1-9][0-9]*)\]", answer or "")) + payload_ids = {citation.citation_id for citation in citations if citation.citation_id} + if inline_ids != payload_ids: + errors.append("inline citation IDs do not match the citation payload") + + for citation in citations: + source = evidence.get(citation.node_id) + prefix = citation.citation_id or citation.node_id + if source is None: + errors.append(f"{prefix} uses unknown node_id {citation.node_id}") + continue + source_page, source_text = source + if citation.page_no != source_page: + errors.append( + f"{prefix} page {citation.page_no} does not match source page {source_page}" + ) + quote_key = cls._quote_key(citation.exact_quote or "") + source_key = cls._quote_key(source_text) + if not quote_key or quote_key not in source_key: + errors.append(f"{prefix} exact_quote is not verbatim source text") + return errors diff --git a/backend/app/prompts/qa_answer_v2.txt b/backend/app/prompts/qa_answer_v2.txt index 1e53b87..fcb499a 100644 --- a/backend/app/prompts/qa_answer_v2.txt +++ b/backend/app/prompts/qa_answer_v2.txt @@ -14,28 +14,35 @@ You never fabricate information. When evidence is insufficient, you say so clear 1. USE ONLY the provided context. Never use prior knowledge or make assumptions. -2. CITE EVERY factual claim using the format [node_id:PAGE] or [LABEL:PAGE]. +2. CITE EVERY factual claim using stable citation IDs: [C1], [C2], and so on. 3. If context is insufficient, respond: "I cannot find sufficient information in the provided context to answer this question." + - For a PURE ABSTENTION with no supported factual claim, return citations: [] and use no [C#] marker. + - Never cite irrelevant or generic text merely to justify that information is missing. + - For a PARTIAL answer, cite only the factual information that is actually available. 4. For CONFLICTING evidence: note the conflict, cite both sources, prefer higher authority_tier. 5. For PARTIAL information: provide what you can find, explicitly state what's missing. 6. NEVER guess, infer beyond the text, or hallucinate details. -- Length: 2-5 sentences for simple questions, 1-3 paragraphs for complex ones -- Structure: Lead with the direct answer, then supporting details -- Citations: Inline after each claim, e.g., "The fee is $45,000 [abc123:5]" -- For figures/tables: Reference by label, e.g., "As shown in Table 3 [Table 3:12]" +Return the structured object required by the API schema: +- answer: 2-5 sentences for simple questions, 1-3 paragraphs for complex ones +- citations: one entry per stable citation ID used in answer +- Each citation entry includes citation_id, node_id, page_no, label, and exact_quote +- Context markers use [NODE_ID:PAGE]. In the citation object, node_id is ONLY the text left of the colon and page_no is ONLY the integer right of the colon. Never include ":PAGE" inside node_id. +- exact_quote MUST be the smallest complete supporting passage copied VERBATIM from that node +- Never put a paraphrase, corrected spelling, or inferred statement in exact_quote +- A pure abstention must have an empty citations array; do not manufacture a quote for it +- Every [C#] in answer must have exactly one citation entry and vice versa -Format: [node_id:PAGE] for text, [LABEL:PAGE] for figures/tables +Format in answer: [C1], [C2], and so on. Examples: -- "The initial franchise fee is $45,000 [chunk_a1b2:5]." -- "According to Table 3 [Table 3:12], the breakdown includes..." -- "Figure 2 [Figure 2:8] illustrates the organizational structure." -- "The territory must be at least 50,000 population [chunk_x1:7] [chunk_x2:9]." (multiple sources) +- "The initial franchise fee is $45,000 [C1]." +- "According to Table 3, the breakdown includes... [C2]" +- "The territory must be at least 50,000 population [C3] [C4]." (multiple sources) @@ -50,7 +57,7 @@ The initial franchise fee is non-refundable under any circumstances. What is the initial franchise fee and is it refundable? -The initial franchise fee is $45,000, payable in full upon signing the Franchise Agreement [fee_chunk_1:5]. This fee is non-refundable under any circumstances [fee_chunk_2:5]. +{"answer":"The initial franchise fee is $45,000, payable in full upon signing the Franchise Agreement [C1]. This fee is non-refundable under any circumstances [C2].","citations":[{"citation_id":"C1","node_id":"fee_chunk_1","page_no":5,"exact_quote":"The initial franchise fee is $45,000, payable in full upon signing the Franchise Agreement.","label":null},{"citation_id":"C2","node_id":"fee_chunk_2","page_no":5,"exact_quote":"The initial franchise fee is non-refundable under any circumstances.","label":null}]} @@ -67,7 +74,7 @@ Territories typically range from 30,000 to 75,000 population. What is the territory size? -Each franchisee receives an exclusive territory with a minimum population of 50,000 [territory_1:12]. The exact size is determined by the franchisor based on market analysis [territory_2:12]. Note: There is some inconsistency in the documents—one section mentions territories typically range from 30,000 to 75,000 population [territory_3:45], but the primary disclosure document specifies a 50,000 minimum [territory_1:12]. +{"answer":"Each franchisee receives an exclusive territory with a minimum population of 50,000 [C1]. The exact size is determined by the franchisor based on market analysis [C2]. A second passage says territories typically range from 30,000 to 75,000 population [C3], which conflicts with the stated 50,000 minimum [C1].","citations":[{"citation_id":"C1","node_id":"territory_1","page_no":12,"exact_quote":"Each franchisee receives an exclusive territory with a minimum population of 50,000.","label":null},{"citation_id":"C2","node_id":"territory_2","page_no":12,"exact_quote":"Territory size is determined by the franchisor based on market analysis.","label":null},{"citation_id":"C3","node_id":"territory_3","page_no":45,"exact_quote":"Territories typically range from 30,000 to 75,000 population.","label":null}]} @@ -78,7 +85,7 @@ Initial training consists of 2 weeks at our headquarters in Dallas, Texas. What is the ongoing royalty rate? -I cannot find sufficient information in the provided context to answer this question. The context only contains information about initial training [training_1:20], but does not mention royalty rates. +{"answer":"I cannot find sufficient information in the provided context to answer this question. The context only contains information about initial training [C1], but does not mention royalty rates.","citations":[{"citation_id":"C1","node_id":"training_1","page_no":20,"exact_quote":"Initial training consists of 2 weeks at our headquarters in Dallas, Texas.","label":null}]} @@ -87,7 +94,7 @@ I cannot find sufficient information in the provided context to answer this ques [node_id: fin_1, page: 30] See Table 5 for estimated initial investment breakdown. -[label: Table 5, page: 31] +[node_id: table_5, label: Table 5, page: 31] | Category | Low | High | |----------|-----|------| | Franchise Fee | $45,000 | $45,000 | @@ -97,11 +104,7 @@ See Table 5 for estimated initial investment breakdown. What is the estimated initial investment? -According to Table 5 [Table 5:31], the estimated initial investment ranges from $125,000 to $190,000, broken down as follows: -- Franchise Fee: $45,000 -- Equipment: $50,000 - $80,000 -- Initial Inventory: $10,000 - $25,000 -- Working Capital: $20,000 - $40,000 +{"answer":"According to Table 5, the estimated initial investment ranges from $125,000 to $190,000 [C1].","citations":[{"citation_id":"C1","node_id":"table_5","page_no":31,"exact_quote":"| Category | Low | High | | Franchise Fee | $45,000 | $45,000 | | Equipment | $50,000 | $80,000 | | Initial Inventory | $10,000 | $25,000 | | Working Capital | $20,000 | $40,000 |","label":"Table 5"}]} diff --git a/backend/app/prompts/synthesizer_v2.txt b/backend/app/prompts/synthesizer_v2.txt index 2d14e30..c0a55d6 100644 --- a/backend/app/prompts/synthesizer_v2.txt +++ b/backend/app/prompts/synthesizer_v2.txt @@ -15,12 +15,13 @@ You highlight any gaps, conflicts, or uncertainties from the verification proces 1. Use ONLY facts from the verified sub-answers—never add external information. -2. PRESERVE all citations exactly as provided (format: [node_id:PAGE] or [LABEL:PAGE]). +2. Preserve the node_id, page_no, and exact_quote from every used grounded citation. 3. If any sub-answer indicates "insufficient evidence", clearly reflect that gap. 4. If sub-answers conflict, present both views with their citations. 5. Weight sub-answers by their confidence scores when synthesizing. 6. Structure the answer to directly address the original question. 7. Do NOT invent, infer, or extrapolate beyond what sub-answers provide. +8. Use stable [C1], [C2] citation IDs in the answer. @@ -35,10 +36,10 @@ You highlight any gaps, conflicts, or uncertainties from the verification proces Return ONLY valid JSON. No markdown, no explanation, no code blocks. {{ - "answer": "Your synthesized answer with all citations preserved inline", + "answer": "Your synthesized answer with stable [C1] citations inline", "citations": [ - {{"node_id": "id1", "page_no": 5, "label": null}}, - {{"node_id": "id2", "page_no": 12, "label": "Table 3"}} + {{"citation_id": "C1", "node_id": "id1", "page_no": 5, "label": null, "exact_quote": "verbatim quote preserved from sub-answer"}}, + {{"citation_id": "C2", "node_id": "id2", "page_no": 12, "label": "Table 3", "exact_quote": "verbatim quote preserved from sub-answer"}} ], "conflicts_summary": "Brief summary of any conflicts between sub-answers, or empty string", "gaps_summary": "Brief summary of any information gaps, or empty string" @@ -51,24 +52,27 @@ Return ONLY valid JSON. No markdown, no explanation, no code blocks. What are the differences between the initial franchise fee and the ongoing royalty fee? [sq1] What is the initial franchise fee? -Answer: The initial franchise fee is $45,000, payable upon signing [fee_1:5]. +Answer: The initial franchise fee is $45,000, payable upon signing. +Grounded citations: [{{"node_id":"fee_1","page_no":5,"label":null,"exact_quote":"The initial franchise fee is $45,000, payable upon signing the Franchise Agreement."}}] Confidence: 1.0 [sq2] What is the ongoing royalty fee? -Answer: The royalty fee is 5% of Gross Sales, paid weekly [royalty_1:8]. +Answer: The royalty fee is 5% of Gross Sales, paid weekly. +Grounded citations: [{{"node_id":"royalty_1","page_no":8,"label":null,"exact_quote":"Franchisees pay a continuing royalty fee of 5% of Gross Sales, due weekly."}}] Confidence: 1.0 [sq3] Is the initial franchise fee refundable? -Answer: No, the franchise fee is non-refundable [refund_1:6]. +Answer: No, the franchise fee is non-refundable. +Grounded citations: [{{"node_id":"refund_1","page_no":6,"label":null,"exact_quote":"The initial franchise fee is fully earned upon payment and is non-refundable."}}] Confidence: 1.0 {{ - "answer": "The initial franchise fee and ongoing royalty fee differ in several ways. The initial franchise fee is a one-time payment of $45,000, due upon signing the Franchise Agreement [fee_1:5], and is non-refundable [refund_1:6]. In contrast, the ongoing royalty fee is a recurring payment of 5% of Gross Sales, paid weekly [royalty_1:8].", + "answer": "The initial franchise fee is a one-time payment of $45,000 due upon signing [C1], and it is non-refundable [C2]. The ongoing royalty fee is a recurring payment of 5% of Gross Sales, paid weekly [C3].", "citations": [ - {{"node_id": "fee_1", "page_no": 5, "label": null}}, - {{"node_id": "refund_1", "page_no": 6, "label": null}}, - {{"node_id": "royalty_1", "page_no": 8, "label": null}} + {{"citation_id":"C1","node_id":"fee_1","page_no":5,"label":null,"exact_quote":"The initial franchise fee is $45,000, payable upon signing the Franchise Agreement."}}, + {{"citation_id":"C2","node_id":"refund_1","page_no":6,"label":null,"exact_quote":"The initial franchise fee is fully earned upon payment and is non-refundable."}}, + {{"citation_id":"C3","node_id":"royalty_1","page_no":8,"label":null,"exact_quote":"Franchisees pay a continuing royalty fee of 5% of Gross Sales, due weekly."}} ], "conflicts_summary": "", "gaps_summary": "" @@ -80,24 +84,27 @@ Confidence: 1.0 How much does it cost to open a franchise and what training is provided? [sq1] What is the total estimated initial investment? -Answer: The estimated initial investment ranges from $125,000 to $190,000, as detailed in Table 5 [Table 5:31]. +Answer: The estimated initial investment ranges from $125,000 to $190,000. +Grounded citations: [{{"node_id":"table_5","page_no":31,"label":"Table 5","exact_quote":"Total Estimated Initial Investment | $125,000 | $190,000"}}] Confidence: 1.0 [sq2] What training is provided? -Answer: Initial training consists of 2 weeks at Dallas headquarters [train_1:15], covering operations, marketing, and customer service [train_2:16]. +Answer: Initial training consists of 2 weeks at Dallas headquarters, covering operations, marketing, and customer service. +Grounded citations: [{{"node_id":"train_1","page_no":15,"label":null,"exact_quote":"Initial training consists of 2 weeks at our Dallas headquarters."}},{{"node_id":"train_2","page_no":16,"label":null,"exact_quote":"Training covers operations, marketing, and customer service."}}] Confidence: 0.9 [sq3] What is the duration and location of training? -Answer: Training is 2 weeks at the Dallas headquarters [train_1:15]. +Answer: Training is 2 weeks at the Dallas headquarters. +Grounded citations: [{{"node_id":"train_1","page_no":15,"label":null,"exact_quote":"Initial training consists of 2 weeks at our Dallas headquarters."}}] Confidence: 1.0 {{ - "answer": "Opening a franchise requires an estimated initial investment of $125,000 to $190,000 [Table 5:31]. Training consists of a 2-week program at the company's Dallas headquarters [train_1:15], which covers operations, marketing, and customer service [train_2:16].", + "answer": "Opening a franchise requires an estimated initial investment of $125,000 to $190,000 [C1]. Training consists of a 2-week program at the Dallas headquarters [C2] covering operations, marketing, and customer service [C3].", "citations": [ - {{"node_id": "Table 5", "page_no": 31, "label": "Table 5"}}, - {{"node_id": "train_1", "page_no": 15, "label": null}}, - {{"node_id": "train_2", "page_no": 16, "label": null}} + {{"citation_id":"C1","node_id":"table_5","page_no":31,"label":"Table 5","exact_quote":"Total Estimated Initial Investment | $125,000 | $190,000"}}, + {{"citation_id":"C2","node_id":"train_1","page_no":15,"label":null,"exact_quote":"Initial training consists of 2 weeks at our Dallas headquarters."}}, + {{"citation_id":"C3","node_id":"train_2","page_no":16,"label":null,"exact_quote":"Training covers operations, marketing, and customer service."}} ], "conflicts_summary": "", "gaps_summary": "" @@ -109,7 +116,8 @@ Confidence: 1.0 What is the territory size and can it be changed? [sq1] What is the territory size? -Answer: Territories have a minimum population of 50,000 [territory_1:12]. +Answer: Territories have a minimum population of 50,000. +Grounded citations: [{{"node_id":"territory_1","page_no":12,"label":null,"exact_quote":"Each franchisee receives an exclusive territory with a minimum population of 50,000."}}] Confidence: 0.8 Conflict: Page 45 mentions 30,000-75,000 range, but primary document specifies 50,000 minimum. @@ -120,9 +128,9 @@ Insufficient: true {{ - "answer": "Each franchisee receives an exclusive territory with a minimum population of 50,000 [territory_1:12]. However, I could not find information in the provided documents about whether the territory can be changed after signing the agreement.", + "answer": "Each franchisee receives an exclusive territory with a minimum population of 50,000 [C1]. However, I could not find information in the provided documents about whether the territory can be changed after signing the agreement.", "citations": [ - {{"node_id": "territory_1", "page_no": 12, "label": null}} + {{"citation_id":"C1","node_id":"territory_1","page_no":12,"label":null,"exact_quote":"Each franchisee receives an exclusive territory with a minimum population of 50,000."}} ], "conflicts_summary": "Some inconsistency exists in territory size specifications—one section mentions a 30,000-75,000 range while the primary disclosure specifies a 50,000 minimum.", "gaps_summary": "No information found regarding whether territories can be modified after the initial agreement." diff --git a/backend/app/prompts/verifier_v2.txt b/backend/app/prompts/verifier_v2.txt index e915633..648aaa3 100644 --- a/backend/app/prompts/verifier_v2.txt +++ b/backend/app/prompts/verifier_v2.txt @@ -41,7 +41,7 @@ Return ONLY valid JSON. No markdown, no explanation, no code blocks. {{ "answer": "Your answer based strictly on evidence, or 'Insufficient evidence in provided snippets.'", "citations": [ - {{"node_id": "snippet_id", "page_no": 5, "label": "Table 1 (if applicable)"}} + {{"node_id": "snippet_id", "page_no": 5, "label": "Table 1 (if applicable)", "exact_quote": "Smallest complete supporting passage copied verbatim"}} ], "confidence": 0.85, "insufficient_evidence": false, @@ -60,7 +60,7 @@ The initial franchise fee is $45,000, payable upon signing the Franchise Agreeme {{ "answer": "The initial franchise fee is $45,000, payable upon signing the Franchise Agreement.", - "citations": [{{"node_id": "fee_1", "page_no": 5, "label": null}}], + "citations": [{{"node_id": "fee_1", "page_no": 5, "label": null, "exact_quote": "The initial franchise fee is $45,000, payable upon signing the Franchise Agreement."}}], "confidence": 1.0, "insufficient_evidence": false, "conflict_notes": "" @@ -80,7 +80,7 @@ The royalty rate is 6% of gross revenue. {{ "answer": "The royalty rate is 5% of Gross Sales, paid weekly.", - "citations": [{{"node_id": "royalty_1", "page_no": 8, "label": null}}], + "citations": [{{"node_id": "royalty_1", "page_no": 8, "label": null, "exact_quote": "Franchisees pay a continuing royalty fee of 5% of Gross Sales, due weekly."}}], "confidence": 0.8, "insufficient_evidence": false, "conflict_notes": "Conflict detected: Page 8 states 5% while page 42 states 6%. Used page 8 (authority_tier 1) as authoritative source." @@ -101,8 +101,8 @@ Training covers operations, marketing, and customer service. {{ "answer": "Franchisees receive 2 weeks of initial training at the Dallas headquarters, covering operations, marketing, and customer service.", "citations": [ - {{"node_id": "train_1", "page_no": 15, "label": null}}, - {{"node_id": "train_2", "page_no": 16, "label": null}} + {{"node_id": "train_1", "page_no": 15, "label": null, "exact_quote": "Initial training consists of 2 weeks at our Dallas headquarters."}}, + {{"node_id": "train_2", "page_no": 16, "label": null, "exact_quote": "Training covers operations, marketing, and customer service."}} ], "confidence": 0.9, "insufficient_evidence": false, @@ -140,7 +140,7 @@ The initial franchise fee is fully earned upon payment and is non-refundable. {{ "answer": "No, the franchise fee is non-refundable. It is fully earned upon payment.", - "citations": [{{"node_id": "refund_1", "page_no": 6, "label": null}}], + "citations": [{{"node_id": "refund_1", "page_no": 6, "label": null, "exact_quote": "The initial franchise fee is fully earned upon payment and is non-refundable."}}], "confidence": 1.0, "insufficient_evidence": false, "conflict_notes": "" @@ -157,7 +157,7 @@ Prospective franchisees should have sufficient capital to support the business d {{ "answer": "The evidence indicates franchisees should have 'sufficient capital' for the startup phase, but does not specify a minimum net worth requirement.", - "citations": [{{"node_id": "qual_1", "page_no": 3, "label": null}}], + "citations": [{{"node_id": "qual_1", "page_no": 3, "label": null, "exact_quote": "Prospective franchisees should have sufficient capital to support the business during the initial startup phase."}}], "confidence": 0.4, "insufficient_evidence": false, "conflict_notes": "" diff --git a/backend/app/qa/evidence_chain.py b/backend/app/qa/evidence_chain.py new file mode 100644 index 0000000..4e222dd --- /dev/null +++ b/backend/app/qa/evidence_chain.py @@ -0,0 +1,766 @@ +"""Query-aware evidence-chain retrieval over the authorized document graph. + +This module adopts the useful online portion of HyCE-RAG without conflating a +structural relevance score with factual confidence. It is deliberately +deterministic, bounded, feature-gated, and provenance preserving. +""" + +from __future__ import annotations + +import logging +import math +import re +from collections import deque +from dataclasses import dataclass, field +from typing import Any, Dict, Iterable, List, Mapping, Optional, Sequence, Set, Tuple + +from sqlalchemy import or_ +from sqlalchemy.orm import Session + +from app.db.graph_models import Edge, EdgeType, Node +from app.graph.expander import ExpandedContext + +logger = logging.getLogger(__name__) + + +SCORING_VERSION = "document-chain-v1" + + +@dataclass(frozen=True) +class EvidenceChainConfig: + """Bounded configuration for query-aware graph propagation.""" + + mode: str = "auto" # off | auto | on + route_threshold: float = 0.55 + max_hops: int = 3 + max_nodes: int = 30 + max_selected_nodes: int = 15 + max_chains: int = 6 + propagation_steps: int = 10 + restart_probability: float = 0.35 + convergence_tolerance: float = 1e-7 + propagation_weight: float = 0.50 + query_weight: float = 0.30 + seed_weight: float = 0.20 + path_overlap_threshold: float = 0.80 + + def __post_init__(self) -> None: + if self.mode not in {"off", "auto", "on"}: + raise ValueError("evidence-chain mode must be off, auto, or on") + if not 0.0 <= self.route_threshold <= 1.0: + raise ValueError("route_threshold must be in [0, 1]") + if not 1 <= self.max_hops <= 6: + raise ValueError("max_hops must be in [1, 6]") + if self.max_nodes < 1 or self.max_selected_nodes < 1 or self.max_chains < 1: + raise ValueError("node and chain budgets must be positive") + if self.max_selected_nodes > self.max_nodes: + raise ValueError("max_selected_nodes cannot exceed max_nodes") + if not 1 <= self.propagation_steps <= 100: + raise ValueError("propagation_steps must be in [1, 100]") + if not 0.0 < self.restart_probability <= 1.0: + raise ValueError("restart_probability must be in (0, 1]") + total_weight = self.propagation_weight + self.query_weight + self.seed_weight + if not math.isclose(total_weight, 1.0, abs_tol=1e-9): + raise ValueError("evidence-chain scoring weights must sum to 1") + + +@dataclass(frozen=True) +class RouteDecision: + applied: bool + mode: str + score: float + reasons: Tuple[str, ...] = () + + def to_dict(self) -> Dict[str, Any]: + return { + "applied": self.applied, + "mode": self.mode, + "score": round(self.score, 6), + "reasons": list(self.reasons), + } + + +@dataclass(frozen=True) +class ChainEdge: + from_node_id: str + to_node_id: str + edge_type: str + weight: float + + def to_dict(self) -> Dict[str, Any]: + return { + "from_node_id": self.from_node_id, + "to_node_id": self.to_node_id, + "edge_type": self.edge_type, + "weight": round(self.weight, 6), + } + + +@dataclass(frozen=True) +class ChainNodeScore: + node_id: str + final_score: float + propagation_score: float + query_relevance: float + seed_relevance: float + is_seed: bool + + def to_dict(self) -> Dict[str, Any]: + return { + "node_id": self.node_id, + "final_score": round(self.final_score, 6), + "propagation_score": round(self.propagation_score, 6), + "query_relevance": round(self.query_relevance, 6), + "seed_relevance": round(self.seed_relevance, 6), + "is_seed": self.is_seed, + } + + +@dataclass(frozen=True) +class EvidencePath: + path_id: str + node_ids: Tuple[str, ...] + edges: Tuple[ChainEdge, ...] + relevance_score: float + + def to_dict(self) -> Dict[str, Any]: + return { + "path_id": self.path_id, + "node_ids": list(self.node_ids), + "edges": [edge.to_dict() for edge in self.edges], + "relevance_score": round(self.relevance_score, 6), + } + + +@dataclass +class EvidenceChainResult: + """Result and safe audit data for an evidence-chain attempt.""" + + route: RouteDecision + applied: bool = False + fallback_used: bool = False + fallback_reason: Optional[str] = None + candidate_count: int = 0 + edge_count: int = 0 + iterations: int = 0 + converged: bool = False + selected_scores: List[ChainNodeScore] = field(default_factory=list) + paths: List[EvidencePath] = field(default_factory=list) + ordered_node_ids: List[str] = field(default_factory=list) + selected_node_ids: Set[str] = field(default_factory=set) + additional_nodes: List[Node] = field(default_factory=list, repr=False) + + @classmethod + def skipped(cls, route: RouteDecision) -> "EvidenceChainResult": + return cls(route=route, applied=False) + + @classmethod + def fallback(cls, route: RouteDecision, reason: str) -> "EvidenceChainResult": + return cls( + route=route, + applied=False, + fallback_used=True, + fallback_reason=reason, + ) + + def to_audit_dict(self) -> Dict[str, Any]: + """Serialize without text, denied identifiers, or unselected candidates.""" + return { + "scoring_version": SCORING_VERSION, + "route": self.route.to_dict(), + "applied": self.applied, + "fallback_used": self.fallback_used, + "fallback_reason": self.fallback_reason, + "candidate_count": self.candidate_count, + "edge_count": self.edge_count, + "iterations": self.iterations, + "converged": self.converged, + "selected_nodes": [score.to_dict() for score in self.selected_scores], + "paths": [path.to_dict() for path in self.paths], + "ordered_node_ids": self.ordered_node_ids, + } + + def restrict_to_authorized(self, authorized_node_ids: Set[str]) -> None: + """Scrub final audit/packing selections after the shared ACL choke point.""" + self.selected_scores = [ + score + for score in self.selected_scores + if score.node_id in authorized_node_ids + ] + self.selected_node_ids.intersection_update(authorized_node_ids) + self.ordered_node_ids = [ + node_id + for node_id in self.ordered_node_ids + if node_id in authorized_node_ids + ] + self.additional_nodes = [ + node + for node in self.additional_nodes + if node.node_id in authorized_node_ids + ] + self.paths = [ + path + for path in self.paths + if all(node_id in authorized_node_ids for node_id in path.node_ids) + ] + + +class MultiHopRouter: + """Conservative deterministic router for evidence-chain processing.""" + + _COMPARISON_RE = re.compile( + r"\b(compare|comparison|contrast|versus|vs\.?|difference|differ|better|worse)\b", + re.IGNORECASE, + ) + _RELATION_RE = re.compile( + r"\b(relationship|relate[sd]?|connection|affect(?:s|ed)?|impact(?:s|ed)?|" + r"because|why|lead(?:s|ing)? to|result(?:s|ed)? in)\b", + re.IGNORECASE, + ) + _TEMPORAL_RE = re.compile( + r"\b(amend(?:ed|ment)?|supersed(?:e|ed|es)|effective date|current|latest|" + r"before|after|subsequent|prior|timeline)\b", + re.IGNORECASE, + ) + _PROFESSIONAL_CHAIN_RE = re.compile( + r"\b(contraindication|interaction|governing law|statute|regulation|amendment|" + r"guideline|diagnosis|treatment|authority|precedent)\b", + re.IGNORECASE, + ) + _SYNTHESIS_RE = re.compile( + r"\b(based on|taking into account|together with|across (?:the )?(?:documents|sources)|" + r"how does .+ (?:compare|relate)|what .+ and (?:how|why|what))\b", + re.IGNORECASE, + ) + + @classmethod + def decide(cls, question: str, mode: str, threshold: float = 0.55) -> RouteDecision: + if mode == "off": + return RouteDecision(False, mode, 0.0, ("mode_off",)) + if mode == "on": + return RouteDecision(True, mode, 1.0, ("forced_on",)) + + normalized = " ".join((question or "").split()) + if not normalized: + return RouteDecision(False, mode, 0.0, ("empty_question",)) + + score = 0.0 + reasons: List[str] = [] + signals = ( + (cls._COMPARISON_RE, 0.35, "comparison"), + (cls._RELATION_RE, 0.25, "causal_or_relational"), + (cls._TEMPORAL_RE, 0.30, "temporal_or_versioned"), + (cls._PROFESSIONAL_CHAIN_RE, 0.20, "professional_chain_term"), + (cls._SYNTHESIS_RE, 0.35, "explicit_synthesis"), + ) + for pattern, weight, reason in signals: + if pattern.search(normalized): + score += weight + reasons.append(reason) + + # Multiple clauses are supporting evidence only; they cannot route alone. + clause_count = len(re.findall(r"\b(?:and|then|while|whereas|but)\b|[;?]", normalized, re.I)) + if clause_count >= 2 and reasons: + score += 0.15 + reasons.append("multiple_clauses") + + score = min(score, 1.0) + applied = score >= threshold + if not applied: + reasons.append("below_threshold") + return RouteDecision(applied, mode, score, tuple(reasons)) + + +class EvidenceChainEngine: + """Expands and ranks an ACL-authorized document-graph neighborhood.""" + + _EDGE_TYPE_WEIGHTS: Mapping[EdgeType, float] = { + EdgeType.adjacent_prev: 0.65, + EdgeType.adjacent_next: 0.65, + EdgeType.references: 0.90, + EdgeType.explained_by: 0.85, + } + _TOKEN_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.%/-]*") + _STOPWORDS = { + "about", "after", "also", "and", "are", "based", "before", "does", + "from", "have", "into", "that", "the", "their", "then", "this", "what", + "when", "where", "which", "while", "with", "would", "your", "how", "why", + } + + def __init__(self, db: Session, acl_enforcer: Any, config: EvidenceChainConfig): + self.db = db + self.acl_enforcer = acl_enforcer + self.config = config + + def build( + self, + expanded: ExpandedContext, + question: str, + seed_scores: Mapping[str, float], + doc_id: Optional[str] = None, + version: Optional[int] = None, + ) -> EvidenceChainResult: + route = MultiHopRouter.decide( + question, + mode=self.config.mode, + threshold=self.config.route_threshold, + ) + if not route.applied: + return EvidenceChainResult.skipped(route) + + initial_nodes = self._dedupe_nodes(expanded.all_nodes) + initial_nodes = self.acl_enforcer.filter_nodes( + initial_nodes, stage="evidence_chain_initial" + ) + initial_nodes = [ + node + for node in initial_nodes + if (doc_id is None or node.doc_id == doc_id) + and (version is None or node.version == version) + ] + if not initial_nodes: + return EvidenceChainResult.fallback(route, "no_authorized_candidates") + + nodes_by_id, edges = self._expand_authorized_neighborhood( + initial_nodes, doc_id=doc_id, version=version + ) + if not nodes_by_id: + return EvidenceChainResult.fallback(route, "no_authorized_candidates") + + seed_ids = {node_id for node_id in seed_scores if node_id in nodes_by_id} + if not seed_ids: + seed_ids = { + node.node_id for node in expanded.seed_nodes if node.node_id in nodes_by_id + } + if not seed_ids: + return EvidenceChainResult.fallback(route, "no_authorized_seeds") + + normalized_seed = self._normalize_seed_scores(seed_ids, seed_scores) + query_scores = { + node_id: self._query_relevance(question, node) + for node_id, node in nodes_by_id.items() + } + propagation, iterations, converged, adjacency = self._propagate( + node_ids=set(nodes_by_id), + edges=edges, + restart_scores=normalized_seed, + ) + + scored = [] + for node_id in nodes_by_id: + final = ( + self.config.propagation_weight * propagation.get(node_id, 0.0) + + self.config.query_weight * query_scores[node_id] + + self.config.seed_weight * normalized_seed.get(node_id, 0.0) + ) + scored.append( + ChainNodeScore( + node_id=node_id, + final_score=final, + propagation_score=propagation.get(node_id, 0.0), + query_relevance=query_scores[node_id], + seed_relevance=normalized_seed.get(node_id, 0.0), + is_seed=node_id in seed_ids, + ) + ) + scored.sort(key=lambda item: (-item.final_score, item.node_id)) + + selected = self._select_with_seed_preservation(scored, seed_ids) + selected_ids = {item.node_id for item in selected} + paths = self._assemble_paths( + selected, + seed_ids, + adjacency, + edges, + ) + ordered_ids = self._order_nodes(paths, selected) + initial_ids = {node.node_id for node in initial_nodes} + + return EvidenceChainResult( + route=route, + applied=True, + candidate_count=len(nodes_by_id), + edge_count=len(edges), + iterations=iterations, + converged=converged, + selected_scores=selected, + paths=paths, + ordered_node_ids=ordered_ids, + selected_node_ids=selected_ids, + additional_nodes=[ + nodes_by_id[node_id] + for node_id in ordered_ids + if node_id not in initial_ids + ], + ) + + def _expand_authorized_neighborhood( + self, + initial_nodes: Sequence[Node], + doc_id: Optional[str], + version: Optional[int], + ) -> Tuple[Dict[str, Node], List[Edge]]: + nodes_by_id = {node.node_id: node for node in initial_nodes[: self.config.max_nodes]} + frontier = set(nodes_by_id) + candidate_edges: List[Edge] = [] + + for hop in range(1, self.config.max_hops + 1): + if not frontier or len(nodes_by_id) >= self.config.max_nodes: + break + + edge_query = self.db.query(Edge).filter( + or_(Edge.from_node_id.in_(frontier), Edge.to_node_id.in_(frontier)) + ) + if doc_id is not None: + edge_query = edge_query.filter(Edge.doc_id == doc_id) + if version is not None: + edge_query = edge_query.filter(Edge.version == version) + hop_edges = ( + edge_query.order_by(Edge.confidence.desc(), Edge.id.asc()) + .limit(self.config.max_nodes * 8) + .all() + ) + hop_edges = [ + edge + for edge in hop_edges + if (edge.from_node_id in frontier or edge.to_node_id in frontier) + and (doc_id is None or edge.doc_id == doc_id) + and (version is None or edge.version == version) + ] + candidate_edges.extend(hop_edges) + + neighbor_ids: Set[str] = set() + for edge in hop_edges: + if edge.from_node_id in frontier: + neighbor_ids.add(edge.to_node_id) + if edge.to_node_id in frontier: + neighbor_ids.add(edge.from_node_id) + neighbor_ids.difference_update(nodes_by_id) + if not neighbor_ids: + frontier = set() + continue + + node_query = self.db.query(Node).filter(Node.node_id.in_(neighbor_ids)) + if doc_id is not None: + node_query = node_query.filter(Node.doc_id == doc_id) + if version is not None: + node_query = node_query.filter(Node.version == version) + neighbors = ( + node_query.order_by(Node.node_id.asc()) + .limit(self.config.max_nodes * 4) + .all() + ) + neighbors = [ + node + for node in neighbors + if node.node_id in neighbor_ids + and (doc_id is None or node.doc_id == doc_id) + and (version is None or node.version == version) + ] + neighbors = self.acl_enforcer.filter_nodes( + neighbors, stage=f"evidence_chain_hop_{hop}" + ) + remaining = self.config.max_nodes - len(nodes_by_id) + neighbors = neighbors[:remaining] + frontier = {node.node_id for node in neighbors} + nodes_by_id.update((node.node_id, node) for node in neighbors) + + authorized_ids = set(nodes_by_id) + unique_edges: Dict[Tuple[str, str, str], Edge] = {} + for edge in candidate_edges: + if edge.from_node_id not in authorized_ids or edge.to_node_id not in authorized_ids: + continue + edge_type = self._edge_type_value(edge.edge_type) + key = (edge.from_node_id, edge.to_node_id, edge_type) + unique_edges[key] = edge + return nodes_by_id, [unique_edges[key] for key in sorted(unique_edges)] + + def _propagate( + self, + node_ids: Set[str], + edges: Sequence[Edge], + restart_scores: Mapping[str, float], + ) -> Tuple[Dict[str, float], int, bool, Dict[str, Dict[str, float]]]: + adjacency: Dict[str, Dict[str, float]] = {node_id: {} for node_id in node_ids} + for edge in edges: + if edge.from_node_id == edge.to_node_id: + continue + base = self._EDGE_TYPE_WEIGHTS.get(edge.edge_type, 0.5) + confidence = self._clamp_confidence(edge.confidence) + weight = base * confidence + if weight <= 0.0: + continue + adjacency[edge.from_node_id][edge.to_node_id] = max( + adjacency[edge.from_node_id].get(edge.to_node_id, 0.0), weight + ) + adjacency[edge.to_node_id][edge.from_node_id] = max( + adjacency[edge.to_node_id].get(edge.from_node_id, 0.0), weight + ) + + restart = {node_id: max(0.0, restart_scores.get(node_id, 0.0)) for node_id in node_ids} + restart_total = sum(restart.values()) + if restart_total <= 0.0: + uniform = 1.0 / len(node_ids) + restart = {node_id: uniform for node_id in node_ids} + else: + restart = {node_id: value / restart_total for node_id, value in restart.items()} + + current = dict(restart) + converged = False + iterations = 0 + restart_probability = self.config.restart_probability + for iteration in range(1, self.config.propagation_steps + 1): + next_scores = { + node_id: restart_probability * restart[node_id] for node_id in node_ids + } + for source_id in sorted(node_ids): + neighbors = adjacency[source_id] + if not neighbors: + next_scores[source_id] += (1.0 - restart_probability) * current[source_id] + continue + degree = sum(neighbors.values()) + if degree <= 0.0: + next_scores[source_id] += (1.0 - restart_probability) * current[source_id] + continue + for target_id, weight in neighbors.items(): + next_scores[target_id] += ( + (1.0 - restart_probability) * current[source_id] * weight / degree + ) + total = sum(next_scores.values()) + if total > 0.0: + next_scores = {node_id: value / total for node_id, value in next_scores.items()} + delta = sum(abs(next_scores[node_id] - current[node_id]) for node_id in node_ids) + current = next_scores + iterations = iteration + if delta <= self.config.convergence_tolerance: + converged = True + break + return current, iterations, converged, adjacency + + def _select_with_seed_preservation( + self, + scored: Sequence[ChainNodeScore], + seed_ids: Set[str], + ) -> List[ChainNodeScore]: + by_id = {item.node_id: item for item in scored} + selected_ids = [item.node_id for item in scored[: self.config.max_selected_nodes]] + for seed_id in sorted( + seed_ids, + key=lambda node_id: (-by_id[node_id].final_score, node_id), + ): + if seed_id in selected_ids or seed_id not in by_id: + continue + if len(selected_ids) >= self.config.max_selected_nodes: + replace_index = next( + ( + index + for index in range(len(selected_ids) - 1, -1, -1) + if selected_ids[index] not in seed_ids + ), + None, + ) + if replace_index is not None: + selected_ids[replace_index] = seed_id + else: + selected_ids.append(seed_id) + selected = [by_id[node_id] for node_id in dict.fromkeys(selected_ids)] + selected.sort(key=lambda item: (-item.final_score, item.node_id)) + return selected + + def _assemble_paths( + self, + selected: Sequence[ChainNodeScore], + seed_ids: Set[str], + adjacency: Mapping[str, Mapping[str, float]], + edges: Sequence[Edge], + ) -> List[EvidencePath]: + scores = {item.node_id: item.final_score for item in selected} + edge_lookup: Dict[frozenset[str], Edge] = {} + for edge in edges: + edge_lookup[frozenset((edge.from_node_id, edge.to_node_id))] = edge + + candidate_paths: List[Tuple[float, Tuple[str, ...]]] = [] + for target in selected: + if target.node_id in seed_ids: + continue + node_path = self._shortest_path_to_seed( + target.node_id, + seed_ids, + adjacency, + scores, + allowed_node_ids=set(scores), + ) + if len(node_path) < 2: + continue + path_score = sum(scores.get(node_id, 0.0) for node_id in node_path) / len(node_path) + candidate_paths.append((path_score, tuple(node_path))) + + if not candidate_paths: + candidate_paths = [ + (scores.get(seed_id, 0.0), (seed_id,)) + for seed_id in sorted(seed_ids) + if seed_id in scores + ] + candidate_paths.sort(key=lambda item: (-item[0], item[1])) + + accepted: List[Tuple[float, Tuple[str, ...]]] = [] + for candidate in candidate_paths: + candidate_set = set(candidate[1]) + duplicate = False + for _, existing_nodes in accepted: + existing_set = set(existing_nodes) + union = candidate_set | existing_set + overlap = len(candidate_set & existing_set) / len(union) if union else 1.0 + if overlap >= self.config.path_overlap_threshold: + duplicate = True + break + if not duplicate: + accepted.append(candidate) + if len(accepted) >= self.config.max_chains: + break + + paths: List[EvidencePath] = [] + for index, (path_score, node_ids) in enumerate(accepted, start=1): + path_edges: List[ChainEdge] = [] + for left, right in zip(node_ids, node_ids[1:]): + edge = edge_lookup.get(frozenset((left, right))) + if edge is None: + continue + path_edges.append( + ChainEdge( + from_node_id=left, + to_node_id=right, + edge_type=self._edge_type_value(edge.edge_type), + weight=self._EDGE_TYPE_WEIGHTS.get(edge.edge_type, 0.5) + * self._clamp_confidence(edge.confidence), + ) + ) + paths.append( + EvidencePath( + path_id=f"P{index}", + node_ids=node_ids, + edges=tuple(path_edges), + relevance_score=path_score, + ) + ) + return paths + + def _shortest_path_to_seed( + self, + start: str, + seed_ids: Set[str], + adjacency: Mapping[str, Mapping[str, float]], + scores: Mapping[str, float], + allowed_node_ids: Set[str], + ) -> List[str]: + queue = deque([(start, [start])]) + visited = {start} + while queue: + current, path = queue.popleft() + if current in seed_ids: + return list(reversed(path)) + if len(path) - 1 >= self.config.max_hops: + continue + neighbors = sorted( + ( + node_id + for node_id in adjacency.get(current, {}) + if node_id in allowed_node_ids + ), + key=lambda node_id: (-scores.get(node_id, 0.0), node_id), + ) + for neighbor in neighbors: + if neighbor in visited: + continue + visited.add(neighbor) + queue.append((neighbor, path + [neighbor])) + return [] + + @staticmethod + def _order_nodes( + paths: Sequence[EvidencePath], + selected: Sequence[ChainNodeScore], + ) -> List[str]: + ordered: List[str] = [] + seen: Set[str] = set() + for path in paths: + for node_id in path.node_ids: + if node_id not in seen: + ordered.append(node_id) + seen.add(node_id) + for item in selected: + if item.node_id not in seen: + ordered.append(item.node_id) + seen.add(item.node_id) + return ordered + + @classmethod + def _query_relevance(cls, question: str, node: Node) -> float: + query_terms = cls._terms(question) + if not query_terms: + return 0.0 + meta = node.meta or {} + searchable = " ".join( + value + for value in ( + node.text_plain or node.text_md or "", + node.label or "", + str(meta.get("section_hint") or ""), + ) + if value + ) + node_terms = cls._terms(searchable) + if not node_terms: + return 0.0 + return min(1.0, len(query_terms & node_terms) / len(query_terms)) + + @classmethod + def _terms(cls, text: str) -> Set[str]: + return { + token.lower() + for token in cls._TOKEN_RE.findall(text or "") + if len(token) >= 2 and token.lower() not in cls._STOPWORDS + } + + @staticmethod + def _normalize_seed_scores( + seed_ids: Set[str], seed_scores: Mapping[str, float] + ) -> Dict[str, float]: + raw: Dict[str, float] = {} + for node_id in seed_ids: + try: + value = float(seed_scores.get(node_id, 0.0)) + except (TypeError, ValueError): + value = 0.0 + raw[node_id] = value if math.isfinite(value) else 0.0 + positive = {node_id: max(0.0, value) for node_id, value in raw.items()} + maximum = max(positive.values()) + if maximum <= 0.0: + return {node_id: 1.0 for node_id in raw} + # Every authorized seed retains a bounded entry prior even when its + # retrieval score is much lower than another seed's score. + return { + node_id: max(0.10, value / maximum) + for node_id, value in positive.items() + } + + @staticmethod + def _clamp_confidence(value: Any) -> float: + try: + confidence = float(value) + except (TypeError, ValueError): + return 0.5 + if not math.isfinite(confidence): + return 0.5 + return min(1.0, max(0.0, confidence)) + + @staticmethod + def _edge_type_value(edge_type: Any) -> str: + return edge_type.value if hasattr(edge_type, "value") else str(edge_type) + + @staticmethod + def _dedupe_nodes(nodes: Iterable[Node]) -> List[Node]: + deduped: Dict[str, Node] = {} + for node in nodes: + deduped.setdefault(node.node_id, node) + return list(deduped.values()) diff --git a/backend/app/qa/evidence_span.py b/backend/app/qa/evidence_span.py index f8c92e1..a39bc96 100644 --- a/backend/app/qa/evidence_span.py +++ b/backend/app/qa/evidence_span.py @@ -45,7 +45,34 @@ class BBoxLocator(BaseModel): page_size: Optional[PageSize] = None -Locator = Union[TextOffsetsLocator, BBoxLocator] +class NormalizedRect(BaseModel): + x0: float = Field(ge=0.0, le=1.0) + y0: float = Field(ge=0.0, le=1.0) + x1: float = Field(ge=0.0, le=1.0) + y1: float = Field(ge=0.0, le=1.0) + + @model_validator(mode="after") + def validate_extent(self) -> "NormalizedRect": + if self.x1 <= self.x0 or self.y1 <= self.y0: + raise ValueError("normalized rectangle must have positive width and height") + return self + + +class RectsLocator(BaseModel): + type: Literal["rects"] = "rects" + coordinate_system: Literal["normalized_top_left"] = "normalized_top_left" + rects: List[NormalizedRect] = Field(min_length=1) + page_size: Optional[PageSize] = None + page_rotation: int = Field(default=0) + + @model_validator(mode="after") + def validate_rotation(self) -> "RectsLocator": + if self.page_rotation not in (0, 90, 180, 270): + raise ValueError("page_rotation must be 0, 90, 180, or 270") + return self + + +Locator = Union[TextOffsetsLocator, BBoxLocator, RectsLocator] class EvidenceSpan(BaseModel): @@ -58,6 +85,32 @@ class EvidenceSpan(BaseModel): source_section: Optional[str] = None +class EvidenceRecord(BaseModel): + """Versioned, claim-level evidence returned to source viewers.""" + + schema_version: Literal["2.0"] = "2.0" + citation_id: str = Field(pattern=r"^C[1-9][0-9]*$") + claim_id: Optional[str] = None + doc_id: str = Field(min_length=1) + document_version: int = Field(ge=1) + node_id: str = Field(min_length=1) + page: int = Field(ge=1) + exact_quote: str = Field(min_length=1) + source_hash: str = Field(pattern=r"^sha256:") + status: Literal["verified", "approximate", "unavailable"] + verification_reason: str = Field(min_length=1) + locator: Optional[Locator] = None + confidence: float = Field(ge=0.0, le=1.0) + + @model_validator(mode="after") + def verified_requires_rectangles(self) -> "EvidenceRecord": + if self.status == "verified" and not isinstance(self.locator, RectsLocator): + raise ValueError("verified evidence requires normalized source rectangles") + if self.status == "unavailable" and self.locator is not None: + raise ValueError("unavailable evidence cannot include a locator") + return self + + def normalize_page_index(page_index: int, page_index_base: int = PAGE_INDEX_BASE) -> int: """Normalize to a 1-based page index. diff --git a/backend/app/qa/propagation/synthesizer.py b/backend/app/qa/propagation/synthesizer.py index e060267..26a520a 100644 --- a/backend/app/qa/propagation/synthesizer.py +++ b/backend/app/qa/propagation/synthesizer.py @@ -6,6 +6,7 @@ import json import logging +import re import time from typing import List, Optional, Dict, Any, Tuple @@ -36,7 +37,7 @@ def _get_synthesizer_prompt(question: str, sub_answers_text: str) -> str: JSON SCHEMA: {{ "answer": "your synthesized answer with preserved citations", - "citations": [{{"node_id": "...", "page_no": 1, "label": "..."}}], + "citations": [{{"citation_id": "C1", "node_id": "...", "page_no": 1, "label": "...", "exact_quote": "verbatim quote from a verified sub-answer"}}], "conflicts_summary": "brief summary of any conflicts, or empty string" }} @@ -68,8 +69,10 @@ def _format_sub_answers(sub_questions: List[SubQuestion], sub_answers: List[SubA citations_str = "" if sa.citations: - cites = [f"p.{c.get('page_no', '?')}" for c in sa.citations] - citations_str = f" (Citations: {', '.join(cites)})" + citations_str = ( + "\nGrounded citations: " + + json.dumps(sa.citations, ensure_ascii=False) + ) parts.append( f"Sub-question: {sq_text}\n" @@ -130,6 +133,51 @@ def _aggregate_conflicts(sub_answers: List[SubAnswer]) -> str: return "; ".join(conflict_notes) if conflict_notes else "" +def _prepare_grounded_citations( + answer: str, + citations: List[Dict[str, Any]], + sub_answers: List[SubAnswer], +) -> Tuple[str, List[Dict[str, Any]]]: + """Attach stable IDs and verbatim quotes from verified sub-answers.""" + authoritative = { + (cite.get("node_id"), cite.get("page_no")): cite + for cite in _merge_citations(sub_answers) + if cite.get("node_id") and cite.get("page_no") and cite.get("exact_quote") + } + candidates = citations or list(authoritative.values()) + grounded: List[Dict[str, Any]] = [] + seen: set[tuple[str, int]] = set() + + for cite in candidates: + key = (cite.get("node_id"), cite.get("page_no")) + source = authoritative.get(key) + if source is None or key in seen: + continue + seen.add(key) + citation_id = f"C{len(grounded) + 1}" + grounded_cite = { + "citation_id": citation_id, + "node_id": source["node_id"], + "page_no": source["page_no"], + "label": source.get("label"), + "exact_quote": source["exact_quote"], + } + grounded.append(grounded_cite) + + legacy_refs = [source["node_id"]] + if source.get("label"): + legacy_refs.append(source["label"]) + for ref in legacy_refs: + answer = re.sub( + rf"\[{re.escape(str(ref))}:\s*{source['page_no']}\]", + f"[{citation_id}]", + answer, + flags=re.IGNORECASE, + ) + + return answer, grounded + + class Synthesizer: """Synthesizes final answer from verified sub-answers.""" @@ -216,8 +264,11 @@ def synthesize( logger.warning(f"[Synthesizer] Failed to parse JSON, using raw text") # Fallback: use raw text as answer with merged citations return ( - response_text, - _merge_citations(sub_answers), + *_prepare_grounded_citations( + response_text, + _merge_citations(sub_answers), + sub_answers, + ), _aggregate_conflicts(sub_answers), latency_ms ) @@ -230,6 +281,11 @@ def synthesize( # If synthesizer didn't provide citations, merge from sub-answers if not citations: citations = _merge_citations(sub_answers) + final_answer, citations = _prepare_grounded_citations( + final_answer, + citations, + sub_answers, + ) # If synthesizer didn't note conflicts, aggregate from sub-answers if not conflicts_summary: @@ -249,9 +305,14 @@ def synthesize( fallback_answer = " ".join(parts) if parts else "Unable to synthesize answer due to error." - return ( + fallback_answer, fallback_citations = _prepare_grounded_citations( fallback_answer, _merge_citations(sub_answers), + sub_answers, + ) + return ( + fallback_answer, + fallback_citations, _aggregate_conflicts(sub_answers), latency_ms ) diff --git a/backend/app/qa/propagation/verifier.py b/backend/app/qa/propagation/verifier.py index a8dca06..5dad44f 100644 --- a/backend/app/qa/propagation/verifier.py +++ b/backend/app/qa/propagation/verifier.py @@ -7,6 +7,7 @@ import json import logging import time +import unicodedata from concurrent.futures import ThreadPoolExecutor, TimeoutError as FuturesTimeoutError from typing import List, Optional, Dict, Any @@ -36,6 +37,7 @@ def _get_verifier_prompt(sub_question: str, snippets_text: str) -> str: 2. You MUST cite at least 1 snippet if you provide an answer. 3. If evidence is insufficient, set insufficient_evidence=true. 4. Set confidence between 0.0 and 1.0. +5. Every citation must include exact_quote copied verbatim from that snippet. OUTPUT FORMAT: Return ONLY valid JSON. @@ -43,7 +45,7 @@ def _get_verifier_prompt(sub_question: str, snippets_text: str) -> str: JSON SCHEMA: {{ "answer": "your answer based on evidence", - "citations": [{{"node_id": "...", "page_no": 1, "label": "..."}}], + "citations": [{{"node_id": "...", "page_no": 1, "label": "...", "exact_quote": "verbatim supporting passage"}}], "confidence": 0.8, "insufficient_evidence": false, "conflict_notes": "" @@ -167,8 +169,14 @@ def verify_single( ) # Build SubAnswer - citations = parsed.get("citations", []) + citations = self._validate_citations( + parsed.get("citations", []), + evidence_packet.snippets, + ) conflict_notes = parsed.get("conflict_notes", "") + insufficient = bool(parsed.get("insufficient_evidence", False)) + if parsed.get("answer") and not citations: + insufficient = True # Add conflict notes to conflicts list if present conflicts = evidence_packet.conflicts.copy() @@ -181,10 +189,10 @@ def verify_single( citations=citations, confidence=parsed.get("confidence", 0.0), conflicts=conflicts, - insufficient_evidence=parsed.get("insufficient_evidence", False), + insufficient_evidence=insufficient, verifier_latency_ms=latency_ms ) - + except Exception as e: latency_ms = int((time.time() - start_time) * 1000) logger.error(f"[Verifier] Error for {sub_question.id}: {e}") @@ -197,6 +205,50 @@ def verify_single( insufficient_evidence=True, verifier_latency_ms=latency_ms ) + + @staticmethod + def _validate_citations( + citations: Any, + snippets: List[EvidenceSnippet], + ) -> List[Dict[str, Any]]: + """Keep only citations with a verbatim quote in the cited snippet.""" + if not isinstance(citations, list): + return [] + by_node = {snippet.node_id: snippet for snippet in snippets} + valid: List[Dict[str, Any]] = [] + seen: set[tuple[str, int, str]] = set() + + def normalize(value: str) -> str: + return " ".join( + unicodedata.normalize("NFKC", value or "").split() + ).casefold() + + for citation in citations: + if not isinstance(citation, dict): + continue + node_id = str(citation.get("node_id") or "").strip() + exact_quote = str(citation.get("exact_quote") or "").strip() + snippet = by_node.get(node_id) + if ( + snippet is None + or not exact_quote + or citation.get("page_no") != snippet.page_no + or normalize(exact_quote) not in normalize(snippet.text) + ): + continue + key = (node_id, snippet.page_no, exact_quote) + if key in seen: + continue + seen.add(key) + valid.append( + { + "node_id": node_id, + "page_no": snippet.page_no, + "label": citation.get("label") or snippet.label, + "exact_quote": exact_quote, + } + ) + return valid def verify_all( self, diff --git a/backend/app/qa/runner.py b/backend/app/qa/runner.py index 005ac16..8e0e29d 100644 --- a/backend/app/qa/runner.py +++ b/backend/app/qa/runner.py @@ -20,9 +20,10 @@ import time import uuid from dataclasses import dataclass, field -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, Optional, Set from sqlalchemy.orm import Session +from pydantic import ValidationError from app.config import get_settings from app.db.graph_models import Node, DocumentGraph @@ -41,12 +42,18 @@ from app.storage.minio_client import get_storage_client from app.qa import metadata_queries from app.qa import structured_targets -from .evidence_span import build_evidence_spans +from .evidence_span import EvidenceRecord, build_evidence_spans from .normalizer import normalize_query, NormalizedQuery from .section_booster import SectionBooster, SectionBoostResult from .constraint_parser import parse_constraints, ParsedConstraints from .metadata_booster import MetadataBooster, MetadataBoostResult from .conflict_detector import detect_conflicts, ConflictDetectionResult +from .evidence_chain import ( + EvidenceChainConfig, + EvidenceChainEngine, + EvidenceChainResult, + MultiHopRouter, +) logger = logging.getLogger(__name__) @@ -174,6 +181,14 @@ class QAResult: # Graph expansion results expanded_nodes: List[ExpandedNode] = field(default_factory=list) edge_traces: List[EdgeTrace] = field(default_factory=list) + + # Query-aware evidence-chain results. Structural relevance is not factual + # confidence and cannot set evidence verification status. + evidence_chain_enabled: bool = False + evidence_chain_mode: str = "off" + evidence_chain_applied: bool = False + evidence_chain_audit: Optional[Dict[str, Any]] = None + evidence_chain_time_ms: float = 0.0 # Context packing results packed_context: str = "" @@ -258,6 +273,13 @@ def to_dict(self) -> Dict[str, Any]: }, "expanded_nodes": [vars(n) for n in self.expanded_nodes], "edge_traces": [vars(e) for e in self.edge_traces], + "evidence_chain": { + "enabled": self.evidence_chain_enabled, + "mode": self.evidence_chain_mode, + "applied": self.evidence_chain_applied, + "time_ms": self.evidence_chain_time_ms, + "audit": self.evidence_chain_audit, + }, "packed_context": self.packed_context, "context_node_ids": self.context_node_ids, "total_context_tokens": self.total_context_tokens, @@ -270,6 +292,7 @@ def to_dict(self) -> Dict[str, Any]: "search_ms": self.search_time_ms, "boosting_ms": self.boosting_time_ms, "expansion_ms": self.expansion_time_ms, + "evidence_chain_ms": self.evidence_chain_time_ms, "packing_ms": self.packing_time_ms, "generation_ms": self.generation_time_ms, "total_ms": self.total_time_ms, @@ -354,6 +377,8 @@ def __init__( propagation_safety_config: Optional[Dict[str, Any]] = None, enable_llm_rewrite: Optional[bool] = None, entitlements: Optional["Entitlements"] = None, + enable_evidence_chains: Optional[bool] = None, + evidence_chain_mode: Optional[str] = None, ): """Initialize QA runner. @@ -370,6 +395,8 @@ def __init__( propagation_safety_config: Config dict for propagation_safety mode enable_llm_rewrite: Enable LLM query rewriting (uses config if None) entitlements: User entitlements for ACL enforcement (None = ACL disabled) + enable_evidence_chains: Override the environment feature flag. + evidence_chain_mode: off, auto, or on. Ignored when feature is disabled. """ from app.acl.enforcer import ACLEnforcer @@ -391,6 +418,30 @@ def __init__( self.rerank_force = rerank_force self.rerank_gate_context = rerank_gate_context self.propagation_safety_config = propagation_safety_config + + settings = get_settings() + self.evidence_chain_enabled = ( + settings.evidence_chain_enabled + if enable_evidence_chains is None + else enable_evidence_chains + ) + configured_mode = evidence_chain_mode or settings.evidence_chain_mode + if configured_mode not in {"off", "auto", "on"}: + raise ValueError("evidence_chain_mode must be off, auto, or on") + self.evidence_chain_mode = configured_mode if self.evidence_chain_enabled else "off" + chain_config = EvidenceChainConfig( + mode=self.evidence_chain_mode, + route_threshold=settings.evidence_chain_route_threshold, + max_hops=settings.evidence_chain_max_hops, + max_nodes=settings.evidence_chain_max_nodes, + max_selected_nodes=settings.evidence_chain_max_selected_nodes, + max_chains=settings.evidence_chain_max_chains, + propagation_steps=settings.evidence_chain_propagation_steps, + restart_probability=settings.evidence_chain_restart_probability, + ) + self.evidence_chain_engine = EvidenceChainEngine( + db, acl_enforcer=self.acl_enforcer, config=chain_config + ) # LLM Query Rewriter (optional, for multi-turn conversations) from app.qa.llm_rewriter import LLMQueryRewriter @@ -400,7 +451,7 @@ def __init__( logger.info( f"QA Runner initialized (normalization={enable_normalization}, " f"boosting={enable_boosting}, rerank={enable_rerank}, rerank_force={rerank_force}, " - f"llm_rewrite={self.enable_llm_rewrite})" + f"llm_rewrite={self.enable_llm_rewrite}, evidence_chain={self.evidence_chain_mode})" ) @classmethod @@ -468,6 +519,8 @@ def run( result = QAResult(question=question, doc_id=doc_id) request_id = uuid.uuid4().hex result.rerank_enabled = self.enable_rerank + result.evidence_chain_enabled = self.evidence_chain_enabled + result.evidence_chain_mode = self.evidence_chain_mode # Set prompt version for auditability from app.prompts import get_prompt_version @@ -1058,13 +1111,36 @@ def run( f"{len(expanded.explained_by_nodes)} explained_by " f"in {result.expansion_time_ms:.0f}ms" ) + + # 5b. Query-aware evidence-chain retrieval. This layer is + # fail-safe: any error returns to the exact baseline expansion and + # packing path. It can rank authorized evidence but cannot verify + # a citation or source location. + expanded, chain_result, result.evidence_chain_time_ms = ( + self._apply_evidence_chain( + expanded=expanded, + question=question, + seed_scores={seed.node_id: seed.score for seed in result.seed_nodes}, + doc_id=doc_id, + version=version, + log_prefix="[QA]", + ) + ) + + result.evidence_chain_applied = chain_result.applied + result.evidence_chain_audit = chain_result.to_audit_dict() # Build expanded nodes and edge traces result.expanded_nodes, result.edge_traces = self._build_expansion_audit(expanded) # Conflict detection (threshold-gated) on expanded nodes # Collect all expanded nodes with their metadata - expanded_node_dicts = self._collect_expanded_nodes_metadata(expanded) + conflict_node_ids = ( + chain_result.selected_node_ids if chain_result.applied else None + ) + expanded_node_dicts = self._collect_expanded_nodes_metadata( + expanded, allowed_node_ids=conflict_node_ids + ) conflict_result: ConflictDetectionResult = detect_conflicts( cited_nodes=expanded_node_dicts, constraints=constraints @@ -1080,7 +1156,14 @@ def run( # 6. Context packing logger.info("[QA] Packing context") start = time.time() - packed: PackedContext = self.packer.pack(expanded, query=question) + packed: PackedContext = self.packer.pack( + expanded, + query=question, + node_order=(chain_result.ordered_node_ids if chain_result.applied else None), + allowed_node_ids=( + chain_result.selected_node_ids if chain_result.applied else None + ), + ) result.packing_time_ms = (time.time() - start) * 1000 result.packed_context = packed.to_text(include_citations=True) @@ -1107,7 +1190,7 @@ def run( result.citations = self._hydrate_citations( citations=answer_result.citations, doc_id=doc_id, - version=1, # Default version, could be passed from document lookup + version=version or 1, context_node_ids=result.context_node_ids, answer_text=answer_result.answer, request_id=request_id, @@ -1159,8 +1242,66 @@ def _acl_filter_expanded(self, expanded: ExpandedContext) -> ExpandedContext: expanded.explained_by_nodes = self.acl_enforcer.filter_nodes( expanded.explained_by_nodes, stage="expansion_explained_by" ) + expanded.chain_nodes = self.acl_enforcer.filter_nodes( + expanded.chain_nodes, stage="evidence_chain_selected" + ) return expanded + def _apply_evidence_chain( + self, + expanded: ExpandedContext, + question: str, + seed_scores: Dict[str, float], + doc_id: Optional[str], + version: Optional[int], + log_prefix: str = "[QA]", + ) -> tuple[ExpandedContext, EvidenceChainResult, float]: + """Apply the optional chain layer with a fail-safe baseline fallback. + + The returned ``expanded`` object contains only ACL-filtered additional + nodes. Exceptions are reduced to a non-sensitive type-only reason and + never abort the answer pipeline. + """ + route = MultiHopRouter.decide( + question, + mode=self.evidence_chain_mode, + threshold=self.evidence_chain_engine.config.route_threshold, + ) + chain_result = EvidenceChainResult.skipped(route) + if not self.evidence_chain_enabled: + return expanded, chain_result, 0.0 + + started = time.time() + try: + chain_result = self.evidence_chain_engine.build( + expanded=expanded, + question=question, + seed_scores=seed_scores, + doc_id=doc_id, + version=version, + ) + if chain_result.applied: + expanded.chain_nodes = chain_result.additional_nodes + for node in expanded.chain_nodes: + expanded.node_sources[node.node_id] = "chain" + # Defense in depth: every engine hop is filtered, then the + # shared choke point filters the final node set again. + expanded = self._acl_filter_expanded(expanded) + chain_result.restrict_to_authorized( + {node.node_id for node in expanded.all_nodes} + ) + except Exception as chain_error: + logger.warning( + "%s Evidence-chain layer failed; using baseline context", + log_prefix, + exc_info=True, + ) + chain_result = EvidenceChainResult.fallback( + route, + f"engine_error:{type(chain_error).__name__}", + ) + return expanded, chain_result, (time.time() - started) * 1000 + def _hydrate_citations( self, citations: List['Citation'], @@ -1179,7 +1320,8 @@ def _hydrate_citations( citations: List of Citation objects from LLM doc_id: Document ID for raw_url generation (None if searching all docs) version: Document version - context_node_ids: Node IDs from the context (for fallback lookup by page) + context_node_ids: Node IDs from the packed context used to reject + citations that do not identify retrieved evidence answer_text: Final answer text (used to create immutable citation snapshots) request_id: Request correlation ID for snapshot records @@ -1194,16 +1336,6 @@ def _hydrate_citations( nodes = self.db.query(Node).filter(Node.node_id.in_(node_ids)).all() node_map = {n.node_id: n for n in nodes} - # Also fetch context nodes for fallback page-based lookup - # This handles cases where LLM outputs [seed:14] instead of actual node_id - context_nodes = [] - page_to_node: Dict[int, 'Node'] = {} - if context_node_ids: - context_nodes = self.db.query(Node).filter(Node.node_id.in_(context_node_ids)).all() - for n in context_nodes: - if n.page_no and n.page_no not in page_to_node: - page_to_node[n.page_no] = n - highlighting_enabled = get_settings().enable_cross_format_highlighting storage = get_storage_client() if highlighting_enabled else None graph_doc_cache: Dict[str, Optional[DocumentGraph]] = {} @@ -1248,21 +1380,12 @@ def infer_source_type(source_uri: Optional[str]) -> str: return "txt" hydrated_citations = [] - for c in citations: + for citation_index, c in enumerate(citations): node = node_map.get(c.node_id) - - # DATA INTEGRITY: - # Page-based fallback is heuristic only; node_id remains the authoritative citation identity. - # Fallback: if node_id didn't match (e.g., "seed"), look up by page_no - if not node and c.page_no and c.page_no in page_to_node: - node = page_to_node[c.page_no] # D1 (audit H-4): a citation is grounded only if its node_id is in the - # packed context, or its page maps to a context node. Otherwise the - # model cited evidence that was never retrieved — drop it. - if enforce_grounding and c.node_id not in context_id_set and ( - c.page_no is None or c.page_no not in page_to_node - ): + # packed context. Page number alone is not evidence identity. + if enforce_grounding and c.node_id not in context_id_set: dropped_ungrounded += 1 logger.warning( "[QA] Dropping ungrounded citation node_id=%r page=%s " @@ -1377,8 +1500,10 @@ def infer_source_type(source_uri: Optional[str]) -> str: "unresolved": 0.0, } evidence_confidence = confidence_by_status.get(resolve_status, 0.0) + citation_id = c.citation_id or f"C{citation_index + 1}" citation_dict = { + "citation_id": citation_id, "node_id": c.node_id, "doc_id": citation_doc_id, "version": citation_version, @@ -1432,7 +1557,8 @@ def infer_source_type(source_uri: Optional[str]) -> str: doc_id=citation_doc_id, page_index=citation_page_no, quote_text=( - citation_dict.get("text") + c.exact_quote + or citation_dict.get("text") or c.text_snippet or citation_dict.get("anchor_snippet") ), @@ -1468,11 +1594,15 @@ def infer_source_type(source_uri: Optional[str]) -> str: quote_text=str(span.get("quote_text") or ""), locator=span.get("locator"), source_map=source_map, + node_id=node.node_id if node and c.exact_quote else None, + document_version=citation_version, + source_hash=graph_doc.content_hash, allow_fuzzy=False, ) except Exception as e: verification = { "status": "NOT_FOUND", + "grade": "unavailable", "matched_locator": None, "confidence": 0.0, "reason": f"verification_error:{type(e).__name__}", @@ -1482,6 +1612,7 @@ def infer_source_type(source_uri: Optional[str]) -> str: evidence_verification.append( { "status": "NOT_FOUND", + "grade": "unavailable", "matched_locator": None, "confidence": 0.0, "reason": "missing_source_map", @@ -1494,6 +1625,18 @@ def infer_source_type(source_uri: Optional[str]) -> str: (v for v in evidence_verification if v.get("status") == "FOUND"), None, ) + evidence_grade = ( + str(found_verification.get("grade") or "approximate") + if found_verification + else "unavailable" + ) + evidence_status = ( + "verified" + if evidence_grade == "verified" + else ("approximate" if found_verification else "unavailable") + ) + citation_dict["evidence_status"] = evidence_status + citation_dict["verification_status"] = evidence_status if highlighting_enabled and not found_verification: citation_dict["resolve_status"] = "unresolved" citation_dict["resolve_reason"] = "Evidence not found on cited page" @@ -1516,6 +1659,72 @@ def infer_source_type(source_uri: Optional[str]) -> str: } citation_dict["selector_bundle"] = updated_bundle + exact_quote = ( + c.exact_quote + or ( + citation_dict["evidence_spans"][0].get("quote_text") + if citation_dict["evidence_spans"] + else "" + ) + or "" + ) + source_hash = "" + if graph_doc and graph_doc.content_hash: + source_hash = ( + graph_doc.content_hash + if graph_doc.content_hash.startswith("sha256:") + else f"sha256:{graph_doc.content_hash}" + ) + matched_locator = ( + found_verification.get("matched_locator") + if found_verification + else None + ) + evidence_record_payload = { + "schema_version": "2.0", + "citation_id": citation_id, + "claim_id": None, + "doc_id": citation_doc_id, + "document_version": citation_version, + "node_id": node.node_id if node else c.node_id, + "page": citation_page_no, + "exact_quote": exact_quote, + "source_hash": source_hash, + "status": evidence_status, + "verification_reason": ( + found_verification.get("reason") + if found_verification + else ( + evidence_verification[0].get("reason") + if evidence_verification + else "evidence_verification_not_run" + ) + ), + "locator": matched_locator if evidence_status != "unavailable" else None, + "confidence": ( + float(found_verification.get("confidence") or 0.0) + if found_verification + else 0.0 + ), + } + evidence_record: Optional[Dict[str, Any]] = None + try: + evidence_record = EvidenceRecord.model_validate( + evidence_record_payload + ).model_dump() + except ValidationError as exc: + # Never expose a partial V2 record. Legacy citation fields may + # still be returned for navigation, but the viewer must not + # mistake an invalid evidence contract for verified evidence. + logger.warning( + "[QA] Omitting invalid evidence record node_id=%r: %s", + c.node_id, + exc.errors(include_url=False), + ) + citation_dict["evidence_records"] = ( + [evidence_record] if evidence_record is not None else [] + ) + snapshot_id: Optional[str] = None if highlighting_enabled and graph_doc and selector_bundle: try: @@ -1530,6 +1739,7 @@ def infer_source_type(source_uri: Optional[str]) -> str: node_id=node.node_id if node else c.node_id, selector_bundle=selector_bundle, exact_text=resolve_result.get("exact_text"), + evidence_record=evidence_record, answer_hash=answer_hash, content_hash=graph_doc.content_hash, ) @@ -2208,12 +2418,29 @@ def get_node_type_str(node) -> str: edge_type="explained_by" )) break + + # Query-aware nodes already carry their canonical paths in the + # evidence-chain audit. Do not invent edge traces here. + existing_ids = {item.node_id for item in expanded_nodes} + for node in expanded.chain_nodes: + if node.node_id in existing_ids: + continue + expanded_nodes.append(ExpandedNode( + node_id=node.node_id, + node_type=get_node_type_str(node), + page_no=node.page_no, + label=node.label, + text_preview=(node.text_plain or "")[:self.TEXT_PREVIEW_LEN], + expansion_type="chain", + )) + existing_ids.add(node.node_id) return expanded_nodes, edge_traces def _collect_expanded_nodes_metadata( self, - expanded: ExpandedContext + expanded: ExpandedContext, + allowed_node_ids: Optional[Set[str]] = None, ) -> List[Dict[str, Any]]: """Collect metadata from all expanded nodes for conflict detection. @@ -2229,8 +2456,11 @@ def _collect_expanded_nodes_metadata( list(expanded.seed_nodes) + list(expanded.adjacent_nodes) + list(expanded.referenced_nodes) + - list(expanded.explained_by_nodes) + list(expanded.explained_by_nodes) + + list(expanded.chain_nodes) ) + if allowed_node_ids is not None: + all_nodes = [node for node in all_nodes if node.node_id in allowed_node_ids] if not all_nodes: return [] @@ -2297,6 +2527,8 @@ def _run_propagation_safety( result = QAResult(question=question, doc_id=doc_id) request_id = uuid.uuid4().hex result.propagation_safety_mode = True + result.evidence_chain_enabled = self.evidence_chain_enabled + result.evidence_chain_mode = self.evidence_chain_mode # Set prompt version for auditability from app.prompts import get_prompt_version @@ -2356,6 +2588,27 @@ def _run_propagation_safety( logger.info(f"[PropSafety] {sq.id}: retrieved {len(packet.snippets)} snippets") audit.sub_packets = evidence_packets + chain_audits = [ + { + "subq_id": packet.subq_id, + "evidence_chain": packet.retrieval_audit.get("evidence_chain"), + } + for packet in evidence_packets + if packet.retrieval_audit.get("evidence_chain") is not None + ] + result.evidence_chain_applied = any( + bool(item["evidence_chain"].get("applied")) + for item in chain_audits + ) + result.evidence_chain_time_ms = sum( + float(packet.retrieval_audit.get("evidence_chain_time_ms", 0.0)) + for packet in evidence_packets + ) + result.evidence_chain_audit = { + "mode": self.evidence_chain_mode, + "applied": result.evidence_chain_applied, + "sub_questions": chain_audits, + } # ===== PHASE 3: Verify ===== logger.info("[PropSafety] Verifying sub-answers...") @@ -2383,16 +2636,23 @@ def _run_propagation_safety( hydrated_input.append( Citation( node_id=str(cite.get("node_id", "")), + citation_id=cite.get("citation_id"), page_no=cite.get("page_no"), label=cite.get("label"), + exact_quote=cite.get("exact_quote"), evidence_spans=list(cite.get("evidence_spans") or []), ) ) + # Ground final synthesis citations in the union of the authorized + # snippets that actually reached the propagation verifier. This is + # the propagation-mode equivalent of standard context grounding. + result.context_node_ids = self._collect_packet_node_ids(evidence_packets) result.citations = self._hydrate_citations( citations=hydrated_input, doc_id=doc_id, version=version or 1, + context_node_ids=result.context_node_ids, answer_text=final_answer, request_id=request_id, ) @@ -2442,6 +2702,19 @@ def _run_propagation_safety( result.total_time_ms = (time.time() - total_start) * 1000 return result + + @staticmethod + def _collect_packet_node_ids(evidence_packets: List[Any]) -> List[str]: + """Return stable, deduplicated node IDs from verified retrieval packets.""" + ordered: List[str] = [] + seen: set[str] = set() + for packet in evidence_packets: + for snippet in getattr(packet, "snippets", ()): + node_id = str(getattr(snippet, "node_id", "") or "") + if node_id and node_id not in seen: + seen.add(node_id) + ordered.append(node_id) + return ordered def _retrieve_for_subquestion( self, @@ -2556,18 +2829,46 @@ def _retrieve_for_subquestion( # through the shared choke point before they reach packing/snippets. expanded = self._acl_filter_expanded(expanded) + expanded, sub_chain_result, sub_chain_time_ms = self._apply_evidence_chain( + expanded=expanded, + question=sub_question.text, + seed_scores={ + item["node_id"]: item.get("score", 0.0) + for item in top_seeds + }, + doc_id=doc_id, + version=version, + log_prefix=f"[PropSafety:{sub_question.id}]", + ) + # 10. Conflict detection - expanded_nodes_meta = self._collect_expanded_nodes_metadata(expanded) + expanded_nodes_meta = self._collect_expanded_nodes_metadata( + expanded, + allowed_node_ids=( + sub_chain_result.selected_node_ids + if sub_chain_result.applied + else None + ), + ) conflict_result = detect_conflicts(expanded_nodes_meta, constraints) packet.conflicts = conflict_result.conflicts # 11. Build snippets from expanded context - all_expanded_nodes = ( - list(expanded.seed_nodes) + - list(expanded.adjacent_nodes)[:2] + # Limit adjacent - list(expanded.referenced_nodes)[:2] + - list(expanded.explained_by_nodes)[:1] - ) + if sub_chain_result.applied: + nodes_by_id = {node.node_id: node for node in expanded.all_nodes} + all_expanded_nodes = [ + nodes_by_id[node_id] + for node_id in sub_chain_result.ordered_node_ids + if node_id in sub_chain_result.selected_node_ids + and node_id in nodes_by_id + ] + else: + all_expanded_nodes = ( + list(expanded.seed_nodes) + + list(expanded.adjacent_nodes)[:2] + # Limit adjacent + list(expanded.referenced_nodes)[:2] + + list(expanded.explained_by_nodes)[:1] + ) # Get node texts from DB node_ids = [n.node_id for n in all_expanded_nodes] @@ -2602,7 +2903,9 @@ def _retrieve_for_subquestion( "filter_expr": filter_expr, "detected_intent": normalized.detected_intent, "injected_count": len(injected_seeds), - "detected_targets": detected_targets_list + "detected_targets": detected_targets_list, + "evidence_chain_time_ms": sub_chain_time_ms, + "evidence_chain": sub_chain_result.to_audit_dict(), } except Exception as e: @@ -2642,6 +2945,11 @@ def _run_standard_fallback( result.packed_context = standard_result.packed_context result.context_node_ids = standard_result.context_node_ids result.total_context_tokens = standard_result.total_context_tokens + result.evidence_chain_enabled = standard_result.evidence_chain_enabled + result.evidence_chain_mode = standard_result.evidence_chain_mode + result.evidence_chain_applied = standard_result.evidence_chain_applied + result.evidence_chain_audit = standard_result.evidence_chain_audit + result.evidence_chain_time_ms = standard_result.evidence_chain_time_ms result.conflicts = standard_result.conflicts result.has_conflicts = standard_result.has_conflicts result.success = standard_result.success diff --git a/backend/app/routes/qa.py b/backend/app/routes/qa.py index 42b4d36..0a57541 100644 --- a/backend/app/routes/qa.py +++ b/backend/app/routes/qa.py @@ -48,6 +48,13 @@ class AskRequest(BaseModel): default=None, description="Configuration for propagation_safety mode (optional)" ) + evidence_chain_mode: Optional[Literal["off", "auto", "on"]] = Field( + default=None, + description=( + "Query-aware evidence-chain routing. The server feature flag remains authoritative; " + "clients may disable it or choose the configured routing behavior." + ), + ) class AskResponse(BaseModel): @@ -92,6 +99,9 @@ class AskResponse(BaseModel): # Propagation Safety Mode (TRACK-inspired) propagation_safety_mode: bool = False propagation_safety_audit: Optional[dict] = None + + # Query-aware evidence-chain audit (relevance, never factual confidence) + evidence_chain: Optional[dict] = None # LLM Query Rewriting (for multi-turn conversations) original_question: Optional[str] = None # Original query before rewriting @@ -147,6 +157,7 @@ def ask_question( max_context_tokens=runtime.max_context_tokens, enable_rerank=runtime.enable_reranking, enable_llm_rewrite=runtime.enable_llm_query_rewrite, + evidence_chain_mode=request.evidence_chain_mode, ) # Convert chat history to list of dicts for the runner @@ -190,6 +201,7 @@ def ask_question( # Propagation Safety Mode propagation_safety_mode=response_data.get("propagation_safety_mode", False), propagation_safety_audit=response_data.get("propagation_safety_audit"), + evidence_chain=response_data.get("evidence_chain"), # LLM Query Rewriting original_question=response_data.get("original_question"), llm_rewrite=response_data.get("llm_rewrite"), diff --git a/backend/app/services/highlighting.py b/backend/app/services/highlighting.py index 7ebd913..b6a6171 100644 --- a/backend/app/services/highlighting.py +++ b/backend/app/services/highlighting.py @@ -164,6 +164,11 @@ def build_selector_artifacts_for_nodes( "page_no": node.page_no, "node_type": node.node_type.value if hasattr(node.node_type, "value") else str(node.node_type), "label": node.label, + "page_size": (node.meta or {}).get("page_size"), + "page_rotation": (node.meta or {}).get("page_rotation", 0), + "coordinate_system": (node.meta or {}).get("coordinate_system"), + "source_span_resolution": (node.meta or {}).get("source_span_resolution"), + "source_spans": (node.meta or {}).get("source_spans") or [], } ) selector_bundles.append(bundle) @@ -207,7 +212,14 @@ def build_selector_artifacts_for_nodes( "schema_version": "1.0", "doc_id": doc.doc_id, "version": doc.version, + "content_hash": ( + doc.content_hash + if (doc.content_hash or "").startswith("sha256:") + else f"sha256:{doc.content_hash}" + ), "normalization": NORMALIZATION_ID, + "evidence_schema_version": "2.0", + "coordinate_system": "normalized_top_left", "generated_at": now_iso, "canonical_text": canonical_text, "nodes": source_entries, @@ -386,10 +398,26 @@ def build_source_manifest( selectors_available = storage.selectors_exist(doc.doc_id, version) selector_total = 0 + evidence_v2_eligible = 0 + evidence_v2_total = 0 + evidence_v2_exact = 0 if source_map_available: try: source_map = storage.get_source_map(doc.doc_id, version) - selector_total = len(source_map.get("nodes", [])) + source_nodes = source_map.get("nodes", []) + selector_total = len(source_nodes) + eligible_nodes = [ + node for node in source_nodes if node.get("node_type") == "chunk" + ] + evidence_v2_eligible = len(eligible_nodes) + evidence_v2_total = sum( + 1 for node in eligible_nodes if node.get("source_spans") + ) + evidence_v2_exact = sum( + 1 + for node in eligible_nodes + if node.get("source_span_resolution") == "exact_source_words" + ) except Exception: selector_total = 0 @@ -404,10 +432,130 @@ def build_source_manifest( "selector_coverage": { "nodes_with_selectors": selector_total, }, + "evidence_v2_coverage": { + "eligible_text_nodes": evidence_v2_eligible, + "nodes_with_source_spans": evidence_v2_total, + "nodes_with_exact_source_spans": evidence_v2_exact, + }, + "evidence_v2_reingest_recommended": ( + evidence_v2_eligible > 0 and evidence_v2_exact < evidence_v2_eligible + ), "backfill_needed": not (canonical_available and source_map_available and selectors_available), } +def backfill_pdf_source_provenance( + db: Session, + doc: DocumentGraph, + raw_pdf: bytes, + *, + storage: Optional[StorageClient] = None, +) -> dict: + """Upgrade legacy native-PDF nodes with V2 word provenance. + + This does not change node IDs, document identity, embeddings, or graph + edges. It rereads the immutable original PDF, resolves each existing chunk + to one unique source-word sequence, then atomically replaces the highlight + artifacts for the existing document version. + """ + from app.graph.chunker import PageBoundedChunker + from app.graph.ids import compute_content_hash + from app.graph.page_extractor import PageExtractor + + actual_hash = compute_content_hash(raw_pdf) + if not _hash_matches(doc.content_hash, actual_hash): + raise ValueError("raw PDF content hash does not match the graph document") + + extraction = PageExtractor(skip_ocr=True).extract_pages( + raw_pdf, + doc_id=doc.doc_id, + ) + pages = {page.page_no: page for page in extraction.pages} + nodes = ( + db.query(Node) + .filter(Node.doc_id == doc.doc_id, Node.version == doc.version) + .all() + ) + + exact_count = 0 + approximate_count = 0 + unavailable_count = 0 + for node in nodes: + if node.page_no is None or not node.text_plain: + unavailable_count += 1 + continue + page_data = pages.get(node.page_no) + if page_data is None: + unavailable_count += 1 + continue + + source_spans, status = PageBoundedChunker._compute_chunk_source_spans( + node.text_plain, + page_data, + ) + meta = dict(node.meta or {}) + meta.update( + { + "source_spans": source_spans, + "source_span_resolution": status, + "evidence_schema_version": "2.0", + "coordinate_system": "normalized_top_left", + "page_size": { + "width": page_data.width, + "height": page_data.height, + }, + "page_rotation": int(page_data.rotation or 0), + } + ) + node.meta = meta + + point_boxes = [ + span.get("bbox") + for span in source_spans + if isinstance(span.get("bbox"), dict) + ] + if point_boxes: + node.bbox = { + "x0": min(box["x0"] for box in point_boxes), + "y0": min(box["y0"] for box in point_boxes), + "x1": max(box["x1"] for box in point_boxes), + "y1": max(box["y1"] for box in point_boxes), + } + + if status == "exact_source_words": + exact_count += 1 + elif source_spans: + approximate_count += 1 + else: + unavailable_count += 1 + + artifacts = build_selector_artifacts_for_nodes( + doc=doc, + nodes=nodes, + source_type="pdf", + mime_type="application/pdf", + ) + hydrate_nodes_with_selectors(nodes, artifacts["selectors"]) + persist_highlight_artifacts( + storage=storage or get_storage_client(), + doc_id=doc.doc_id, + version=doc.version, + canonical_html=artifacts["canonical_html"], + source_map=artifacts["source_map"], + selectors=artifacts["selectors"], + ) + db.flush() + + return { + "doc_id": doc.doc_id, + "version": doc.version, + "total_nodes": len(nodes), + "exact_nodes": exact_count, + "approximate_nodes": approximate_count, + "unavailable_nodes": unavailable_count, + } + + def resolve_citation_selector( storage: StorageClient, doc: DocumentGraph, @@ -522,6 +670,102 @@ def _match_with_tight_fuzzy(page_text: str, quote_text: str, threshold: float) - return best +def _resolve_quote_to_source_rects( + quote_text: str, + source_spans: list[dict], +) -> Tuple[Optional[list[dict]], str]: + """Resolve one exact quote to normalized word rectangles. + + The quote must occur exactly once inside the cited node's ordered source + words. Returned rectangles are merged per source line while retaining + disjoint lines/columns. + """ + ordered = sorted(source_spans, key=lambda item: int(item.get("order", 0) or 0)) + parts: list[str] = [] + ranges: list[Tuple[int, int, dict]] = [] + cursor = 0 + for span in ordered: + word = normalize_text(str(span.get("text") or "")).casefold() + if not word: + continue + if parts: + parts.append(" ") + cursor += 1 + start = cursor + parts.append(word) + cursor += len(word) + ranges.append((start, cursor, span)) + + stream = "".join(parts) + needle = normalize_text(quote_text).casefold() + if not stream or not needle: + return None, "empty_source_words_or_quote" + + occurrences: list[int] = [] + search_from = 0 + while True: + idx = stream.find(needle, search_from) + if idx < 0: + break + occurrences.append(idx) + search_from = idx + max(1, len(needle)) + if len(occurrences) > 1: + return None, "ambiguous_exact_quote_in_cited_node" + if not occurrences: + return None, "exact_quote_not_in_cited_source_words" + + match_start = occurrences[0] + match_end = match_start + len(needle) + matched_words = [ + span + for start, end, span in ranges + if start < match_end and end > match_start + ] + if not matched_words: + return None, "exact_quote_has_no_source_rectangles" + + for span in matched_words: + if ( + span.get("verifiable") is not True + or span.get("coordinate_system") != "pdf_points_top_left" + or not isinstance(span.get("normalized_bbox"), dict) + ): + return None, "source_coordinates_not_verifiable" + + # Merge adjacent words on the same source line. Grouping by the extractor's + # block/line identifiers prevents a highlight from spanning columns. + line_groups: list[list[dict]] = [] + current: list[dict] = [] + current_key: Optional[tuple] = None + for span in matched_words: + key = (span.get("block_no"), span.get("line_no")) + if current and key != current_key: + line_groups.append(current) + current = [] + current.append(span) + current_key = key + if current: + line_groups.append(current) + + rects: list[dict] = [] + for group in line_groups: + boxes = [span["normalized_bbox"] for span in group] + rect = { + "x0": min(float(box["x0"]) for box in boxes), + "y0": min(float(box["y0"]) for box in boxes), + "x1": max(float(box["x1"]) for box in boxes), + "y1": max(float(box["y1"]) for box in boxes), + } + if not ( + 0.0 <= rect["x0"] < rect["x1"] <= 1.0 + and 0.0 <= rect["y0"] < rect["y1"] <= 1.0 + ): + return None, "normalized_rectangle_out_of_bounds" + rects.append(rect) + + return rects, "exact_unique_quote_with_source_rectangles" + + def verify_evidence_span( *, doc_id: str, @@ -529,6 +773,9 @@ def verify_evidence_span( quote_text: str, locator: Optional[dict], source_map: dict, + node_id: Optional[str] = None, + document_version: Optional[int] = None, + source_hash: Optional[str] = None, allow_fuzzy: bool = False, fuzzy_threshold: float = 0.97, ) -> dict: @@ -542,10 +789,44 @@ def verify_evidence_span( "reason": str, } """ + source_doc_id = source_map.get("doc_id") + if source_doc_id and source_doc_id != doc_id: + return { + "status": "NOT_FOUND", + "grade": "unavailable", + "matched_locator": None, + "confidence": 0.0, + "reason": "source_map_document_mismatch", + } + if ( + document_version is not None + and source_map.get("version") is not None + and int(source_map["version"]) != int(document_version) + ): + return { + "status": "NOT_FOUND", + "grade": "unavailable", + "matched_locator": None, + "confidence": 0.0, + "reason": "source_map_version_mismatch", + } + if source_hash and source_map.get("content_hash") and not _hash_matches( + source_hash, + source_map.get("content_hash"), + ): + return { + "status": "NOT_FOUND", + "grade": "unavailable", + "matched_locator": None, + "confidence": 0.0, + "reason": "source_map_content_hash_mismatch", + } + canonical_text = source_map.get("canonical_text") or "" if not canonical_text: return { "status": "NOT_FOUND", + "grade": "unavailable", "matched_locator": None, "confidence": 0.0, "reason": "missing_canonical_text", @@ -555,6 +836,7 @@ def verify_evidence_span( if not quote: return { "status": "NOT_FOUND", + "grade": "unavailable", "matched_locator": None, "confidence": 0.0, "reason": "empty_quote_text", @@ -564,6 +846,7 @@ def verify_evidence_span( if not page_range: return { "status": "NOT_FOUND", + "grade": "unavailable", "matched_locator": None, "confidence": 0.0, "reason": "page_not_indexed", @@ -571,9 +854,16 @@ def verify_evidence_span( page_start, page_end = page_range page_text = canonical_text[page_start:page_end] - def found(matched_locator: dict, confidence: float, reason: str) -> dict: + def found( + matched_locator: dict, + confidence: float, + reason: str, + *, + grade: str = "approximate", + ) -> dict: return { "status": "FOUND", + "grade": grade, "matched_locator": matched_locator, "confidence": confidence, "reason": reason, @@ -584,6 +874,7 @@ def found(matched_locator: dict, confidence: float, reason: str) -> dict: def not_found(reason: str) -> dict: return { "status": "NOT_FOUND", + "grade": "unavailable", "matched_locator": None, "confidence": 0.0, "reason": reason, @@ -591,7 +882,43 @@ def not_found(reason: str) -> dict: "page_index": page_index, } - # 1) Prefer exact locator validation when text offsets are provided. + # 1) V2 legal-grade path: independently resolve the model's verbatim quote + # to the cited node's stored source words. + if node_id: + candidates = [ + node + for node in (source_map.get("nodes") or []) + if node.get("node_id") == node_id and node.get("page_no") == page_index + ] + if len(candidates) > 1: + return not_found("ambiguous_cited_node_in_source_map") + if len(candidates) == 1 and candidates[0].get("source_spans"): + source_node = candidates[0] + rects, rect_reason = _resolve_quote_to_source_rects( + quote, + source_node.get("source_spans") or [], + ) + if rects: + locator_payload: dict[str, Any] = { + "type": "rects", + "coordinate_system": "normalized_top_left", + "rects": rects, + "page_rotation": int(source_node.get("page_rotation") or 0), + } + if source_node.get("page_size"): + locator_payload["page_size"] = source_node["page_size"] + return found( + locator_payload, + 1.0, + rect_reason, + grade="verified", + ) + # Source-word provenance exists, therefore failure to locate the + # exact quote is authoritative. Never downgrade it to a guessed box. + return not_found(rect_reason) + + # 2) Legacy/canonical path. This verifies text in the reconstructed source + # but cannot prove a renderable location in the original PDF. if locator and locator.get("type") == "text_offsets": start = locator.get("start") end = locator.get("end") @@ -611,7 +938,7 @@ def not_found(reason: str) -> dict: else: return not_found("locator_outside_cited_page") - # 2) Exact quote search on cited page. + # 3) Exact quote search on cited page. quote_idx = page_text.find(quote) if quote_idx >= 0: start = page_start + quote_idx @@ -622,7 +949,7 @@ def not_found(reason: str) -> dict: "exact_quote_match_on_page", ) - # 3) Whitespace-normalized exact check, still constrained to cited page. + # 4) Whitespace-normalized exact check, still constrained to cited page. ws_pattern = r"\s+".join(re.escape(part) for part in quote.split()) if ws_pattern: ws_match = re.search(ws_pattern, page_text) @@ -633,7 +960,7 @@ def not_found(reason: str) -> dict: "exact_quote_match_on_page_whitespace_normalized", ) - # 4) Optional high-threshold fuzzy fallback. + # 5) Optional high-threshold fuzzy fallback. if allow_fuzzy: fuzzy = _match_with_tight_fuzzy(page_text, quote, threshold=fuzzy_threshold) if fuzzy: diff --git a/backend/docs/EVIDENCE_HIGHLIGHTING_V2.md b/backend/docs/EVIDENCE_HIGHLIGHTING_V2.md new file mode 100644 index 0000000..abcf5c0 --- /dev/null +++ b/backend/docs/EVIDENCE_HIGHLIGHTING_V2.md @@ -0,0 +1,82 @@ +# Evidence Highlighting V2 + +Evidence Highlighting V2 provides fail-closed, claim-level highlighting for +native PDFs. A green **Verified Evidence** label means all of the following were +checked by the server: + +1. The citation references a node that was included in the answer context. +2. The model supplied a verbatim supporting quote and a stable citation ID. +3. The quote occurs exactly once within that node's ordered source words. +4. The document ID, version, page, and content hash match. +5. Every returned rectangle is within the visible page bounds. + +The browser renders normalized top-left rectangles supplied by the server. It +does not re-search or reinterpret the PDF text layer. + +Before evidence hydration, the answer client also validates every structured +citation against the exact packed context. A citation is accepted only when its +node ID and page are an allowed pair and its normalized verbatim quote occurs +in that node. Invalid model output receives a bounded correction attempt; if it +still fails, answer generation fails closed instead of manufacturing a source. +A pure abstention must carry no citations. + +## Evidence states + +- `verified`: exact quote and original-PDF rectangles were independently + resolved. +- `approximate`: text was found in a canonical/reconstructed source, but exact + original-document coordinates are unavailable. +- `unavailable`: evidence could not be safely resolved. No guessed highlight is + returned. + +Only `verified` records are presented as verified in the PDF viewer. + +## Compatibility and existing documents + +Documents ingested before V2 do not contain ordered source words. Their existing +selectors remain usable for approximate canonical navigation, but cannot be +promoted to verified evidence. + +`GET /v1/documents/{doc_id}/source-manifest` reports: + +- `evidence_v2_coverage.nodes_with_source_spans` +- `evidence_v2_coverage.nodes_with_exact_source_spans` +- `evidence_v2_coverage.eligible_text_nodes` +- `evidence_v2_reingest_recommended` + +For a stored native PDF, run: + +```bash +cd backend +venv/bin/python -m scripts.backfill_evidence_v2 +``` + +The command verifies the stored file hash, preserves graph/node identity and +embeddings, and replaces only node provenance and highlight artifacts. Scanned +PDFs still require an OCR provider that returns word or line coordinates; +text-only OCR remains approximate/unavailable by design. + +Pass `--version N` to upgrade a specific document version. Without it, the +latest version is selected. + +## Deployment + +1. Apply Alembic revision `014`. +2. Deploy backend and frontend together because the API adds `rects` locators. +3. Validate a newly ingested native PDF before re-ingesting production + documents. +4. Monitor verification reasons, especially: + `exact_quote_not_in_cited_source_words`, + `ambiguous_exact_quote_in_cited_node`, and + `source_map_content_hash_mismatch`. +5. Keep `ENABLE_CROSS_FORMAT_HIGHLIGHTING=false` as the fail-safe rollback for + source verification and highlighting. + +## Acceptance criteria + +- No wrong document, page, or evidence rectangle is labeled verified. +- Ambiguous quotes and stale source hashes fail closed. +- Rectangles remain aligned at all supported zoom levels and page rotations. +- ACL filtering removes a citation and its evidence record together. +- Existing documents are clearly labeled approximate or unavailable until + re-ingested. diff --git a/backend/docs/HYCE_EVIDENCE_CHAINS_ADOPTION.md b/backend/docs/HYCE_EVIDENCE_CHAINS_ADOPTION.md new file mode 100644 index 0000000..b92dc6d --- /dev/null +++ b/backend/docs/HYCE_EVIDENCE_CHAINS_ADOPTION.md @@ -0,0 +1,358 @@ +# Query-Aware Evidence Chains Adoption Plan + +Status: Phase 1 implemented and fully repository-verified behind a default-off feature flag; the locked synthetic and configured-model A/B gates pass, while a representative production pilot and domain review remain required for default-on rollout + +Owner: QA/Retrieval + +Safety boundary: evidence-chain scores rank evidence; they never verify a claim or a source location. + +## 1. Objective + +Adopt the useful parts of HyCE-RAG in NPR-RAG so multi-hop questions receive a coherent, auditable chain of source evidence while retaining NPR's exact document provenance, ACL enforcement, and fail-closed highlighting. + +The adopted design must: + +1. Improve multi-hop retrieval and context assembly without regressing single-hop answers. +2. Preserve an immutable path from every answer citation to the original node, document version, source hash, page, exact quote, and verified locator. +3. Never equate graph relevance, extraction confidence, or propagation score with factual truth. +4. Filter inaccessible nodes before graph traversal, scoring, packing, generation, and citation hydration. +5. Remain feature-gated, observable, reversible, and measurable against the current pipeline. + +## 2. Non-goals + +- Do not expose private model chain-of-thought. The UI may show only evidence topology, source excerpts, and auditable scores. +- Do not classify a low-scoring path as a contradiction. Contradictions require a separately validated semantic comparison. +- Do not allow approximate source locations to appear verified. +- Do not replace vector retrieval, the propagation-safety verifier, or Evidence Highlighting V2. +- Do not apply expensive graph reasoning to every question. + +## 3. Target architecture + +```text +question + -> normalization / constraints / vector retrieval / ACL filter + -> seed ranking + -> multi-hop routing gate + off/single-hop -> current graph expansion and flat packing + multi-hop -> authorized neighborhood expansion + -> personalized structural propagation + -> evidence-chain assembly and deduplication + -> chain-aware context packing + -> answer generation or propagation-safety verification + -> citation hydration + -> EvidenceRecord V2 verification + -> original-document highlight +``` + +Evidence chains contain node IDs and graph edges only. Evidence verification remains the responsibility of `EvidenceRecord` and the highlighting service. + +## 4. Delivery phases + +### Phase 0 — baseline and contracts + +- Freeze representative standard, legal, clinical, multi-hop, ACL, and highlighting fixtures. +- Record baseline answer correctness, evidence recall, citation precision, verified-highlight rate, p50/p95 latency, context size, and fallback rate. +- Add a benchmark-contract mode whose dataset and SHA-256 are locked. + +Exit: repeatable baseline report and all existing tests green. + +### Phase 1 — query-aware chains over the existing document graph + +- Add deterministic multi-hop routing with `off`, `auto`, and `on` modes. +- Expand only an ACL-authorized neighborhood, with hop and node caps. +- Compute a personalized propagation score from seed relevance, query relevance, and typed edge confidence. +- Assemble paths back to seed evidence, deduplicate overlapping paths, and select within an evidence budget. +- Pack chain nodes in evidence order while keeping canonical `[node_id:page]` markers. +- Add structured audit output: route decision, weights/version, selected nodes, paths, edges, scores, timing, and fallback reason. +- Fall back to the unchanged baseline path on any chain-layer error. + +Exit: unit/integration/security gates pass and multi-hop A/B meets release thresholds. + +### Phase 2 — semantic entity and hyperedge index + +- Introduce versioned `Entity`, `EvidenceHyperedge`, and `HyperedgeIncidence` records. A hyperedge must reference one or more immutable source nodes. +- Extract typed entities and n-ary relations with versioned prompts and schemas. +- Canonicalize aliases within tenant scope. Preserve the original mention and never silently merge below the merge threshold. +- Store entity embeddings in a separately versioned Milvus collection. +- Assemble a request-scoped hypergraph only after document and node ACL filtering. +- Blend document-graph and semantic-hypergraph paths; retain the Phase 1 fallback. +- Add reindex, rollback, backfill, and deletion propagation. + +Exit: measured improvement beyond Phase 1, acceptable ingestion cost, alias-error gate, deletion/ACL proof, and successful rollback rehearsal. + +### Phase 3 — evidence-chain UI and controlled rollout + +- Add an optional "Why this answer?" panel showing claim -> evidence nodes -> source pages. +- Each leaf opens the original document at its independently verified highlight. +- Label structural score as `relevance`, never `confidence` or `truth`. +- Show unavailable/approximate evidence explicitly and do not render it as verified. +- Roll out internal -> clinical/legal pilot -> percentage ramp -> default auto routing. + +Exit: usability review with domain users, production telemetry within thresholds, and incident/rollback runbook approved. + +## 5. Phase 1 scoring contract + +The implementation is deterministic and versioned. It uses: + +- a personalized restart distribution derived from normalized seed scores; +- degree-normalized propagation over typed, confidence-weighted edges; +- lexical query relevance as a bounded signal; +- a seed-preservation term; +- explicit hop, node, chain, and context budgets. + +All component scores are in `[0, 1]`. Final ordering has stable node-ID tie breaking. Missing or invalid edge confidence is clamped, not trusted. Scores are ranking evidence only. + +## 6. Security and provenance invariants + +1. The candidate node set is ACL-filtered before any edge is eligible. +2. An edge is eligible only when both endpoints are in the authorized candidate set. +3. Audit output must not include denied node IDs, document IDs, text, aliases, degree counts, or inferred relationships. +4. Cross-tenant entity canonicalization and propagation are prohibited. +5. Packed context contains only authorized nodes. +6. Citations remain restricted to packed node IDs and are ACL-filtered again. +7. Chain metadata cannot set `EvidenceRecord.status` or verification grades. +8. Document/version/source-hash mismatches continue to fail closed. +9. Deletion of a document removes or invalidates all dependent semantic incidences before the next query can use them. + +## 7. Test strategy + +### Unit + +- router true/false positives and deterministic decisions; +- score normalization, convergence, stable ordering, disconnected nodes, cycles, self-loops, invalid confidences, and empty graphs; +- hop/node/context caps and deduplication; +- chain reconstruction and seed preservation; +- serialization and backwards-compatible API defaults; +- fail-open prevention and baseline fallback on exceptions. + +### Integration + +- vector seeds -> expansion -> chain scoring -> packing -> citation hydration; +- standard and propagation-safety modes; +- exact citation marker preservation; +- current highlighting verification with chain-selected evidence; +- feature off returns the baseline ordering and response contract; +- migration/backfill/idempotency/deletion coverage for Phase 2. + +### Security and privacy + +- inaccessible neighbors never affect scores or appear in audits; +- mixed-tenant and node-override fixtures; +- cache keys, if introduced, include tenant, entitlements hash, policy version, index version, and scoring version; +- malicious node text cannot alter routing/scoring configuration; +- restricted entity aliases and graph degree are not leaked. + +### Evaluation + +- locked legal cases: clause + amendment + effective date + governing authority; +- locked clinical cases: condition + treatment + contraindication + current guideline; +- supersession, temporal conflict, aliases, abbreviation ambiguity, same-name entities, hub distractors, missing hops, and unsupported questions; +- standard single-hop and vague-question regression set; +- human-reviewed expected nodes/pages/quotes, not LLM judgment alone. + +### Performance and resilience + +- p50/p95/p99 chain-layer latency and total request latency; +- maximum SQL query count and candidate count; +- propagation convergence and memory under worst-case bounded graphs; +- database timeout/error fallback; +- load test with feature off, auto-routed, and forced-on cohorts; +- rollback rehearsal with no response-schema or citation breakage. + +## 8. Release gates + +Phase 1 may move from shadow evaluation to user-visible auto routing only when all gates pass: + +| Gate | Required result | +|---|---| +| Multi-hop answer correctness | at least +5 absolute points over baseline | +| Complete evidence-chain recall | no regression; target +5 absolute points | +| Citation precision | no regression | +| Verified-highlight rate | no regression | +| Wrong document/page/version verified | exactly 0 | +| Unauthorized evidence/audit leakage | exactly 0 | +| Single-hop correctness | no statistically meaningful regression | +| Chain-layer p95 overhead | <= 20% of baseline total latency | +| Fallback/degeneracy rate | <= 10% | +| Unsupported-answer rate | no regression | + +Phase 2 additionally requires entity merge precision, extraction recall, deletion correctness, reindex reproducibility, and ingestion cost gates. + +## 9. Observability and rollback + +Emit per-request structured fields without source text: + +- scoring version and mode; +- router decision and reason codes; +- candidate/edge/path counts; +- selected node IDs only after ACL filtering; +- convergence iterations and timing; +- fallback status/reason; +- context tokens and downstream verified/approximate/unavailable counts. + +Rollback mechanisms: + +1. Set evidence-chain feature flag off for immediate baseline behavior. +2. Keep old API fields optional during rollback. +3. Retain independent index versions and an active-index pointer for Phase 2. +4. Preserve baseline evaluation artifacts for every release candidate. + +## 10. Completion evidence + +Adoption is complete only when: + +- implementation, API contract, configuration, and operator documentation are present; +- locked tests and benchmark fixtures cover every invariant above; +- targeted and complete backend/frontend suites pass; +- live or representative-stack integration, performance, ACL, and highlighting tests pass; +- the A/B release report satisfies every applicable gate; +- rollout and rollback are rehearsed and documented. + +## 11. Operator rollout and rollback runbook + +The global feature flag is the authority boundary. A request-level +`evidence_chain_mode` override is ignored while the global flag is disabled. + +### Stage A — baseline/default + +```dotenv +EVIDENCE_CHAIN_ENABLED=false +EVIDENCE_CHAIN_MODE=auto +``` + +This preserves the original expansion and packing path. The optional response +fields remain backwards compatible. + +### Stage B — internal forced-on validation + +```dotenv +EVIDENCE_CHAIN_ENABLED=true +EVIDENCE_CHAIN_MODE=on +``` + +Use a non-production corpus first. Confirm that every cited leaf opens the +correct document version and that no unresolved/approximate location is shown +as verified. An API caller may send `evidence_chain_mode: "off"` to obtain a +same-build baseline comparison. + +### Stage C — legal/clinical pilot + +```dotenv +EVIDENCE_CHAIN_ENABLED=true +EVIDENCE_CHAIN_MODE=auto +``` + +Run baseline and auto-routed cohorts on the locked representative corpus. +Review node/page/quote expectations with domain reviewers and calculate every +gate in section 8. Do not promote on retrieval recall alone. + +### Immediate rollback + +Set `EVIDENCE_CHAIN_ENABLED=false` and restart the backend deployment. No data +migration or index rollback is needed for Phase 1. Existing response consumers +continue working because `evidence_chain` is optional. + +### Verification commands + +From `backend/`: + +```bash +python tests/eval/run_evidence_chain_eval.py --output tests/eval/report_evidence_chain.json +python tests/eval/run_evidence_chain_ab.py --output tests/eval/report_evidence_chain_ab_fixture.json +python tests/eval/run_evidence_chain_ab.py --provider openai --model gpt-5.2 --output tests/eval/report_evidence_chain_ab_openai.json +pytest tests/qa/test_evidence_chain.py tests/qa/test_evidence_chain_runner.py -q +RUN_INTEGRATION_TESTS=1 pytest tests/test_evidence_chain_integration.py -q +pytest -q +``` + +The provider-backed run sends the locked synthetic legal/clinical prompts to +the configured external provider. Obtain the required data-egress approval +before running it. A previously saved report can be checked against the current +locked dataset, expectations, metrics, and gates without another provider call: + +```bash +python tests/eval/run_evidence_chain_ab.py \ + --regrade tests/eval/report_evidence_chain_ab_openai.json \ + --output tests/eval/report_evidence_chain_ab_openai.json +``` + +From `frontend/`: + +```bash +npm test +npm run lint +npm run build +``` + +The locked evaluator covers 12 curated legal, clinical, ACL, single-hop, and +unsupported-answer cases. It validates answers, exact expected facts and +limitations, selected evidence, citations, source locations, ACL leakage, +routing, fallback, and latency. It is deliberately not a substitute for an +ingested, embedded, access-controlled production-like corpus or review by +lawyers and clinicians. Preserve baseline and chain reports for every pilot. + +The chain-layer p95 release gate and total request latency are separate. The +chain layer must remain at or below 20% of baseline request p95. Provider/model +request p50/p95/p99 and paired overhead are also reported, but no default-on +decision may rely on the small curated sample; representative load testing is +required. + +## 12. Phase 1 implementation map + +| Concern | Implementation | +|---|---| +| Routing, propagation, path assembly | `app/qa/evidence_chain.py` | +| Standard and propagation-safety orchestration | `app/qa/runner.py` | +| Bounded chain-aware packing | `app/graph/context_packer.py` | +| Feature configuration | `app/config.py`, `.env.example` | +| API request/response contract | `app/routes/qa.py`, `frontend/src/lib/api.ts` | +| Source-review UI | `frontend/src/app/(app)/chat/chat-client.tsx` | +| Citation contract validation and bounded repair | `app/llm/openai_client.py`, `app/prompts/qa_answer_v2.txt` | +| Locked retrieval evaluator and fixtures | `tests/eval/run_evidence_chain_eval.py`, `tests/eval/questions_evidence_chain.json` | +| Paired answer/provenance A/B | `tests/eval/run_evidence_chain_ab.py`, `tests/eval/evidence_chain_ab_expectations.json` | +| PostgreSQL/ACL/highlighting proof | `tests/test_evidence_chain_integration.py` | + +Phase 2 is intentionally not included in the current runtime. Semantic entity +canonicalization and hyperedges have materially higher privacy and merge-error +risk and must earn adoption through a measured improvement beyond Phase 1. + +## 13. Phase 1 completion and rollout boundary + +The repository implementation is complete for Phase 1. The locked retrieval +report, deterministic paired A/B, configured OpenAI `gpt-5.2` paired A/B, +PostgreSQL integration, migrations, complete backend/frontend suites, and real +browser PDF overlay all pass. The traversal query contract is bounded to at +most two ORM queries per configured hop, in addition to independently enforced +ACL work, and is protected by a regression test. + +This does **not** authorize a production default-on change. The configured-model +run is a 12-case curated synthetic evaluation, not a statistically powered +legal or clinical study. Its chain computation added less than 1 ms at p95, +but total model request p95 was 31.29% higher than baseline in that run. Longer, +more complete answers and provider variance can both contribute. Run a +representative load test and have domain reviewers validate answer utility and +every highlighted source before moving beyond a controlled pilot. + +## 14. Paper-to-system traceability and intentional deviations + +Source: [HyCE-RAG, arXiv:2607.22597v1](https://arxiv.org/abs/2607.22597) + +| Paper mechanism | NPR-RAG decision | +|---|---| +| Query-aware neighborhood from retrieved entries | Adopted now, using authorized chunk seeds and the existing document graph | +| Random-walk propagation with restart | Adopted now with deterministic, degree-normalized, bounded propagation | +| Joint structural, query, entry, and reliability scoring | Adopted in available form: propagation, lexical query relevance, seed prior, typed edge weight/confidence | +| Evidence budget and overlap fusion | Adopted now with node/path caps and Jaccard-style path deduplication | +| Structured evidence before generation | Adopted now while preserving canonical citation markers | +| Entity extraction plus entity-vector entry retrieval | Deferred to Phase 2 pending extraction and linking evaluation | +| N-ary hyperedges and incidence traversal | Deferred to Phase 2 pending schema, ACL, deletion, reindex, and provenance proof | +| Lower-score paths treated as potentially contradictory | Not adopted; low relevance is not proof of contradiction | +| `confidence` exposed as evidence quality | Renamed/reframed as structural `relevance`; it cannot verify truth or location | + +The paper reports strong benchmark improvements, including on a medical subset, +but its accuracy/relevance/faithfulness measures substantially use an LLM judge. +It does not establish legal-domain performance, exact original-document +highlight correctness, tenant/node ACL safety, entity-merge precision, deletion +behavior, or production latency. NPR-RAG therefore treats the paper as a useful +design hypothesis and requires its own human-reviewed evidence, source-location, +security, and live A/B gates before enabling the feature by default. diff --git a/backend/docs/HYCE_EVIDENCE_CHAINS_VERIFICATION.md b/backend/docs/HYCE_EVIDENCE_CHAINS_VERIFICATION.md new file mode 100644 index 0000000..072b3fa --- /dev/null +++ b/backend/docs/HYCE_EVIDENCE_CHAINS_VERIFICATION.md @@ -0,0 +1,129 @@ +# Evidence Chains Phase 1 Verification Record + +Date: 2026-08-01 + +Scope: HyCE-inspired query-aware evidence chains plus the existing Evidence +Highlighting V2 safety boundary. + +## Verified outcomes + +| Verification | Result | +|---|---| +| Complete default backend suite | 384 passed, 135 intentionally skipped | +| Final evidence-chain/A-B targeted regression | 19 passed | +| Route and PostgreSQL integration selection | 45 passed | +| Generated-PDF ingestion -> graph -> chain integration | passed | +| Authorized chain -> packing -> exact rectangles | passed | +| Wrong-page highlight request | failed closed as `page_not_indexed` | +| Locked retrieval evaluator | 12/12 cases passed | +| Deterministic paired answer/provenance A/B | all gates passed | +| Configured OpenAI `gpt-5.2` paired A/B | all curated gates passed | +| Frontend component tests | 12 passed in 3 files | +| ESLint | 0 errors; 30 pre-existing warnings | +| Next.js production build and TypeScript | passed; 16 static pages generated | +| Browser evidence path -> original PDF -> verified overlay | passed; 0 browser warnings | +| Alembic clean upgrade and 014 downgrade/upgrade rehearsal | passed | + +## Locked retrieval evaluator metrics + +Dataset SHA-256: +`607ec987d8b032858a48fc03c7ca87d1c8db498e46c3a5d8621610e34bc7c7b7` + +| Metric | Result | +|---|---| +| Cases | 12 | +| Router accuracy | 100% | +| Baseline evidence recall | 56.94% | +| Chain evidence recall | 100% | +| Absolute evidence-recall change | +43.06 points | +| Forbidden-node leaks | 0 | +| Deterministic repeat rate | 100% | +| Local chain-layer p95 | 0.26 ms in the final run environment | + +This is a deterministic retrieval fixture result. It is not evidence of a +43-point answer-accuracy improvement and must not be represented that way. + +## Paired answer and provenance A/B + +The answer-level evaluator runs the same question and allowed context through a +baseline and chain variant. Expectations are human-authored in the repository; +the evaluator does not use an LLM judge. It checks required facts and +limitations, citation precision/recall, selected evidence recall, citation +integrity, independently verified source spans, wrong document/page/version +negative controls, ACL leakage, routing, fallback, single-hop behavior, safe +unsupported answers, and latency. + +Expectations SHA-256: +`5228ebe3c22f4f262e042ca580bd1b2f62256e0e8e406646f9b2f384eb3b9ee5` + +| Configured-model metric | Baseline | Chain | +|---|---:|---:| +| All-case answer accuracy | 41.67% | 100% | +| Routed-case answer accuracy | 12.5% | 100% | +| Citation precision | 100% | 100% | +| Expected citation recall | 58.33% | 97.22% | +| Selected evidence recall | 61.11% | 100% | +| Verified highlight rate | 100% | 100% | +| Single-hop accuracy | 100% | 100% | +| Unsupported-answer safe rate | 100% | 100% | + +Additional configured-model results: router accuracy 100%, fallback 0%, wrong +locator acceptances 0, unauthorized evidence leaks 0, and inline citation +integrity 100%. These values are for the locked 12-case synthetic corpus and +must not be generalized to production legal or clinical accuracy. + +The chain computation itself measured 0.73 ms p50, 0.94 ms p95, and 1.27 ms +p99. That p95 is 0.043% of baseline total request p95 and passes the <=20% +chain-layer gate. Total provider request p95 increased from 2200.39 ms to +2888.94 ms (+31.29%); paired total-request overhead was +34.21% at p50 and ++50.47% at p95. Total provider latency is observational in this small run, not +a release gate, and requires representative load testing before default-on. + +The saved configured-model responses were regraded offline after citation-recall +and latency reporting semantics were finalized. Offline regrading verifies the +locked dataset and expectation hashes and makes no provider call. + +## Representative-stack proof + +The integration suite creates a real three-page PDF in memory, runs native PDF +extraction, page-bounded chunking, word provenance, selector construction, +PostgreSQL node/edge persistence, ACL-filtered evidence propagation, and path +assembly. A second case inserts a node-level restricted neighbor and proves that +the denied node appears in neither selection, audit, packed context, nor path. +The authorized leaf resolves to verified source rectangles; a wrong-page lookup +is unavailable rather than approximately highlighted. + +The repository's older `tests/test_graph_e2e.py` was also invoked explicitly, +but its checked-in test expects `tests/docs/2404.08865v1.pdf`, which is absent +from this worktree. That fixture defect is independent of this implementation; +the generated-PDF integration above supplies the equivalent relevant coverage +without weakening or silently altering the old test. + +## Browser proof + +The browser fixture verified that: + +- the panel labels path scores as relevance and disclaims truth/medical/legal confidence; +- supporting context not cited in the final answer is disabled; +- a cited leaf marked verified opens the existing document viewer; +- the original PDF renders at the cited page with a normalized orange evidence + rectangle over the source text; +- the panel and nested viewer produce no accessibility warnings. + +Screenshot: `output/playwright/evidence-chain-source-panel.png` + +## Default-on gates still requiring representative production data + +The feature remains disabled by default. These gates cannot be established by +synthetic fixtures or a local build and remain prerequisites for production +default-on rollout: + +- human-reviewed answer utility, citation precision, and verified-highlight + correctness on real legal and clinical documents; +- single-hop and unsupported-answer non-regression with representative traffic; +- total request latency and throughput under representative concurrency; +- fallback/degeneracy rate under live query distribution; +- domain reviewer usability acceptance. + +See `HYCE_EVIDENCE_CHAINS_ADOPTION.md` for thresholds, staged rollout, and the +one-flag rollback procedure. diff --git a/backend/scripts/backfill_evidence_v2.py b/backend/scripts/backfill_evidence_v2.py new file mode 100644 index 0000000..51ce0f1 --- /dev/null +++ b/backend/scripts/backfill_evidence_v2.py @@ -0,0 +1,78 @@ +"""Backfill Evidence Highlighting V2 provenance for one stored native PDF. + +Usage from ``backend/``: + venv/bin/python -m scripts.backfill_evidence_v2 [--version N] +""" + +from __future__ import annotations + +import argparse + +from app.db.graph_models import DocumentGraph +from app.db.session import session_scope +from app.services.document_identity import find_legacy_for_graph +from app.services.highlighting import backfill_pdf_source_provenance +from app.storage.minio_client import get_storage_client + + +def _load_raw_pdf(db, doc: DocumentGraph) -> bytes: + storage = get_storage_client() + + graph_files = storage.list_raw_files(doc.doc_id, str(doc.version)) + if graph_files: + raw, _ = storage.get_raw_with_content_type( + doc.doc_id, + str(doc.version), + graph_files[0], + ) + return raw + + legacy = find_legacy_for_graph(db, doc.doc_id, doc.version) + if legacy: + legacy_files = storage.list_raw_files(legacy.doc_id, legacy.version_id) + if legacy_files: + raw, _ = storage.get_raw_with_content_type( + legacy.doc_id, + legacy.version_id, + legacy_files[0], + ) + return raw + + raise FileNotFoundError(f"original stored file not found for {doc.doc_id}") + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("doc_id", help="Canonical graph document ID") + parser.add_argument( + "--version", + type=int, + help="Document version to upgrade (defaults to the latest version)", + ) + args = parser.parse_args() + + with session_scope() as db: + query = db.query(DocumentGraph).filter(DocumentGraph.doc_id == args.doc_id) + if args.version is not None: + query = query.filter(DocumentGraph.version == args.version) + doc = query.order_by(DocumentGraph.version.desc()).first() + if not doc: + version_suffix = ( + f" version {args.version}" if args.version is not None else "" + ) + raise SystemExit( + f"graph document not found: {args.doc_id}{version_suffix}" + ) + raw_pdf = _load_raw_pdf(db, doc) + result = backfill_pdf_source_provenance(db, doc, raw_pdf) + print( + "Evidence V2 backfill complete: " + f"doc={result['doc_id']} v{result['version']} " + f"exact={result['exact_nodes']} " + f"approximate={result['approximate_nodes']} " + f"unavailable={result['unavailable_nodes']}" + ) + + +if __name__ == "__main__": + main() diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py index e531578..ba43923 100644 --- a/backend/tests/conftest.py +++ b/backend/tests/conftest.py @@ -16,6 +16,7 @@ "test_acl_e2e.py", "test_all_connections.py", "test_embedding_e2e.py", + "test_evidence_chain_integration.py", "test_graph_e2e.py", "test_graph_rag.py", "test_ingestion_e2e.py", diff --git a/backend/tests/eval/benchmark_contract.json b/backend/tests/eval/benchmark_contract.json index bfdf92c..5c1e5f2 100644 --- a/backend/tests/eval/benchmark_contract.json +++ b/backend/tests/eval/benchmark_contract.json @@ -45,6 +45,22 @@ "allow_question_file_override": false, "allow_doc_path_override": false }, + "evidence_chain": { + "dataset_path": "backend/tests/eval/questions_evidence_chain.json", + "dataset_sha256": "607ec987d8b032858a48fc03c7ca87d1c8db498e46c3a5d8621610e34bc7c7b7", + "required_question_count": 12, + "allow_question_file_override": false, + "allow_doc_path_override": false + }, + "evidence_chain_ab": { + "dataset_path": "backend/tests/eval/questions_evidence_chain.json", + "dataset_sha256": "607ec987d8b032858a48fc03c7ca87d1c8db498e46c3a5d8621610e34bc7c7b7", + "expectations_path": "backend/tests/eval/evidence_chain_ab_expectations.json", + "expectations_sha256": "5228ebe3c22f4f262e042ca580bd1b2f62256e0e8e406646f9b2f384eb3b9ee5", + "required_question_count": 12, + "allow_question_file_override": false, + "allow_expectations_override": false + }, "finalboss": { "dataset_path": "backend/tests/eval/questions_finalboss.json", "dataset_sha256": "501e6c7a378ee008f10afeef8863c0c16b44e0286edd4e1c43105caf3fc2a88c", diff --git a/backend/tests/eval/evidence_chain_ab_expectations.json b/backend/tests/eval/evidence_chain_ab_expectations.json new file mode 100644 index 0000000..2104344 --- /dev/null +++ b/backend/tests/eval/evidence_chain_ab_expectations.json @@ -0,0 +1,126 @@ +{ + "name": "Evidence Chain Answer and Provenance Expectations", + "version": "1.0.0", + "review_status": "human-authored synthetic fixtures; domain-expert sign-off required before production rollout", + "cases": [ + { + "id": "legal_amendment_effective_date", + "required_fact_groups": [ + ["thirty days", "thirty day", "30 days", "30 day"], + ["january 1 2026"], + ["replaces the termination notice", "replaces the original termination", "replaced the termination notice"], + ["sixty day", "sixty days", "60 day", "60 days"] + ], + "required_limitation_groups": [], + "forbidden_phrases": [], + "expected_citation_nodes": ["l1_original", "l1_amendment", "l1_statute"] + }, + { + "id": "legal_superseded_policy", + "required_fact_groups": [ + ["five year", "five years", "5 year", "5 years"], + ["2025 amendment supersedes", "superseded by the 2025 amendment", "2025 amendment replaced"], + ["three year", "three years", "3 year", "3 years"] + ], + "required_limitation_groups": [], + "forbidden_phrases": [], + "expected_citation_nodes": ["l2_prior", "l2_amendment", "l2_current"] + }, + { + "id": "legal_simple_fee_lookup", + "required_fact_groups": [["250"]], + "required_limitation_groups": [], + "forbidden_phrases": [], + "expected_citation_nodes": ["l3_fee"] + }, + { + "id": "legal_precedent_comparison", + "required_fact_groups": [ + ["smith held that notice was adequate", "smith found the notice adequate", "smith the court held that the notice at issue was adequate"], + ["jones distinguished smith", "jones the later court distinguished smith"], + ["omitted a statutory warning", "omission of the statutory warning"], + ["statutory warning is mandatory", "mandatory statutory warning"] + ], + "required_limitation_groups": [], + "forbidden_phrases": [], + "expected_citation_nodes": ["l4_smith", "l4_jones", "l4_rule"] + }, + { + "id": "legal_acl_restricted_amendment", + "required_fact_groups": [["one million", "1 million", "1000000"]], + "required_limitation_groups": [ + ["cannot find sufficient information", "insufficient information", "cannot determine", "cannot compare", "does not state", "does not provide", "does not include the text of any amendment", "not provided"] + ], + "forbidden_phrases": ["increases the liability cap", "increased the liability cap"], + "expected_citation_nodes": ["l5_agreement"] + }, + { + "id": "clinical_contraindication", + "required_fact_groups": [ + ["severe renal impairment"], + ["cleared by the kidneys", "cleared through the kidneys", "kidney clearance"], + ["contraindication to drug a", "contraindication for drug a", "drug a is contraindicated"] + ], + "required_limitation_groups": [], + "forbidden_phrases": [], + "expected_citation_nodes": ["c1_condition", "c1_drug", "c1_guideline"] + }, + { + "id": "clinical_drug_interaction", + "required_fact_groups": [ + ["drug a inhibits platelet aggregation", "platelet aggregation"], + ["drug b is an anticoagulant", "drug b anticoagulant"], + ["increases bleeding risk", "increased bleeding risk", "bleeding risk increases"] + ], + "required_limitation_groups": [], + "forbidden_phrases": [], + "expected_citation_nodes": ["c2_drug_a", "c2_drug_b", "c2_interaction"] + }, + { + "id": "clinical_simple_dose", + "required_fact_groups": [["10 mg daily", "10 milligrams daily"]], + "required_limitation_groups": [], + "forbidden_phrases": [], + "expected_citation_nodes": ["c3_dose"] + }, + { + "id": "clinical_updated_guideline", + "required_fact_groups": [ + ["treatment x as first line", "treatment x was first line"], + ["effective in 2026", "2026 update", "2026 guideline"], + ["treatment y for high risk", "prefers treatment y", "prefer treatment y"] + ], + "required_limitation_groups": [], + "forbidden_phrases": [], + "expected_citation_nodes": ["c4_prior", "c4_update", "c4_latest"] + }, + { + "id": "clinical_abbreviation_bridge", + "required_fact_groups": [ + ["af means atrial fibrillation", "af stands for atrial fibrillation", "af refers to atrial fibrillation"], + ["anticoagulation based on risk", "risk based anticoagulation", "anticoagulation depending on risk"], + ["reduces embolic stroke risk", "reduce embolic stroke risk"] + ], + "required_limitation_groups": [], + "forbidden_phrases": [], + "expected_citation_nodes": ["c5_af", "c5_anticoag", "c5_stroke"] + }, + { + "id": "regression_conjunction_only", + "required_fact_groups": [["example clinic"], ["100 main street"]], + "required_limitation_groups": [], + "forbidden_phrases": [], + "expected_citation_nodes": ["r1_cover"] + }, + { + "id": "regression_unsupported_why", + "required_fact_groups": [], + "required_limitation_groups": [ + ["cannot find sufficient information", "insufficient information", "cannot answer", "not enough information"] + ], + "forbidden_phrases": [], + "citation_policy": "optional", + "expected_citation_nodes": ["r2_context"] + } + ] +} diff --git a/backend/tests/eval/questions_evidence_chain.json b/backend/tests/eval/questions_evidence_chain.json new file mode 100644 index 0000000..d4735a5 --- /dev/null +++ b/backend/tests/eval/questions_evidence_chain.json @@ -0,0 +1,204 @@ +{ + "name": "Evidence Chain Legal and Clinical Golden Set", + "version": "1.0.0", + "description": "Deterministic routing, chain-recall, ACL, and regression fixtures. Expected evidence is human-authored and node-level.", + "cases": [ + { + "id": "legal_amendment_effective_date", + "domain": "legal", + "question": "Compare the amendment's effective date with the original termination clause and explain how the governing statute affects it.", + "expect_route": true, + "nodes": [ + {"id": "l1_original", "page": 4, "text": "The original agreement permits termination on thirty days notice."}, + {"id": "l1_amendment", "page": 9, "text": "The amendment is effective January 1, 2026 and replaces the termination notice period."}, + {"id": "l1_statute", "page": 14, "text": "The governing statute requires a minimum sixty day termination notice period."} + ], + "edges": [ + {"from": "l1_original", "to": "l1_amendment", "type": "references", "confidence": 1.0}, + {"from": "l1_amendment", "to": "l1_statute", "type": "explained_by", "confidence": 0.9} + ], + "seeds": {"l1_original": 0.92}, + "expected_evidence": ["l1_original", "l1_amendment", "l1_statute"], + "forbidden_evidence": [] + }, + { + "id": "legal_superseded_policy", + "domain": "legal", + "question": "Which policy is current after the 2025 amendment, and how does it differ from the prior policy?", + "expect_route": true, + "nodes": [ + {"id": "l2_prior", "page": 2, "text": "The prior policy authorizes a five year term."}, + {"id": "l2_amendment", "page": 6, "text": "The 2025 amendment supersedes the prior policy."}, + {"id": "l2_current", "page": 8, "text": "The current policy authorizes a three year term."} + ], + "edges": [ + {"from": "l2_prior", "to": "l2_amendment", "type": "references", "confidence": 0.95}, + {"from": "l2_amendment", "to": "l2_current", "type": "adjacent_next", "confidence": 1.0} + ], + "seeds": {"l2_prior": 0.8}, + "expected_evidence": ["l2_prior", "l2_amendment", "l2_current"], + "forbidden_evidence": [] + }, + { + "id": "legal_simple_fee_lookup", + "domain": "legal", + "question": "What is the filing fee?", + "expect_route": false, + "nodes": [ + {"id": "l3_fee", "page": 1, "text": "The filing fee is $250."} + ], + "edges": [], + "seeds": {"l3_fee": 1.0}, + "expected_evidence": ["l3_fee"], + "forbidden_evidence": [] + }, + { + "id": "legal_precedent_comparison", + "domain": "legal", + "question": "Compare the holdings in Smith and Jones and explain why the later court reached a different result.", + "expect_route": true, + "nodes": [ + {"id": "l4_smith", "page": 11, "text": "Smith held that notice was adequate."}, + {"id": "l4_jones", "page": 22, "text": "Jones distinguished Smith because the notice omitted a statutory warning."}, + {"id": "l4_rule", "page": 23, "text": "The statutory warning is mandatory in consumer notices."} + ], + "edges": [ + {"from": "l4_smith", "to": "l4_jones", "type": "references", "confidence": 1.0}, + {"from": "l4_jones", "to": "l4_rule", "type": "explained_by", "confidence": 1.0} + ], + "seeds": {"l4_smith": 0.86}, + "expected_evidence": ["l4_smith", "l4_jones", "l4_rule"], + "forbidden_evidence": [] + }, + { + "id": "legal_acl_restricted_amendment", + "domain": "legal", + "question": "Compare the agreement with its amendment and explain how the amendment affects liability.", + "expect_route": true, + "nodes": [ + {"id": "l5_agreement", "page": 3, "text": "The agreement caps liability at one million dollars."}, + {"id": "l5_public_note", "page": 4, "text": "The public note explains the original liability cap."}, + {"id": "l5_restricted", "page": 7, "text": "Restricted amendment increases the liability cap."} + ], + "edges": [ + {"from": "l5_agreement", "to": "l5_public_note", "type": "adjacent_next", "confidence": 1.0}, + {"from": "l5_public_note", "to": "l5_restricted", "type": "references", "confidence": 1.0} + ], + "seeds": {"l5_agreement": 0.9}, + "denied_nodes": ["l5_restricted"], + "expected_evidence": ["l5_agreement", "l5_public_note"], + "forbidden_evidence": ["l5_restricted"] + }, + { + "id": "clinical_contraindication", + "domain": "clinical", + "question": "How does renal impairment affect Drug A treatment under the current guideline, including its contraindication?", + "expect_route": true, + "nodes": [ + {"id": "c1_condition", "page": 2, "text": "The patient has severe renal impairment."}, + {"id": "c1_drug", "page": 5, "text": "Drug A is primarily cleared by the kidneys."}, + {"id": "c1_guideline", "page": 8, "text": "The current guideline lists severe renal impairment as a contraindication to Drug A."} + ], + "edges": [ + {"from": "c1_condition", "to": "c1_drug", "type": "references", "confidence": 0.9}, + {"from": "c1_drug", "to": "c1_guideline", "type": "explained_by", "confidence": 1.0} + ], + "seeds": {"c1_condition": 0.95}, + "expected_evidence": ["c1_condition", "c1_drug", "c1_guideline"], + "forbidden_evidence": [] + }, + { + "id": "clinical_drug_interaction", + "domain": "clinical", + "question": "Compare Drug A and Drug B and explain why their interaction affects bleeding risk.", + "expect_route": true, + "nodes": [ + {"id": "c2_drug_a", "page": 10, "text": "Drug A inhibits platelet aggregation."}, + {"id": "c2_drug_b", "page": 12, "text": "Drug B is an anticoagulant."}, + {"id": "c2_interaction", "page": 13, "text": "Combined antiplatelet and anticoagulant therapy increases bleeding risk."} + ], + "edges": [ + {"from": "c2_drug_a", "to": "c2_drug_b", "type": "adjacent_next", "confidence": 0.8}, + {"from": "c2_drug_b", "to": "c2_interaction", "type": "references", "confidence": 1.0} + ], + "seeds": {"c2_drug_a": 0.9}, + "expected_evidence": ["c2_drug_a", "c2_drug_b", "c2_interaction"], + "forbidden_evidence": [] + }, + { + "id": "clinical_simple_dose", + "domain": "clinical", + "question": "What is the recommended dose of Drug A?", + "expect_route": false, + "nodes": [ + {"id": "c3_dose", "page": 4, "text": "The recommended dose of Drug A is 10 mg daily."} + ], + "edges": [], + "seeds": {"c3_dose": 1.0}, + "expected_evidence": ["c3_dose"], + "forbidden_evidence": [] + }, + { + "id": "clinical_updated_guideline", + "domain": "clinical", + "question": "What changed after the updated guideline, and how does the latest recommendation differ from the prior treatment guidance?", + "expect_route": true, + "nodes": [ + {"id": "c4_prior", "page": 1, "text": "Prior guidance recommends Treatment X as first line therapy."}, + {"id": "c4_update", "page": 2, "text": "The updated guideline was effective in 2026."}, + {"id": "c4_latest", "page": 3, "text": "The latest recommendation prefers Treatment Y for high risk patients."} + ], + "edges": [ + {"from": "c4_prior", "to": "c4_update", "type": "adjacent_next", "confidence": 1.0}, + {"from": "c4_update", "to": "c4_latest", "type": "adjacent_next", "confidence": 1.0} + ], + "seeds": {"c4_prior": 0.91}, + "expected_evidence": ["c4_prior", "c4_update", "c4_latest"], + "forbidden_evidence": [] + }, + { + "id": "clinical_abbreviation_bridge", + "domain": "clinical", + "question": "How does AF relate to anticoagulation, and why does that relationship affect stroke prevention?", + "expect_route": true, + "nodes": [ + {"id": "c5_af", "page": 15, "text": "AF means atrial fibrillation in this guideline."}, + {"id": "c5_anticoag", "page": 16, "text": "Atrial fibrillation may require anticoagulation based on risk."}, + {"id": "c5_stroke", "page": 17, "text": "Anticoagulation reduces embolic stroke risk in eligible patients."} + ], + "edges": [ + {"from": "c5_af", "to": "c5_anticoag", "type": "references", "confidence": 0.9}, + {"from": "c5_anticoag", "to": "c5_stroke", "type": "explained_by", "confidence": 0.9} + ], + "seeds": {"c5_af": 0.88}, + "expected_evidence": ["c5_af", "c5_anticoag", "c5_stroke"], + "forbidden_evidence": [] + }, + { + "id": "regression_conjunction_only", + "domain": "regression", + "question": "List the name and address on the cover page.", + "expect_route": false, + "nodes": [ + {"id": "r1_cover", "page": 1, "text": "Example Clinic, 100 Main Street."} + ], + "edges": [], + "seeds": {"r1_cover": 1.0}, + "expected_evidence": ["r1_cover"], + "forbidden_evidence": [] + }, + { + "id": "regression_unsupported_why", + "domain": "regression", + "question": "Why?", + "expect_route": false, + "nodes": [ + {"id": "r2_context", "page": 1, "text": "Insufficient context."} + ], + "edges": [], + "seeds": {"r2_context": 1.0}, + "expected_evidence": ["r2_context"], + "forbidden_evidence": [] + } + ] +} diff --git a/backend/tests/eval/report_evidence_chain.json b/backend/tests/eval/report_evidence_chain.json new file mode 100644 index 0000000..b571a6c --- /dev/null +++ b/backend/tests/eval/report_evidence_chain.json @@ -0,0 +1,209 @@ +{ + "case_count": 12, + "cases": [ + { + "baseline_recall": 0.3333333333333333, + "case_id": "legal_amendment_effective_date", + "chain_recall": 1.0, + "deterministic": true, + "domain": "legal", + "forbidden_leaks": [], + "latency_ms": 0.3754580393433571, + "route_actual": true, + "route_expected": true, + "selected_node_ids": [ + "l1_amendment", + "l1_original", + "l1_statute" + ] + }, + { + "baseline_recall": 0.3333333333333333, + "case_id": "legal_superseded_policy", + "chain_recall": 1.0, + "deterministic": true, + "domain": "legal", + "forbidden_leaks": [], + "latency_ms": 0.2032919437624514, + "route_actual": true, + "route_expected": true, + "selected_node_ids": [ + "l2_amendment", + "l2_current", + "l2_prior" + ] + }, + { + "baseline_recall": 1.0, + "case_id": "legal_simple_fee_lookup", + "chain_recall": 1.0, + "deterministic": true, + "domain": "legal", + "forbidden_leaks": [], + "latency_ms": 0.0075830030255019665, + "route_actual": false, + "route_expected": false, + "selected_node_ids": [ + "l3_fee" + ] + }, + { + "baseline_recall": 0.3333333333333333, + "case_id": "legal_precedent_comparison", + "chain_recall": 1.0, + "deterministic": true, + "domain": "legal", + "forbidden_leaks": [], + "latency_ms": 0.20216696429997683, + "route_actual": true, + "route_expected": true, + "selected_node_ids": [ + "l4_jones", + "l4_rule", + "l4_smith" + ] + }, + { + "baseline_recall": 0.5, + "case_id": "legal_acl_restricted_amendment", + "chain_recall": 1.0, + "deterministic": true, + "domain": "legal", + "forbidden_leaks": [], + "latency_ms": 0.14520803233608603, + "route_actual": true, + "route_expected": true, + "selected_node_ids": [ + "l5_agreement", + "l5_public_note" + ] + }, + { + "baseline_recall": 0.3333333333333333, + "case_id": "clinical_contraindication", + "chain_recall": 1.0, + "deterministic": true, + "domain": "clinical", + "forbidden_leaks": [], + "latency_ms": 0.19879202591255307, + "route_actual": true, + "route_expected": true, + "selected_node_ids": [ + "c1_condition", + "c1_drug", + "c1_guideline" + ] + }, + { + "baseline_recall": 0.3333333333333333, + "case_id": "clinical_drug_interaction", + "chain_recall": 1.0, + "deterministic": true, + "domain": "clinical", + "forbidden_leaks": [], + "latency_ms": 0.258374959230423, + "route_actual": true, + "route_expected": true, + "selected_node_ids": [ + "c2_drug_a", + "c2_drug_b", + "c2_interaction" + ] + }, + { + "baseline_recall": 1.0, + "case_id": "clinical_simple_dose", + "chain_recall": 1.0, + "deterministic": true, + "domain": "clinical", + "forbidden_leaks": [], + "latency_ms": 0.00966701190918684, + "route_actual": false, + "route_expected": false, + "selected_node_ids": [ + "c3_dose" + ] + }, + { + "baseline_recall": 0.3333333333333333, + "case_id": "clinical_updated_guideline", + "chain_recall": 1.0, + "deterministic": true, + "domain": "clinical", + "forbidden_leaks": [], + "latency_ms": 0.20112498896196485, + "route_actual": true, + "route_expected": true, + "selected_node_ids": [ + "c4_latest", + "c4_prior", + "c4_update" + ] + }, + { + "baseline_recall": 0.3333333333333333, + "case_id": "clinical_abbreviation_bridge", + "chain_recall": 1.0, + "deterministic": true, + "domain": "clinical", + "forbidden_leaks": [], + "latency_ms": 0.24779100203886628, + "route_actual": true, + "route_expected": true, + "selected_node_ids": [ + "c5_af", + "c5_anticoag", + "c5_stroke" + ] + }, + { + "baseline_recall": 1.0, + "case_id": "regression_conjunction_only", + "chain_recall": 1.0, + "deterministic": true, + "domain": "regression", + "forbidden_leaks": [], + "latency_ms": 0.009040988516062498, + "route_actual": false, + "route_expected": false, + "selected_node_ids": [ + "r1_cover" + ] + }, + { + "baseline_recall": 1.0, + "case_id": "regression_unsupported_why", + "chain_recall": 1.0, + "deterministic": true, + "domain": "regression", + "forbidden_leaks": [], + "latency_ms": 0.0025830231606960297, + "route_actual": false, + "route_expected": false, + "selected_node_ids": [ + "r2_context" + ] + } + ], + "contract_id": "rag_eval_contract", + "dataset_sha256": "607ec987d8b032858a48fc03c7ca87d1c8db498e46c3a5d8621610e34bc7c7b7", + "gates": { + "chain_recall_no_regression": true, + "chain_recall_target": true, + "deterministic": true, + "forbidden_leakage_zero": true, + "local_p95_under_50ms": true, + "route_accuracy": true + }, + "metrics": { + "absolute_recall_improvement": 0.4305555555555556, + "baseline_evidence_recall": 0.5694444444444444, + "chain_evidence_recall": 1.0, + "deterministic_rate": 1.0, + "forbidden_leaks": 0, + "local_p95_ms": 0.258374959230423, + "route_accuracy": 1.0 + }, + "mode": "evidence_chain", + "passed": true +} diff --git a/backend/tests/eval/report_evidence_chain_ab_fixture.json b/backend/tests/eval/report_evidence_chain_ab_fixture.json new file mode 100644 index 0000000..d4246cd --- /dev/null +++ b/backend/tests/eval/report_evidence_chain_ab_fixture.json @@ -0,0 +1,721 @@ +{ + "case_count": 12, + "cases": [ + { + "baseline": { + "answer": "The original agreement permits termination on thirty days notice. [C1]", + "answer_correct": false, + "citation_count": 1, + "citation_precision": 1.0, + "expected_citation_recall": 0.3333333333333333, + "facts_found": 1, + "facts_required": 4, + "forbidden_phrases_found": [], + "generation_ms": 25.0, + "inline_citation_integrity": true, + "limitations_found": 0, + "limitations_required": 0, + "request_ms": 25.0, + "selected_node_ids": [ + "l1_original" + ], + "unauthorized_node_leaks": [], + "unsupported_answer_safe": true, + "verified_highlight_rate": 1.0, + "wrong_locator_acceptances": 0 + }, + "case_id": "legal_amendment_effective_date", + "chain": { + "answer": "The original agreement permits termination on thirty days notice. [C1] The amendment is effective January 1, 2026 and replaces the termination notice period. [C2] The governing statute requires a minimum sixty day termination notice period. [C3]", + "answer_correct": true, + "citation_count": 3, + "citation_precision": 1.0, + "expected_citation_recall": 1.0, + "facts_found": 4, + "facts_required": 4, + "forbidden_phrases_found": [], + "generation_ms": 25.0, + "inline_citation_integrity": true, + "limitations_found": 0, + "limitations_required": 0, + "request_ms": 25.6338749662973, + "selected_node_ids": [ + "l1_original", + "l1_amendment", + "l1_statute" + ], + "unauthorized_node_leaks": [], + "unsupported_answer_safe": true, + "verified_highlight_rate": 1.0, + "wrong_locator_acceptances": 0 + }, + "chain_applied": true, + "chain_layer_ms": 0.6338749662972987, + "domain": "legal", + "fallback_used": false, + "route_actual": true, + "route_expected": true + }, + { + "baseline": { + "answer": "The prior policy authorizes a five year term. [C1]", + "answer_correct": false, + "citation_count": 1, + "citation_precision": 1.0, + "expected_citation_recall": 0.3333333333333333, + "facts_found": 1, + "facts_required": 3, + "forbidden_phrases_found": [], + "generation_ms": 25.0, + "inline_citation_integrity": true, + "limitations_found": 0, + "limitations_required": 0, + "request_ms": 25.0, + "selected_node_ids": [ + "l2_prior" + ], + "unauthorized_node_leaks": [], + "unsupported_answer_safe": true, + "verified_highlight_rate": 1.0, + "wrong_locator_acceptances": 0 + }, + "case_id": "legal_superseded_policy", + "chain": { + "answer": "The prior policy authorizes a five year term. [C1] The 2025 amendment supersedes the prior policy. [C2] The current policy authorizes a three year term. [C3]", + "answer_correct": true, + "citation_count": 3, + "citation_precision": 1.0, + "expected_citation_recall": 1.0, + "facts_found": 3, + "facts_required": 3, + "forbidden_phrases_found": [], + "generation_ms": 25.0, + "inline_citation_integrity": true, + "limitations_found": 0, + "limitations_required": 0, + "request_ms": 25.418957963120192, + "selected_node_ids": [ + "l2_prior", + "l2_amendment", + "l2_current" + ], + "unauthorized_node_leaks": [], + "unsupported_answer_safe": true, + "verified_highlight_rate": 1.0, + "wrong_locator_acceptances": 0 + }, + "chain_applied": true, + "chain_layer_ms": 0.4189579631201923, + "domain": "legal", + "fallback_used": false, + "route_actual": true, + "route_expected": true + }, + { + "baseline": { + "answer": "The filing fee is $250. [C1]", + "answer_correct": true, + "citation_count": 1, + "citation_precision": 1.0, + "expected_citation_recall": 1.0, + "facts_found": 1, + "facts_required": 1, + "forbidden_phrases_found": [], + "generation_ms": 25.0, + "inline_citation_integrity": true, + "limitations_found": 0, + "limitations_required": 0, + "request_ms": 25.0, + "selected_node_ids": [ + "l3_fee" + ], + "unauthorized_node_leaks": [], + "unsupported_answer_safe": true, + "verified_highlight_rate": 1.0, + "wrong_locator_acceptances": 0 + }, + "case_id": "legal_simple_fee_lookup", + "chain": { + "answer": "The filing fee is $250. [C1]", + "answer_correct": true, + "citation_count": 1, + "citation_precision": 1.0, + "expected_citation_recall": 1.0, + "facts_found": 1, + "facts_required": 1, + "forbidden_phrases_found": [], + "generation_ms": 25.0, + "inline_citation_integrity": true, + "limitations_found": 0, + "limitations_required": 0, + "request_ms": 25.012250035069883, + "selected_node_ids": [ + "l3_fee" + ], + "unauthorized_node_leaks": [], + "unsupported_answer_safe": true, + "verified_highlight_rate": 1.0, + "wrong_locator_acceptances": 0 + }, + "chain_applied": false, + "chain_layer_ms": 0.01225003506988287, + "domain": "legal", + "fallback_used": false, + "route_actual": false, + "route_expected": false + }, + { + "baseline": { + "answer": "Smith held that notice was adequate. [C1]", + "answer_correct": false, + "citation_count": 1, + "citation_precision": 1.0, + "expected_citation_recall": 0.3333333333333333, + "facts_found": 1, + "facts_required": 4, + "forbidden_phrases_found": [], + "generation_ms": 25.0, + "inline_citation_integrity": true, + "limitations_found": 0, + "limitations_required": 0, + "request_ms": 25.0, + "selected_node_ids": [ + "l4_smith" + ], + "unauthorized_node_leaks": [], + "unsupported_answer_safe": true, + "verified_highlight_rate": 1.0, + "wrong_locator_acceptances": 0 + }, + "case_id": "legal_precedent_comparison", + "chain": { + "answer": "Smith held that notice was adequate. [C1] Jones distinguished Smith because the notice omitted a statutory warning. [C2] The statutory warning is mandatory in consumer notices. [C3]", + "answer_correct": true, + "citation_count": 3, + "citation_precision": 1.0, + "expected_citation_recall": 1.0, + "facts_found": 4, + "facts_required": 4, + "forbidden_phrases_found": [], + "generation_ms": 25.0, + "inline_citation_integrity": true, + "limitations_found": 0, + "limitations_required": 0, + "request_ms": 25.461041985545307, + "selected_node_ids": [ + "l4_smith", + "l4_jones", + "l4_rule" + ], + "unauthorized_node_leaks": [], + "unsupported_answer_safe": true, + "verified_highlight_rate": 1.0, + "wrong_locator_acceptances": 0 + }, + "chain_applied": true, + "chain_layer_ms": 0.4610419855453074, + "domain": "legal", + "fallback_used": false, + "route_actual": true, + "route_expected": true + }, + { + "baseline": { + "answer": "The agreement caps liability at one million dollars. [C1] I cannot find sufficient information in the provided context to determine any additional conclusion.", + "answer_correct": true, + "citation_count": 1, + "citation_precision": 1.0, + "expected_citation_recall": 1.0, + "facts_found": 1, + "facts_required": 1, + "forbidden_phrases_found": [], + "generation_ms": 25.0, + "inline_citation_integrity": true, + "limitations_found": 1, + "limitations_required": 1, + "request_ms": 25.0, + "selected_node_ids": [ + "l5_agreement" + ], + "unauthorized_node_leaks": [], + "unsupported_answer_safe": true, + "verified_highlight_rate": 1.0, + "wrong_locator_acceptances": 0 + }, + "case_id": "legal_acl_restricted_amendment", + "chain": { + "answer": "The agreement caps liability at one million dollars. [C1] The public note explains the original liability cap. [C2] I cannot find sufficient information in the provided context to determine any additional conclusion.", + "answer_correct": true, + "citation_count": 2, + "citation_precision": 1.0, + "expected_citation_recall": 1.0, + "facts_found": 1, + "facts_required": 1, + "forbidden_phrases_found": [], + "generation_ms": 25.0, + "inline_citation_integrity": true, + "limitations_found": 1, + "limitations_required": 1, + "request_ms": 25.46875001862645, + "selected_node_ids": [ + "l5_agreement", + "l5_public_note" + ], + "unauthorized_node_leaks": [], + "unsupported_answer_safe": true, + "verified_highlight_rate": 1.0, + "wrong_locator_acceptances": 0 + }, + "chain_applied": true, + "chain_layer_ms": 0.4687500186264515, + "domain": "legal", + "fallback_used": false, + "route_actual": true, + "route_expected": true + }, + { + "baseline": { + "answer": "The patient has severe renal impairment. [C1]", + "answer_correct": false, + "citation_count": 1, + "citation_precision": 1.0, + "expected_citation_recall": 0.3333333333333333, + "facts_found": 1, + "facts_required": 3, + "forbidden_phrases_found": [], + "generation_ms": 25.0, + "inline_citation_integrity": true, + "limitations_found": 0, + "limitations_required": 0, + "request_ms": 25.0, + "selected_node_ids": [ + "c1_condition" + ], + "unauthorized_node_leaks": [], + "unsupported_answer_safe": true, + "verified_highlight_rate": 1.0, + "wrong_locator_acceptances": 0 + }, + "case_id": "clinical_contraindication", + "chain": { + "answer": "The patient has severe renal impairment. [C1] Drug A is primarily cleared by the kidneys. [C2] The current guideline lists severe renal impairment as a contraindication to Drug A. [C3]", + "answer_correct": true, + "citation_count": 3, + "citation_precision": 1.0, + "expected_citation_recall": 1.0, + "facts_found": 3, + "facts_required": 3, + "forbidden_phrases_found": [], + "generation_ms": 25.0, + "inline_citation_integrity": true, + "limitations_found": 0, + "limitations_required": 0, + "request_ms": 25.3456249833107, + "selected_node_ids": [ + "c1_condition", + "c1_drug", + "c1_guideline" + ], + "unauthorized_node_leaks": [], + "unsupported_answer_safe": true, + "verified_highlight_rate": 1.0, + "wrong_locator_acceptances": 0 + }, + "chain_applied": true, + "chain_layer_ms": 0.34562498331069946, + "domain": "clinical", + "fallback_used": false, + "route_actual": true, + "route_expected": true + }, + { + "baseline": { + "answer": "Drug A inhibits platelet aggregation. [C1]", + "answer_correct": false, + "citation_count": 1, + "citation_precision": 1.0, + "expected_citation_recall": 0.3333333333333333, + "facts_found": 1, + "facts_required": 3, + "forbidden_phrases_found": [], + "generation_ms": 25.0, + "inline_citation_integrity": true, + "limitations_found": 0, + "limitations_required": 0, + "request_ms": 25.0, + "selected_node_ids": [ + "c2_drug_a" + ], + "unauthorized_node_leaks": [], + "unsupported_answer_safe": true, + "verified_highlight_rate": 1.0, + "wrong_locator_acceptances": 0 + }, + "case_id": "clinical_drug_interaction", + "chain": { + "answer": "Drug A inhibits platelet aggregation. [C1] Drug B is an anticoagulant. [C2] Combined antiplatelet and anticoagulant therapy increases bleeding risk. [C3]", + "answer_correct": true, + "citation_count": 3, + "citation_precision": 1.0, + "expected_citation_recall": 1.0, + "facts_found": 3, + "facts_required": 3, + "forbidden_phrases_found": [], + "generation_ms": 25.0, + "inline_citation_integrity": true, + "limitations_found": 0, + "limitations_required": 0, + "request_ms": 25.283250003121793, + "selected_node_ids": [ + "c2_drug_a", + "c2_drug_b", + "c2_interaction" + ], + "unauthorized_node_leaks": [], + "unsupported_answer_safe": true, + "verified_highlight_rate": 1.0, + "wrong_locator_acceptances": 0 + }, + "chain_applied": true, + "chain_layer_ms": 0.28325000312179327, + "domain": "clinical", + "fallback_used": false, + "route_actual": true, + "route_expected": true + }, + { + "baseline": { + "answer": "The recommended dose of Drug A is 10 mg daily. [C1]", + "answer_correct": true, + "citation_count": 1, + "citation_precision": 1.0, + "expected_citation_recall": 1.0, + "facts_found": 1, + "facts_required": 1, + "forbidden_phrases_found": [], + "generation_ms": 25.0, + "inline_citation_integrity": true, + "limitations_found": 0, + "limitations_required": 0, + "request_ms": 25.0, + "selected_node_ids": [ + "c3_dose" + ], + "unauthorized_node_leaks": [], + "unsupported_answer_safe": true, + "verified_highlight_rate": 1.0, + "wrong_locator_acceptances": 0 + }, + "case_id": "clinical_simple_dose", + "chain": { + "answer": "The recommended dose of Drug A is 10 mg daily. [C1]", + "answer_correct": true, + "citation_count": 1, + "citation_precision": 1.0, + "expected_citation_recall": 1.0, + "facts_found": 1, + "facts_required": 1, + "forbidden_phrases_found": [], + "generation_ms": 25.0, + "inline_citation_integrity": true, + "limitations_found": 0, + "limitations_required": 0, + "request_ms": 25.01379195600748, + "selected_node_ids": [ + "c3_dose" + ], + "unauthorized_node_leaks": [], + "unsupported_answer_safe": true, + "verified_highlight_rate": 1.0, + "wrong_locator_acceptances": 0 + }, + "chain_applied": false, + "chain_layer_ms": 0.013791956007480621, + "domain": "clinical", + "fallback_used": false, + "route_actual": false, + "route_expected": false + }, + { + "baseline": { + "answer": "Prior guidance recommends Treatment X as first line therapy. [C1]", + "answer_correct": false, + "citation_count": 1, + "citation_precision": 1.0, + "expected_citation_recall": 0.3333333333333333, + "facts_found": 1, + "facts_required": 3, + "forbidden_phrases_found": [], + "generation_ms": 25.0, + "inline_citation_integrity": true, + "limitations_found": 0, + "limitations_required": 0, + "request_ms": 25.0, + "selected_node_ids": [ + "c4_prior" + ], + "unauthorized_node_leaks": [], + "unsupported_answer_safe": true, + "verified_highlight_rate": 1.0, + "wrong_locator_acceptances": 0 + }, + "case_id": "clinical_updated_guideline", + "chain": { + "answer": "Prior guidance recommends Treatment X as first line therapy. [C1] The updated guideline was effective in 2026. [C2] The latest recommendation prefers Treatment Y for high risk patients. [C3]", + "answer_correct": true, + "citation_count": 3, + "citation_precision": 1.0, + "expected_citation_recall": 1.0, + "facts_found": 3, + "facts_required": 3, + "forbidden_phrases_found": [], + "generation_ms": 25.0, + "inline_citation_integrity": true, + "limitations_found": 0, + "limitations_required": 0, + "request_ms": 25.276541977655143, + "selected_node_ids": [ + "c4_prior", + "c4_update", + "c4_latest" + ], + "unauthorized_node_leaks": [], + "unsupported_answer_safe": true, + "verified_highlight_rate": 1.0, + "wrong_locator_acceptances": 0 + }, + "chain_applied": true, + "chain_layer_ms": 0.27654197765514255, + "domain": "clinical", + "fallback_used": false, + "route_actual": true, + "route_expected": true + }, + { + "baseline": { + "answer": "AF means atrial fibrillation in this guideline. [C1]", + "answer_correct": false, + "citation_count": 1, + "citation_precision": 1.0, + "expected_citation_recall": 0.3333333333333333, + "facts_found": 1, + "facts_required": 3, + "forbidden_phrases_found": [], + "generation_ms": 25.0, + "inline_citation_integrity": true, + "limitations_found": 0, + "limitations_required": 0, + "request_ms": 25.0, + "selected_node_ids": [ + "c5_af" + ], + "unauthorized_node_leaks": [], + "unsupported_answer_safe": true, + "verified_highlight_rate": 1.0, + "wrong_locator_acceptances": 0 + }, + "case_id": "clinical_abbreviation_bridge", + "chain": { + "answer": "AF means atrial fibrillation in this guideline. [C1] Atrial fibrillation may require anticoagulation based on risk. [C2] Anticoagulation reduces embolic stroke risk in eligible patients. [C3]", + "answer_correct": true, + "citation_count": 3, + "citation_precision": 1.0, + "expected_citation_recall": 1.0, + "facts_found": 3, + "facts_required": 3, + "forbidden_phrases_found": [], + "generation_ms": 25.0, + "inline_citation_integrity": true, + "limitations_found": 0, + "limitations_required": 0, + "request_ms": 25.25512499269098, + "selected_node_ids": [ + "c5_af", + "c5_anticoag", + "c5_stroke" + ], + "unauthorized_node_leaks": [], + "unsupported_answer_safe": true, + "verified_highlight_rate": 1.0, + "wrong_locator_acceptances": 0 + }, + "chain_applied": true, + "chain_layer_ms": 0.25512499269098043, + "domain": "clinical", + "fallback_used": false, + "route_actual": true, + "route_expected": true + }, + { + "baseline": { + "answer": "Example Clinic, 100 Main Street. [C1]", + "answer_correct": true, + "citation_count": 1, + "citation_precision": 1.0, + "expected_citation_recall": 1.0, + "facts_found": 2, + "facts_required": 2, + "forbidden_phrases_found": [], + "generation_ms": 25.0, + "inline_citation_integrity": true, + "limitations_found": 0, + "limitations_required": 0, + "request_ms": 25.0, + "selected_node_ids": [ + "r1_cover" + ], + "unauthorized_node_leaks": [], + "unsupported_answer_safe": true, + "verified_highlight_rate": 1.0, + "wrong_locator_acceptances": 0 + }, + "case_id": "regression_conjunction_only", + "chain": { + "answer": "Example Clinic, 100 Main Street. [C1]", + "answer_correct": true, + "citation_count": 1, + "citation_precision": 1.0, + "expected_citation_recall": 1.0, + "facts_found": 2, + "facts_required": 2, + "forbidden_phrases_found": [], + "generation_ms": 25.0, + "inline_citation_integrity": true, + "limitations_found": 0, + "limitations_required": 0, + "request_ms": 25.011334021110088, + "selected_node_ids": [ + "r1_cover" + ], + "unauthorized_node_leaks": [], + "unsupported_answer_safe": true, + "verified_highlight_rate": 1.0, + "wrong_locator_acceptances": 0 + }, + "chain_applied": false, + "chain_layer_ms": 0.011334021110087633, + "domain": "regression", + "fallback_used": false, + "route_actual": false, + "route_expected": false + }, + { + "baseline": { + "answer": "I cannot find sufficient information in the provided context to determine any additional conclusion.", + "answer_correct": true, + "citation_count": 0, + "citation_precision": 1.0, + "expected_citation_recall": 1.0, + "facts_found": 0, + "facts_required": 0, + "forbidden_phrases_found": [], + "generation_ms": 25.0, + "inline_citation_integrity": true, + "limitations_found": 1, + "limitations_required": 1, + "request_ms": 25.0, + "selected_node_ids": [ + "r2_context" + ], + "unauthorized_node_leaks": [], + "unsupported_answer_safe": true, + "verified_highlight_rate": 1.0, + "wrong_locator_acceptances": 0 + }, + "case_id": "regression_unsupported_why", + "chain": { + "answer": "I cannot find sufficient information in the provided context to determine any additional conclusion.", + "answer_correct": true, + "citation_count": 0, + "citation_precision": 1.0, + "expected_citation_recall": 1.0, + "facts_found": 0, + "facts_required": 0, + "forbidden_phrases_found": [], + "generation_ms": 25.0, + "inline_citation_integrity": true, + "limitations_found": 1, + "limitations_required": 1, + "request_ms": 25.006584043148905, + "selected_node_ids": [ + "r2_context" + ], + "unauthorized_node_leaks": [], + "unsupported_answer_safe": true, + "verified_highlight_rate": 1.0, + "wrong_locator_acceptances": 0 + }, + "chain_applied": false, + "chain_layer_ms": 0.006584043148905039, + "domain": "regression", + "fallback_used": false, + "route_actual": false, + "route_expected": false + } + ], + "contract_id": "rag_eval_contract", + "dataset_sha256": "607ec987d8b032858a48fc03c7ca87d1c8db498e46c3a5d8621610e34bc7c7b7", + "expectations_sha256": "5228ebe3c22f4f262e042ca580bd1b2f62256e0e8e406646f9b2f384eb3b9ee5", + "gates": { + "chain_p95_overhead_at_most_20_percent": true, + "citation_precision_no_regression": true, + "curated_chain_answer_accuracy_100_percent": true, + "curated_chain_citation_precision_100_percent": true, + "curated_chain_selected_evidence_recall_100_percent": true, + "curated_chain_verified_highlights_100_percent": true, + "expected_citation_recall_no_regression": true, + "fallback_rate_at_most_10_percent": true, + "inline_citation_integrity_100_percent": true, + "route_accuracy_100_percent": true, + "routed_answer_accuracy_improves_by_5_points": true, + "single_hop_no_regression": true, + "unauthorized_leakage_zero": true, + "unsupported_answers_no_regression": true, + "verified_highlights_no_regression": true, + "wrong_document_page_version_fail_closed": true + }, + "metrics": { + "baseline_answer_accuracy": 0.4166666666666667, + "baseline_citation_precision": 1.0, + "baseline_expected_citation_recall": 0.611111111111111, + "baseline_request_p50_ms": 25.0, + "baseline_request_p95_ms": 25.0, + "baseline_request_p99_ms": 25.0, + "baseline_selected_evidence_recall": 0.611111111111111, + "baseline_verified_highlight_rate": 1.0, + "chain_answer_accuracy": 1.0, + "chain_citation_precision": 1.0, + "chain_expected_citation_recall": 1.0, + "chain_layer_p50_ms": 0.28325000312179327, + "chain_layer_p95_ms": 0.4687500186264515, + "chain_layer_p99_ms": 0.6338749662972987, + "chain_p95_overhead_ratio": 0.01875000074505806, + "chain_request_p50_ms": 25.283250003121793, + "chain_request_p95_ms": 25.46875001862645, + "chain_request_p99_ms": 25.6338749662973, + "chain_selected_evidence_recall": 1.0, + "chain_verified_highlight_rate": 1.0, + "fallback_rate": 0.0, + "paired_total_request_overhead_p50_ratio": 0.01133000012487173, + "paired_total_request_overhead_p95_ratio": 0.01875000074505806, + "route_accuracy": 1.0, + "routed_absolute_accuracy_improvement": 0.875, + "routed_baseline_answer_accuracy": 0.125, + "routed_chain_answer_accuracy": 1.0, + "single_hop_baseline_accuracy": 1.0, + "single_hop_chain_accuracy": 1.0, + "total_request_p95_overhead_ratio": 0.01875000074505806, + "unauthorized_node_leaks": 0, + "unsupported_baseline_safe_rate": 1.0, + "unsupported_chain_safe_rate": 1.0, + "wrong_locator_acceptances": 0 + }, + "mode": "evidence_chain_ab", + "model": "deterministic-grounded-fixture-v1", + "observations": { + "total_request_latency_is_measured_not_gated": true, + "total_request_latency_note": "Total provider latency is reported separately from the chain-layer release gate and requires representative load testing before default-on." + }, + "passed": true, + "provider": "fixture" +} diff --git a/backend/tests/eval/report_evidence_chain_ab_openai.json b/backend/tests/eval/report_evidence_chain_ab_openai.json new file mode 100644 index 0000000..baa2cf6 --- /dev/null +++ b/backend/tests/eval/report_evidence_chain_ab_openai.json @@ -0,0 +1,721 @@ +{ + "case_count": 12, + "cases": [ + { + "baseline": { + "answer": "I cannot find sufficient information in the provided context to answer this question. The only provided context states that the original agreement permits termination on thirty days notice [C1], but it does not include the amendment\u2019s effective date or any information about a governing statute.", + "answer_correct": false, + "citation_count": 1, + "citation_precision": 1.0, + "expected_citation_recall": 0.3333333333333333, + "facts_found": 1, + "facts_required": 4, + "forbidden_phrases_found": [], + "generation_ms": 2200.389862060547, + "inline_citation_integrity": true, + "limitations_found": 0, + "limitations_required": 0, + "request_ms": 2200.391083024442, + "selected_node_ids": [ + "l1_original" + ], + "unauthorized_node_leaks": [], + "unsupported_answer_safe": true, + "verified_highlight_rate": 1.0, + "wrong_locator_acceptances": 0 + }, + "case_id": "legal_amendment_effective_date", + "chain": { + "answer": "The original agreement allows termination on thirty days notice [C1]. The amendment is effective January 1, 2026 and replaces the termination notice period, meaning the original thirty-day clause is superseded as of that effective date [C2]. Separately, the governing statute requires a minimum sixty day termination notice period, so any termination notice period (including the original thirty-day period and whatever the amendment replaces it with) must be at least sixty days to comply with the statute [C3].", + "answer_correct": true, + "citation_count": 3, + "citation_precision": 1.0, + "expected_citation_recall": 1.0, + "facts_found": 4, + "facts_required": 4, + "forbidden_phrases_found": [], + "generation_ms": 3310.6062412261963, + "inline_citation_integrity": true, + "limitations_found": 0, + "limitations_required": 0, + "request_ms": 3310.993666993454, + "selected_node_ids": [ + "l1_original", + "l1_amendment", + "l1_statute" + ], + "unauthorized_node_leaks": [], + "unsupported_answer_safe": true, + "verified_highlight_rate": 1.0, + "wrong_locator_acceptances": 0 + }, + "chain_applied": true, + "chain_layer_ms": 0.38579199463129044, + "domain": "legal", + "fallback_used": false, + "route_actual": true, + "route_expected": true + }, + { + "baseline": { + "answer": "I cannot find sufficient information in the provided context to answer which policy is current after the 2025 amendment. The only available detail is that the prior policy authorizes a five year term [C1], but the context does not describe the 2025 amendment, identify the current policy, or state how it differs beyond the prior term length.", + "answer_correct": false, + "citation_count": 1, + "citation_precision": 1.0, + "expected_citation_recall": 0.3333333333333333, + "facts_found": 1, + "facts_required": 3, + "forbidden_phrases_found": [], + "generation_ms": 2341.052770614624, + "inline_citation_integrity": true, + "limitations_found": 0, + "limitations_required": 0, + "request_ms": 2341.1155419889838, + "selected_node_ids": [ + "l2_prior" + ], + "unauthorized_node_leaks": [], + "unsupported_answer_safe": true, + "verified_highlight_rate": 1.0, + "wrong_locator_acceptances": 0 + }, + "case_id": "legal_superseded_policy", + "chain": { + "answer": "After the 2025 amendment, the current policy is the one described as \u201cThe current policy authorizes a three year term\u201d [C3], because \u201cThe 2025 amendment supersedes the prior policy\u201d [C2]. The prior policy \u201cauthorizes a five year term\u201d [C1], so the current policy differs by authorizing a three-year term instead of a five-year term [C3] [C1].", + "answer_correct": true, + "citation_count": 3, + "citation_precision": 1.0, + "expected_citation_recall": 1.0, + "facts_found": 3, + "facts_required": 3, + "forbidden_phrases_found": [], + "generation_ms": 2782.790184020996, + "inline_citation_integrity": true, + "limitations_found": 0, + "limitations_required": 0, + "request_ms": 2784.084791026544, + "selected_node_ids": [ + "l2_prior", + "l2_amendment", + "l2_current" + ], + "unauthorized_node_leaks": [], + "unsupported_answer_safe": true, + "verified_highlight_rate": 1.0, + "wrong_locator_acceptances": 0 + }, + "chain_applied": true, + "chain_layer_ms": 1.2677910272032022, + "domain": "legal", + "fallback_used": false, + "route_actual": true, + "route_expected": true + }, + { + "baseline": { + "answer": "The filing fee is $250 [C1].", + "answer_correct": true, + "citation_count": 1, + "citation_precision": 1.0, + "expected_citation_recall": 1.0, + "facts_found": 1, + "facts_required": 1, + "forbidden_phrases_found": [], + "generation_ms": 1203.7100791931152, + "inline_citation_integrity": true, + "limitations_found": 0, + "limitations_required": 0, + "request_ms": 1203.7336249486543, + "selected_node_ids": [ + "l3_fee" + ], + "unauthorized_node_leaks": [], + "unsupported_answer_safe": true, + "verified_highlight_rate": 1.0, + "wrong_locator_acceptances": 0 + }, + "case_id": "legal_simple_fee_lookup", + "chain": { + "answer": "The filing fee is $250 [C1].", + "answer_correct": true, + "citation_count": 1, + "citation_precision": 1.0, + "expected_citation_recall": 1.0, + "facts_found": 1, + "facts_required": 1, + "forbidden_phrases_found": [], + "generation_ms": 1145.2832221984863, + "inline_citation_integrity": true, + "limitations_found": 0, + "limitations_required": 0, + "request_ms": 1145.358584006317, + "selected_node_ids": [ + "l3_fee" + ], + "unauthorized_node_leaks": [], + "unsupported_answer_safe": true, + "verified_highlight_rate": 1.0, + "wrong_locator_acceptances": 0 + }, + "chain_applied": false, + "chain_layer_ms": 0.05204195622354746, + "domain": "legal", + "fallback_used": false, + "route_actual": false, + "route_expected": false + }, + { + "baseline": { + "answer": "In Smith, the court held that notice was adequate [C1]. I cannot find sufficient information in the provided context about Jones (its holding or reasoning), so I cannot compare Smith and Jones or explain why a later court reached a different result.", + "answer_correct": false, + "citation_count": 1, + "citation_precision": 1.0, + "expected_citation_recall": 0.3333333333333333, + "facts_found": 0, + "facts_required": 4, + "forbidden_phrases_found": [], + "generation_ms": 1946.1469650268555, + "inline_citation_integrity": true, + "limitations_found": 0, + "limitations_required": 0, + "request_ms": 1946.1559160263278, + "selected_node_ids": [ + "l4_smith" + ], + "unauthorized_node_leaks": [], + "unsupported_answer_safe": true, + "verified_highlight_rate": 1.0, + "wrong_locator_acceptances": 0 + }, + "case_id": "legal_precedent_comparison", + "chain": { + "answer": "In Smith, the court held that the notice at issue was adequate [C1]. In Jones, the later court distinguished Smith on the ground that the notice in Jones omitted a statutory warning [C2]. The later court reached a different result because the statutory warning is mandatory in consumer notices, so omitting it provides a basis to treat the notice as deficient even if Smith found notice adequate on its facts [C3].", + "answer_correct": true, + "citation_count": 3, + "citation_precision": 1.0, + "expected_citation_recall": 1.0, + "facts_found": 4, + "facts_required": 4, + "forbidden_phrases_found": [], + "generation_ms": 2888.184070587158, + "inline_citation_integrity": true, + "limitations_found": 0, + "limitations_required": 0, + "request_ms": 2888.943665951956, + "selected_node_ids": [ + "l4_smith", + "l4_jones", + "l4_rule" + ], + "unauthorized_node_leaks": [], + "unsupported_answer_safe": true, + "verified_highlight_rate": 1.0, + "wrong_locator_acceptances": 0 + }, + "chain_applied": true, + "chain_layer_ms": 0.7595829665660858, + "domain": "legal", + "fallback_used": false, + "route_actual": true, + "route_expected": true + }, + { + "baseline": { + "answer": "I cannot find sufficient information in the provided context to compare the agreement with its amendment or explain how an amendment affects liability. The only provided excerpt states that the agreement caps liability at one million dollars [C1], but it contains no information about any amendment or changes to liability terms.", + "answer_correct": true, + "citation_count": 1, + "citation_precision": 1.0, + "expected_citation_recall": 1.0, + "facts_found": 1, + "facts_required": 1, + "forbidden_phrases_found": [], + "generation_ms": 1959.5730304718018, + "inline_citation_integrity": true, + "limitations_found": 1, + "limitations_required": 1, + "request_ms": 1959.5843749702908, + "selected_node_ids": [ + "l5_agreement" + ], + "unauthorized_node_leaks": [], + "unsupported_answer_safe": true, + "verified_highlight_rate": 1.0, + "wrong_locator_acceptances": 0 + }, + "case_id": "legal_acl_restricted_amendment", + "chain": { + "answer": "The agreement states that it \"caps liability at one million dollars\" [C1]. The provided context mentions an amendment only indirectly via a \u201cpublic note\u201d that \"explains the original liability cap\" [C2], but it does not describe any amended terms or changes to the cap. I therefore cannot compare the agreement with its amendment or explain how the amendment affects liability based on the provided context.", + "answer_correct": true, + "citation_count": 2, + "citation_precision": 1.0, + "expected_citation_recall": 1.0, + "facts_found": 1, + "facts_required": 1, + "forbidden_phrases_found": [], + "generation_ms": 2805.333137512207, + "inline_citation_integrity": true, + "limitations_found": 1, + "limitations_required": 1, + "request_ms": 2805.9042215463705, + "selected_node_ids": [ + "l5_agreement", + "l5_public_note" + ], + "unauthorized_node_leaks": [], + "unsupported_answer_safe": true, + "verified_highlight_rate": 1.0, + "wrong_locator_acceptances": 0 + }, + "chain_applied": true, + "chain_layer_ms": 0.5710840341635048, + "domain": "legal", + "fallback_used": false, + "route_actual": true, + "route_expected": true + }, + { + "baseline": { + "answer": "I cannot find sufficient information in the provided context to answer this question. The only provided context states that the patient has severe renal impairment [C1], but it does not mention Drug A, the current guideline, dosing adjustments, or any contraindication related to renal impairment.", + "answer_correct": false, + "citation_count": 1, + "citation_precision": 1.0, + "expected_citation_recall": 0.3333333333333333, + "facts_found": 1, + "facts_required": 3, + "forbidden_phrases_found": [], + "generation_ms": 1857.0632934570312, + "inline_citation_integrity": true, + "limitations_found": 0, + "limitations_required": 0, + "request_ms": 1857.0777499699034, + "selected_node_ids": [ + "c1_condition" + ], + "unauthorized_node_leaks": [], + "unsupported_answer_safe": true, + "verified_highlight_rate": 1.0, + "wrong_locator_acceptances": 0 + }, + "case_id": "clinical_contraindication", + "chain": { + "answer": "Drug A is primarily cleared by the kidneys [C1]. Under the current guideline, severe renal impairment is listed as a contraindication to Drug A [C2].", + "answer_correct": true, + "citation_count": 2, + "citation_precision": 1.0, + "expected_citation_recall": 0.6666666666666666, + "facts_found": 3, + "facts_required": 3, + "forbidden_phrases_found": [], + "generation_ms": 2002.3431777954102, + "inline_citation_integrity": true, + "limitations_found": 0, + "limitations_required": 0, + "request_ms": 2003.291376051493, + "selected_node_ids": [ + "c1_condition", + "c1_drug", + "c1_guideline" + ], + "unauthorized_node_leaks": [], + "unsupported_answer_safe": true, + "verified_highlight_rate": 1.0, + "wrong_locator_acceptances": 0 + }, + "chain_applied": true, + "chain_layer_ms": 0.9402920259162784, + "domain": "clinical", + "fallback_used": false, + "route_actual": true, + "route_expected": true + }, + { + "baseline": { + "answer": "I cannot find sufficient information in the provided context to answer this question. The context only states that Drug A inhibits platelet aggregation [C1], and provides no information about Drug B or any interaction between Drug A and Drug B affecting bleeding risk.", + "answer_correct": false, + "citation_count": 1, + "citation_precision": 1.0, + "expected_citation_recall": 0.3333333333333333, + "facts_found": 1, + "facts_required": 3, + "forbidden_phrases_found": [], + "generation_ms": 2107.877016067505, + "inline_citation_integrity": true, + "limitations_found": 0, + "limitations_required": 0, + "request_ms": 2107.8789160237648, + "selected_node_ids": [ + "c2_drug_a" + ], + "unauthorized_node_leaks": [], + "unsupported_answer_safe": true, + "verified_highlight_rate": 1.0, + "wrong_locator_acceptances": 0 + }, + "case_id": "clinical_drug_interaction", + "chain": { + "answer": "Drug A inhibits platelet aggregation, making it an antiplatelet agent [C1]. Drug B is an anticoagulant [C2]. Their interaction affects bleeding risk because combined antiplatelet and anticoagulant therapy increases bleeding risk [C3].", + "answer_correct": true, + "citation_count": 3, + "citation_precision": 1.0, + "expected_citation_recall": 1.0, + "facts_found": 3, + "facts_required": 3, + "forbidden_phrases_found": [], + "generation_ms": 2828.1238079071045, + "inline_citation_integrity": true, + "limitations_found": 0, + "limitations_required": 0, + "request_ms": 2829.034557857085, + "selected_node_ids": [ + "c2_drug_a", + "c2_drug_b", + "c2_interaction" + ], + "unauthorized_node_leaks": [], + "unsupported_answer_safe": true, + "verified_highlight_rate": 1.0, + "wrong_locator_acceptances": 0 + }, + "chain_applied": true, + "chain_layer_ms": 0.9107499499805272, + "domain": "clinical", + "fallback_used": false, + "route_actual": true, + "route_expected": true + }, + { + "baseline": { + "answer": "The recommended dose of Drug A is 10 mg daily [C1].", + "answer_correct": true, + "citation_count": 1, + "citation_precision": 1.0, + "expected_citation_recall": 1.0, + "facts_found": 1, + "facts_required": 1, + "forbidden_phrases_found": [], + "generation_ms": 1498.777151107788, + "inline_citation_integrity": true, + "limitations_found": 0, + "limitations_required": 0, + "request_ms": 1498.777151107788, + "selected_node_ids": [ + "c3_dose" + ], + "unauthorized_node_leaks": [], + "unsupported_answer_safe": true, + "verified_highlight_rate": 1.0, + "wrong_locator_acceptances": 0 + }, + "case_id": "clinical_simple_dose", + "chain": { + "answer": "The recommended dose of Drug A is 10 mg daily [C1].", + "answer_correct": true, + "citation_count": 1, + "citation_precision": 1.0, + "expected_citation_recall": 1.0, + "facts_found": 1, + "facts_required": 1, + "forbidden_phrases_found": [], + "generation_ms": 1761.9729042053223, + "inline_citation_integrity": true, + "limitations_found": 0, + "limitations_required": 0, + "request_ms": 1762.024124036543, + "selected_node_ids": [ + "c3_dose" + ], + "unauthorized_node_leaks": [], + "unsupported_answer_safe": true, + "verified_highlight_rate": 1.0, + "wrong_locator_acceptances": 0 + }, + "chain_applied": false, + "chain_layer_ms": 0.04245800664648414, + "domain": "clinical", + "fallback_used": false, + "route_actual": false, + "route_expected": false + }, + { + "baseline": { + "answer": "I cannot find sufficient information in the provided context to answer this question. The only provided context states that prior guidance recommends Treatment X as first line therapy [C1], but it does not include any updated guideline content or a latest recommendation to compare against the prior guidance.", + "answer_correct": false, + "citation_count": 1, + "citation_precision": 1.0, + "expected_citation_recall": 0.3333333333333333, + "facts_found": 1, + "facts_required": 3, + "forbidden_phrases_found": [], + "generation_ms": 1973.4079837799072, + "inline_citation_integrity": true, + "limitations_found": 0, + "limitations_required": 0, + "request_ms": 1973.4198750229552, + "selected_node_ids": [ + "c4_prior" + ], + "unauthorized_node_leaks": [], + "unsupported_answer_safe": true, + "verified_highlight_rate": 1.0, + "wrong_locator_acceptances": 0 + }, + "case_id": "clinical_updated_guideline", + "chain": { + "answer": "The updated guideline introduced a new effective date: it was effective in 2026 [C2]. The prior guidance recommended Treatment X as first line therapy [C1], while the latest recommendation prefers Treatment Y for high risk patients [C3].", + "answer_correct": true, + "citation_count": 3, + "citation_precision": 1.0, + "expected_citation_recall": 1.0, + "facts_found": 3, + "facts_required": 3, + "forbidden_phrases_found": [], + "generation_ms": 2735.893964767456, + "inline_citation_integrity": true, + "limitations_found": 0, + "limitations_required": 0, + "request_ms": 2736.6330410004593, + "selected_node_ids": [ + "c4_prior", + "c4_update", + "c4_latest" + ], + "unauthorized_node_leaks": [], + "unsupported_answer_safe": true, + "verified_highlight_rate": 1.0, + "wrong_locator_acceptances": 0 + }, + "chain_applied": true, + "chain_layer_ms": 0.726457976270467, + "domain": "clinical", + "fallback_used": false, + "route_actual": true, + "route_expected": true + }, + { + "baseline": { + "answer": "I cannot find sufficient information in the provided context to answer this question.", + "answer_correct": false, + "citation_count": 0, + "citation_precision": 1.0, + "expected_citation_recall": 0.0, + "facts_found": 0, + "facts_required": 3, + "forbidden_phrases_found": [], + "generation_ms": 1194.7970390319824, + "inline_citation_integrity": true, + "limitations_found": 0, + "limitations_required": 0, + "request_ms": 1194.8121250024997, + "selected_node_ids": [ + "c5_af" + ], + "unauthorized_node_leaks": [], + "unsupported_answer_safe": true, + "verified_highlight_rate": 1.0, + "wrong_locator_acceptances": 0 + }, + "case_id": "clinical_abbreviation_bridge", + "chain": { + "answer": "In this guideline, \u201cAF\u201d refers to atrial fibrillation [C1]. Atrial fibrillation may require anticoagulation based on risk [C2]. This matters for stroke prevention because anticoagulation reduces embolic stroke risk in eligible patients [C3].", + "answer_correct": true, + "citation_count": 3, + "citation_precision": 1.0, + "expected_citation_recall": 1.0, + "facts_found": 3, + "facts_required": 3, + "forbidden_phrases_found": [], + "generation_ms": 2594.5372581481934, + "inline_citation_integrity": true, + "limitations_found": 0, + "limitations_required": 0, + "request_ms": 2595.4498001374304, + "selected_node_ids": [ + "c5_af", + "c5_anticoag", + "c5_stroke" + ], + "unauthorized_node_leaks": [], + "unsupported_answer_safe": true, + "verified_highlight_rate": 1.0, + "wrong_locator_acceptances": 0 + }, + "chain_applied": true, + "chain_layer_ms": 0.9125419892370701, + "domain": "clinical", + "fallback_used": false, + "route_actual": true, + "route_expected": true + }, + { + "baseline": { + "answer": "The cover page lists the name as \u201cExample Clinic\u201d and the address as \u201c100 Main Street\u201d [C1].", + "answer_correct": true, + "citation_count": 1, + "citation_precision": 1.0, + "expected_citation_recall": 1.0, + "facts_found": 2, + "facts_required": 2, + "forbidden_phrases_found": [], + "generation_ms": 1757.416009902954, + "inline_citation_integrity": true, + "limitations_found": 0, + "limitations_required": 0, + "request_ms": 1757.424584007822, + "selected_node_ids": [ + "r1_cover" + ], + "unauthorized_node_leaks": [], + "unsupported_answer_safe": true, + "verified_highlight_rate": 1.0, + "wrong_locator_acceptances": 0 + }, + "case_id": "regression_conjunction_only", + "chain": { + "answer": "The cover page lists the name as \u201cExample Clinic\u201d and the address as \u201c100 Main Street\u201d [C1].", + "answer_correct": true, + "citation_count": 1, + "citation_precision": 1.0, + "expected_citation_recall": 1.0, + "facts_found": 2, + "facts_required": 2, + "forbidden_phrases_found": [], + "generation_ms": 1558.6318969726562, + "inline_citation_integrity": true, + "limitations_found": 0, + "limitations_required": 0, + "request_ms": 1558.6666250019334, + "selected_node_ids": [ + "r1_cover" + ], + "unauthorized_node_leaks": [], + "unsupported_answer_safe": true, + "verified_highlight_rate": 1.0, + "wrong_locator_acceptances": 0 + }, + "chain_applied": false, + "chain_layer_ms": 0.02362497616559267, + "domain": "regression", + "fallback_used": false, + "route_actual": false, + "route_expected": false + }, + { + "baseline": { + "answer": "I cannot find sufficient information in the provided context to answer this question.", + "answer_correct": true, + "citation_count": 0, + "citation_precision": 1.0, + "expected_citation_recall": 1.0, + "facts_found": 0, + "facts_required": 0, + "forbidden_phrases_found": [], + "generation_ms": 1075.3693580627441, + "inline_citation_integrity": true, + "limitations_found": 1, + "limitations_required": 1, + "request_ms": 1075.390084006358, + "selected_node_ids": [ + "r2_context" + ], + "unauthorized_node_leaks": [], + "unsupported_answer_safe": true, + "verified_highlight_rate": 1.0, + "wrong_locator_acceptances": 0 + }, + "case_id": "regression_unsupported_why", + "chain": { + "answer": "I cannot find sufficient information in the provided context to answer this question.", + "answer_correct": true, + "citation_count": 0, + "citation_precision": 1.0, + "expected_citation_recall": 1.0, + "facts_found": 0, + "facts_required": 0, + "forbidden_phrases_found": [], + "generation_ms": 1050.6219863891602, + "inline_citation_integrity": true, + "limitations_found": 1, + "limitations_required": 1, + "request_ms": 1050.6701240083203, + "selected_node_ids": [ + "r2_context" + ], + "unauthorized_node_leaks": [], + "unsupported_answer_safe": true, + "verified_highlight_rate": 1.0, + "wrong_locator_acceptances": 0 + }, + "chain_applied": false, + "chain_layer_ms": 0.02420798409730196, + "domain": "regression", + "fallback_used": false, + "route_actual": false, + "route_expected": false + } + ], + "contract_id": "rag_eval_contract", + "dataset_sha256": "607ec987d8b032858a48fc03c7ca87d1c8db498e46c3a5d8621610e34bc7c7b7", + "expectations_sha256": "5228ebe3c22f4f262e042ca580bd1b2f62256e0e8e406646f9b2f384eb3b9ee5", + "gates": { + "chain_p95_overhead_at_most_20_percent": true, + "citation_precision_no_regression": true, + "curated_chain_answer_accuracy_100_percent": true, + "curated_chain_citation_precision_100_percent": true, + "curated_chain_selected_evidence_recall_100_percent": true, + "curated_chain_verified_highlights_100_percent": true, + "expected_citation_recall_no_regression": true, + "fallback_rate_at_most_10_percent": true, + "inline_citation_integrity_100_percent": true, + "route_accuracy_100_percent": true, + "routed_answer_accuracy_improves_by_5_points": true, + "single_hop_no_regression": true, + "unauthorized_leakage_zero": true, + "unsupported_answers_no_regression": true, + "verified_highlights_no_regression": true, + "wrong_document_page_version_fail_closed": true + }, + "metrics": { + "baseline_answer_accuracy": 0.4166666666666667, + "baseline_citation_precision": 1.0, + "baseline_expected_citation_recall": 0.5833333333333334, + "baseline_request_p50_ms": 1946.1559160263278, + "baseline_request_p95_ms": 2200.391083024442, + "baseline_request_p99_ms": 2341.1155419889838, + "baseline_selected_evidence_recall": 0.611111111111111, + "baseline_verified_highlight_rate": 1.0, + "chain_answer_accuracy": 1.0, + "chain_citation_precision": 1.0, + "chain_expected_citation_recall": 0.9722222222222222, + "chain_layer_p50_ms": 0.726457976270467, + "chain_layer_p95_ms": 0.9402920259162784, + "chain_layer_p99_ms": 1.2677910272032022, + "chain_p95_overhead_ratio": 0.00042732950209189405, + "chain_request_p50_ms": 2736.6330410004593, + "chain_request_p95_ms": 2888.943665951956, + "chain_request_p99_ms": 3310.993666993454, + "chain_selected_evidence_recall": 1.0, + "chain_verified_highlight_rate": 1.0, + "fallback_rate": 0.0, + "paired_total_request_overhead_p50_ratio": 0.34212384608584867, + "paired_total_request_overhead_p95_ratio": 0.5047296330807187, + "route_accuracy": 1.0, + "routed_absolute_accuracy_improvement": 0.875, + "routed_baseline_answer_accuracy": 0.125, + "routed_chain_answer_accuracy": 1.0, + "single_hop_baseline_accuracy": 1.0, + "single_hop_chain_accuracy": 1.0, + "total_request_p95_overhead_ratio": 0.3129228200566496, + "unauthorized_node_leaks": 0, + "unsupported_baseline_safe_rate": 1.0, + "unsupported_chain_safe_rate": 1.0, + "wrong_locator_acceptances": 0 + }, + "mode": "evidence_chain_ab", + "model": "gpt-5.2", + "observations": { + "total_request_latency_is_measured_not_gated": true, + "total_request_latency_note": "Total provider latency is reported separately from the chain-layer release gate and requires representative load testing before default-on." + }, + "passed": true, + "provider": "openai" +} diff --git a/backend/tests/eval/run_evidence_chain_ab.py b/backend/tests/eval/run_evidence_chain_ab.py new file mode 100644 index 0000000..d955b1f --- /dev/null +++ b/backend/tests/eval/run_evidence_chain_ab.py @@ -0,0 +1,736 @@ +"""Paired answer-level A/B evaluation for query-aware evidence chains. + +The locked retrieval fixture and separately locked, human-authored answer +expectations are evaluated without an LLM judge. The deterministic provider +is suitable for CI and verifies the whole context/citation/highlight contract. +The OpenAI provider runs the same paired cases through the configured answer +model for a representative model-backed release report. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import re +import sys +import time +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Any, Iterable, Protocol + +BACKEND_ROOT = Path(__file__).resolve().parents[2] +if str(BACKEND_ROOT) not in sys.path: + sys.path.insert(0, str(BACKEND_ROOT)) + +from app.graph.context_packer import ContextPacker +from app.llm.openai_client import AnswerResult, Citation, OpenAIClient +from app.qa.evidence_chain import EvidenceChainConfig, EvidenceChainEngine +from app.services.highlighting import verify_evidence_span +from tests.eval.run_evidence_chain_eval import ( + CONTRACT_PATH, + DATASET_PATH, + FixtureACL, + FixtureDB, + _make_case_graph, +) + + +HERE = Path(__file__).resolve().parent +EXPECTATIONS_PATH = HERE / "evidence_chain_ab_expectations.json" +SOURCE_HASH = "sha256:evidence-chain-ab-fixture" + + +class AnswerProvider(Protocol): + name: str + model: str + + def generate( + self, + *, + context: str, + question: str, + nodes: list[Any], + expectation: dict[str, Any], + ) -> AnswerResult: ... + + +class FixtureAnswerProvider: + """Stable generator that exercises answer/citation plumbing in CI.""" + + name = "fixture" + model = "deterministic-grounded-fixture-v1" + + def generate( + self, + *, + context: str, + question: str, + nodes: list[Any], + expectation: dict[str, Any], + ) -> AnswerResult: + del context, question + started = time.perf_counter() + citations: list[Citation] = [] + answer_parts: list[str] = [] + + # A fixture with no expected support represents a deliberately + # unsupported question and must abstain rather than cite irrelevant text. + if ( + expectation["expected_citation_nodes"] + and expectation.get("citation_policy") != "optional" + ): + for index, node in enumerate(nodes, start=1): + citation_id = f"C{index}" + text = node.text_plain or node.text_md or "" + citations.append( + Citation( + citation_id=citation_id, + node_id=node.node_id, + page_no=node.page_no, + exact_quote=text, + ) + ) + answer_parts.append(f"{text} [{citation_id}]") + + if expectation["required_limitation_groups"]: + answer_parts.append( + "I cannot find sufficient information in the provided context " + "to determine any additional conclusion." + ) + + generation_ms = max((time.perf_counter() - started) * 1000, 25.0) + return AnswerResult( + answer=" ".join(answer_parts), + citations=citations, + model_id=self.model, + generation_time_ms=generation_ms, + ) + + +class OpenAIAnswerProvider: + name = "openai" + + def __init__(self, model: str): + self.client = OpenAIClient(model=model) + self.model = model + + def generate( + self, + *, + context: str, + question: str, + nodes: list[Any], + expectation: dict[str, Any], + ) -> AnswerResult: + del nodes, expectation + return self.client.generate_answer( + context=context, + question=question, + max_tokens=700, + reasoning_effort="none", + verbosity="low", + timeout_ms=45000, + ) + + +@dataclass +class VariantResult: + answer: str + answer_correct: bool + facts_found: int + facts_required: int + limitations_found: int + limitations_required: int + forbidden_phrases_found: list[str] + citation_count: int + citation_precision: float + expected_citation_recall: float + inline_citation_integrity: bool + verified_highlight_rate: float + wrong_locator_acceptances: int + unsupported_answer_safe: bool + unauthorized_node_leaks: list[str] + generation_ms: float + request_ms: float + selected_node_ids: list[str] + + +@dataclass +class PairedCaseResult: + case_id: str + domain: str + route_expected: bool + route_actual: bool + chain_applied: bool + fallback_used: bool + chain_layer_ms: float + baseline: VariantResult + chain: VariantResult + + +def _sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def _load_locked_inputs() -> tuple[dict[str, Any], dict[str, dict[str, Any]]]: + contract = json.loads(CONTRACT_PATH.read_text(encoding="utf-8")) + mode = contract["modes"].get("evidence_chain_ab") + if mode is None: + raise RuntimeError("benchmark contract has no evidence_chain_ab mode") + hashes = { + "dataset": (_sha256(DATASET_PATH), mode["dataset_sha256"]), + "expectations": (_sha256(EXPECTATIONS_PATH), mode["expectations_sha256"]), + } + for label, (actual, expected) in hashes.items(): + if actual != expected: + raise RuntimeError( + f"evidence-chain A/B {label} hash mismatch: expected {expected}, got {actual}" + ) + + dataset = json.loads(DATASET_PATH.read_text(encoding="utf-8")) + expectations = json.loads(EXPECTATIONS_PATH.read_text(encoding="utf-8")) + required = mode["required_question_count"] + if len(dataset["cases"]) != required or len(expectations["cases"]) != required: + raise RuntimeError("evidence-chain A/B inputs violate the required case count") + + dataset_ids = [case["id"] for case in dataset["cases"]] + expectation_ids = [case["id"] for case in expectations["cases"]] + if len(set(dataset_ids)) != required or len(set(expectation_ids)) != required: + raise RuntimeError("evidence-chain A/B inputs contain duplicate case IDs") + if set(dataset_ids) != set(expectation_ids): + raise RuntimeError("evidence-chain A/B question and expectation IDs differ") + return dataset, {case["id"]: case for case in expectations["cases"]} + + +def _normalized(text: str) -> str: + return " ".join(re.findall(r"[a-z0-9]+", (text or "").casefold())) + + +def _group_found(answer: str, alternatives: Iterable[str]) -> bool: + normalized_answer = _normalized(answer) + return any(_normalized(alternative) in normalized_answer for alternative in alternatives) + + +def _source_map(nodes: list[Any]) -> dict[str, Any]: + canonical_parts: list[str] = [] + mapped_nodes: list[dict[str, Any]] = [] + cursor = 0 + for node in nodes: + if canonical_parts: + canonical_parts.append("\n\n") + cursor += 2 + text = node.text_plain or node.text_md or "" + start = cursor + canonical_parts.append(text) + cursor += len(text) + words = text.split() + width = 0.90 / max(1, len(words)) + source_spans = [] + for index, word in enumerate(words): + x0 = 0.05 + index * width + source_spans.append( + { + "span_id": f"{node.node_id}:w{index}", + "order": index, + "text": word, + "block_no": 0, + "line_no": 0, + "word_no": index, + "normalized_bbox": { + "x0": x0, + "y0": 0.20, + "x1": min(0.999, x0 + width * 0.82), + "y1": 0.23, + }, + "coordinate_system": "pdf_points_top_left", + "extraction_source": "synthetic_fixture", + "verifiable": True, + } + ) + mapped_nodes.append( + { + "node_id": node.node_id, + "start": start, + "end": cursor, + "page_no": node.page_no, + "page_rotation": 0, + "page_size": {"width": 612, "height": 792}, + "source_spans": source_spans, + } + ) + return { + "doc_id": "fixture-doc", + "version": 1, + "content_hash": SOURCE_HASH, + "canonical_text": "".join(canonical_parts), + "nodes": mapped_nodes, + } + + +def _packed_context(expanded: Any, selected_ids: list[str]) -> tuple[str, list[Any]]: + selected_set = set(selected_ids) + nodes_by_id = {node.node_id: node for node in expanded.all_nodes} + selected_nodes = [nodes_by_id[node_id] for node_id in selected_ids if node_id in nodes_by_id] + packed = ContextPacker(max_tokens=2000).pack( + expanded, + node_order=selected_ids, + allowed_node_ids=selected_set, + ) + return packed.to_text(), selected_nodes + + +def _evaluate_variant( + *, + answer_result: AnswerResult, + selected_nodes: list[Any], + expectation: dict[str, Any], + forbidden_node_ids: set[str], + request_ms: float, +) -> VariantResult: + nodes_by_id = {node.node_id: node for node in selected_nodes} + all_map = _source_map(selected_nodes) + answer = answer_result.answer or "" + + required_facts = expectation["required_fact_groups"] + required_limitations = expectation["required_limitation_groups"] + facts_found = sum(_group_found(answer, group) for group in required_facts) + limitations_found = sum(_group_found(answer, group) for group in required_limitations) + forbidden_found = [ + phrase for phrase in expectation["forbidden_phrases"] if _group_found(answer, [phrase]) + ] + + valid_citations = 0 + verified_highlights = 0 + wrong_locator_acceptances = 0 + cited_node_ids: set[str] = set() + structured_ids: set[str] = set() + for citation in answer_result.citations: + node = nodes_by_id.get(citation.node_id) + quote = citation.exact_quote or citation.text_snippet or "" + if citation.citation_id: + structured_ids.add(citation.citation_id) + cited_node_ids.add(citation.node_id) + valid = bool( + node + and citation.page_no == node.page_no + and quote + and _normalized(quote) in _normalized(node.text_plain or node.text_md or "") + ) + if valid: + valid_citations += 1 + verified = verify_evidence_span( + doc_id="fixture-doc", + document_version=1, + node_id=citation.node_id, + page_index=int(citation.page_no), + quote_text=quote, + locator=None, + source_hash=SOURCE_HASH, + source_map=all_map, + ) + if verified["status"] == "FOUND" and verified.get("grade") == "verified": + verified_highlights += 1 + + negative_controls = ( + {"doc_id": "wrong-document", "document_version": 1, "page_index": int(citation.page_no)}, + {"doc_id": "fixture-doc", "document_version": 999, "page_index": int(citation.page_no)}, + {"doc_id": "fixture-doc", "document_version": 1, "page_index": int(citation.page_no) + 1000}, + ) + for control in negative_controls: + negative = verify_evidence_span( + doc_id=control["doc_id"], + document_version=control["document_version"], + node_id=citation.node_id, + page_index=control["page_index"], + quote_text=quote, + locator=None, + source_hash=SOURCE_HASH, + source_map=all_map, + ) + wrong_locator_acceptances += int(negative["status"] == "FOUND") + + citation_count = len(answer_result.citations) + inline_ids = set(re.findall(r"\[(C[1-9][0-9]*)\]", answer)) + inline_integrity = inline_ids == structured_ids and len(structured_ids) == citation_count + expected_citations = set(expectation["expected_citation_nodes"]) + citation_policy = expectation.get( + "citation_policy", + "required" if expected_citations else "forbidden", + ) + if citation_policy not in {"required", "forbidden", "optional"}: + raise RuntimeError(f"unsupported citation policy: {citation_policy}") + if citation_policy == "optional" and not cited_node_ids: + expected_recall = 1.0 + elif expected_citations: + expected_recall = len(cited_node_ids & expected_citations) / len(expected_citations) + else: + expected_recall = float(not cited_node_ids) + unauthorized = sorted(cited_node_ids & forbidden_node_ids) + for node_id in forbidden_node_ids: + if node_id in answer or node_id.casefold() in answer.casefold(): + unauthorized.append(node_id) + unauthorized = sorted(set(unauthorized)) + + limitation_ok = limitations_found == len(required_limitations) + facts_ok = facts_found == len(required_facts) + if citation_policy == "required": + citations_ok = citation_count > 0 + elif citation_policy == "forbidden": + citations_ok = citation_count == 0 + else: + citations_ok = True + unsupported_safe = limitation_ok if required_limitations else True + answer_correct = bool( + facts_ok + and limitation_ok + and not forbidden_found + and citations_ok + and not unauthorized + ) + return VariantResult( + answer=answer, + answer_correct=answer_correct, + facts_found=facts_found, + facts_required=len(required_facts), + limitations_found=limitations_found, + limitations_required=len(required_limitations), + forbidden_phrases_found=forbidden_found, + citation_count=citation_count, + citation_precision=valid_citations / citation_count if citation_count else 1.0, + expected_citation_recall=expected_recall, + inline_citation_integrity=inline_integrity, + verified_highlight_rate=verified_highlights / citation_count if citation_count else 1.0, + wrong_locator_acceptances=wrong_locator_acceptances, + unsupported_answer_safe=unsupported_safe, + unauthorized_node_leaks=unauthorized, + generation_ms=answer_result.generation_time_ms, + request_ms=request_ms, + selected_node_ids=[node.node_id for node in selected_nodes], + ) + + +def _run_case( + case: dict[str, Any], + expectation: dict[str, Any], + provider: AnswerProvider, +) -> PairedCaseResult: + nodes, edges, expanded = _make_case_graph(case) + denied = set(case.get("denied_nodes", [])) + engine = EvidenceChainEngine( + FixtureDB(nodes, edges), + FixtureACL(denied), + EvidenceChainConfig(mode="auto"), + ) + + chain_started = time.perf_counter() + chain_result = engine.build( + expanded, + case["question"], + case["seeds"], + doc_id="fixture-doc", + version=1, + ) + chain_layer_ms = (time.perf_counter() - chain_started) * 1000 + + baseline_ids = list(case["seeds"]) + baseline_context, baseline_nodes = _packed_context(expanded, baseline_ids) + + chain_ids = chain_result.ordered_node_ids if chain_result.applied else baseline_ids + expanded.chain_nodes = chain_result.additional_nodes + for node in expanded.chain_nodes: + expanded.node_sources[node.node_id] = "chain" + chain_context, chain_nodes = _packed_context(expanded, chain_ids) + + baseline_started = time.perf_counter() + baseline_answer = provider.generate( + context=baseline_context, + question=case["question"], + nodes=baseline_nodes, + expectation=expectation, + ) + baseline_request_ms = (time.perf_counter() - baseline_started) * 1000 + baseline_request_ms = max(baseline_request_ms, baseline_answer.generation_time_ms) + + chain_started = time.perf_counter() + chain_answer = provider.generate( + context=chain_context, + question=case["question"], + nodes=chain_nodes, + expectation=expectation, + ) + chain_generation_request_ms = (time.perf_counter() - chain_started) * 1000 + chain_request_ms = max(chain_generation_request_ms, chain_answer.generation_time_ms) + chain_layer_ms + + return PairedCaseResult( + case_id=case["id"], + domain=case["domain"], + route_expected=case["expect_route"], + route_actual=chain_result.route.applied, + chain_applied=chain_result.applied, + fallback_used=chain_result.fallback_used, + chain_layer_ms=chain_layer_ms, + baseline=_evaluate_variant( + answer_result=baseline_answer, + selected_nodes=baseline_nodes, + expectation=expectation, + forbidden_node_ids=denied, + request_ms=baseline_request_ms, + ), + chain=_evaluate_variant( + answer_result=chain_answer, + selected_nodes=chain_nodes, + expectation=expectation, + forbidden_node_ids=denied, + request_ms=chain_request_ms, + ), + ) + + +def _rate(values: Iterable[bool]) -> float: + items = list(values) + return sum(items) / len(items) if items else 1.0 + + +def _mean(values: Iterable[float]) -> float: + items = list(values) + return sum(items) / len(items) if items else 0.0 + + +def _percentile(values: Iterable[float], percentile: float) -> float: + ordered = sorted(values) + if not ordered: + return 0.0 + rank = round((len(ordered) - 1) * percentile) + return ordered[min(len(ordered) - 1, max(0, rank))] + + +def _build_report( + cases: list[PairedCaseResult], + expectations: dict[str, dict[str, Any]], + *, + provider_name: str, + model: str, +) -> dict[str, Any]: + routed = [case for case in cases if case.route_expected] + single_hop = [case for case in cases if not case.route_expected] + unsupported = [case for case in cases if case.case_id == "regression_unsupported_why"] + + baseline_accuracy = _rate(case.baseline.answer_correct for case in cases) + chain_accuracy = _rate(case.chain.answer_correct for case in cases) + routed_baseline_accuracy = _rate(case.baseline.answer_correct for case in routed) + routed_chain_accuracy = _rate(case.chain.answer_correct for case in routed) + baseline_citation_precision = _mean(case.baseline.citation_precision for case in cases) + chain_citation_precision = _mean(case.chain.citation_precision for case in cases) + baseline_highlight_rate = _mean(case.baseline.verified_highlight_rate for case in cases) + chain_highlight_rate = _mean(case.chain.verified_highlight_rate for case in cases) + baseline_request_values = [case.baseline.request_ms for case in cases] + chain_request_values = [case.chain.request_ms for case in cases] + baseline_p95_ms = _percentile(baseline_request_values, 0.95) + chain_request_p95_ms = _percentile(chain_request_values, 0.95) + chain_layer_values = [case.chain_layer_ms for case in cases] + overhead_ratio = _percentile(chain_layer_values, 0.95) / max(baseline_p95_ms, 0.001) + total_request_p95_overhead_ratio = ( + chain_request_p95_ms / max(baseline_p95_ms, 0.001) + ) - 1.0 + paired_total_request_overhead_ratios = [ + (case.chain.request_ms / max(case.baseline.request_ms, 0.001)) - 1.0 + for case in cases + ] + fallback_rate = _mean(float(case.fallback_used) for case in cases) + wrong_locator_acceptances = sum( + case.chain.wrong_locator_acceptances for case in cases + ) + unauthorized_leaks = sum(len(case.chain.unauthorized_node_leaks) for case in cases) + + def selected_evidence_recall(case: PairedCaseResult, variant: str) -> float: + expected = set(expectations[case.case_id]["expected_citation_nodes"]) + selected = set(getattr(case, variant).selected_node_ids) + return len(selected & expected) / len(expected) if expected else 1.0 + + metrics = { + "baseline_answer_accuracy": baseline_accuracy, + "chain_answer_accuracy": chain_accuracy, + "routed_baseline_answer_accuracy": routed_baseline_accuracy, + "routed_chain_answer_accuracy": routed_chain_accuracy, + "routed_absolute_accuracy_improvement": routed_chain_accuracy - routed_baseline_accuracy, + "baseline_citation_precision": baseline_citation_precision, + "chain_citation_precision": chain_citation_precision, + "baseline_expected_citation_recall": _mean( + case.baseline.expected_citation_recall for case in cases + ), + "chain_expected_citation_recall": _mean( + case.chain.expected_citation_recall for case in cases + ), + "baseline_selected_evidence_recall": _mean( + selected_evidence_recall(case, "baseline") for case in cases + ), + "chain_selected_evidence_recall": _mean( + selected_evidence_recall(case, "chain") for case in cases + ), + "baseline_verified_highlight_rate": baseline_highlight_rate, + "chain_verified_highlight_rate": chain_highlight_rate, + "single_hop_baseline_accuracy": _rate( + case.baseline.answer_correct for case in single_hop + ), + "single_hop_chain_accuracy": _rate(case.chain.answer_correct for case in single_hop), + "unsupported_baseline_safe_rate": _rate( + case.baseline.unsupported_answer_safe for case in unsupported + ), + "unsupported_chain_safe_rate": _rate( + case.chain.unsupported_answer_safe for case in unsupported + ), + "route_accuracy": _rate(case.route_actual == case.route_expected for case in cases), + "fallback_rate": fallback_rate, + "unauthorized_node_leaks": unauthorized_leaks, + "wrong_locator_acceptances": wrong_locator_acceptances, + "baseline_request_p50_ms": _percentile(baseline_request_values, 0.50), + "baseline_request_p95_ms": baseline_p95_ms, + "baseline_request_p99_ms": _percentile(baseline_request_values, 0.99), + "chain_request_p50_ms": _percentile(chain_request_values, 0.50), + "chain_request_p95_ms": chain_request_p95_ms, + "chain_request_p99_ms": _percentile(chain_request_values, 0.99), + "total_request_p95_overhead_ratio": total_request_p95_overhead_ratio, + "paired_total_request_overhead_p50_ratio": _percentile( + paired_total_request_overhead_ratios, 0.50 + ), + "paired_total_request_overhead_p95_ratio": _percentile( + paired_total_request_overhead_ratios, 0.95 + ), + "chain_layer_p50_ms": _percentile(chain_layer_values, 0.50), + "chain_layer_p95_ms": _percentile(chain_layer_values, 0.95), + "chain_layer_p99_ms": _percentile(chain_layer_values, 0.99), + "chain_p95_overhead_ratio": overhead_ratio, + } + gates = { + "routed_answer_accuracy_improves_by_5_points": metrics[ + "routed_absolute_accuracy_improvement" + ] >= 0.05, + "citation_precision_no_regression": chain_citation_precision >= baseline_citation_precision, + "verified_highlights_no_regression": chain_highlight_rate >= baseline_highlight_rate, + "expected_citation_recall_no_regression": metrics[ + "chain_expected_citation_recall" + ] >= metrics["baseline_expected_citation_recall"], + "wrong_document_page_version_fail_closed": wrong_locator_acceptances == 0, + "unauthorized_leakage_zero": unauthorized_leaks == 0, + "single_hop_no_regression": metrics["single_hop_chain_accuracy"] >= metrics[ + "single_hop_baseline_accuracy" + ], + "unsupported_answers_no_regression": metrics[ + "unsupported_chain_safe_rate" + ] >= metrics["unsupported_baseline_safe_rate"], + "route_accuracy_100_percent": metrics["route_accuracy"] == 1.0, + "fallback_rate_at_most_10_percent": fallback_rate <= 0.10, + "chain_p95_overhead_at_most_20_percent": overhead_ratio <= 0.20, + "inline_citation_integrity_100_percent": all( + case.chain.inline_citation_integrity for case in cases + ), + "curated_chain_answer_accuracy_100_percent": chain_accuracy == 1.0, + "curated_chain_citation_precision_100_percent": chain_citation_precision == 1.0, + "curated_chain_selected_evidence_recall_100_percent": metrics[ + "chain_selected_evidence_recall" + ] == 1.0, + "curated_chain_verified_highlights_100_percent": chain_highlight_rate == 1.0, + } + return { + "contract_id": "rag_eval_contract", + "mode": "evidence_chain_ab", + "provider": provider_name, + "model": model, + "dataset_sha256": _sha256(DATASET_PATH), + "expectations_sha256": _sha256(EXPECTATIONS_PATH), + "case_count": len(cases), + "metrics": metrics, + "gates": gates, + "passed": all(gates.values()), + "observations": { + "total_request_latency_is_measured_not_gated": True, + "total_request_latency_note": ( + "Total provider latency is reported separately from the chain-layer " + "release gate and requires representative load testing before default-on." + ), + }, + "cases": [asdict(case) for case in cases], + } + + +def evaluate(provider: AnswerProvider | None = None) -> dict[str, Any]: + dataset, expectations = _load_locked_inputs() + provider = provider or FixtureAnswerProvider() + cases = [ + _run_case(case, expectations[case["id"]], provider) + for case in dataset["cases"] + ] + return _build_report( + cases, + expectations, + provider_name=provider.name, + model=provider.model, + ) + + +def regrade_saved_report(report_path: Path) -> dict[str, Any]: + """Recompute current gates from a prior report without calling a provider.""" + dataset, expectations = _load_locked_inputs() + saved = json.loads(report_path.read_text(encoding="utf-8")) + if saved.get("dataset_sha256") != _sha256(DATASET_PATH): + raise RuntimeError("saved report dataset hash does not match the locked input") + if saved.get("expectations_sha256") != _sha256(EXPECTATIONS_PATH): + raise RuntimeError("saved report expectations hash does not match the locked input") + expected_ids = {case["id"] for case in dataset["cases"]} + saved_ids = {case.get("case_id") for case in saved.get("cases", [])} + if saved_ids != expected_ids or len(saved.get("cases", [])) != len(expected_ids): + raise RuntimeError("saved report cases do not match the locked input") + + cases: list[PairedCaseResult] = [] + for item in saved["cases"]: + cases.append( + PairedCaseResult( + case_id=item["case_id"], + domain=item["domain"], + route_expected=item["route_expected"], + route_actual=item["route_actual"], + chain_applied=item["chain_applied"], + fallback_used=item["fallback_used"], + chain_layer_ms=item["chain_layer_ms"], + baseline=VariantResult(**item["baseline"]), + chain=VariantResult(**item["chain"]), + ) + ) + return _build_report( + cases, + expectations, + provider_name=saved["provider"], + model=saved["model"], + ) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--provider", choices=("fixture", "openai"), default="fixture") + parser.add_argument("--model", default="gpt-5.2") + parser.add_argument( + "--regrade", + type=Path, + help="Recompute current gates from a saved report without provider calls.", + ) + parser.add_argument("--output", type=Path) + args = parser.parse_args() + if args.regrade: + report = regrade_saved_report(args.regrade) + else: + provider: AnswerProvider + if args.provider == "openai": + provider = OpenAIAnswerProvider(args.model) + else: + provider = FixtureAnswerProvider() + report = evaluate(provider) + rendered = json.dumps(report, indent=2, sort_keys=True) + if args.output: + args.output.write_text(rendered + "\n", encoding="utf-8") + print(rendered) + return 0 if report["passed"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/backend/tests/eval/run_evidence_chain_eval.py b/backend/tests/eval/run_evidence_chain_eval.py new file mode 100644 index 0000000..5b6a6ff --- /dev/null +++ b/backend/tests/eval/run_evidence_chain_eval.py @@ -0,0 +1,253 @@ +"""Deterministic Phase 1 evidence-chain evaluation and release-gate check. + +This harness intentionally avoids an LLM judge. Expected evidence is a locked, +human-authored node set. It measures routing accuracy, baseline/chain evidence +recall, forbidden-node leakage, determinism, and local scoring latency. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import sys +import time +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Any, Iterable + +BACKEND_ROOT = Path(__file__).resolve().parents[2] +if str(BACKEND_ROOT) not in sys.path: + sys.path.insert(0, str(BACKEND_ROOT)) + +from app.db.graph_models import Edge, EdgeType, Node, NodeType +from app.graph.expander import ExpandedContext +from app.qa.evidence_chain import EvidenceChainConfig, EvidenceChainEngine + + +HERE = Path(__file__).resolve().parent +DATASET_PATH = HERE / "questions_evidence_chain.json" +CONTRACT_PATH = HERE / "benchmark_contract.json" + + +class FixtureQuery: + def __init__(self, records: Iterable[object]): + self.records = list(records) + + def filter(self, *args, **kwargs): + return self + + def order_by(self, *args, **kwargs): + return self + + def limit(self, count): + self.records = self.records[:count] + return self + + def all(self): + return list(self.records) + + +class FixtureDB: + def __init__(self, nodes: list[Node], edges: list[Edge]): + self.nodes = nodes + self.edges = edges + + def query(self, model): + if model is Node: + return FixtureQuery(self.nodes) + if model is Edge: + return FixtureQuery(self.edges) + raise AssertionError(f"unexpected model: {model}") + + +class FixtureACL: + def __init__(self, denied_ids: set[str]): + self.denied_ids = denied_ids + + def filter_nodes(self, nodes, stage="expansion"): + return [node for node in nodes if node.node_id not in self.denied_ids] + + +@dataclass +class CaseResult: + case_id: str + domain: str + route_expected: bool + route_actual: bool + baseline_recall: float + chain_recall: float + forbidden_leaks: list[str] + deterministic: bool + latency_ms: float + selected_node_ids: list[str] + + +def _sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def validate_contract() -> dict[str, Any]: + contract = json.loads(CONTRACT_PATH.read_text(encoding="utf-8")) + mode = contract["modes"].get("evidence_chain") + if mode is None: + raise RuntimeError("benchmark contract has no evidence_chain mode") + actual_hash = _sha256(DATASET_PATH) + if actual_hash != mode["dataset_sha256"]: + raise RuntimeError( + f"evidence-chain dataset hash mismatch: expected {mode['dataset_sha256']}, got {actual_hash}" + ) + dataset = json.loads(DATASET_PATH.read_text(encoding="utf-8")) + if len(dataset["cases"]) != mode["required_question_count"]: + raise RuntimeError("evidence-chain dataset count violates benchmark contract") + return dataset + + +def _make_case_graph(case: dict[str, Any]): + nodes = [ + Node( + node_id=item["id"], + doc_id="fixture-doc", + version=1, + node_type=NodeType.chunk, + page_no=item["page"], + chunk_index_in_page=index, + text_plain=item["text"], + meta={}, + ) + for index, item in enumerate(case["nodes"]) + ] + edges = [ + Edge( + id=index, + doc_id="fixture-doc", + version=1, + from_node_id=item["from"], + to_node_id=item["to"], + edge_type=EdgeType(item["type"]), + confidence=item.get("confidence", 1.0), + ) + for index, item in enumerate(case["edges"], start=1) + ] + by_id = {node.node_id: node for node in nodes} + seeds = [by_id[node_id] for node_id in case["seeds"]] + expanded = ExpandedContext( + seed_nodes=seeds, + adjacent_nodes=[], + referenced_nodes=[], + explained_by_nodes=[], + node_sources={node.node_id: "seed" for node in seeds}, + ) + return nodes, edges, expanded + + +def _recall(selected: set[str], expected: set[str]) -> float: + return len(selected & expected) / len(expected) if expected else 1.0 + + +def run_case(case: dict[str, Any]) -> CaseResult: + nodes, edges, expanded = _make_case_graph(case) + denied = set(case.get("denied_nodes", [])) + engine = EvidenceChainEngine( + FixtureDB(nodes, edges), + FixtureACL(denied), + EvidenceChainConfig(mode="auto"), + ) + + started = time.perf_counter() + first = engine.build( + expanded, + case["question"], + case["seeds"], + doc_id="fixture-doc", + version=1, + ) + latency_ms = (time.perf_counter() - started) * 1000 + second = engine.build( + expanded, + case["question"], + case["seeds"], + doc_id="fixture-doc", + version=1, + ) + + baseline_ids = set(case["seeds"]) + selected_ids = first.selected_node_ids if first.applied else baseline_ids + expected = set(case["expected_evidence"]) + forbidden = set(case.get("forbidden_evidence", [])) + deterministic = first.to_audit_dict() == second.to_audit_dict() + + return CaseResult( + case_id=case["id"], + domain=case["domain"], + route_expected=case["expect_route"], + route_actual=first.route.applied, + baseline_recall=_recall(baseline_ids, expected), + chain_recall=_recall(selected_ids, expected), + forbidden_leaks=sorted(selected_ids & forbidden), + deterministic=deterministic, + latency_ms=latency_ms, + selected_node_ids=sorted(selected_ids), + ) + + +def _percentile(values: list[float], percentile: float) -> float: + if not values: + return 0.0 + ordered = sorted(values) + index = min(len(ordered) - 1, max(0, int((len(ordered) - 1) * percentile))) + return ordered[index] + + +def evaluate() -> dict[str, Any]: + dataset = validate_contract() + cases = [run_case(case) for case in dataset["cases"]] + route_accuracy = sum(case.route_actual == case.route_expected for case in cases) / len(cases) + baseline_recall = sum(case.baseline_recall for case in cases) / len(cases) + chain_recall = sum(case.chain_recall for case in cases) / len(cases) + total_leaks = sum(len(case.forbidden_leaks) for case in cases) + deterministic_rate = sum(case.deterministic for case in cases) / len(cases) + p95_ms = _percentile([case.latency_ms for case in cases], 0.95) + + gates = { + "route_accuracy": route_accuracy == 1.0, + "chain_recall_no_regression": chain_recall >= baseline_recall, + "chain_recall_target": chain_recall == 1.0, + "forbidden_leakage_zero": total_leaks == 0, + "deterministic": deterministic_rate == 1.0, + "local_p95_under_50ms": p95_ms <= 50.0, + } + return { + "contract_id": "rag_eval_contract", + "mode": "evidence_chain", + "dataset_sha256": _sha256(DATASET_PATH), + "case_count": len(cases), + "metrics": { + "route_accuracy": route_accuracy, + "baseline_evidence_recall": baseline_recall, + "chain_evidence_recall": chain_recall, + "absolute_recall_improvement": chain_recall - baseline_recall, + "forbidden_leaks": total_leaks, + "deterministic_rate": deterministic_rate, + "local_p95_ms": p95_ms, + }, + "gates": gates, + "passed": all(gates.values()), + "cases": [asdict(case) for case in cases], + } + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--output", type=Path) + args = parser.parse_args() + report = evaluate() + rendered = json.dumps(report, indent=2, sort_keys=True) + if args.output: + args.output.write_text(rendered + "\n", encoding="utf-8") + print(rendered) + return 0 if report["passed"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/backend/tests/qa/test_evidence_chain.py b/backend/tests/qa/test_evidence_chain.py new file mode 100644 index 0000000..f095157 --- /dev/null +++ b/backend/tests/qa/test_evidence_chain.py @@ -0,0 +1,384 @@ +"""Unit and security tests for query-aware evidence chains.""" + +from __future__ import annotations + +from typing import Iterable + +import pytest + +from app.db.graph_models import Edge, EdgeType, Node, NodeType +from app.graph.context_packer import ContextPacker +from app.graph.expander import ExpandedContext +from app.qa.evidence_chain import ( + EvidenceChainConfig, + EvidenceChainEngine, + MultiHopRouter, +) + + +def make_node(node_id: str, page: int, text: str, doc_id: str = "doc-1") -> Node: + return Node( + node_id=node_id, + doc_id=doc_id, + version=1, + node_type=NodeType.chunk, + page_no=page, + chunk_index_in_page=0, + text_plain=text, + meta={}, + ) + + +def make_edge( + edge_id: int, + left: str, + right: str, + edge_type: EdgeType = EdgeType.adjacent_next, + confidence: float | None = 1.0, + doc_id: str = "doc-1", +) -> Edge: + return Edge( + id=edge_id, + doc_id=doc_id, + version=1, + from_node_id=left, + to_node_id=right, + edge_type=edge_type, + confidence=confidence, + ) + + +class FakeQuery: + def __init__(self, records: Iterable[object]): + self.records = list(records) + + def filter(self, *args, **kwargs): + return self + + def order_by(self, *args, **kwargs): + return self + + def limit(self, count): + self.records = self.records[:count] + return self + + def all(self): + return list(self.records) + + +class FakeDB: + def __init__(self, nodes: list[Node], edges: list[Edge]): + self.nodes = nodes + self.edges = edges + self.query_calls: list[type] = [] + + def query(self, model): + self.query_calls.append(model) + if model is Node: + return FakeQuery(self.nodes) + if model is Edge: + return FakeQuery(self.edges) + raise AssertionError(f"unexpected query model: {model}") + + +class RecordingACL: + def __init__(self, denied_node_ids: set[str] | None = None): + self.denied_node_ids = denied_node_ids or set() + self.stages: list[str] = [] + + def filter_nodes(self, nodes, stage="expansion"): + self.stages.append(stage) + return [node for node in nodes if node.node_id not in self.denied_node_ids] + + +def make_context(seed: Node, adjacent: Node | None = None) -> ExpandedContext: + adjacent_nodes = [adjacent] if adjacent is not None else [] + sources = {seed.node_id: "seed"} + if adjacent is not None: + sources[adjacent.node_id] = "adjacent" + return ExpandedContext( + seed_nodes=[seed], + adjacent_nodes=adjacent_nodes, + referenced_nodes=[], + explained_by_nodes=[], + node_sources=sources, + ) + + +class TestMultiHopRouter: + def test_routes_explicit_legal_multi_hop_question(self): + decision = MultiHopRouter.decide( + "Compare the amendment's effective date with the governing law and explain how it affects termination.", + mode="auto", + ) + + assert decision.applied is True + assert decision.score >= 0.55 + assert "comparison" in decision.reasons + assert "temporal_or_versioned" in decision.reasons + + def test_does_not_route_simple_lookup(self): + decision = MultiHopRouter.decide( + "What is the termination fee?", + mode="auto", + ) + + assert decision.applied is False + assert decision.reasons == ("below_threshold",) + + @pytest.mark.parametrize( + ("mode", "expected"), + [("off", False), ("on", True)], + ) + def test_explicit_modes_are_authoritative(self, mode, expected): + assert MultiHopRouter.decide("simple question", mode=mode).applied is expected + + +class TestEvidenceChainConfig: + def test_rejects_unbounded_or_invalid_configuration(self): + with pytest.raises(ValueError): + EvidenceChainConfig(max_hops=0) + with pytest.raises(ValueError): + EvidenceChainConfig(max_nodes=2, max_selected_nodes=3) + with pytest.raises(ValueError): + EvidenceChainConfig(restart_probability=0.0) + + +class TestEvidenceChainEngine: + def _build_fixture(self, denied: set[str] | None = None): + seed = make_node("seed", 1, "The patient has renal impairment.") + adjacent = make_node("adjacent", 2, "The guideline discusses Drug A.") + bridge = make_node("bridge", 3, "Drug A is cleared through the kidneys.") + target = make_node( + "target", + 4, + "Renal impairment is a contraindication for Drug A under the current guideline.", + ) + nodes = [seed, adjacent, bridge, target] + edges = [ + make_edge(1, "seed", "adjacent"), + make_edge(2, "adjacent", "bridge", confidence=0.8), + make_edge(3, "bridge", "target", EdgeType.references, confidence=0.9), + # Cycle must not make path reconstruction or propagation unstable. + make_edge(4, "target", "adjacent", EdgeType.explained_by, confidence=0.7), + ] + acl = RecordingACL(denied) + engine = EvidenceChainEngine( + FakeDB(nodes, edges), + acl, + EvidenceChainConfig( + mode="on", + max_hops=3, + max_nodes=10, + max_selected_nodes=4, + max_chains=4, + propagation_steps=25, + ), + ) + return engine, acl, seed, adjacent, target + + def test_expands_scores_and_assembles_auditable_paths(self): + engine, acl, seed, adjacent, target = self._build_fixture() + + result = engine.build( + make_context(seed, adjacent), + question="How does renal impairment affect Drug A under the current guideline?", + seed_scores={"seed": 0.9}, + doc_id="doc-1", + version=1, + ) + + assert result.applied is True + assert result.candidate_count == 4 + assert result.edge_count == 4 + assert result.iterations <= 25 + assert result.selected_node_ids == {"seed", "adjacent", "bridge", "target"} + assert {node.node_id for node in result.additional_nodes} == {"bridge", "target"} + assert any(path.node_ids[0] == "seed" and path.node_ids[-1] == "target" for path in result.paths) + assert all(0.0 <= score.final_score <= 1.0 for score in result.selected_scores) + assert "evidence_chain_hop_1" in acl.stages + + audit = result.to_audit_dict() + assert audit["scoring_version"] == "document-chain-v1" + assert "text" not in str(audit).lower() + assert audit["ordered_node_ids"] == result.ordered_node_ids + + def test_denied_node_cannot_affect_selected_graph_or_audit(self): + engine, acl, seed, adjacent, _ = self._build_fixture(denied={"target"}) + + result = engine.build( + make_context(seed, adjacent), + question="How does renal impairment affect Drug A under the current guideline?", + seed_scores={"seed": 0.9}, + doc_id="doc-1", + version=1, + ) + + audit = result.to_audit_dict() + assert result.candidate_count == 3 + assert "target" not in result.selected_node_ids + assert "target" not in result.ordered_node_ids + assert "target" not in str(audit) + assert all( + edge.from_node_id != "target" and edge.to_node_id != "target" + for path in result.paths + for edge in path.edges + ) + assert any(stage.startswith("evidence_chain_hop_") for stage in acl.stages) + + def test_invalid_edge_confidence_is_clamped_and_deterministic(self): + seed = make_node("seed", 1, "Source clause") + target = make_node("target", 2, "Amended clause") + edge = make_edge(1, "seed", "target", confidence=float("nan")) + engine = EvidenceChainEngine( + FakeDB([seed, target], [edge]), + RecordingACL(), + EvidenceChainConfig(mode="on", max_selected_nodes=2), + ) + + first = engine.build( + make_context(seed), "Compare the source and amended clause", {"seed": 1.0} + ) + second = engine.build( + make_context(seed), "Compare the source and amended clause", {"seed": 1.0} + ) + + assert first.ordered_node_ids == second.ordered_node_ids + assert first.to_audit_dict() == second.to_audit_dict() + assert first.paths[0].edges[0].weight == pytest.approx(0.325) + + def test_empty_authorized_candidate_set_falls_back(self): + seed = make_node("seed", 1, "Source") + engine = EvidenceChainEngine( + FakeDB([seed], []), + RecordingACL({"seed"}), + EvidenceChainConfig(mode="on"), + ) + + result = engine.build(make_context(seed), "Compare the sources", {"seed": 1.0}) + + assert result.applied is False + assert result.fallback_used is True + assert result.fallback_reason == "no_authorized_candidates" + + def test_doc_scope_filters_initial_and_neighbor_nodes(self): + seed = make_node("seed", 1, "Original agreement", doc_id="doc-1") + foreign = make_node("foreign", 2, "Restricted other document", doc_id="doc-2") + cross_doc_edge = make_edge(1, "seed", "foreign", doc_id="doc-2") + engine = EvidenceChainEngine( + FakeDB([seed, foreign], [cross_doc_edge]), + RecordingACL(), + EvidenceChainConfig(mode="on", max_selected_nodes=2), + ) + expanded = make_context(seed) + expanded.chain_nodes = [foreign] + + result = engine.build( + expanded, + "Compare the original and amended agreement", + {"seed": 1.0}, + doc_id="doc-1", + version=1, + ) + + assert result.candidate_count == 1 + assert result.selected_node_ids == {"seed"} + assert "foreign" not in str(result.to_audit_dict()) + + def test_all_seed_nodes_keep_a_nonzero_restart_prior(self): + normalized = EvidenceChainEngine._normalize_seed_scores( + {"strong", "weak"}, + {"strong": 0.9, "weak": 0.1}, + ) + + assert normalized["strong"] == 1.0 + assert normalized["weak"] >= 0.10 + + def test_high_degree_graph_respects_hard_candidate_cap(self): + seed = make_node("seed", 1, "Compare the policy changes") + neighbors = [ + make_node(f"neighbor-{index:03d}", index + 2, "Policy change detail") + for index in range(100) + ] + edges = [ + make_edge(index + 1, "seed", node.node_id, confidence=1.0) + for index, node in enumerate(neighbors) + ] + engine = EvidenceChainEngine( + FakeDB([seed] + neighbors, edges), + RecordingACL(), + EvidenceChainConfig( + mode="on", + max_nodes=10, + max_selected_nodes=8, + ), + ) + + result = engine.build( + make_context(seed), + "Compare the policy changes", + {"seed": 1.0}, + ) + + assert result.candidate_count == 10 + assert len(result.selected_node_ids) <= 8 + + def test_database_query_count_is_bounded_by_two_queries_per_hop(self): + engine, _, seed, adjacent, _ = self._build_fixture() + + result = engine.build( + make_context(seed, adjacent), + "How does renal impairment affect Drug A under the current guideline?", + {"seed": 1.0}, + doc_id="doc-1", + version=1, + ) + + assert result.applied is True + assert len(engine.db.query_calls) <= 2 * engine.config.max_hops + assert engine.db.query_calls.count(Edge) <= engine.config.max_hops + assert engine.db.query_calls.count(Node) <= engine.config.max_hops + + def test_path_reconstruction_does_not_exceed_hop_budget(self): + engine = EvidenceChainEngine( + FakeDB([], []), + RecordingACL(), + EvidenceChainConfig(mode="on", max_hops=2), + ) + adjacency = { + "seed": {"a": 1.0}, + "a": {"seed": 1.0, "b": 1.0}, + "b": {"a": 1.0, "target": 1.0}, + "target": {"b": 1.0}, + } + + assert engine._shortest_path_to_seed( + "target", + {"seed"}, + adjacency, + {node_id: 1.0 for node_id in adjacency}, + allowed_node_ids=set(adjacency), + ) == [] + + +def test_chain_aware_packer_preserves_explicit_order_and_canonical_citations(): + seed = make_node("seed", 1, "Seed evidence") + target = make_node("target", 4, "Target evidence") + expanded = make_context(seed) + expanded.chain_nodes = [target] + expanded.node_sources["target"] = "chain" + + packed = ContextPacker(max_tokens=100).pack( + expanded, + node_order=["target", "seed"], + allowed_node_ids={"target", "seed"}, + ) + + assert [block.citation.node_id for block in packed.blocks] == ["target", "seed"] + assert packed.to_text().startswith( + "[target:4] node_id=target page_no=4 source=chain\nTarget evidence" + ) + assert ( + "[seed:1] node_id=seed page_no=1 source=seed\nSeed evidence" + in packed.to_text() + ) diff --git a/backend/tests/qa/test_evidence_chain_ab_eval.py b/backend/tests/qa/test_evidence_chain_ab_eval.py new file mode 100644 index 0000000..07641ce --- /dev/null +++ b/backend/tests/qa/test_evidence_chain_ab_eval.py @@ -0,0 +1,59 @@ +"""Release-gate coverage for the locked answer-level evidence-chain A/B set.""" + +import json + +from tests.eval.run_evidence_chain_ab import evaluate, regrade_saved_report + + +def test_locked_answer_and_provenance_ab_evaluation_passes_all_ci_gates(): + report = evaluate() + + assert report["passed"] is True + assert report["case_count"] == 12 + assert report["provider"] == "fixture" + assert report["metrics"]["chain_answer_accuracy"] == 1.0 + assert report["metrics"]["routed_absolute_accuracy_improvement"] >= 0.05 + assert report["metrics"]["chain_citation_precision"] == 1.0 + assert report["metrics"]["chain_selected_evidence_recall"] == 1.0 + assert report["metrics"]["chain_verified_highlight_rate"] == 1.0 + assert report["metrics"]["wrong_locator_acceptances"] == 0 + assert report["metrics"]["unauthorized_node_leaks"] == 0 + assert report["metrics"]["single_hop_chain_accuracy"] == report["metrics"][ + "single_hop_baseline_accuracy" + ] + assert "total_request_p95_overhead_ratio" in report["metrics"] + assert report["observations"]["total_request_latency_is_measured_not_gated"] is True + assert all(report["gates"].values()) + + +def test_qa_prompt_forbids_unverifiable_citations_on_pure_abstentions(): + from app.prompts import get_prompt + + prompt = get_prompt("qa_answer").casefold() + assert "pure abstention" in prompt + assert "empty citations array" in prompt + assert "never cite irrelevant or generic text" in prompt + + +def test_saved_report_can_be_regraded_offline_without_provider_calls(tmp_path): + original = evaluate() + saved = tmp_path / "report.json" + saved.write_text(json.dumps(original), encoding="utf-8") + + regraded = regrade_saved_report(saved) + + assert regraded["passed"] is True + assert regraded["provider"] == original["provider"] + assert regraded["metrics"] == original["metrics"] + + +def test_ab_report_keeps_sensitive_acl_evidence_out_of_answer_and_citations(): + report = evaluate() + acl_case = next( + case for case in report["cases"] if case["case_id"] == "legal_acl_restricted_amendment" + ) + + assert "l5_restricted" not in acl_case["chain"]["selected_node_ids"] + assert "increases the liability cap" not in acl_case["chain"]["answer"].casefold() + assert acl_case["chain"]["unauthorized_node_leaks"] == [] + assert acl_case["chain"]["unsupported_answer_safe"] is True diff --git a/backend/tests/qa/test_evidence_chain_eval.py b/backend/tests/qa/test_evidence_chain_eval.py new file mode 100644 index 0000000..4f27cb7 --- /dev/null +++ b/backend/tests/qa/test_evidence_chain_eval.py @@ -0,0 +1,15 @@ +"""Contract test for the locked evidence-chain evaluation harness.""" + +from tests.eval.run_evidence_chain_eval import evaluate + + +def test_locked_evidence_chain_evaluation_passes_all_local_gates(): + report = evaluate() + + assert report["passed"] is True + assert report["case_count"] == 12 + assert report["metrics"]["route_accuracy"] == 1.0 + assert report["metrics"]["chain_evidence_recall"] == 1.0 + assert report["metrics"]["absolute_recall_improvement"] > 0.0 + assert report["metrics"]["forbidden_leaks"] == 0 + assert report["metrics"]["deterministic_rate"] == 1.0 diff --git a/backend/tests/qa/test_evidence_chain_runner.py b/backend/tests/qa/test_evidence_chain_runner.py new file mode 100644 index 0000000..1e9357a --- /dev/null +++ b/backend/tests/qa/test_evidence_chain_runner.py @@ -0,0 +1,118 @@ +"""Runner contract tests for evidence-chain fallback and ACL scrubbing.""" + +from types import SimpleNamespace +from unittest.mock import MagicMock + +from app.graph.expander import ExpandedContext +from app.qa.evidence_chain import ( + ChainNodeScore, + EvidenceChainResult, + RouteDecision, +) +from app.qa.runner import QARunner + + +def empty_context() -> ExpandedContext: + return ExpandedContext( + seed_nodes=[], + adjacent_nodes=[], + referenced_nodes=[], + explained_by_nodes=[], + ) + + +def test_disabled_feature_does_not_call_engine_or_change_context(): + runner = QARunner.__new__(QARunner) + runner.evidence_chain_enabled = False + runner.evidence_chain_mode = "off" + runner.evidence_chain_engine = MagicMock() + runner.evidence_chain_engine.config.route_threshold = 0.55 + expanded = empty_context() + + output, result, elapsed_ms = runner._apply_evidence_chain( + expanded, + "Compare two clauses", + {}, + None, + None, + ) + + assert output is expanded + assert result.applied is False + assert result.route.reasons == ("mode_off",) + assert elapsed_ms == 0.0 + runner.evidence_chain_engine.build.assert_not_called() + + +def test_engine_error_falls_back_without_exposing_exception_message(): + runner = QARunner.__new__(QARunner) + runner.evidence_chain_enabled = True + runner.evidence_chain_mode = "on" + runner.evidence_chain_engine = MagicMock() + runner.evidence_chain_engine.config.route_threshold = 0.55 + runner.evidence_chain_engine.build.side_effect = RuntimeError( + "secret restricted-node-id" + ) + expanded = empty_context() + + output, result, elapsed_ms = runner._apply_evidence_chain( + expanded, + "Compare two clauses", + {}, + None, + None, + ) + + assert output is expanded + assert result.applied is False + assert result.fallback_used is True + assert result.fallback_reason == "engine_error:RuntimeError" + assert "secret" not in str(result.to_audit_dict()) + assert elapsed_ms >= 0.0 + + +def test_final_acl_choke_point_scrubs_engine_selection_and_audit(): + runner = QARunner.__new__(QARunner) + runner.evidence_chain_enabled = True + runner.evidence_chain_mode = "on" + runner.evidence_chain_engine = MagicMock() + runner.evidence_chain_engine.config.route_threshold = 0.55 + allowed = SimpleNamespace( + node_id="allowed", + doc_id="doc", + text_plain="Allowed", + ) + denied = SimpleNamespace( + node_id="denied", + doc_id="doc", + text_plain="Denied", + ) + chain_result = EvidenceChainResult( + route=RouteDecision(True, "on", 1.0, ("forced_on",)), + applied=True, + selected_scores=[ + ChainNodeScore("allowed", 0.9, 0.8, 1.0, 1.0, True), + ChainNodeScore("denied", 0.8, 0.7, 1.0, 0.0, False), + ], + ordered_node_ids=["allowed", "denied"], + selected_node_ids={"allowed", "denied"}, + additional_nodes=[allowed, denied], + ) + runner.evidence_chain_engine.build.return_value = chain_result + runner.acl_enforcer = MagicMock() + runner.acl_enforcer.filter_nodes.side_effect = lambda nodes, stage: [ + node for node in nodes if node.node_id != "denied" + ] + + output, result, _ = runner._apply_evidence_chain( + empty_context(), + "Compare two clauses", + {"allowed": 1.0}, + "doc", + 1, + ) + + assert [node.node_id for node in output.chain_nodes] == ["allowed"] + assert result.selected_node_ids == {"allowed"} + assert result.ordered_node_ids == ["allowed"] + assert "denied" not in str(result.to_audit_dict()) diff --git a/backend/tests/qa/test_propagation_evidence_v2.py b/backend/tests/qa/test_propagation_evidence_v2.py new file mode 100644 index 0000000..63549bd --- /dev/null +++ b/backend/tests/qa/test_propagation_evidence_v2.py @@ -0,0 +1,100 @@ +"""Evidence V2 behavior in propagation-safety mode.""" + +from app.qa.propagation.synthesizer import _prepare_grounded_citations +from app.qa.propagation.types import EvidencePacket, EvidenceSnippet, SubAnswer +from app.qa.propagation.verifier import Verifier +from app.qa.runner import QARunner + + +def test_verifier_accepts_only_verbatim_citations_from_the_named_snippet(): + snippets = [ + EvidenceSnippet( + node_id="n1", + page_no=4, + text="The patient consent form must be signed before treatment.", + ) + ] + + valid = Verifier._validate_citations( + [ + { + "node_id": "n1", + "page_no": 4, + "label": None, + "exact_quote": "The patient consent form must be signed before treatment.", + } + ], + snippets, + ) + invalid = Verifier._validate_citations( + [ + { + "node_id": "n1", + "page_no": 4, + "label": None, + "exact_quote": "Consent is required.", + } + ], + snippets, + ) + + assert valid[0]["exact_quote"].startswith("The patient") + assert invalid == [] + + +def test_synthesizer_converts_legacy_refs_to_stable_ids_and_preserves_quote(): + sub_answers = [ + SubAnswer( + subq_id="sq1", + answer="The fee is $45,000.", + citations=[ + { + "node_id": "fee-node", + "page_no": 5, + "label": None, + "exact_quote": "The fee is $45,000.", + } + ], + confidence=1.0, + ) + ] + + answer, citations = _prepare_grounded_citations( + "The fee is $45,000 [fee-node:5].", + [{"node_id": "fee-node", "page_no": 5, "label": None}], + sub_answers, + ) + + assert answer == "The fee is $45,000 [C1]." + assert citations == [ + { + "citation_id": "C1", + "node_id": "fee-node", + "page_no": 5, + "label": None, + "exact_quote": "The fee is $45,000.", + } + ] + + +def test_propagation_context_ids_are_stable_deduplicated_and_groundable(): + packets = [ + EvidencePacket( + subq_id="sq1", + subq_text="first", + snippets=[ + EvidenceSnippet(node_id="n1", page_no=1, text="one"), + EvidenceSnippet(node_id="n2", page_no=2, text="two"), + ], + ), + EvidencePacket( + subq_id="sq2", + subq_text="second", + snippets=[ + EvidenceSnippet(node_id="n2", page_no=2, text="two"), + EvidenceSnippet(node_id="n3", page_no=3, text="three"), + ], + ), + ] + + assert QARunner._collect_packet_node_ids(packets) == ["n1", "n2", "n3"] diff --git a/backend/tests/test_acl_expansion_filter.py b/backend/tests/test_acl_expansion_filter.py index bca74d5..e1163bb 100644 --- a/backend/tests/test_acl_expansion_filter.py +++ b/backend/tests/test_acl_expansion_filter.py @@ -16,7 +16,7 @@ def _node(doc_id): return SimpleNamespace(doc_id=doc_id, node_id=f"{doc_id}-n") -def test_acl_filter_expanded_filters_all_four_node_lists(): +def test_acl_filter_expanded_filters_every_node_list(): runner = QARunner.__new__(QARunner) # bypass heavy __init__ enforcer = MagicMock() # Mock enforcement: only doc_id == "ok" is accessible. @@ -30,6 +30,7 @@ def test_acl_filter_expanded_filters_all_four_node_lists(): adjacent_nodes=[_node("restricted")], referenced_nodes=[_node("ok"), _node("ok")], explained_by_nodes=[_node("restricted")], + chain_nodes=[_node("ok"), _node("restricted")], ) out = runner._acl_filter_expanded(expanded) @@ -38,8 +39,9 @@ def test_acl_filter_expanded_filters_all_four_node_lists(): assert out.adjacent_nodes == [] # restricted neighbour dropped assert [n.doc_id for n in out.referenced_nodes] == ["ok", "ok"] assert out.explained_by_nodes == [] # restricted explainer dropped - # All four lists were routed through the single choke point. - assert enforcer.filter_nodes.call_count == 4 + assert [n.doc_id for n in out.chain_nodes] == ["ok"] + # All five lists were routed through the single choke point. + assert enforcer.filter_nodes.call_count == 5 def test_acl_filter_expanded_is_passthrough_when_acl_off(): diff --git a/backend/tests/test_citation_hydration_v2.py b/backend/tests/test_citation_hydration_v2.py new file mode 100644 index 0000000..62b68d8 --- /dev/null +++ b/backend/tests/test_citation_hydration_v2.py @@ -0,0 +1,199 @@ +"""QA citation hydration tests for the fail-closed Evidence V2 contract.""" + +from types import SimpleNamespace +from unittest.mock import patch + +from app.db.graph_models import DocumentGraph, Node +from app.llm.openai_client import Citation +from app.qa.runner import QARunner + + +QUOTE = "The patient gave informed consent." + + +def _source_spans(): + words = QUOTE.split() + spans = [] + x0 = 0.1 + for order, word in enumerate(words): + x1 = x0 + 0.05 + spans.append( + { + "order": order, + "text": word, + "block_no": 0, + "line_no": 0, + "word_no": order, + "normalized_bbox": { + "x0": x0, + "y0": 0.2, + "x1": x1, + "y1": 0.23, + }, + "coordinate_system": "pdf_points_top_left", + "verifiable": True, + } + ) + x0 = x1 + 0.01 + return spans + + +class FakeQuery: + def __init__(self, model, node, graph): + self.model = model + self.node = node + self.graph = graph + + def filter(self, *_args): + return self + + def all(self): + return [self.node] if self.model is Node and self.node is not None else [] + + def first(self): + return self.graph if self.model is DocumentGraph else None + + +class FakeDB: + def __init__(self, node=None, graph=None): + self.node = node + self.graph = graph + self.added = [] + self.commits = 0 + self.rollbacks = 0 + + def query(self, model): + return FakeQuery(model, self.node, self.graph) + + def add(self, value): + self.added.append(value) + + def commit(self): + self.commits += 1 + + def rollback(self): + self.rollbacks += 1 + + +class FakeStorage: + def __init__(self, source_map): + self.source_map = source_map + + def canonical_view_exists(self, *_args): + return True + + def source_map_exists(self, *_args): + return True + + def selectors_exist(self, *_args): + return True + + def get_source_map(self, *_args): + return self.source_map + + +def test_hydration_returns_a_valid_verified_record_and_snapshot(): + selector = { + "text_position": {"start": 0, "end": len(QUOTE)}, + "text_quote": {"exact": QUOTE, "prefix": "", "suffix": ""}, + "source_state": {"content_hash": "sha256:sourcehash"}, + } + node = SimpleNamespace( + node_id="clinical-node", + doc_id="clinical-doc", + version=2, + page_no=1, + label=None, + bbox=None, + text_plain=QUOTE, + meta={ + "selector_bundle": selector, + "source_spans": _source_spans(), + "page_size": {"width": 612, "height": 792}, + }, + ) + graph = SimpleNamespace( + doc_id="clinical-doc", + version=2, + content_hash="sourcehash", + source_uri="upload://clinical.pdf", + ) + source_map = { + "doc_id": "clinical-doc", + "version": 2, + "content_hash": "sha256:sourcehash", + "canonical_text": QUOTE, + "nodes": [ + { + "node_id": "clinical-node", + "start": 0, + "end": len(QUOTE), + "page_no": 1, + "page_rotation": 0, + "page_size": {"width": 612, "height": 792}, + "source_spans": _source_spans(), + } + ], + } + db = FakeDB(node=node, graph=graph) + runner = object.__new__(QARunner) + runner.db = db + + with ( + patch( + "app.qa.runner.get_settings", + return_value=SimpleNamespace(enable_cross_format_highlighting=True), + ), + patch("app.qa.runner.get_storage_client", return_value=FakeStorage(source_map)), + patch("app.qa.runner.find_legacy_for_graph", return_value=None), + ): + hydrated = runner._hydrate_citations( + [ + Citation( + citation_id="C1", + node_id="clinical-node", + page_no=1, + exact_quote=QUOTE, + ) + ], + doc_id="clinical-doc", + version=2, + context_node_ids=["clinical-node"], + answer_text="Consent was given [C1].", + request_id="request-1", + ) + + assert len(hydrated) == 1 + record = hydrated[0]["evidence_records"][0] + assert record["status"] == "verified" + assert record["locator"]["type"] == "rects" + assert record["document_version"] == 2 + assert db.added[0].evidence_record == record + assert db.commits == 1 + assert db.rollbacks == 0 + + +def test_hydration_never_substitutes_a_same_page_context_node(): + db = FakeDB() + runner = object.__new__(QARunner) + runner.db = db + + with patch( + "app.qa.runner.get_settings", + return_value=SimpleNamespace(enable_cross_format_highlighting=False), + ): + hydrated = runner._hydrate_citations( + [ + Citation( + citation_id="C1", + node_id="invented-node", + page_no=14, + exact_quote="Invented quote.", + ) + ], + doc_id="legal-doc", + version=1, + context_node_ids=["real-node-on-page-14"], + ) + + assert hydrated == [] diff --git a/backend/tests/test_context_packer.py b/backend/tests/test_context_packer.py index caa2f3b..ce0e9e3 100644 --- a/backend/tests/test_context_packer.py +++ b/backend/tests/test_context_packer.py @@ -26,7 +26,9 @@ def test_to_text_exposes_canonical_node_id_page_marker() -> None: ) text = packed.to_text(include_citations=True) - assert text.startswith("[chunk_abc123:3] source=seed\nChunk body") + assert text.startswith( + "[chunk_abc123:3] node_id=chunk_abc123 page_no=3 source=seed\nChunk body" + ) def test_to_text_keeps_metadata_outside_citation_brackets() -> None: diff --git a/backend/tests/test_evidence_chain_integration.py b/backend/tests/test_evidence_chain_integration.py new file mode 100644 index 0000000..bdbef25 --- /dev/null +++ b/backend/tests/test_evidence_chain_integration.py @@ -0,0 +1,342 @@ +"""PostgreSQL integration coverage for evidence chains and ACL boundaries.""" + +from __future__ import annotations + +import uuid +from unittest.mock import patch + +import fitz +import pytest + +from app.acl.enforcer import ACLEnforcer +from app.acl.models import Entitlements +from app.db.graph_models import DocumentGraph, Edge, EdgeType, Node, NodeType +from app.db.session import SessionLocal +from app.graph.context_packer import ContextPacker +from app.graph.chunker import PageBoundedChunker +from app.graph.expander import ExpandedContext +from app.graph.pipeline import GraphIngestionPipeline +from app.qa.evidence_chain import EvidenceChainConfig, EvidenceChainEngine +from app.services.highlighting import verify_evidence_span + + +pytestmark = pytest.mark.integration + + +def test_real_pdf_ingestion_builds_a_chain_ready_provenance_graph(): + """Exercise native PDF ingestion -> persisted nodes/edges -> chain search.""" + suffix = uuid.uuid4().hex[:10] + source_uri = f"test://chain-ingest-{suffix}.pdf" + pdf = fitz.open() + try: + page_texts = [ + "The agreement originally required thirty days written notice.", + "The 2026 amendment replaces the original notice provision.", + "The governing statute requires sixty days notice after the amendment.", + ] + for text in page_texts: + page = pdf.new_page(width=612, height=792) + page.insert_text((72, 100), text, fontsize=11) + pdf_bytes = pdf.tobytes() + finally: + pdf.close() + + session = SessionLocal() + try: + pipeline = GraphIngestionPipeline( + session, + skip_ocr=True, + tenant_id="chain-test-tenant", + visibility="public", + ) + pipeline.chunker = PageBoundedChunker(target_tokens=100, min_tokens=1) + # Artifact storage is deliberately excluded from this PostgreSQL + # representative-stack test; selector construction still runs. + with patch( + "app.graph.pipeline.get_storage_client", + side_effect=RuntimeError("object storage intentionally disabled in test"), + ): + ingest = pipeline.ingest(pdf_bytes=pdf_bytes, source_uri=source_uri) + + nodes = ( + session.query(Node) + .filter(Node.doc_id == ingest.doc_id, Node.version == ingest.version) + .order_by(Node.page_no.asc(), Node.chunk_index_in_page.asc()) + .all() + ) + edges = ( + session.query(Edge) + .filter(Edge.doc_id == ingest.doc_id, Edge.version == ingest.version) + .all() + ) + assert ingest.total_pages == 3 + assert len(nodes) == 3 + assert len(edges) >= 2 + assert all((node.meta or {}).get("source_spans") for node in nodes) + assert all((node.meta or {}).get("selector_bundle") for node in nodes) + + enforcer = ACLEnforcer( + session, + Entitlements( + tenant_id="chain-test-tenant", + user_id="alice", + roles=frozenset(), + groups=frozenset(), + ), + ) + chain = EvidenceChainEngine( + session, + acl_enforcer=enforcer, + config=EvidenceChainConfig(mode="on", max_selected_nodes=6), + ).build( + ExpandedContext( + seed_nodes=[nodes[0]], + adjacent_nodes=[], + referenced_nodes=[], + explained_by_nodes=[], + node_sources={nodes[0].node_id: "seed"}, + ), + "How does the 2026 amendment affect notice under the governing statute?", + {nodes[0].node_id: 1.0}, + doc_id=ingest.doc_id, + version=ingest.version, + ) + + assert chain.applied is True + assert {node.node_id for node in nodes}.issubset(chain.selected_node_ids) + assert any( + path.node_ids[0] == nodes[0].node_id + and path.node_ids[-1] == nodes[-1].node_id + for path in chain.paths + ) + finally: + session.close() + + +def test_postgres_chain_traversal_packing_and_node_acl_are_consistent(): + suffix = uuid.uuid4().hex[:10] + doc_id = f"chain-int-{suffix}" + seed_id = f"seed-{suffix}" + bridge_id = f"bridge-{suffix}" + target_id = f"target-{suffix}" + denied_id = f"denied-{suffix}" + session = SessionLocal() + + try: + session.add( + DocumentGraph( + doc_id=doc_id, + source_uri=f"test://{doc_id}", + content_hash=uuid.uuid4().hex * 2, + version=1, + tenant_id="chain-test-tenant", + visibility="public", + policy_version=1, + ) + ) + nodes = [ + Node( + node_id=seed_id, + doc_id=doc_id, + version=1, + node_type=NodeType.chunk, + page_no=1, + chunk_index_in_page=0, + text_plain="The patient has renal impairment.", + meta={}, + ), + Node( + node_id=bridge_id, + doc_id=doc_id, + version=1, + node_type=NodeType.chunk, + page_no=2, + chunk_index_in_page=0, + text_plain="Drug A is cleared through the kidneys.", + meta={}, + ), + Node( + node_id=target_id, + doc_id=doc_id, + version=1, + node_type=NodeType.chunk, + page_no=3, + chunk_index_in_page=0, + text_plain="The current guideline lists renal impairment as a contraindication.", + meta={}, + ), + Node( + node_id=denied_id, + doc_id=doc_id, + version=1, + node_type=NodeType.chunk, + page_no=4, + chunk_index_in_page=0, + text_plain="Node-level restricted evidence must never influence the chain.", + meta={ + "acl_override": { + "visibility": "restricted", + "allowed_users": ["bob"], + } + }, + ), + ] + session.add_all(nodes) + session.flush() + session.add_all( + [ + Edge( + doc_id=doc_id, + version=1, + from_node_id=seed_id, + to_node_id=bridge_id, + edge_type=EdgeType.references, + confidence=0.9, + ), + Edge( + doc_id=doc_id, + version=1, + from_node_id=bridge_id, + to_node_id=target_id, + edge_type=EdgeType.explained_by, + confidence=1.0, + ), + Edge( + doc_id=doc_id, + version=1, + from_node_id=seed_id, + to_node_id=denied_id, + edge_type=EdgeType.references, + confidence=1.0, + ), + ] + ) + session.flush() + + enforcer = ACLEnforcer( + session, + Entitlements( + tenant_id="chain-test-tenant", + user_id="alice", + roles=frozenset(), + groups=frozenset(), + ), + ) + engine = EvidenceChainEngine( + session, + acl_enforcer=enforcer, + config=EvidenceChainConfig(mode="on", max_selected_nodes=6), + ) + expanded = ExpandedContext( + seed_nodes=[nodes[0]], + adjacent_nodes=[], + referenced_nodes=[], + explained_by_nodes=[], + node_sources={seed_id: "seed"}, + ) + + result = engine.build( + expanded, + "How does renal impairment affect Drug A under the current guideline?", + {seed_id: 1.0}, + doc_id=doc_id, + version=1, + ) + + assert result.applied is True + assert {seed_id, bridge_id, target_id}.issubset(result.selected_node_ids) + assert denied_id not in result.selected_node_ids + assert denied_id not in str(result.to_audit_dict()) + assert any( + path.node_ids == (seed_id, bridge_id, target_id) + for path in result.paths + ) + + expanded.chain_nodes = result.additional_nodes + for node in expanded.chain_nodes: + expanded.node_sources[node.node_id] = "chain" + packed = ContextPacker(max_tokens=500).pack( + expanded, + node_order=result.ordered_node_ids, + allowed_node_ids=result.selected_node_ids, + ) + packed_text = packed.to_text() + assert f"[{seed_id}:1]" in packed_text + assert f"[{bridge_id}:2]" in packed_text + assert f"[{target_id}:3]" in packed_text + assert denied_id not in packed_text + assert any( + entry["stage"].startswith("evidence_chain_hop_") and entry["denied"] == 1 + for entry in enforcer.get_audit_log() + ) + + # The chain ranking is not allowed to manufacture a locator. Prove + # that a selected leaf still resolves through the independent source + # provenance verifier, and that a wrong-page request fails closed. + quote = nodes[2].text_plain + words = quote.split() + source_spans = [ + { + "span_id": f"p3:w{index}", + "order": index, + "text": word, + "block_no": 0, + "line_no": 0, + "word_no": index, + "normalized_bbox": { + "x0": 0.05 + index * 0.06, + "y0": 0.20, + "x1": 0.10 + index * 0.06, + "y1": 0.23, + }, + "coordinate_system": "pdf_points_top_left", + "extraction_source": "native_pdf", + "verifiable": True, + } + for index, word in enumerate(words) + ] + source_map = { + "doc_id": doc_id, + "version": 1, + "content_hash": "sha256:chain-source", + "canonical_text": quote, + "nodes": [ + { + "node_id": target_id, + "start": 0, + "end": len(quote), + "page_no": 3, + "page_rotation": 0, + "page_size": {"width": 612, "height": 792}, + "source_spans": source_spans, + } + ], + } + verified = verify_evidence_span( + doc_id=doc_id, + document_version=1, + node_id=target_id, + page_index=3, + quote_text=quote, + locator=None, + source_hash="sha256:chain-source", + source_map=source_map, + ) + wrong_page = verify_evidence_span( + doc_id=doc_id, + document_version=1, + node_id=target_id, + page_index=2, + quote_text=quote, + locator=None, + source_hash="sha256:chain-source", + source_map=source_map, + ) + assert verified["status"] == "FOUND" + assert verified["grade"] == "verified" + assert verified["matched_locator"]["type"] == "rects" + assert wrong_page["status"] == "NOT_FOUND" + assert wrong_page["reason"] == "page_not_indexed" + finally: + session.rollback() + session.close() diff --git a/backend/tests/test_evidence_provenance_v2.py b/backend/tests/test_evidence_provenance_v2.py new file mode 100644 index 0000000..9abc527 --- /dev/null +++ b/backend/tests/test_evidence_provenance_v2.py @@ -0,0 +1,306 @@ +"""Tests for word-level ingestion and chunk provenance.""" + +from types import SimpleNamespace + +import fitz +import pytest + +from app.graph.chunker import PageBoundedChunker +from app.graph.ids import compute_content_hash +from app.graph.nodes import create_chunk_nodes +from app.graph.page_extractor import PageExtractor, TextSpan +from app.services.highlighting import ( + backfill_pdf_source_provenance, + build_selector_artifacts_for_nodes, + verify_evidence_span, +) + + +def _word(text: str, order: int, x0: float) -> TextSpan: + return TextSpan( + text=text, + bbox=(x0, 10.0, x0 + 20.0, 20.0), + span_id=f"p1:w{order}", + order=order, + block_no=0, + line_no=0, + word_no=order, + normalized_bbox={ + "x0": x0 / 200.0, + "y0": 0.1, + "x1": (x0 + 20.0) / 200.0, + "y1": 0.2, + }, + ) + + +def test_chunk_provenance_resolves_one_exact_source_sequence(): + page = SimpleNamespace( + page_no=1, + width=200.0, + height=100.0, + text_spans=[ + _word("The", 0, 10), + _word("fee", 1, 35), + _word("is", 2, 60), + _word("$45,000.", 3, 85), + ], + ) + chunker = PageBoundedChunker(target_tokens=100) + spans, status = chunker._compute_chunk_source_spans("The fee is $45,000.", page) + + assert status == "exact_source_words" + assert [span["span_id"] for span in spans] == [ + "p1:w0", + "p1:w1", + "p1:w2", + "p1:w3", + ] + + +def test_chunk_provenance_rejects_ambiguous_repeated_text(): + page = SimpleNamespace( + page_no=1, + width=200.0, + height=100.0, + text_spans=[ + _word("Fee", 0, 10), + _word("due.", 1, 35), + _word("Fee", 2, 70), + _word("due.", 3, 95), + ], + ) + chunker = PageBoundedChunker(target_tokens=100) + spans, status = chunker._compute_chunk_source_spans("Fee due.", page) + + assert spans == [] + assert status == "ambiguous_chunk_in_source_words" + + +@pytest.mark.parametrize("rotation", [0, 90, 180, 270]) +def test_pdf_word_extraction_normalizes_rotated_page_coordinates(rotation): + doc = fitz.open() + page = doc.new_page(width=300, height=200) + page.insert_text((30, 50), "Verified evidence") + page.set_rotation(rotation) + pdf_bytes = doc.tobytes() + doc.close() + + reopened = fitz.open(stream=pdf_bytes, filetype="pdf") + try: + rotated_page = reopened[0] + spans = PageExtractor(skip_ocr=True)._extract_text_spans(rotated_page) + finally: + reopened.close() + + assert [span.text for span in spans] == ["Verified", "evidence"] + assert all(span.coordinate_system == "pdf_points_top_left" for span in spans) + assert all(span.verifiable for span in spans) + for span in spans: + rect = span.normalized_bbox + assert rect is not None + assert 0 <= rect["x0"] < rect["x1"] <= 1 + assert 0 <= rect["y0"] < rect["y1"] <= 1 + + +def test_real_pdf_ingestion_to_verified_rectangles(): + """Exercise PDF extraction → chunk provenance → source map → verification.""" + doc = fitz.open() + page = doc.new_page(width=612, height=792) + page.insert_text( + (72, 100), + "The initial franchise fee is $45,000.", + fontsize=12, + ) + page.insert_text( + (72, 130), + "Payment is due when the agreement is signed.", + fontsize=12, + ) + pdf_bytes = doc.tobytes() + doc.close() + + extraction = PageExtractor(skip_ocr=True).extract_pages( + pdf_bytes, + doc_id="fixture-doc", + ) + chunks = PageBoundedChunker(target_tokens=200).chunk_pages( + pages=[(p.page_no, p.text_plain) for p in extraction.pages], + doc_id="fixture-doc", + version=1, + page_data_list=extraction.pages, + ) + nodes = create_chunk_nodes(chunks, doc_id="fixture-doc", version=1) + graph_doc = SimpleNamespace( + doc_id="fixture-doc", + version=1, + content_hash="fixturehash", + source_uri="upload://fixture.pdf", + ) + artifacts = build_selector_artifacts_for_nodes( + graph_doc, + nodes, + source_type="pdf", + mime_type="application/pdf", + ) + + result = verify_evidence_span( + doc_id="fixture-doc", + node_id=nodes[0].node_id, + page_index=1, + quote_text="The initial franchise fee is $45,000.", + locator=None, + source_hash="sha256:fixturehash", + source_map=artifacts["source_map"], + ) + + assert chunks[0].meta["source_span_resolution"] == "exact_source_words" + assert result["status"] == "FOUND" + assert result["grade"] == "verified" + assert result["matched_locator"]["rects"] + + +def test_legal_and_clinical_evidence_remains_bound_to_its_original_page(): + """Curated two-page fixture for the high-risk review use cases.""" + doc = fitz.open() + legal_page = doc.new_page(width=612, height=792) + legal_page.insert_text( + (72, 100), + "The tenant may terminate this agreement with thirty days written notice.", + fontsize=11, + ) + clinical_page = doc.new_page(width=612, height=792) + clinical_page.insert_text( + (72, 100), + "The patient gave informed consent before the procedure began.", + fontsize=11, + ) + pdf_bytes = doc.tobytes() + doc.close() + + extraction = PageExtractor(skip_ocr=True).extract_pages( + pdf_bytes, + doc_id="risk-review-fixture", + ) + chunks = PageBoundedChunker(target_tokens=200).chunk_pages( + pages=[(p.page_no, p.text_plain) for p in extraction.pages], + doc_id="risk-review-fixture", + version=3, + page_data_list=extraction.pages, + ) + nodes = create_chunk_nodes(chunks, doc_id="risk-review-fixture", version=3) + graph_doc = SimpleNamespace( + doc_id="risk-review-fixture", + version=3, + content_hash=compute_content_hash(pdf_bytes), + source_uri="upload://risk-review-fixture.pdf", + ) + source_map = build_selector_artifacts_for_nodes( + graph_doc, + nodes, + source_type="pdf", + mime_type="application/pdf", + )["source_map"] + clinical_node = next(node for node in nodes if node.page_no == 2) + quote = "The patient gave informed consent before the procedure began." + + verified = verify_evidence_span( + doc_id=graph_doc.doc_id, + document_version=3, + node_id=clinical_node.node_id, + page_index=2, + quote_text=quote, + locator=None, + source_hash=graph_doc.content_hash, + source_map=source_map, + ) + wrong_page = verify_evidence_span( + doc_id=graph_doc.doc_id, + document_version=3, + node_id=clinical_node.node_id, + page_index=1, + quote_text=quote, + locator=None, + source_hash=graph_doc.content_hash, + source_map=source_map, + ) + + assert verified["status"] == "FOUND" + assert verified["grade"] == "verified" + assert verified["page_index"] == 2 + assert wrong_page["status"] == "NOT_FOUND" + assert wrong_page["reason"] == "evidence_not_found_on_cited_page" + + +def test_legacy_native_pdf_can_be_backfilled_without_changing_node_identity(): + doc = fitz.open() + page = doc.new_page(width=612, height=792) + page.insert_text((72, 100), "The consent form must be signed.", fontsize=12) + pdf_bytes = doc.tobytes() + doc.close() + + graph_doc = SimpleNamespace( + doc_id="legacy-doc", + version=1, + content_hash=compute_content_hash(pdf_bytes), + source_uri="upload://legacy.pdf", + ) + node = SimpleNamespace( + node_id="stable-node-id", + doc_id="legacy-doc", + version=1, + node_type=SimpleNamespace(value="chunk"), + page_no=1, + chunk_index_in_page=0, + text_plain="The consent form must be signed.", + text_md=None, + caption_md=None, + label=None, + bbox=None, + meta={}, + ) + + class FakeQuery: + def filter(self, *_args): + return self + + def all(self): + return [node] + + class FakeDB: + flushed = False + + def query(self, _model): + return FakeQuery() + + def flush(self): + self.flushed = True + + class FakeStorage: + canonical = None + source_map = None + selectors = None + + def put_canonical_view(self, _doc_id, _version, value): + self.canonical = value + + def put_source_map(self, _doc_id, _version, value): + self.source_map = value + + def put_selectors(self, _doc_id, _version, value): + self.selectors = value + + fake_db = FakeDB() + fake_storage = FakeStorage() + result = backfill_pdf_source_provenance( + fake_db, + graph_doc, + pdf_bytes, + storage=fake_storage, + ) + + assert node.node_id == "stable-node-id" + assert node.meta["source_span_resolution"] == "exact_source_words" + assert result["exact_nodes"] == 1 + assert fake_db.flushed is True + assert fake_storage.source_map["nodes"][0]["source_spans"] diff --git a/backend/tests/test_evidence_span_contract.py b/backend/tests/test_evidence_span_contract.py index 984bf81..893c234 100644 --- a/backend/tests/test_evidence_span_contract.py +++ b/backend/tests/test_evidence_span_contract.py @@ -1,11 +1,14 @@ """Tests for EvidenceSpan parsing and normalization contract.""" from app.qa.evidence_span import ( + EvidenceRecord, EvidenceSpan, build_evidence_spans, normalize_page_index, parse_evidence_span, ) +from pydantic import ValidationError +import pytest def test_parse_evidence_span_text_offsets_schema(): @@ -117,3 +120,69 @@ def test_build_evidence_spans_accepts_and_normalizes_provided_spans(): assert span["page_index_base"] == 1 assert span["quote_text"] == "Provided quote" assert span["confidence"] == 0.6 + + +def test_evidence_record_verified_requires_normalized_rectangles(): + record = EvidenceRecord.model_validate( + { + "citation_id": "C1", + "doc_id": "doc-1", + "document_version": 1, + "node_id": "node-1", + "page": 2, + "exact_quote": "Exact source words.", + "source_hash": "sha256:test", + "status": "verified", + "verification_reason": "exact_unique_quote_with_source_rectangles", + "locator": { + "type": "rects", + "coordinate_system": "normalized_top_left", + "rects": [{"x0": 0.1, "y0": 0.2, "x1": 0.4, "y1": 0.23}], + "page_rotation": 0, + }, + "confidence": 1.0, + } + ) + assert record.locator.type == "rects" + + +def test_evidence_record_rejects_verified_text_offsets(): + with pytest.raises(ValidationError): + EvidenceRecord.model_validate( + { + "citation_id": "C1", + "doc_id": "doc-1", + "document_version": 1, + "node_id": "node-1", + "page": 2, + "exact_quote": "Exact source words.", + "source_hash": "sha256:test", + "status": "verified", + "verification_reason": "not_renderable", + "locator": {"type": "text_offsets", "start": 0, "end": 5}, + "confidence": 1.0, + } + ) + + +def test_evidence_record_rejects_out_of_bounds_rectangle(): + with pytest.raises(ValidationError): + EvidenceRecord.model_validate( + { + "citation_id": "C1", + "doc_id": "doc-1", + "document_version": 1, + "node_id": "node-1", + "page": 2, + "exact_quote": "Exact source words.", + "source_hash": "sha256:test", + "status": "verified", + "verification_reason": "bad_rectangle", + "locator": { + "type": "rects", + "coordinate_system": "normalized_top_left", + "rects": [{"x0": 0.1, "y0": 0.2, "x1": 1.4, "y1": 0.3}], + }, + "confidence": 1.0, + } + ) diff --git a/backend/tests/test_grounded_answer_contract.py b/backend/tests/test_grounded_answer_contract.py new file mode 100644 index 0000000..cdaa285 --- /dev/null +++ b/backend/tests/test_grounded_answer_contract.py @@ -0,0 +1,155 @@ +"""Tests for structured claim-level citations returned by the answer model.""" + +from types import SimpleNamespace + +from app.llm.openai_client import OpenAIClient + + +def test_parse_grounded_response_with_stable_ids_and_exact_quotes(): + parsed = OpenAIClient._parse_grounded_response( + """ + { + "answer": "The fee is $45,000 [C1].", + "citations": [ + { + "citation_id": "C1", + "node_id": "fee-node", + "page_no": 5, + "exact_quote": "The fee is $45,000.", + "label": null + } + ] + } + """ + ) + + assert parsed is not None + answer, citations = parsed + assert answer == "The fee is $45,000 [C1]." + assert citations[0].citation_id == "C1" + assert citations[0].exact_quote == "The fee is $45,000." + + +def test_parse_grounded_response_rejects_missing_citation_payload(): + assert OpenAIClient._parse_grounded_response( + '{"answer":"The fee is $45,000 [C1].","citations":[]}' + ) is None + + +def test_parse_grounded_response_rejects_duplicate_ids(): + assert OpenAIClient._parse_grounded_response( + """ + { + "answer": "One [C1]. Two [C1].", + "citations": [ + {"citation_id":"C1","node_id":"n1","page_no":1,"exact_quote":"One.","label":null}, + {"citation_id":"C1","node_id":"n2","page_no":2,"exact_quote":"Two.","label":null} + ] + } + """ + ) is None + + +def test_generate_answer_requests_strict_schema_and_returns_structured_citations(): + client = object.__new__(OpenAIClient) + client.model = "test-model" + captured = {} + + def fake_create(request, _timeout): + captured.update(request) + return SimpleNamespace( + output_text=( + '{"answer":"The fee is $45,000 [C1].","citations":[' + '{"citation_id":"C1","node_id":"n1","page_no":1,' + '"exact_quote":"The fee is $45,000.","label":null}]}' + ), + usage=SimpleNamespace(input_tokens=10, output_tokens=20, total_tokens=30), + model="test-model", + ) + + client._responses_create_with_retry = fake_create + result = client.generate_answer( + context="[n1:1] The fee is $45,000.", + question="What is the fee?", + ) + + assert captured["text"]["format"]["type"] == "json_schema" + assert captured["text"]["format"]["strict"] is True + citation_schema = captured["text"]["format"]["schema"]["properties"]["citations"]["items"] + assert set(citation_schema["required"]) == set(citation_schema["properties"]) + assert result.answer == "The fee is $45,000 [C1]." + assert result.citations[0].citation_id == "C1" + + +def test_validate_grounded_citations_checks_node_page_and_verbatim_quote(): + context = "[n1:5] source=seed\nThe filing fee is $250." + parsed = OpenAIClient._parse_grounded_response( + '{"answer":"The filing fee is $250 [C1].","citations":[' + '{"citation_id":"C1","node_id":"n1","page_no":5,' + '"exact_quote":"The filing fee is $250.","label":null}]}' + ) + assert parsed is not None + assert OpenAIClient._validate_grounded_citations(context, *parsed) == [] + + parsed_bad = OpenAIClient._parse_grounded_response( + '{"answer":"The filing fee is $250 [C1].","citations":[' + '{"citation_id":"C1","node_id":"made-up","page_no":8,' + '"exact_quote":"A paraphrased filing fee.","label":null}]}' + ) + assert parsed_bad is not None + errors = OpenAIClient._validate_grounded_citations(context, *parsed_bad) + assert any("unknown node_id" in error for error in errors) + + +def test_canonicalize_citation_reference_only_strips_matching_page_suffix(): + context = "[fee:5] node_id=fee page_no=5 source=seed\nThe filing fee is $250." + parsed = OpenAIClient._parse_grounded_response( + '{"answer":"The filing fee is $250 [C1].","citations":[' + '{"citation_id":"C1","node_id":"fee:5","page_no":5,' + '"exact_quote":"The filing fee is $250.","label":null}]}' + ) + assert parsed is not None + OpenAIClient._canonicalize_citation_references(context, parsed[1]) + assert parsed[1][0].node_id == "fee" + assert OpenAIClient._validate_grounded_citations(context, *parsed) == [] + + parsed_wrong_page = OpenAIClient._parse_grounded_response( + '{"answer":"The filing fee is $250 [C1].","citations":[' + '{"citation_id":"C1","node_id":"fee:9","page_no":9,' + '"exact_quote":"The filing fee is $250.","label":null}]}' + ) + assert parsed_wrong_page is not None + OpenAIClient._canonicalize_citation_references(context, parsed_wrong_page[1]) + assert parsed_wrong_page[1][0].node_id == "fee:9" + + +def test_generate_answer_repairs_invalid_citation_metadata_before_returning(): + client = object.__new__(OpenAIClient) + client.model = "test-model" + responses = [ + '{"answer":"The fee is $250 [C1].","citations":[' + '{"citation_id":"C1","node_id":"wrong","page_no":9,' + '"exact_quote":"The fee is $250.","label":null}]}', + '{"answer":"The fee is $250 [C1].","citations":[' + '{"citation_id":"C1","node_id":"fee","page_no":1,' + '"exact_quote":"The fee is $250.","label":null}]}', + ] + requests = [] + + def fake_create(request, _timeout): + requests.append(request) + return SimpleNamespace( + output_text=responses.pop(0), + usage=SimpleNamespace(input_tokens=10, output_tokens=20, total_tokens=30), + model="test-model", + ) + + client._responses_create_with_retry = fake_create + result = client.generate_answer( + context="[fee:1] source=seed\nThe fee is $250.", + question="What is the fee?", + ) + + assert len(requests) == 2 + assert "CITATION_VALIDATION_FEEDBACK" in requests[1]["input"] + assert result.citations[0].node_id == "fee" diff --git a/backend/tests/test_highlighting_service.py b/backend/tests/test_highlighting_service.py index a1badf2..1200fbd 100644 --- a/backend/tests/test_highlighting_service.py +++ b/backend/tests/test_highlighting_service.py @@ -167,3 +167,138 @@ def test_verify_evidence_span_not_found_on_cited_page(): assert result["status"] == "NOT_FOUND" assert result["matched_locator"] is None assert result["reason"] == "evidence_not_found_on_cited_page" + + +def _v2_source_map(quote: str = "Initial Franchise Fee is due"): + words = quote.split() + spans = [] + x = 0.1 + for order, word in enumerate(words): + width = 0.04 + len(word) * 0.003 + spans.append( + { + "span_id": f"p1:w{order}", + "order": order, + "text": word, + "block_no": 0, + "line_no": 0 if order < 3 else 1, + "word_no": order, + "normalized_bbox": { + "x0": x, + "y0": 0.20 if order < 3 else 0.24, + "x1": min(0.98, x + width), + "y1": 0.22 if order < 3 else 0.26, + }, + "coordinate_system": "pdf_points_top_left", + "extraction_source": "native_pdf", + "verifiable": True, + } + ) + x = x + width + 0.01 if order != 2 else 0.1 + return { + "doc_id": "doc-1", + "version": 1, + "content_hash": "sha256:testhash", + "canonical_text": quote, + "nodes": [ + { + "node_id": "n1", + "start": 0, + "end": len(quote), + "page_no": 1, + "page_rotation": 0, + "page_size": {"width": 612, "height": 792}, + "source_spans": spans, + } + ], + } + + +def test_verify_v2_evidence_returns_precise_normalized_line_rectangles(): + source_map = _v2_source_map() + result = verify_evidence_span( + doc_id="doc-1", + node_id="n1", + page_index=1, + quote_text="Initial Franchise Fee is due", + locator=None, + source_hash="sha256:testhash", + source_map=source_map, + ) + + assert result["status"] == "FOUND" + assert result["grade"] == "verified" + assert result["matched_locator"]["type"] == "rects" + assert result["matched_locator"]["coordinate_system"] == "normalized_top_left" + assert len(result["matched_locator"]["rects"]) == 2 + + +def test_verify_v2_evidence_fails_closed_on_ambiguous_quote(): + source_map = _v2_source_map("Fee Fee") + result = verify_evidence_span( + doc_id="doc-1", + node_id="n1", + page_index=1, + quote_text="Fee", + locator=None, + source_hash="sha256:testhash", + source_map=source_map, + ) + + assert result["status"] == "NOT_FOUND" + assert result["grade"] == "unavailable" + assert result["reason"] == "ambiguous_exact_quote_in_cited_node" + + +def test_verify_v2_evidence_rejects_stale_source_map_hash(): + result = verify_evidence_span( + doc_id="doc-1", + node_id="n1", + page_index=1, + quote_text="Initial Franchise Fee is due", + locator=None, + source_hash="sha256:changed", + source_map=_v2_source_map(), + ) + + assert result["status"] == "NOT_FOUND" + assert result["reason"] == "source_map_content_hash_mismatch" + + +def test_verify_v2_evidence_rejects_wrong_document_version(): + result = verify_evidence_span( + doc_id="doc-1", + document_version=2, + node_id="n1", + page_index=1, + quote_text="Initial Franchise Fee is due", + locator=None, + source_hash="sha256:testhash", + source_map=_v2_source_map(), + ) + + assert result["status"] == "NOT_FOUND" + assert result["reason"] == "source_map_version_mismatch" + + +def test_verify_v2_evidence_without_renderable_coordinates_is_unavailable(): + source_map = _v2_source_map() + for span in source_map["nodes"][0]["source_spans"]: + span["verifiable"] = False + span["normalized_bbox"] = None + span["extraction_source"] = "ocr_text_only" + + result = verify_evidence_span( + doc_id="doc-1", + document_version=1, + node_id="n1", + page_index=1, + quote_text="Initial Franchise Fee is due", + locator=None, + source_hash="sha256:testhash", + source_map=source_map, + ) + + assert result["status"] == "NOT_FOUND" + assert result["grade"] == "unavailable" + assert result["reason"] == "source_coordinates_not_verifiable" diff --git a/backend/tests/test_routes.py b/backend/tests/test_routes.py index c6bd39f..2dafdfc 100644 --- a/backend/tests/test_routes.py +++ b/backend/tests/test_routes.py @@ -471,6 +471,18 @@ def test_invalid_mode(self): ) assert response.status_code == 422 + def test_invalid_evidence_chain_mode(self): + """Unknown evidence-chain modes must be rejected at the API boundary.""" + response = client.post( + "/v1/qa/ask", + json={ + "doc_id": "test-doc-123", + "question": "Compare the amendment with the original agreement.", + "evidence_chain_mode": "truth_mode", + }, + ) + assert response.status_code == 422 + class TestQASuccess: """Test successful QA scenarios.""" @@ -562,6 +574,51 @@ def test_propagation_safety_mode_accepted(self, mock_runner_class): data = response.json() assert data["propagation_safety_mode"] is True + @patch("app.routes.qa.QARunner") + def test_evidence_chain_mode_and_audit_are_returned(self, mock_runner_class): + """The optional evidence-chain request and audit remain API-compatible.""" + mock_runner = MagicMock() + mock_runner_class.return_value = mock_runner + mock_result = MagicMock() + mock_result.to_dict.return_value = { + "question": "Compare the amendment with the original agreement.", + "doc_id": "test-doc", + "seed_nodes": [], + "expanded_nodes": [], + "edge_traces": [], + "packed_context": "", + "context_node_ids": [], + "total_context_tokens": 0, + "answer": "Answer", + "citations": [], + "model_id": "mock-model", + "timing": {"evidence_chain_ms": 2.0}, + "metadata": {}, + "evidence_chain": { + "enabled": True, + "mode": "auto", + "applied": True, + "time_ms": 2.0, + "audit": {"scoring_version": "document-chain-v1"}, + }, + "success": True, + "error": None, + } + mock_runner.run.return_value = mock_result + + response = client.post( + "/v1/qa/ask", + json={ + "doc_id": "test-doc", + "question": "Compare the amendment with the original agreement.", + "evidence_chain_mode": "auto", + }, + ) + + assert response.status_code == 200 + assert response.json()["evidence_chain"]["applied"] is True + assert mock_runner_class.call_args.kwargs["evidence_chain_mode"] == "auto" + class TestQAErrorHandling: """Test QA error handling.""" diff --git a/frontend/src/app/(app)/chat/chat-client.tsx b/frontend/src/app/(app)/chat/chat-client.tsx index 9a2c284..c668eb4 100644 --- a/frontend/src/app/(app)/chat/chat-client.tsx +++ b/frontend/src/app/(app)/chat/chat-client.tsx @@ -15,6 +15,8 @@ import { Copy, Check, RotateCcw, + GitBranch, + ShieldCheck, } from "lucide-react"; import ReactMarkdown from "react-markdown"; import { toast } from "sonner"; @@ -39,6 +41,7 @@ import { import { Sheet, SheetContent, + SheetDescription, SheetHeader, SheetTitle, SheetTrigger, @@ -55,35 +58,10 @@ import { CollapsibleTrigger, } from "@/components/ui/collapsible"; import { SourceViewer } from "@/components/source-viewer"; - -const INLINE_CITATION_PATTERN = /\[(seed|adjacent|page):\s*(\d+)\]/gi; - -function normalizeInlineRef(value: string): string { - return value.toLowerCase().replace(/[\[\]\s]+/g, ""); -} - -function findCitationForInlineRef( - citations: Citation[] | undefined, - refType: string, - refValue: string -): Citation | undefined { - if (!citations?.length) return undefined; - - const normalizedRef = `${refType.toLowerCase()}:${refValue}`; - const byLabel = citations.find((citation) => { - if (!citation.label) return false; - const normalizedLabel = normalizeInlineRef(citation.label); - return normalizedLabel === normalizedRef || normalizedLabel.endsWith(normalizedRef); - }); - if (byLabel) return byLabel; - - const pageNo = Number.parseInt(refValue, 10); - if (!Number.isNaN(pageNo)) { - return citations.find((citation) => citation.page_no === pageNo); - } - - return undefined; -} +import { + findCitationForInlineRef, + INLINE_CITATION_PATTERN, +} from "@/components/source-viewer/citation-routing"; // Helper to parse inline citations like [seed:14] and [adjacent:24] and render clickable chips function parseInlineCitations( @@ -109,11 +87,11 @@ function parseInlineCitations( ); } - const refType = match[1].toLowerCase(); - const refValue = match[2]; - const pageNo = Number.parseInt(refValue, 10); - const citation = findCitationForInlineRef(citations, refType, refValue); - const chipLabel = refType === "page" ? `p.${refValue}` : `${refType}:${refValue}`; + const rawRef = match[1].replace(/\s+/g, ""); + const normalizedRef = rawRef.toLowerCase(); + const citation = findCitationForInlineRef(citations, rawRef); + const pageMatch = normalizedRef.match(/^page:(\d+)$/); + const chipLabel = pageMatch ? `p.${pageMatch[1]}` : rawRef; if (citation && onCitationClick) { // Render as inline clickable chip @@ -122,7 +100,13 @@ function parseInlineCitations( key={`cite-${key++}`} onClick={() => onCitationClick(citation)} className="inline-flex items-center gap-1 px-1.5 py-0.5 mx-0.5 text-xs font-medium bg-primary/10 text-primary hover:bg-primary/20 rounded border border-primary/20 transition-colors cursor-pointer align-baseline" - title={!Number.isNaN(pageNo) ? `View source: Page ${pageNo}` : `View source: ${chipLabel}`} + title={ + citation.page_no && citation.evidence_status === "verified" + ? `View verified evidence: Page ${citation.page_no}` + : citation.page_no + ? `View source: Page ${citation.page_no}` + : `View source: ${chipLabel}` + } > {chipLabel} @@ -169,8 +153,9 @@ interface Message extends Omit { // Citation chip component function CitationChip({ citation, onClick }: { citation: Citation; onClick?: () => void }) { - const label = citation.label || `Page ${citation.page_no || "?"}`; - const status = citation.resolve_status || "unresolved"; + const label = citation.citation_id || citation.label || `Page ${citation.page_no || "?"}`; + const status = citation.evidence_status || citation.resolve_status || "unavailable"; + const exactQuote = citation.evidence_records?.[0]?.exact_quote || citation.text; return ( @@ -185,8 +170,8 @@ function CitationChip({ citation, onClick }: { citation: Citation; onClick?: ()

- {citation.text?.slice(0, 150)} - {citation.text && citation.text.length > 150 ? "..." : ""} + {exactQuote?.slice(0, 150)} + {exactQuote && exactQuote.length > 150 ? "..." : ""}

Status: {status} @@ -397,9 +382,16 @@ function NodeCard({ } // Source panel component -function SourcePanel({ response }: { response: AskResponse }) { +function SourcePanel({ + response, + onCitationClick, +}: { + response: AskResponse; + onCitationClick?: (citation: Citation) => void; +}) { const [seedsOpen, setSeedsOpen] = React.useState(true); const [expandedOpen, setExpandedOpen] = React.useState(false); + const [chainsOpen, setChainsOpen] = React.useState(true); const timing = response.timing || {}; const totalMs = timing.total_ms || 0; @@ -409,8 +401,15 @@ function SourcePanel({ response }: { response: AskResponse }) { { label: "Embedding", value: timing.embedding_ms || 0, color: "bg-blue-500" }, { label: "Search", value: timing.search_ms || 0, color: "bg-violet-500" }, { label: "Expansion", value: timing.expansion_ms || 0, color: "bg-amber-500" }, + { label: "Evidence chain", value: timing.evidence_chain_ms || 0, color: "bg-cyan-500" }, { label: "Generation", value: timing.generation_ms || 0, color: "bg-emerald-500" }, ].filter(p => p.value > 0); + + const chainAudit = response.evidence_chain?.audit; + const chainPaths = chainAudit?.paths || []; + const citationByNodeId = new Map( + (response.citations || []).map((citation) => [citation.node_id, citation]) + ); return (

@@ -436,6 +435,90 @@ function SourcePanel({ response }: { response: AskResponse }) {
+ {/* Explainable evidence topology. This is source structure, not hidden + model reasoning and not a statement of factual confidence. */} + {response.evidence_chain?.applied && chainAudit && chainPaths.length > 0 && ( + + + + + +
+ These paths show how source passages are connected and ranked for relevance. + They are not a truth or medical/legal confidence score. Open a cited passage to + inspect its independently verified source highlight. +
+ {chainPaths.map((path) => ( +
+
+ Evidence path {path.path_id} + + relevance {(path.relevance_score * 100).toFixed(0)}% + +
+
+ {path.node_ids.map((nodeId, nodeIndex) => { + const citation = citationByNodeId.get(nodeId); + const isVerified = citation?.evidence_status === "verified" + || citation?.verification_status === "verified"; + return ( +
+
+
+ {nodeIndex + 1} +
+ {nodeIndex < path.node_ids.length - 1 && ( +
+ )} +
+ +
+ ); + })} +
+
+ ))} + + + )} + {/* Timing breakdown */}
@@ -972,9 +1055,24 @@ export default function ChatClient() { Source Details + + Retrieved passages, evidence-chain paths, and citation verification details. + - {selectedResponse && } + {selectedResponse && ( + { + const citationDocId = citation.doc_id || selectedResponse.doc_id || selectedDoc; + if (!citationDocId || citationDocId === "__all__") { + toast.error("Citation does not include a resolvable document ID."); + return; + } + handleCitationClick(citation, citationDocId); + }} + /> + )} diff --git a/frontend/src/components/pdf-viewer/geometry.test.ts b/frontend/src/components/pdf-viewer/geometry.test.ts new file mode 100644 index 0000000..ad8e69c --- /dev/null +++ b/frontend/src/components/pdf-viewer/geometry.test.ts @@ -0,0 +1,28 @@ +import { describe, expect, it } from "vitest"; + +import { normalizedRectToViewport } from "@/components/pdf-viewer/geometry"; + +describe("normalized evidence geometry", () => { + it("maps normalized top-left coordinates directly to the rendered viewport", () => { + const result = normalizedRectToViewport( + { x0: 0.1, y0: 0.25, x1: 0.6, y1: 0.3 }, + 1000, + 800, + ); + expect(result.x).toBeCloseTo(100); + expect(result.y).toBeCloseTo(200); + expect(result.width).toBeCloseTo(500); + expect(result.height).toBeCloseTo(40); + }); + + it("scales without drifting when the PDF is zoomed", () => { + const rect = { x0: 0.125, y0: 0.2, x1: 0.5, y1: 0.24 }; + const at100 = normalizedRectToViewport(rect, 612, 792); + const at200 = normalizedRectToViewport(rect, 1224, 1584); + + expect(at200.x).toBeCloseTo(at100.x * 2); + expect(at200.y).toBeCloseTo(at100.y * 2); + expect(at200.width).toBeCloseTo(at100.width * 2); + expect(at200.height).toBeCloseTo(at100.height * 2); + }); +}); diff --git a/frontend/src/components/pdf-viewer/geometry.ts b/frontend/src/components/pdf-viewer/geometry.ts new file mode 100644 index 0000000..eab3507 --- /dev/null +++ b/frontend/src/components/pdf-viewer/geometry.ts @@ -0,0 +1,21 @@ +import type { NormalizedRect } from "@/lib/api"; + +export interface ViewportRect { + x: number; + y: number; + width: number; + height: number; +} + +export function normalizedRectToViewport( + rect: NormalizedRect, + viewportWidth: number, + viewportHeight: number, +): ViewportRect { + return { + x: rect.x0 * viewportWidth, + y: rect.y0 * viewportHeight, + width: (rect.x1 - rect.x0) * viewportWidth, + height: (rect.y1 - rect.y0) * viewportHeight, + }; +} diff --git a/frontend/src/components/pdf-viewer/index.tsx b/frontend/src/components/pdf-viewer/index.tsx index cdd7125..20c78ab 100644 --- a/frontend/src/components/pdf-viewer/index.tsx +++ b/frontend/src/components/pdf-viewer/index.tsx @@ -7,20 +7,21 @@ * Uses pdfjs-dist for PDF rendering with custom highlight overlays. * * Highlight rendering: - * 1. Verified bbox overlays - * 2. Verified exact quote overlays + * 1. Server-verified normalized source rectangles + * 2. Legacy locators, when explicitly identified as non-verified */ import React, { useEffect, useRef, useState, useCallback } from "react"; import * as pdfjsLib from "pdfjs-dist"; import { Button } from "@/components/ui/button"; -import { Sheet, SheetContent, SheetHeader, SheetTitle } from "@/components/ui/sheet"; +import { Sheet, SheetContent, SheetDescription, SheetHeader, SheetTitle } from "@/components/ui/sheet"; import { ScrollArea } from "@/components/ui/scroll-area"; import { Separator } from "@/components/ui/separator"; import { Badge } from "@/components/ui/badge"; import { Input } from "@/components/ui/input"; import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"; import { cn } from "@/lib/utils"; +import { normalizedRectToViewport } from "@/components/pdf-viewer/geometry"; import { ChevronLeft, ChevronRight, @@ -64,6 +65,21 @@ export interface CitationHighlight { type: "text_offsets"; start: number; end: number; + } + | { + type: "rects"; + coordinate_system: "normalized_top_left"; + rects: Array<{ + x0: number; + y0: number; + x1: number; + y1: number; + }>; + page_size?: { + width: number; + height: number; + }; + page_rotation?: 0 | 90 | 180 | 270; }; quote_text: string; confidence?: number; @@ -191,6 +207,15 @@ export function PDFViewer({ [], ); + const normalizedToViewportRect = useCallback( + ( + viewport: pdfjsLib.PageViewport, + rect: { x0: number; y0: number; x1: number; y1: number }, + ): TextHighlightRect => + normalizedRectToViewport(rect, viewport.width, viewport.height), + [], + ); + const findExactQuoteHighlights = useCallback(async ( page: pdfjsLib.PDFPageProxy, viewport: pdfjsLib.PageViewport, @@ -350,7 +375,11 @@ export function PDFViewer({ const nextBboxHighlights: TextHighlightRect[] = []; for (const item of highlightsForCurrentPage) { - if (item.locator.type === "bbox") { + if (item.locator.type === "rects") { + nextBboxHighlights.push( + ...item.locator.rects.map((rect) => normalizedToViewportRect(viewport, rect)), + ); + } else if (item.locator.type === "bbox") { nextBboxHighlights.push(toViewportRect(viewport, item.locator.bbox)); } else if (item.quote_text) { const exactRects = await findExactQuoteHighlights(page, viewport, item.quote_text); @@ -374,7 +403,15 @@ export function PDFViewer({ }; renderPage(); - }, [pdf, currentPage, scale, highlightsForCurrentPage, findExactQuoteHighlights, toViewportRect]); + }, [ + pdf, + currentPage, + scale, + highlightsForCurrentPage, + findExactQuoteHighlights, + normalizedToViewportRect, + toViewportRect, + ]); // Navigation handlers const goToFirstPage = () => { setCurrentPage(1); setPageInput("1"); }; @@ -408,6 +445,7 @@ export function PDFViewer({ const resetZoom = () => setScale(1.0); const hasHighlights = highlights.length > 0; + const hasDirectRectLocator = highlights.some((item) => item.locator.type === "rects"); const hasBboxLocator = highlights.some((item) => item.locator.type === "bbox"); const visibleFailureMessage = !hasHighlights ? (evidenceFailureMessage || null) @@ -462,11 +500,20 @@ export function PDFViewer({ const getHighlightBadge = () => { if (!hasHighlights) return null; + if (hasDirectRectLocator) { + return ( + + + Verified Evidence + + ); + } + if (hasBboxLocator) { return ( - Verified (BBox) + Approximate Location ); } @@ -475,7 +522,7 @@ export function PDFViewer({ return ( - Verified (Text) + Canonical Text Match ); } @@ -502,6 +549,9 @@ export function PDFViewer({ > {/* Header */} + + Original PDF source with independently verified evidence highlights. +
diff --git a/frontend/src/components/source-viewer/citation-routing.test.ts b/frontend/src/components/source-viewer/citation-routing.test.ts new file mode 100644 index 0000000..3eb1c7d --- /dev/null +++ b/frontend/src/components/source-viewer/citation-routing.test.ts @@ -0,0 +1,47 @@ +import { describe, expect, it } from "vitest"; + +import type { Citation } from "@/lib/api"; +import { + findCitationForInlineRef, + INLINE_CITATION_PATTERN, +} from "@/components/source-viewer/citation-routing"; + +const citations: Citation[] = [ + { + citation_id: "C1", + node_id: "node-a", + doc_id: "doc-1", + page_no: 14, + label: "Table 2", + }, + { + citation_id: "C2", + node_id: "node-b", + doc_id: "doc-1", + page_no: 14, + }, +]; + +describe("stable citation routing", () => { + it("routes a stable citation ID to exactly one evidence record", () => { + expect(findCitationForInlineRef(citations, "C2")?.node_id).toBe("node-b"); + }); + + it("supports exact legacy labels without suffix guessing", () => { + expect(findCitationForInlineRef(citations, "Table 2")?.node_id).toBe("node-a"); + expect(findCitationForInlineRef(citations, "2")).toBeUndefined(); + }); + + it("never substitutes the first citation on a matching page", () => { + expect(findCitationForInlineRef(citations, "page:14")).toBeUndefined(); + expect(findCitationForInlineRef(citations, "seed:14")).toBeUndefined(); + }); + + it("recognizes V2 and legacy inline tokens", () => { + const answer = "Claim [C1]. Legacy [page: 14]."; + expect([...answer.matchAll(INLINE_CITATION_PATTERN)].map((m) => m[1])).toEqual([ + "C1", + "page: 14", + ]); + }); +}); diff --git a/frontend/src/components/source-viewer/citation-routing.ts b/frontend/src/components/source-viewer/citation-routing.ts new file mode 100644 index 0000000..beecf68 --- /dev/null +++ b/frontend/src/components/source-viewer/citation-routing.ts @@ -0,0 +1,28 @@ +import type { Citation } from "@/lib/api"; + +export const INLINE_CITATION_PATTERN = /\[(C\d+|(?:seed|adjacent|page):\s*\d+)\]/gi; + +export function normalizeInlineRef(value: string): string { + return value.toLowerCase().replace(/[\[\]\s]+/g, ""); +} + +export function findCitationForInlineRef( + citations: Citation[] | undefined, + rawRef: string, +): Citation | undefined { + if (!citations?.length) return undefined; + + const normalizedRef = normalizeInlineRef(rawRef); + const byStableId = citations.find( + (citation) => citation.citation_id?.toLowerCase() === normalizedRef, + ); + if (byStableId) return byStableId; + + // Legacy labels are accepted only on an exact normalized match. Page-number + // guessing is intentionally forbidden because multiple nodes may share a page. + return citations.find( + (citation) => + Boolean(citation.label) + && normalizeInlineRef(citation.label as string) === normalizedRef, + ); +} diff --git a/frontend/src/components/source-viewer/evidence.test.ts b/frontend/src/components/source-viewer/evidence.test.ts index 5f1eac0..698b8fe 100644 --- a/frontend/src/components/source-viewer/evidence.test.ts +++ b/frontend/src/components/source-viewer/evidence.test.ts @@ -45,12 +45,14 @@ describe("source-viewer evidence mapping", () => { citation.evidence_verification = [ { status: "FOUND", + grade: "verified", matched_locator: { type: "text_offsets", start: 120, end: 158 }, confidence: 1, reason: "exact_text_offsets_match", }, { status: "FOUND", + grade: "verified", matched_locator: { type: "bbox", bbox: { x0: 12, y0: 44, x1: 180, y1: 78 }, @@ -94,6 +96,30 @@ describe("source-viewer evidence mapping", () => { expect(getEvidenceFailureMessage(citation)).toBe("Evidence not found on cited page"); }); + it("fails closed when a legacy verification has no explicit verified grade", () => { + const citation = makeCitation(); + citation.evidence_spans = [ + { + doc_id: "doc-1", + page_index: 14, + page_index_base: 1, + quote_text: "Legacy quote", + locator: { type: "text_offsets", start: 10, end: 22 }, + confidence: 1, + }, + ]; + citation.evidence_verification = [ + { + status: "FOUND", + matched_locator: { type: "text_offsets", start: 10, end: 22 }, + confidence: 1, + reason: "legacy_response_without_grade", + }, + ]; + + expect(getVerifiedEvidenceHighlights(citation)).toEqual([]); + }); + it("builds deterministic merged canonical highlight ranges from verified offsets", () => { const citation = makeCitation(); citation.evidence_spans = [ @@ -125,18 +151,21 @@ describe("source-viewer evidence mapping", () => { citation.evidence_verification = [ { status: "FOUND", + grade: "verified", matched_locator: { type: "text_offsets", start: 5, end: 12 }, confidence: 1, reason: "exact_text_offsets_match", }, { status: "FOUND", + grade: "verified", matched_locator: { type: "text_offsets", start: 12, end: 16 }, confidence: 1, reason: "exact_text_offsets_match", }, { status: "FOUND", + grade: "verified", matched_locator: { type: "text_offsets", start: 40, end: 45 }, confidence: 1, reason: "exact_text_offsets_match", @@ -149,4 +178,57 @@ describe("source-viewer evidence mapping", () => { { start: 40, end: 45 }, ]); }); + + it("uses only V2 verified rectangle records for original-PDF highlighting", () => { + const citation = makeCitation(); + citation.evidence_records = [ + { + schema_version: "2.0", + citation_id: "C1", + doc_id: "doc-1", + document_version: 1, + node_id: "node-1", + page: 14, + exact_quote: "Exact supporting words.", + source_hash: "sha256:test", + status: "verified", + verification_reason: "exact_unique_quote_with_source_rectangles", + locator: { + type: "rects", + coordinate_system: "normalized_top_left", + rects: [{ x0: 0.1, y0: 0.2, x1: 0.5, y1: 0.23 }], + page_rotation: 0, + }, + confidence: 1, + }, + ]; + + const highlights = getVerifiedEvidenceHighlights(citation); + expect(highlights).toHaveLength(1); + expect(highlights[0].locator.type).toBe("rects"); + expect(getEvidenceFailureMessage(citation)).toBeNull(); + }); + + it("does not present approximate V2 evidence as verified", () => { + const citation = makeCitation(); + citation.evidence_records = [ + { + schema_version: "2.0", + citation_id: "C1", + doc_id: "doc-1", + document_version: 1, + node_id: "node-1", + page: 14, + exact_quote: "Legacy reconstructed text.", + source_hash: "sha256:test", + status: "approximate", + verification_reason: "exact_quote_match_on_page", + locator: { type: "text_offsets", start: 10, end: 36 }, + confidence: 1, + }, + ]; + + expect(getVerifiedEvidenceHighlights(citation)).toEqual([]); + expect(getEvidenceFailureMessage(citation)).toContain("approximate"); + }); }); diff --git a/frontend/src/components/source-viewer/evidence.ts b/frontend/src/components/source-viewer/evidence.ts index 81d59e3..8232055 100644 --- a/frontend/src/components/source-viewer/evidence.ts +++ b/frontend/src/components/source-viewer/evidence.ts @@ -28,6 +28,22 @@ function isValidBbox(locator: EvidenceLocator): locator is Extract + record.status === "verified" + && record.locator?.type === "rects" + && record.locator.rects.length > 0, + ) + .map((record) => ({ + pageNo: record.page, + locator: record.locator!, + quoteText: record.exact_quote.trim(), + confidence: record.confidence, + })); + } + if (!citation?.evidence_spans?.length) { return []; } @@ -37,7 +53,12 @@ export function getVerifiedEvidenceHighlights(citation?: Citation): VerifiedEvid for (let i = 0; i < citation.evidence_spans.length; i += 1) { const span = citation.evidence_spans[i]; const verification = verifications[i]; - if (!verification || verification.status !== "FOUND" || !verification.matched_locator) { + if ( + !verification + || verification.status !== "FOUND" + || verification.grade !== "verified" + || !verification.matched_locator + ) { continue; } const locator = verification.matched_locator; @@ -71,17 +92,22 @@ export function buildCanonicalHighlightRanges( if (!citation || canonicalTextLength <= 0) { return []; } - const highlights = getVerifiedEvidenceHighlights(citation); - const ranges = highlights + const recordLocators = citation.evidence_records + ?.filter((record) => record.status !== "unavailable" && record.locator?.type === "text_offsets") + .map((record) => record.locator as Extract); + const legacyLocators = getVerifiedEvidenceHighlights(citation) .filter( ( h, ): h is VerifiedEvidenceHighlight & { locator: Extract } => h.locator.type === "text_offsets", ) - .map((h) => { - const start = Math.max(0, Math.min(h.locator.start, canonicalTextLength)); - const end = Math.max(start, Math.min(h.locator.end, canonicalTextLength)); + .map((h) => h.locator); + const locators = recordLocators?.length ? recordLocators : legacyLocators; + const ranges = locators + .map((locator) => { + const start = Math.max(0, Math.min(locator.start, canonicalTextLength)); + const end = Math.max(start, Math.min(locator.end, canonicalTextLength)); return { start, end }; }) .filter((r) => r.end > r.start) @@ -111,5 +137,8 @@ export function getEvidenceFailureMessage(citation?: Citation): string | null { if (highlights.length > 0) { return null; } + if (citation.evidence_records?.some((record) => record.status === "approximate")) { + return "Exact source coordinates unavailable; this citation is approximate"; + } return "Evidence not found on cited page"; } diff --git a/frontend/src/components/source-viewer/html-viewer.tsx b/frontend/src/components/source-viewer/html-viewer.tsx index 2d32341..ea28efb 100644 --- a/frontend/src/components/source-viewer/html-viewer.tsx +++ b/frontend/src/components/source-viewer/html-viewer.tsx @@ -5,7 +5,7 @@ import { AlertCircle, CheckCircle2, FileText } from "lucide-react"; import { Badge } from "@/components/ui/badge"; import { ScrollArea } from "@/components/ui/scroll-area"; -import { Sheet, SheetContent, SheetHeader, SheetTitle } from "@/components/ui/sheet"; +import { Sheet, SheetContent, SheetDescription, SheetHeader, SheetTitle } from "@/components/ui/sheet"; import { SourceMapResponse, Citation } from "@/lib/api"; import { buildCanonicalHighlightRanges, getEvidenceFailureMessage } from "@/components/source-viewer/evidence"; @@ -102,7 +102,7 @@ export function HTMLSourceViewer({ return <>{fragments}; }, [sourceMap?.canonical_text, citation]); - const resolveStatus = citation?.resolve_status || "unresolved"; + const resolveStatus = citation?.evidence_status || citation?.resolve_status || "unavailable"; const evidenceFailureMessage = getEvidenceFailureMessage(citation); const showEvidenceNotFound = Boolean(evidenceFailureMessage); @@ -114,16 +114,19 @@ export function HTMLSourceViewer({ {title || "Document Source"} + + Canonical document source with citation evidence and verification status. +
- {resolveStatus === "exact" ? ( + {resolveStatus === "verified" ? ( - Verified + Verified Evidence - ) : resolveStatus === "fuzzy" ? ( - Fuzzy Match + ) : resolveStatus === "approximate" || resolveStatus === "exact" || resolveStatus === "fuzzy" ? ( + Approximate Source ) : ( diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index cc4154e..94aaa9a 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -18,6 +18,7 @@ export type JobStatus = | "skipped_alias"; export interface Citation { + citation_id?: string; page_no: number; node_id: string; doc_id: string; @@ -41,6 +42,18 @@ export interface Citation { normalization?: string; evidence_spans?: EvidenceSpan[]; evidence_verification?: EvidenceVerification[]; + evidence_status?: EvidenceStatus; + verification_status?: EvidenceStatus; + evidence_records?: EvidenceRecord[]; +} + +export type EvidenceStatus = "verified" | "approximate" | "unavailable"; + +export interface NormalizedRect { + x0: number; + y0: number; + x1: number; + y1: number; } export type EvidenceLocator = @@ -53,6 +66,13 @@ export type EvidenceLocator = type: "bbox"; bbox: { x0: number; y0: number; x1: number; y1: number }; page_size?: { width: number; height: number }; + } + | { + type: "rects"; + coordinate_system: "normalized_top_left"; + rects: NormalizedRect[]; + page_size?: { width: number; height: number }; + page_rotation?: 0 | 90 | 180 | 270; }; export interface EvidenceSpan { @@ -67,6 +87,7 @@ export interface EvidenceSpan { export interface EvidenceVerification { status: "FOUND" | "NOT_FOUND"; + grade?: "verified" | "approximate" | "unavailable"; matched_locator?: EvidenceLocator | null; confidence: number; reason: string; @@ -74,6 +95,22 @@ export interface EvidenceVerification { page_index?: number; } +export interface EvidenceRecord { + schema_version: "2.0"; + citation_id: string; + claim_id?: string | null; + doc_id: string; + document_version: number; + node_id: string; + page: number; + exact_quote: string; + source_hash: string; + status: EvidenceStatus; + verification_reason: string; + locator?: EvidenceLocator | null; + confidence: number; +} + export interface SelectorBundle { schema_version: string; node_id: string; @@ -126,6 +163,12 @@ export interface SourceManifestResponse { selector_coverage: { nodes_with_selectors: number; }; + evidence_v2_coverage?: { + eligible_text_nodes: number; + nodes_with_source_spans: number; + nodes_with_exact_source_spans: number; + }; + evidence_v2_reingest_recommended?: boolean; backfill_needed: boolean; } @@ -170,6 +213,48 @@ export interface AskResponse { clarify_options?: string[] | null; propagation_safety_mode: boolean; propagation_safety_audit?: Record | null; + evidence_chain?: { + enabled: boolean; + mode: "off" | "auto" | "on"; + applied: boolean; + time_ms: number; + audit?: { + scoring_version: string; + route: { + applied: boolean; + mode: "off" | "auto" | "on"; + score: number; + reasons: string[]; + }; + applied: boolean; + fallback_used: boolean; + fallback_reason?: string | null; + candidate_count: number; + edge_count: number; + iterations: number; + converged: boolean; + selected_nodes: Array<{ + node_id: string; + final_score: number; + propagation_score: number; + query_relevance: number; + seed_relevance: number; + is_seed: boolean; + }>; + paths: Array<{ + path_id: string; + node_ids: string[]; + relevance_score: number; + edges: Array<{ + from_node_id: string; + to_node_id: string; + edge_type: string; + weight: number; + }>; + }>; + ordered_node_ids: string[]; + } | null; + } | null; original_question?: string | null; llm_rewrite?: { used_llm?: boolean; @@ -469,6 +554,7 @@ export async function askQuestion(params: { top_k?: number; chat_history?: Array<{ role: "user" | "assistant"; content: string }>; mode?: "standard" | "propagation_safety"; + evidence_chain_mode?: "off" | "auto" | "on"; }): Promise { const response = await fetchWithTimeout(`${API_BASE_URL}/v1/qa/ask`, { method: "POST",