feat: add verified evidence chains and source highlighting

README.md

@@ -172,6 +172,8 @@ rag/
 - [Setup Guide](SETUP.md) - **Complete setup instructions with troubleshooting**
 - [Quick Start Guide](backend/docs/QUICK_START.md) - Condensed setup steps
 - [Deployment Guide](backend/docs/DEPLOYMENT_GUIDE.md) - Production deployment
+- [Evidence Highlighting V2](backend/docs/EVIDENCE_HIGHLIGHTING_V2.md) - Fail-closed source provenance and viewer contract
+- [Query-Aware Evidence Chains](backend/docs/HYCE_EVIDENCE_CHAINS_ADOPTION.md) - HyCE-inspired adoption, testing, rollout, and rollback plan
 - [System Specification](lighthouse.md) - Full architecture spec
 - [Prompting Guide](PROMPTING_GUIDE.md) - Prompt engineering practices
 

backend/.env.example

@@ -93,6 +93,23 @@ OPENAI_API_KEY=your-openai-api-key-here  # REQUIRED - get from https://platform.
 # LLM_REWRITE_MODEL=gpt-4o-mini
 # LLM_REWRITE_TIMEOUT=5
 
+# =============================================================================
+# QUERY-AWARE EVIDENCE CHAINS (CONTROLLED ROLLOUT)
+# =============================================================================
+# HyCE-inspired online graph propagation for multi-hop evidence assembly.
+# Structural scores rank evidence only; exact citation/highlight verification
+# remains independent and fail-closed. Start in shadow/evaluation environments.
+
+# EVIDENCE_CHAIN_ENABLED=false
+# EVIDENCE_CHAIN_MODE=auto                   # off | auto | on
+# EVIDENCE_CHAIN_ROUTE_THRESHOLD=0.55
+# EVIDENCE_CHAIN_MAX_HOPS=3
+# EVIDENCE_CHAIN_MAX_NODES=30
+# EVIDENCE_CHAIN_MAX_SELECTED_NODES=15
+# EVIDENCE_CHAIN_MAX_CHAINS=6
+# EVIDENCE_CHAIN_PROPAGATION_STEPS=10
+# EVIDENCE_CHAIN_RESTART_PROBABILITY=0.35
+
 # =============================================================================
 # OCR SETTINGS (for scanned PDFs and image-based documents)
 # =============================================================================

backend/alembic/versions/014_evidence_record_v2.py

@@ -0,0 +1,28 @@
+"""Persist claim-level evidence records on citation snapshots.
+
+Revision ID: 014
+Revises: 013
+"""
+
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+
+revision: str = "014"
+down_revision: Union[str, None] = "013"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.add_column(
+        "citation_snapshots",
+        sa.Column("evidence_record", postgresql.JSONB(astext_type=sa.Text()), nullable=True),
+    )
+
+
+def downgrade() -> None:
+    op.drop_column("citation_snapshots", "evidence_record")

backend/app/config.py

@@ -115,6 +115,19 @@ class Settings(BaseSettings):
     # Cross-format selector highlighting
     enable_cross_format_highlighting: bool = True
 
+    # Query-aware evidence chains (HyCE-inspired online retrieval layer).
+    # Feature-gated for controlled A/B rollout; scores rank evidence only and
+    # never set citation/highlight verification status.
+    evidence_chain_enabled: bool = False
+    evidence_chain_mode: str = "auto"  # off | auto | on
+    evidence_chain_route_threshold: float = 0.55
+    evidence_chain_max_hops: int = 3
+    evidence_chain_max_nodes: int = 30
+    evidence_chain_max_selected_nodes: int = 15
+    evidence_chain_max_chains: int = 6
+    evidence_chain_propagation_steps: int = 10
+    evidence_chain_restart_probability: float = 0.35
+
     # Access Control Layer (ACL)
     # When enabled, enforces tenant-scoped ABAC/RBAC at every pipeline stage.
     # Identity resolved from headers: X-Tenant-Id, X-User-Id, X-Roles, X-Groups
@@ -149,6 +162,34 @@ def validate_acl_disclosure_mode(cls, v: str) -> str:
             raise ValueError(f"acl_disclosure_mode must be one of {valid}, got '{v}'")
         return v
 
+    @field_validator("evidence_chain_mode")
+    @classmethod
+    def validate_evidence_chain_mode(cls, v: str) -> str:
+        valid = {"off", "auto", "on"}
+        if v not in valid:
+            raise ValueError(f"evidence_chain_mode must be one of {valid}, got '{v}'")
+        return v
+
+    @field_validator("evidence_chain_route_threshold", "evidence_chain_restart_probability")
+    @classmethod
+    def validate_evidence_chain_probability(cls, v: float) -> float:
+        if not 0.0 < v <= 1.0:
+            raise ValueError("evidence-chain probabilities must be in (0, 1]")
+        return v
+
+    @field_validator(
+        "evidence_chain_max_hops",
+        "evidence_chain_max_nodes",
+        "evidence_chain_max_selected_nodes",
+        "evidence_chain_max_chains",
+        "evidence_chain_propagation_steps",
+    )
+    @classmethod
+    def validate_evidence_chain_positive_ints(cls, v: int) -> int:
+        if v <= 0:
+            raise ValueError("evidence-chain limits must be positive")
+        return v
+
     @field_validator("jwt_algorithms")
     @classmethod
     def validate_jwt_algorithms(cls, v: list[str]) -> list[str]:
@@ -261,6 +302,10 @@ def log_configuration_summary(self) -> None:
         logger.info(f"  OpenAI API Key: {'configured' if self.openai_api_key else 'NOT SET (required for embeddings)'}")
         logger.info(f"  Docling: enabled={self.docling_enabled_default}, mode={self.docling_mode}")
         logger.info(f"  Cross-format highlighting: enabled={self.enable_cross_format_highlighting}")
+        logger.info(
+            f"  Evidence chains: enabled={self.evidence_chain_enabled}, "
+            f"mode={self.evidence_chain_mode}"
+        )
         logger.info(f"  ACL: enabled={self.acl_enabled}, strict={self.acl_strict_mode}, disclosure={self.acl_disclosure_mode}")
         if not self.acl_enabled:
             logger.warning(

backend/app/db/models.py

@@ -177,6 +177,7 @@ class CitationSnapshot(Base):
     node_id = Column(String(64), nullable=False)
     selector_bundle = Column(JSONB, nullable=False)
     exact_text = Column(Text, nullable=True)
+    evidence_record = Column(JSONB, nullable=True)
     answer_hash = Column(String(128), nullable=False)
     content_hash = Column(String(128), nullable=False)
     created_at = Column(DateTime(timezone=True), server_default=func.now())

backend/app/graph/chunker.py

@@ -11,6 +11,7 @@
 
 import re
 import logging
+import unicodedata
 from dataclasses import dataclass, field
 from typing import List, Optional, Tuple, Any, TYPE_CHECKING
 
@@ -520,15 +521,27 @@ def _create_chunk(
                 "width": page_data.width,
                 "height": page_data.height
             }
+            meta["page_rotation"] = int(getattr(page_data, "rotation", 0) or 0)
+            meta["coordinate_system"] = "normalized_top_left"
 
             # Preserve docling structural provenance where available.
             if getattr(page_data, "meta", None):
                 for key in ("docling_self_ref", "block_id", "sheet", "slide"):
                     if page_data.meta.get(key) is not None:
                         meta[key] = page_data.meta.get(key)
 
-            # Compute merged bbox from matching text spans
-            bbox = self._compute_chunk_bbox(text_plain, page_data)
+            # Retain ordered word provenance. The legacy merged bbox remains
+            # available for approximate navigation, but V2 evidence is resolved
+            # from source_spans into precise line rectangles.
+            source_spans, provenance_status = self._compute_chunk_source_spans(
+                text_plain,
+                page_data,
+            )
+            meta["source_spans"] = source_spans
+            meta["source_span_resolution"] = provenance_status
+            meta["evidence_schema_version"] = "2.0"
+
+            bbox = self._compute_chunk_bbox(text_plain, page_data, source_spans=source_spans)
             if bbox:
                 meta["bbox"] = bbox
 
@@ -559,7 +572,9 @@ def _count_tokens(self, text: str) -> int:
     def _compute_chunk_bbox(
         self,
         chunk_text: str,
-        page_data: Any
+        page_data: Any,
+        *,
+        source_spans: Optional[List[dict]] = None,
     ) -> Optional[dict]:
         """Compute merged bounding box for chunk text from page spans.
 
@@ -576,7 +591,22 @@ def _compute_chunk_bbox(
         if not page_data or not hasattr(page_data, 'text_spans') or not page_data.text_spans:
             return None
 
-        # Normalize chunk text for matching
+        if source_spans:
+            point_boxes = [
+                span.get("bbox")
+                for span in source_spans
+                if isinstance(span.get("bbox"), dict)
+            ]
+            if point_boxes:
+                return {
+                    "x0": round(min(box["x0"] for box in point_boxes), 2),
+                    "y0": round(min(box["y0"] for box in point_boxes), 2),
+                    "x1": round(max(box["x1"] for box in point_boxes), 2),
+                    "y1": round(max(box["y1"] for box in point_boxes), 2),
+                }
+
+        # Legacy approximate fallback for documents ingested without V2 word
+        # provenance.
         chunk_text_normalized = ' '.join(chunk_text.split()).lower()
 
         # Find spans that appear in the chunk text
@@ -613,3 +643,89 @@ def _compute_chunk_bbox(
             "x1": round(x1, 2),
             "y1": round(y1, 2)
         }
+
+    @staticmethod
+    def _normalize_provenance_text(value: str) -> str:
+        return " ".join(unicodedata.normalize("NFKC", value or "").split()).casefold()
+
+    @classmethod
+    def _compute_chunk_source_spans(
+        cls,
+        chunk_text: str,
+        page_data: Any,
+    ) -> Tuple[List[dict], str]:
+        """Resolve a chunk to one unique contiguous sequence of source words.
+
+        Ambiguous and inexact matches deliberately return no verifiable spans.
+        That makes downstream evidence fail closed instead of highlighting a
+        plausible-looking but potentially incorrect region.
+        """
+        spans = list(getattr(page_data, "text_spans", None) or [])
+        if not spans:
+            return [], "missing_source_words"
+
+        ordered = sorted(spans, key=lambda item: int(getattr(item, "order", 0) or 0))
+        stream_parts: List[str] = []
+        ranges: List[Tuple[int, int, Any]] = []
+        cursor = 0
+        for span in ordered:
+            normalized = cls._normalize_provenance_text(getattr(span, "text", ""))
+            if not normalized:
+                continue
+            if stream_parts:
+                stream_parts.append(" ")
+                cursor += 1
+            start = cursor
+            stream_parts.append(normalized)
+            cursor += len(normalized)
+            ranges.append((start, cursor, span))
+
+        page_stream = "".join(stream_parts)
+        needle = cls._normalize_provenance_text(chunk_text)
+        if not page_stream or not needle:
+            return [], "empty_normalized_text"
+
+        occurrences: List[int] = []
+        search_from = 0
+        while True:
+            idx = page_stream.find(needle, search_from)
+            if idx < 0:
+                break
+            occurrences.append(idx)
+            search_from = idx + max(1, len(needle))
+            if len(occurrences) > 1:
+                break
+
+        if not occurrences:
+            return [], "chunk_not_exact_in_source_words"
+        if len(occurrences) > 1:
+            return [], "ambiguous_chunk_in_source_words"
+
+        match_start = occurrences[0]
+        match_end = match_start + len(needle)
+        matched: List[dict] = []
+        for start, end, span in ranges:
+            if start >= match_end or end <= match_start:
+                continue
+            payload = span.to_dict() if hasattr(span, "to_dict") else {
+                "text": getattr(span, "text", ""),
+                "bbox": {
+                    "x0": span.bbox[0],
+                    "y0": span.bbox[1],
+                    "x1": span.bbox[2],
+                    "y1": span.bbox[3],
+                },
+            }
+            payload["page_no"] = int(page_data.page_no)
+            matched.append(payload)
+
+        if not matched:
+            return [], "chunk_match_has_no_source_words"
+        if not all(
+            span.get("verifiable") is True
+            and isinstance(span.get("normalized_bbox"), dict)
+            and span.get("coordinate_system") == "pdf_points_top_left"
+            for span in matched
+        ):
+            return matched, "approximate_source_provenance"
+        return matched, "exact_source_words"

backend/app/graph/context_packer.py

@@ -6,7 +6,7 @@
 
 import logging
 from dataclasses import dataclass, field
-from typing import List, Dict, Optional, Any
+from typing import List, Dict, Optional, Any, Set
 
 from app.db.graph_models import Node, NodeType
 from .expander import ExpandedContext
@@ -78,7 +78,11 @@ def to_text(self, include_citations: bool = True) -> str:
                 marker = f"[{citation.node_id}:{page_no}]"
 
                 # Keep auxiliary metadata outside [] so citation regexes stay simple.
-                meta_parts = [f"source={citation.source_type}"]
+                meta_parts = [
+                    f"node_id={citation.node_id}",
+                    f"page_no={page_no}",
+                    f"source={citation.source_type}",
+                ]
                 if citation.label:
                     meta_parts.append(f"label={citation.label}")
 
@@ -121,7 +125,9 @@ def __init__(
     def pack(
         self,
         expanded: ExpandedContext,
-        query: Optional[str] = None
+        query: Optional[str] = None,
+        node_order: Optional[List[str]] = None,
+        allowed_node_ids: Optional[Set[str]] = None,
     ) -> PackedContext:
         """Pack expanded context into ordered blocks.
 
@@ -143,6 +149,36 @@ def pack(
         total_chars = 0
         max_chars = self.max_tokens * self.CHARS_PER_TOKEN
 
+        # Query-aware evidence chains provide an explicit, bounded evidence
+        # order.  This branch preserves canonical citations while avoiding
+        # accidental inclusion of unselected candidate nodes.
+        if node_order is not None:
+            nodes_by_id = {node.node_id: node for node in expanded.all_nodes}
+            for node_id in node_order:
+                if allowed_node_ids is not None and node_id not in allowed_node_ids:
+                    continue
+                node = nodes_by_id.get(node_id)
+                if node is None:
+                    continue
+                source_type = expanded.node_sources.get(node_id, "chain")
+                block = self._create_block(node, source_type, order)
+                if block and (self.include_empty or block.text.strip()):
+                    if total_chars + len(block.text) <= max_chars:
+                        blocks.append(block)
+                        total_chars += len(block.text)
+                        order += 1
+
+            tokens_estimate = total_chars // self.CHARS_PER_TOKEN
+            logger.info(
+                f"Packed {len(blocks)} chain-ordered blocks, ~{tokens_estimate} tokens "
+                f"(max={self.max_tokens})"
+            )
+            return PackedContext(
+                blocks=blocks,
+                total_chars=total_chars,
+                total_tokens_estimate=tokens_estimate,
+            )
+
         # 1. Seed chunks
         for node in self._sort_chunks(expanded.seed_nodes):
             block = self._create_block(node, 'seed', order)

backend/app/graph/docling_adapter.py

@@ -283,7 +283,16 @@ def _get_text_span(item: Any) -> Optional[TextSpan]:
         t = getattr(bbox, "t", None) or getattr(bbox, "y0", None) or 0
         r = getattr(bbox, "r", None) or getattr(bbox, "x1", None) or 0
         b = getattr(bbox, "b", None) or getattr(bbox, "y1", None) or 0
-        return TextSpan(text=text, bbox=(l, t, r, b))
+        # Docling coordinate origins vary by source adapter. Preserve this
+        # provenance for approximate navigation, but do not mark it as
+        # legal-grade renderable evidence until the origin is normalized.
+        return TextSpan(
+            text=text,
+            bbox=(l, t, r, b),
+            coordinate_system="docling_points_unspecified",
+            extraction_source="docling",
+            verifiable=False,
+        )
     except Exception:
         return None