From bb9c6c76dfe03869237e0561f73999aad13b32a3 Mon Sep 17 00:00:00 2001 From: Ashish-dwi99 Date: Thu, 2 Jul 2026 22:53:16 +0530 Subject: [PATCH 1/3] Release Dhee 7.2.4 memory enrichment --- CHANGELOG.md | 9 + dhee/__init__.py | 2 +- dhee/core/engram_extractor.py | 20 +- dhee/db/sqlite.py | 88 +++++++- dhee/hooks/claude_code/signal.py | 4 + dhee/memory/main.py | 13 ++ dhee/memory/quality.py | 6 +- dhee/memory/write_pipeline.py | 286 +++++++++++++++++++++----- dhee/simple.py | 27 ++- install.sh | 2 +- pyproject.toml | 2 +- tests/test_deferred_enrichment.py | 152 ++++++++++++++ tests/test_memory_quality_contract.py | 34 +-- tests/test_packaging.py | 2 +- 14 files changed, 576 insertions(+), 71 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e7fe953..ec59fb4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,15 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/), and this project adheres to [Semantic Versioning](https://semver.org/). +## [7.2.4] - 2026-07-02 - Clear-water memory enrichment + +- Deferred enrichment now runs structured engram extraction and marks honest + `failed` and `degraded` states instead of silently completing factless rows. +- Operational tool, screen, and trajectory noise is stored as episodic events + instead of polluting semantic memory recall. +- Added re-extraction support for complete memories that missed structured + facts, plus deterministic preference and plan extraction paths. + ## [7.2.2] - 2026-06-06 - Split provider memory profiles - Added provider runtime profiles that let agent LLMs use OpenAI, Anthropic, diff --git a/dhee/__init__.py b/dhee/__init__.py index 9ca9a3e..6325fb6 100644 --- a/dhee/__init__.py +++ b/dhee/__init__.py @@ -50,7 +50,7 @@ # Default import remains model-free for backwards compatibility. Memory = CoreMemory -__version__ = "7.2.3" +__version__ = "7.2.4" __all__ = [ # Memory classes "Engram", diff --git a/dhee/core/engram_extractor.py b/dhee/core/engram_extractor.py index 73093b4..ad6c303 100644 --- a/dhee/core/engram_extractor.py +++ b/dhee/core/engram_extractor.py @@ -113,6 +113,7 @@ def extract( user_profile: Optional[Dict[str, Any]] = None, existing_metadata: Optional[Dict[str, Any]] = None, user_id: str = "default", + strict_llm_failure: bool = False, ) -> UniversalEngram: """Extract structured engram with contextual anchoring. @@ -129,6 +130,8 @@ def extract( ) if engram: return engram + if strict_llm_failure: + raise RuntimeError("Engram LLM extraction produced no structured result") # Rule-based extraction return self._extract_rule_based( @@ -330,13 +333,17 @@ def _extract_facts_rules(self, content: str) -> List[Fact]: (r"(?:I|user)\s+(?:prefer|like|love|enjoy|use)\s+(.+?)(?:\.|$|,)", "prefers"), (r"(?:I|user)\s+(?:switched to|moved to|changed to)\s+(.+?)(?:\.|$|,)", "switched_to"), (r"(?:I|user)\s+(?:work at|work for|employed at)\s+(.+?)(?:\.|$|,)", "works_at"), + (r"(?:I|user|owner)\s+(?:live|lives|reside|resides)\s+in\s+(.+?)(?:\.|$|,)", "lives_in"), + (r"(?:my|user's|owner's)\s+(?:home|city|location)\s+is\s+(.+?)(?:\.|$|,)", "lives_in"), + (r"(?:my|user's|owner's)\s+(?:plan|goal)\s+is\s+to\s+(.+?)(?:\.|$|,)", "plans_to"), + (r"(?:I|user|owner)\s+(?:plan|plans|intend|intends)\s+to\s+(.+?)(?:\.|$|,)", "plans_to"), (r"(?:I|user)\s+(?:visited|went to|traveled to)\s+(.+?)(?:\.|$|,)", "visited"), (r"(?:I|user)\s+(?:bought|purchased)\s+(.+?)(?:\s+for\s+)?([\d,.]+)?\s*(\w+)?", "bought"), ] for pattern, predicate in preference_patterns: matches = re.finditer(pattern, content, re.IGNORECASE) for match in matches: - value = match.group(1).strip().rstrip(".") + value = self._clean_rule_value(match.group(1)) if len(value) > 100: continue fact = Fact( @@ -360,6 +367,17 @@ def _extract_facts_rules(self, content: str) -> List[Fact]: return facts + @staticmethod + def _clean_rule_value(value: str) -> str: + text = str(value or "").strip().rstrip(".") + text = re.split( + r"\s+(?:and|but|while)\s+(?=(?:I|user|owner|my|user's|owner's)\b)", + text, + maxsplit=1, + flags=re.IGNORECASE, + )[0] + return text.strip().rstrip(".") + def _extract_entities_rules(self, content: str) -> List[EntityRef]: """Rule-based entity extraction.""" entities = [] diff --git a/dhee/db/sqlite.py b/dhee/db/sqlite.py index d8fb928..a570839 100644 --- a/dhee/db/sqlite.py +++ b/dhee/db/sqlite.py @@ -2192,21 +2192,97 @@ def update_strength_bulk(self, updates: Dict[str, float]) -> None: ) def get_pending_enrichment(self, user_id: Optional[str] = None, limit: int = 50) -> List[Dict[str, Any]]: - """Return memories with enrichment_status='pending', ordered oldest first.""" + """Return pending and retryable failed enrichment rows, oldest first.""" + fetch_limit = max(1, int(limit)) * 4 with self._get_connection() as conn: if user_id: rows = conn.execute( - "SELECT * FROM memories WHERE enrichment_status = 'pending' AND user_id = ? " + "SELECT * FROM memories WHERE enrichment_status IN ('pending', 'failed') AND user_id = ? " "AND tombstone = 0 ORDER BY created_at ASC LIMIT ?", - (user_id, limit), + (user_id, fetch_limit), ).fetchall() else: rows = conn.execute( - "SELECT * FROM memories WHERE enrichment_status = 'pending' " + "SELECT * FROM memories WHERE enrichment_status IN ('pending', 'failed') " "AND tombstone = 0 ORDER BY created_at ASC LIMIT ?", - (limit,), + (fetch_limit,), ).fetchall() - return [self._row_to_dict(row) for row in rows] + retryable: List[Dict[str, Any]] = [] + for row in rows: + memory = self._row_to_dict(row) + status = str(memory.get("enrichment_status") or "").lower() + metadata = memory.get("metadata") if isinstance(memory.get("metadata"), dict) else {} + attempts_raw = metadata.get("enrichment_attempts", 0) + try: + attempts = int(attempts_raw or 0) + except (TypeError, ValueError): + attempts = 0 + if status == "pending" or attempts < 3: + retryable.append(memory) + if len(retryable) >= max(1, int(limit)): + break + return retryable + + def requeue_complete_without_engram_facts( + self, + user_id: Optional[str] = None, + limit: int = 100, + ) -> Dict[str, Any]: + """Reopen complete semantic memories that have no linked engram facts.""" + try: + with self._get_connection() as conn: + conn.execute("SELECT 1 FROM engram_facts LIMIT 0") + except Exception: + return {"requeued_count": 0, "reason": "engram_facts_unavailable"} + + params: List[Any] = [] + query = ( + "SELECT m.* FROM memories m " + "LEFT JOIN engram_facts f ON f.memory_id = m.id " + "WHERE m.tombstone = 0 " + "AND m.enrichment_status = 'complete' " + "AND COALESCE(m.memory_type, 'semantic') != 'operational_event' " + "GROUP BY m.id HAVING COUNT(f.id) = 0 " + "ORDER BY m.created_at ASC LIMIT ?" + ) + if user_id: + query = ( + "SELECT m.* FROM memories m " + "LEFT JOIN engram_facts f ON f.memory_id = m.id " + "WHERE m.tombstone = 0 " + "AND m.enrichment_status = 'complete' " + "AND COALESCE(m.memory_type, 'semantic') != 'operational_event' " + "AND m.user_id = ? " + "GROUP BY m.id HAVING COUNT(f.id) = 0 " + "ORDER BY m.created_at ASC LIMIT ?" + ) + params.append(user_id) + params.append(max(1, int(limit))) + + rows: List[Dict[str, Any]] = [] + with self._get_connection() as conn: + for row in conn.execute(query, params).fetchall(): + rows.append(self._row_to_dict(row)) + + updates: List[Dict[str, Any]] = [] + for memory in rows: + metadata = memory.get("metadata") if isinstance(memory.get("metadata"), dict) else {} + metadata = dict(metadata) + metadata["enrichment_attempts"] = 0 + metadata.pop("last_enrichment_error", None) + metadata["reextract_queued_at"] = _utcnow_iso() + updates.append( + { + "id": memory["id"], + "metadata": metadata, + "enrichment_status": "pending", + } + ) + self.update_enrichment_bulk(updates) + return { + "requeued_count": len(updates), + "memory_ids": [item["id"] for item in updates], + } def update_enrichment_status(self, memory_id: str, status: str) -> None: """Mark a memory's enrichment_status (e.g. 'complete').""" diff --git a/dhee/hooks/claude_code/signal.py b/dhee/hooks/claude_code/signal.py index bffe161..ad79c4b 100644 --- a/dhee/hooks/claude_code/signal.py +++ b/dhee/hooks/claude_code/signal.py @@ -111,6 +111,8 @@ def _write_signal( "tool": tool_name, "path": path, "success": success, + "memory_type": "operational_event", + "source_app": "claude_code", } @@ -149,6 +151,8 @@ def _shell_signal( "kind": "failure", "tool": tool_name, "success": False, + "memory_type": "operational_event", + "source_app": "claude_code", } diff --git a/dhee/memory/main.py b/dhee/memory/main.py index e01255a..7d8a27c 100644 --- a/dhee/memory/main.py +++ b/dhee/memory/main.py @@ -1165,6 +1165,19 @@ def enrich_pending( max_batches=max_batches, ) + def reextract( + self, + user_id: str = "default", + limit: int = 100, + ) -> Dict[str, Any]: + """Requeue complete memories that have no linked engram facts.""" + if not hasattr(self.db, "requeue_complete_without_engram_facts"): + return {"requeued_count": 0, "reason": "db_requeue_unavailable"} + return self.db.requeue_complete_without_engram_facts( + user_id=user_id, + limit=limit, + ) + _normalize_bitemporal_value = staticmethod(normalize_bitemporal_value) _parse_bitemporal_datetime = classmethod(lambda cls, v: parse_bitemporal_datetime(v)) _attach_bitemporal_metadata = classmethod( diff --git a/dhee/memory/quality.py b/dhee/memory/quality.py index 5c4711b..d9a219c 100644 --- a/dhee/memory/quality.py +++ b/dhee/memory/quality.py @@ -141,7 +141,8 @@ r"(?:/[\w .@%+=:,~/-]+|~/?[\w .@%+=:,~/-]*|\./[\w .@%+=:,~/-]+)\s+" r"(?:-m\s+)?(?:python[0-9.]*|pip[0-9.]*|pytest|uv|npm|pnpm|yarn|git|gh|sqlite3|curl|claude|codex|dhee|ruff|mypy|sed|rg|grep|ls|cat|echo|export)\b|" r"[A-Z_][A-Z0-9_]*=.*\b(?:python[0-9.]*|pytest|sqlite3|curl|git|echo)\b|" - r"(?:python[0-9.]*|pip[0-9.]*|pytest|ruff|mypy|sqlite3|curl|echo|export)\b|" + r"(?:pytest|ruff|mypy|sqlite3|curl|echo|export)\b|" + r"(?:python[0-9.]*|pip[0-9.]*)\s+(?:-|/|~|\.|[A-Za-z0-9_./-]*\.py\b|install\b|uninstall\b|freeze\b|list\b|show\b|check\b)|" r"time\s+\(|" r"uv\s+(?:run|sync|pip|tool|python)\b|" r"(?:npm|pnpm|yarn)\s+(?:run|test|install|build|lint|exec)\b|" @@ -382,11 +383,14 @@ def _is_artifact_like_metadata(metadata: Mapping[str, Any]) -> bool: def _is_operational_event(content: str, metadata: Mapping[str, Any]) -> bool: """True for transport-level agent/tool events that are evidence, not belief.""" kind = str(metadata.get("kind") or metadata.get("type") or "").strip().lower() + memory_type = str(metadata.get("memory_type") or "").strip().lower() tool = str(metadata.get("tool") or metadata.get("tool_name") or metadata.get("native_tool") or "").strip().lower() source = str(metadata.get("source") or metadata.get("source_type") or metadata.get("source_app") or "").strip().lower() success = metadata.get("success") text = " ".join(str(content or "").strip().split()) + if memory_type in {"operational_event", "trajectory", "tool_observation"}: + return True if kind in {"file_touched", "tool_event", "tool_failure", "failure", "session_log", "codex_event", "claude_code_event"}: return True if source in {"claude_code_hook", "codex_hook", "session_log"}: diff --git a/dhee/memory/write_pipeline.py b/dhee/memory/write_pipeline.py index 57015ac..f755dd4 100644 --- a/dhee/memory/write_pipeline.py +++ b/dhee/memory/write_pipeline.py @@ -8,6 +8,7 @@ from __future__ import annotations +import hashlib import json import logging import re @@ -46,6 +47,8 @@ logger = logging.getLogger(__name__) +_MAX_ENRICHMENT_ATTEMPTS = 3 + class MemoryWritePipeline: """Handles the full memory write path: processing, enrichment, indexing. @@ -316,6 +319,163 @@ def _apply_profile_updates_batch( except Exception as exc: logger.warning("%s for %s: %s", warning_prefix, memory_id, exc) + @staticmethod + def _context_messages_from_memory(memory: Dict[str, Any]) -> Optional[List[Dict[str, Any]]]: + raw = memory.get("conversation_context") + if isinstance(raw, list): + return [item for item in raw if isinstance(item, dict)] + if not isinstance(raw, str) or not raw.strip(): + return None + try: + parsed = json.loads(raw) + except json.JSONDecodeError: + return None + if not isinstance(parsed, list): + return None + return [item for item in parsed if isinstance(item, dict)] + + @staticmethod + def _enrichment_attempts(metadata: Dict[str, Any]) -> int: + try: + return max(0, int(metadata.get("enrichment_attempts") or 0)) + except (TypeError, ValueError): + return 0 + + def _run_engram_extraction( + self, + *, + memory_id: str, + content: str, + mem_metadata: Dict[str, Any], + user_id: Optional[str], + context_messages: Optional[List[Dict[str, Any]]] = None, + strict_llm_failure: bool = False, + ) -> Tuple[bool, bool, Optional[Any]]: + """Run structured engram extraction and store resolver rows. + + Returns ``(attempted, succeeded, engram)`` so callers can distinguish a + disabled extractor from an extractor that failed. + """ + engram_extractor = self._engram_extractor + if not engram_extractor: + return False, False, None + + try: + session_ctx = None + if context_messages: + session_ctx = {"recent_messages": context_messages[-5:]} + engram = engram_extractor.extract( + content=content, + session_context=session_ctx, + existing_metadata=mem_metadata, + user_id=user_id or "default", + strict_llm_failure=strict_llm_failure, + ) + context_resolver = self._context_resolver + if context_resolver and engram: + context_resolver.store_engram(engram, memory_id) + if ( + engram.prospective_scenes + and self._config.prospective_scene.enable_prospective_scenes + and self._store_prospective_scenes_fn + ): + self._store_prospective_scenes_fn( + engram.prospective_scenes, + memory_id, + user_id or "default", + ) + return True, True, engram + except Exception as exc: + logger.warning("Engram extraction failed for %s: %s", memory_id, exc) + return True, False, None + + def _store_operational_event( + self, + *, + content: str, + mem_metadata: Dict[str, Any], + user_id: Optional[str], + agent_id: Optional[str], + source_app: Optional[str], + memory_id: Optional[str], + ) -> Dict[str, Any]: + """Store transport/tool signal as an episodic event without a memory row.""" + owner_user_id = user_id or str(mem_metadata.get("user_id") or "default") + event_time = str(mem_metadata.get("event_time") or datetime.now(timezone.utc).isoformat()) + event_seed = "|".join( + [ + owner_user_id, + str(mem_metadata.get("source_event_id") or ""), + str(mem_metadata.get("kind") or ""), + content, + ] + ) + digest = hashlib.sha256(event_seed.encode("utf-8")).hexdigest() + event_memory_id = memory_id or f"operational:{digest[:32]}" + tool_name = str(mem_metadata.get("tool") or mem_metadata.get("tool_name") or "").strip() + path = str(mem_metadata.get("path") or mem_metadata.get("source_path") or "").strip() + actor_id = str( + mem_metadata.get("actor_id") + or agent_id + or tool_name + or source_app + or mem_metadata.get("source_app") + or "agent" + ).strip() + actor_role = str(mem_metadata.get("actor_role") or ("tool" if tool_name else "agent")).strip() + event_kind = str(mem_metadata.get("kind") or mem_metadata.get("type") or "operational_event").strip() + entity_key = path or tool_name or actor_id + canonical_key = f"operational:{actor_id}:{event_kind}:{entity_key or digest[:12]}" + value_norm = " ".join(content.lower().split())[:500] + + self._db.add_episodic_events( + [ + { + "id": f"{event_memory_id}:event", + "memory_id": event_memory_id, + "user_id": owner_user_id, + "conversation_id": mem_metadata.get("conversation_id"), + "session_id": mem_metadata.get("session_id"), + "turn_id": mem_metadata.get("turn_id", 0), + "actor_id": actor_id, + "actor_role": actor_role, + "event_time": event_time, + "event_type": "operational_event", + "canonical_key": canonical_key, + "value_text": content, + "value_num": None, + "value_unit": None, + "currency": None, + "normalized_time_start": event_time, + "normalized_time_end": event_time, + "time_granularity": "instant", + "entity_key": entity_key, + "value_norm": value_norm, + "confidence": mem_metadata.get("confidence", 0.8), + "superseded_by": None, + } + ] + ) + self._record_cost( + phase="write", + user_id=owner_user_id, + llm_calls=0.0, + input_tokens=0.0, + output_tokens=0.0, + embed_calls=0.0, + ) + return { + "id": event_memory_id, + "memory": content, + "event": "EVENT", + "layer": "episodic", + "strength": 0.0, + "namespace": "operational", + "memory_type": "operational_event", + "episodic": True, + "stored_as": "episodic_event", + } + # ------------------------------------------------------------------ # Extracted public methods # ------------------------------------------------------------------ @@ -491,6 +651,16 @@ def _add_llm_cost(input_tokens: float) -> None: if quality.is_canonical: expiration_date = None + if quality.is_operational: + return self._store_operational_event( + content=content, + mem_metadata=mem_metadata, + user_id=user_id, + agent_id=agent_id, + source_app=source_app, + memory_id=memory_id, + ) + admission = evaluate_memory_candidate( content, mem_metadata, @@ -526,6 +696,15 @@ def _add_llm_cost(input_tokens: float) -> None: initial_strength = enforce_quality_strength(initial_strength, quality) if quality.is_canonical: expiration_date = None + if quality.is_operational: + return self._store_operational_event( + content=content, + mem_metadata=mem_metadata, + user_id=user_id, + agent_id=agent_id, + source_app=source_app, + memory_id=memory_id, + ) blocked = detect_sensitive_categories(content) # allow_sensitive: explicit caller opt-in, or caller explicitly provided @@ -1051,29 +1230,14 @@ def _do_category(): ) # Dhee: Universal Engram extraction - engram_extractor = self._engram_extractor - if engram_extractor: - try: - session_ctx = None - if context_messages: - session_ctx = {"recent_messages": context_messages[-5:]} - engram = engram_extractor.extract( - content=content, - session_context=session_ctx, - existing_metadata=mem_metadata, - user_id=user_id or "default", - ) - context_resolver = self._context_resolver - if context_resolver: - context_resolver.store_engram(engram, effective_memory_id) - if engram.prospective_scenes and self._config.prospective_scene.enable_prospective_scenes: - self._store_prospective_scenes_fn( - engram.prospective_scenes, - effective_memory_id, - user_id or "default", - ) - except Exception as e: - logger.warning("Engram extraction failed for %s: %s", effective_memory_id, e) + _engram_attempted, _engram_succeeded, engram = self._run_engram_extraction( + memory_id=effective_memory_id, + content=content, + mem_metadata=mem_metadata, + user_id=user_id, + context_messages=context_messages, + strict_llm_failure=False, + ) # Dhee: Self-evolution — record extraction quality signal evolution_layer = self._evolution_layer @@ -1081,7 +1245,7 @@ def _do_category(): try: engram_facts = None engram_context = None - if engram_extractor and 'engram' in dir() and engram: # noqa: F821 + if _engram_succeeded and engram: engram_facts = [f.to_dict() if hasattr(f, 'to_dict') else f for f in getattr(engram, 'facts', [])] engram_context = getattr(engram, 'context', None) if engram_context and hasattr(engram_context, '__dict__'): @@ -1867,8 +2031,6 @@ def process_memory_batch( warning_prefix="Batch fact embedding/insert failed", ) - engram_extractor = self._engram_extractor - context_resolver = self._context_resolver for entry in record_entries: idx = entry["source_index"] record = entry["record"] @@ -1886,22 +2048,14 @@ def process_memory_batch( warning_prefix="Profile update failed", ) - if engram_extractor: - try: - engram = engram_extractor.extract( - content=record.get("memory", ""), - session_context=None, - existing_metadata=record.get("metadata"), - user_id=record.get("user_id") or user_id or "default", - ) - if context_resolver and engram: - context_resolver.store_engram(engram, record["id"]) - except Exception as exc: - logger.warning( - "Engram extraction failed for %s: %s", - record["id"], - exc, - ) + self._run_engram_extraction( + memory_id=record["id"], + content=record.get("memory", ""), + mem_metadata=record.get("metadata") or {}, + user_id=record.get("user_id") or user_id, + context_messages=None, + strict_llm_failure=False, + ) if episodic_rows: sample_count = float(len(episodic_rows)) @@ -1932,9 +2086,17 @@ def enrich_pending( limit = batch_size * max_batches pending = self._db.get_pending_enrichment(user_id=user_id, limit=limit) if not pending: - return {"enriched_count": 0, "batches": 0, "remaining": 0} + return { + "enriched_count": 0, + "failed_count": 0, + "degraded_count": 0, + "batches": 0, + "remaining": 0, + } enriched_count = 0 + failed_count = 0 + degraded_count = 0 batches_processed = 0 unified = self._unified_enrichment @@ -1983,6 +2145,9 @@ def enrich_pending( mem_id = memory["id"] mem_meta = memory.get("metadata", {}) or {} mem_cats = memory.get("categories", []) or [] + attempts = self._enrichment_attempts(mem_meta) + 1 + mem_meta["enrichment_attempts"] = attempts + mem_meta["last_enrichment_attempt_at"] = datetime.now(timezone.utc).isoformat() if enrichment: if enrichment.echo_result: @@ -2026,16 +2191,42 @@ def enrich_pending( } ) - mem_meta["enrichment_status"] = "complete" + extraction_attempted, extraction_succeeded, _engram = self._run_engram_extraction( + memory_id=mem_id, + content=memory.get("memory", ""), + mem_metadata=mem_meta, + user_id=memory.get("user_id") or user_id, + context_messages=self._context_messages_from_memory(memory), + strict_llm_failure=True, + ) + if extraction_attempted: + status_ready = extraction_succeeded + else: + status_ready = enrichment is not None + + if status_ready: + mem_meta["enrichment_status"] = "complete" + mem_meta.pop("last_enrichment_error", None) + next_status = "complete" + enriched_count += 1 + else: + if attempts >= _MAX_ENRICHMENT_ATTEMPTS: + next_status = "degraded" + degraded_count += 1 + else: + next_status = "failed" + failed_count += 1 + mem_meta["enrichment_status"] = next_status + mem_meta["last_enrichment_error"] = "deferred_enrichment_produced_no_structured_result" + db_updates.append( { "id": mem_id, "metadata": mem_meta, "categories": mem_cats, - "enrichment_status": "complete", + "enrichment_status": next_status, } ) - enriched_count += 1 self._insert_fact_vectors( fact_entries=fact_entries, @@ -2050,6 +2241,8 @@ def enrich_pending( return { "enriched_count": enriched_count, + "failed_count": failed_count, + "degraded_count": degraded_count, "batches": batches_processed, "remaining": remaining_count, } @@ -2126,6 +2319,7 @@ def classify_memory_type(self, metadata: Dict[str, Any], role: str) -> str: "project", "project_status", "project_tag", "warroom", "warroom_message", "test_fixture", "operational_event", + "tool_observation", "trajectory", "utterance", "screen_observation", "profile", "goal", "constraint", "preference", "decision", "style", "product", "product_philosophy"): diff --git a/dhee/simple.py b/dhee/simple.py index 5b7edbc..e58b042 100644 --- a/dhee/simple.py +++ b/dhee/simple.py @@ -356,6 +356,7 @@ def add( scope: Optional[str] = None, source_app: Optional[str] = None, infer: bool = False, + context_messages: Optional[List[Dict[str, str]]] = None, ) -> Dict[str, Any]: """Add a memory. @@ -391,6 +392,7 @@ def add( scope=scope, source_app=source_app, infer=infer, + context_messages=context_messages, ) def search( @@ -673,6 +675,14 @@ def enrich_pending( max_batches=max_batches, ) + def reextract( + self, + user_id: str = "default", + limit: int = 100, + ) -> Dict[str, Any]: + """Requeue complete memories that have no structured engram facts.""" + return self.memory.reextract(user_id=user_id, limit=limit) + def close(self) -> None: """Release runtime resources held by the underlying memory engine.""" self._memory.close() @@ -743,6 +753,7 @@ def remember( content: str, user_id: Optional[str] = None, metadata: Optional[Dict[str, Any]] = None, + context_messages: Optional[List[Dict[str, str]]] = None, ) -> Dict[str, Any]: """Store a fact, preference, or observation. @@ -766,7 +777,13 @@ def remember( if tier != "smriti": meta["tier"] = tier - result = self._engram.add(content, user_id=uid, infer=False, metadata=meta or None) + result = self._engram.add( + content, + user_id=uid, + infer=False, + metadata=meta or None, + context_messages=context_messages, + ) response: Dict[str, Any] = {"stored": True} memory_id = None if isinstance(result, dict): @@ -800,6 +817,14 @@ def remember( response["detected_intention"] = intention.to_dict() return response + def reextract( + self, + user_id: Optional[str] = None, + limit: int = 100, + ) -> Dict[str, Any]: + """Requeue complete memories that have no structured engram facts.""" + return self._engram.reextract(user_id=user_id or self._user_id, limit=limit) + def sweep_admission( self, user_id: Optional[str] = None, diff --git a/install.sh b/install.sh index 0097483..42869a3 100755 --- a/install.sh +++ b/install.sh @@ -24,7 +24,7 @@ DHEE_HOME="$HOME/.dhee" VENV_DIR="$DHEE_HOME/.venv" BIN_DIR="$HOME/.local/bin" MIN_PYTHON="3.9" -DEFAULT_PACKAGE="dhee>=7.2.2" +DEFAULT_PACKAGE="dhee>=7.2.4" PACKAGE="${DHEE_INSTALL_PACKAGE:-$DEFAULT_PACKAGE}" FALLBACK_PACKAGE="${DHEE_FALLBACK_PACKAGE:-git+https://github.com/Sankhya-AI/Dhee.git@main}" diff --git a/pyproject.toml b/pyproject.toml index b019389..32f2a22 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "dhee" -version = "7.2.3" +version = "7.2.4" description = "Dhee - world memory layer and context compiler for AI agents" readme = "README.md" requires-python = ">=3.9" diff --git a/tests/test_deferred_enrichment.py b/tests/test_deferred_enrichment.py index 1ee79ec..3c5201d 100644 --- a/tests/test_deferred_enrichment.py +++ b/tests/test_deferred_enrichment.py @@ -26,6 +26,7 @@ CategoryMemConfig, EchoMemConfig, EmbedderConfig, + EngramExtractionConfig, EnrichmentConfig, KnowledgeGraphConfig, LLMConfig, @@ -58,6 +59,7 @@ def _make_deferred_memory(tmpdir, defer=True, echo=False, categories=False): defer_enrichment=defer, context_window_turns=5, ), + engram_extraction=EngramExtractionConfig(use_llm_extraction=False), batch=BatchConfig(enable_batch=False), ) return Memory(config) @@ -421,6 +423,156 @@ def test_enrich_pending_marks_complete(self): assert len(pending_after) == 0 m.close() + def test_enrich_pending_writes_structured_facts(self): + """Deferred enrichment runs the same engram fact writer as inline adds.""" + with tempfile.TemporaryDirectory() as tmpdir: + m = _make_deferred_memory(tmpdir) + result = m.add( + messages="I prefer Python for backend development", + user_id="test_user", + infer=False, + ) + memory_id = result["results"][0]["id"] + + enrich_result = m.enrich_pending(user_id="test_user", batch_size=10) + assert enrich_result["enriched_count"] == 1 + + with m.db._get_connection() as conn: + facts = [ + dict(row) + for row in conn.execute( + "SELECT subject, predicate, value FROM engram_facts WHERE memory_id = ?", + (memory_id,), + ).fetchall() + ] + preferences = [ + dict(row) + for row in conn.execute( + "SELECT subject, topic, value FROM engram_preferences WHERE memory_id = ?", + (memory_id,), + ).fetchall() + ] + assert {"subject": "user", "predicate": "prefers", "value": "Python for backend development"} in facts + assert {"subject": "user", "topic": "prefers", "value": "Python for backend development"} in preferences + m.close() + + def test_enrich_pending_splits_multiple_personal_facts(self): + """Rule extraction keeps adjacent facts atomic instead of swallowing clauses.""" + with tempfile.TemporaryDirectory() as tmpdir: + m = _make_deferred_memory(tmpdir) + result = m.add( + messages="I live in Pune and I prefer Python for backend development.", + user_id="test_user", + infer=False, + ) + memory_id = result["results"][0]["id"] + + enrich_result = m.enrich_pending(user_id="test_user", batch_size=10) + assert enrich_result["enriched_count"] == 1 + + with m.db._get_connection() as conn: + facts = [ + dict(row) + for row in conn.execute( + "SELECT subject, predicate, value FROM engram_facts WHERE memory_id = ?", + (memory_id,), + ).fetchall() + ] + assert {"subject": "user", "predicate": "lives_in", "value": "Pune"} in facts + assert {"subject": "user", "predicate": "prefers", "value": "Python for backend development"} in facts + m.close() + + def test_enrich_pending_marks_failed_then_retries(self): + """A failed extraction is not recorded as complete and retries later.""" + from dhee.core.engram import Fact, UniversalEngram + + class FailingExtractor: + def extract(self, **_kwargs): + raise RuntimeError("provider down") + + class WorkingExtractor: + def extract(self, **_kwargs): + fact = Fact(subject="user", predicate="lives_in", value="Bangalore") + fact.canonical_key = fact.make_canonical_key() + return UniversalEngram(raw_content="Owner lives in Bangalore", user_id="test_user", facts=[fact]) + + with tempfile.TemporaryDirectory() as tmpdir: + m = _make_deferred_memory(tmpdir) + result = m.add( + messages="Owner lives in Bangalore", + user_id="test_user", + infer=False, + ) + memory_id = result["results"][0]["id"] + m._engram_extractor = FailingExtractor() + + first = m.enrich_pending(user_id="test_user", batch_size=10) + assert first["enriched_count"] == 0 + assert first["failed_count"] == 1 + failed = m.db.get_memory(memory_id) + assert failed["enrichment_status"] == "failed" + assert failed["metadata"]["enrichment_attempts"] == 1 + assert m.db.get_pending_enrichment(user_id="test_user", limit=10)[0]["id"] == memory_id + + m._engram_extractor = WorkingExtractor() + second = m.enrich_pending(user_id="test_user", batch_size=10) + assert second["enriched_count"] == 1 + complete = m.db.get_memory(memory_id) + assert complete["enrichment_status"] == "complete" + + with m.db._get_connection() as conn: + fact_count = conn.execute( + "SELECT COUNT(*) AS cnt FROM engram_facts WHERE memory_id = ?", + (memory_id,), + ).fetchone()["cnt"] + assert fact_count == 1 + m.close() + + def test_enrich_pending_degrades_after_three_failed_attempts(self): + """Failed rows drain to degraded after the bounded retry budget.""" + class FailingExtractor: + def extract(self, **_kwargs): + raise RuntimeError("provider down") + + with tempfile.TemporaryDirectory() as tmpdir: + m = _make_deferred_memory(tmpdir) + result = m.add( + messages="Owner lives in Bangalore", + user_id="test_user", + infer=False, + ) + memory_id = result["results"][0]["id"] + m._engram_extractor = FailingExtractor() + + for _ in range(3): + m.enrich_pending(user_id="test_user", batch_size=10) + + memory = m.db.get_memory(memory_id) + assert memory["enrichment_status"] == "degraded" + assert memory["metadata"]["enrichment_attempts"] == 3 + assert m.db.get_pending_enrichment(user_id="test_user", limit=10) == [] + m.close() + + def test_reextract_requeues_complete_memories_without_facts(self): + """Backlog repair reopens complete rows that have no engram_facts.""" + with tempfile.TemporaryDirectory() as tmpdir: + m = _make_deferred_memory(tmpdir) + result = m.add( + messages="Owner lives in Bangalore", + user_id="test_user", + infer=False, + ) + memory_id = result["results"][0]["id"] + m.db.update_enrichment_status(memory_id, "complete") + + reextract = m.reextract(user_id="test_user", limit=10) + assert reextract["requeued_count"] == 1 + assert reextract["memory_ids"] == [memory_id] + memory = m.db.get_memory(memory_id) + assert memory["enrichment_status"] == "pending" + assert memory["metadata"]["enrichment_attempts"] == 0 + m.close() + def test_enrich_pending_respects_batch_size(self): """enrich_pending processes in batches respecting max_batches.""" with tempfile.TemporaryDirectory() as tmpdir: diff --git a/tests/test_memory_quality_contract.py b/tests/test_memory_quality_contract.py index 995c6b7..d9e9d84 100644 --- a/tests/test_memory_quality_contract.py +++ b/tests/test_memory_quality_contract.py @@ -108,11 +108,10 @@ def test_operational_file_touch_is_evidence_not_personal_recall(tmp_path): infer=False, )["results"][0] - loaded = memory.get(operational["id"]) - assert loaded["namespace"] == "operational" - assert loaded["memory_type"] == "operational_event" - assert loaded["strength"] <= 0.05 - assert loaded["metadata"]["suppress_from_default_recall"] is True + assert operational["event"] == "EVENT" + assert operational["stored_as"] == "episodic_event" + assert operational["memory_type"] == "operational_event" + assert memory.get(operational["id"]) is None personal_results = memory.memory.search( "What do you remember about Chotu assistant goals preferences decisions?", @@ -127,7 +126,15 @@ def test_operational_file_touch_is_evidence_not_personal_recall(tmp_path): limit=5, min_strength=0.0, )["results"] - assert operational["id"] in [row["id"] for row in operational_results] + assert operational["id"] not in [row["id"] for row in operational_results] + + events = memory.memory.db.get_episodic_events(user_id="default", limit=20) + assert any( + row["memory_id"] == operational["id"] + and row["event_type"] == "operational_event" + and "/Users/example/project/src/app.py" in row["value_text"] + for row in events + ) memory.close() @@ -749,12 +756,15 @@ def test_weirdly_phrased_command_failure_is_quarantined(tmp_path): ) memory_id = result["results"][0]["id"] - loaded = memory.get(memory_id) - - assert loaded["namespace"] == "operational" - assert loaded["memory_type"] == "operational_event" - assert loaded["metadata"]["dhee_memory_class"] == "operational_event" - assert loaded["metadata"]["suppress_from_default_recall"] is True + assert result["results"][0]["event"] == "EVENT" + assert memory.get(memory_id) is None + events = memory.memory.db.get_episodic_events(user_id="default", limit=20) + assert any( + row["memory_id"] == memory_id + and row["event_type"] == "operational_event" + and "pytest tests/test_login.py" in row["value_text"] + for row in events + ) memory.close() diff --git a/tests/test_packaging.py b/tests/test_packaging.py index 5331a20..0df03ab 100644 --- a/tests/test_packaging.py +++ b/tests/test_packaging.py @@ -62,7 +62,7 @@ def test_curl_installer_verifies_handoff_bus(): assert "Cross-agent handoff bus ready" in installer assert "from dhee.core.kernel import _get_bus" in installer assert "for bin_name in dhee dhee-mcp engram-bus" in installer - assert 'DEFAULT_PACKAGE="dhee>=7.2.2"' in installer + assert 'DEFAULT_PACKAGE="dhee>=7.2.4"' in installer assert "DHEE_INSTALL_PACKAGE" in installer assert "FALLBACK_PACKAGE" in installer assert "DHEE_INIT_REPO" in installer From 5c665387ef96ea6e39acca9d446fb03f563f271c Mon Sep 17 00:00:00 2001 From: Ashish-dwi99 Date: Fri, 3 Jul 2026 06:52:57 +0530 Subject: [PATCH 2/3] Release Dhee 7.2.5 lifecycle repair --- CHANGELOG.md | 11 ++ dhee/__init__.py | 2 +- dhee/configs/base.py | 12 +- dhee/contract_runtime.py | 9 +- dhee/core/scene.py | 89 +++++++++++-- dhee/doctor.py | 71 +++++++++++ dhee/llms/nvidia.py | 36 +++++- dhee/memory/main.py | 220 +++++++++++++++++++++++++++++++- dhee/provider_defaults.py | 2 +- dhee/router/handlers.py | 27 ++++ dhee/simple.py | 20 ++- dhee/verification_runner.py | 21 ++- install.sh | 2 +- pyproject.toml | 2 +- tests/conftest.py | 4 +- tests/test_data_dir_paths.py | 29 +++++ tests/test_nvidia_llm.py | 59 +++++++++ tests/test_packaging.py | 2 +- tests/test_scene_maintenance.py | 73 +++++++++++ 19 files changed, 662 insertions(+), 29 deletions(-) create mode 100644 tests/test_data_dir_paths.py create mode 100644 tests/test_nvidia_llm.py create mode 100644 tests/test_scene_maintenance.py diff --git a/CHANGELOG.md b/CHANGELOG.md index ec59fb4..8eb697f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,17 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/), and this project adheres to [Semantic Versioning](https://semver.org/). +## [7.2.5] - 2026-07-03 - Clear-water lifecycle repair + +- Updated the default NVIDIA extraction/generation model to + `moonshotai/kimi-k2.6` and added a live provider model ping in Dhee doctor. +- Enabled scene summarization by default and wired lifecycle enrichment to fill + missing scene summaries with deterministic fallback summaries. +- Added scene-noise audit and repair for operational/test prompt scenes, + tombstoning active noise while preserving it as episodic events. +- Expanded `DHEE_DATA_DIR` tildes consistently and hardened pytest data-dir + isolation so fixture users do not write into the live vault by accident. + ## [7.2.4] - 2026-07-02 - Clear-water memory enrichment - Deferred enrichment now runs structured engram extraction and marks honest diff --git a/dhee/__init__.py b/dhee/__init__.py index 6325fb6..099aac2 100644 --- a/dhee/__init__.py +++ b/dhee/__init__.py @@ -50,7 +50,7 @@ # Default import remains model-free for backwards compatibility. Memory = CoreMemory -__version__ = "7.2.4" +__version__ = "7.2.5" __all__ = [ # Memory classes "Engram", diff --git a/dhee/configs/base.py b/dhee/configs/base.py index e7b2080..c53c29b 100644 --- a/dhee/configs/base.py +++ b/dhee/configs/base.py @@ -1,4 +1,5 @@ import os +from pathlib import Path from typing import Any, Dict, List, Optional from pydantic import BaseModel, Field, field_validator @@ -12,12 +13,17 @@ ) +def resolve_dhee_data_dir(path: str | os.PathLike[str]) -> str: + """Return an absolute Dhee data directory with user markers expanded.""" + return os.path.abspath(os.path.expanduser(os.fspath(path))) + + def _dhee_data_dir() -> str: """Resolve data directory: DHEE_DATA_DIR > ~/.dhee.""" env = os.environ.get("DHEE_DATA_DIR") if env: - return env - return os.path.join(os.path.expanduser("~"), ".dhee") + return resolve_dhee_data_dir(env) + return resolve_dhee_data_dir(Path.home() / ".dhee") _VALID_VECTOR_PROVIDERS = {"memory", "sqlite_vec", "zvec"} @@ -180,7 +186,7 @@ class SceneConfig(BaseModel): scene_topic_threshold: float = 0.55 # cosine sim below this = topic shift auto_close_inactive_minutes: int = 120 max_scene_memories: int = 50 - use_llm_summarization: bool = False + use_llm_summarization: bool = True summary_regenerate_threshold: int = 5 diff --git a/dhee/contract_runtime.py b/dhee/contract_runtime.py index 007aefe..32d4f0d 100644 --- a/dhee/contract_runtime.py +++ b/dhee/contract_runtime.py @@ -741,17 +741,14 @@ def guard_router_call(tool_name: str, arguments: Dict[str, Any]) -> Dict[str, An "repo": str(repo_root), }) if mode == "deny": - # Even in deny mode a supervisor outage must leave a recovery - # path open: read-only tools cannot violate a contract, and the - # dhee CLI is how the agent repairs the runtime state. command = str(arguments.get("command") or "").strip() - recoverable = tool_name in _READ_TOOL_NAMES or tool_name in _GREP_TOOL_NAMES or ( - tool_name in _BASH_TOOL_NAMES and command.startswith("dhee ") + recoverable = tool_name in {"Read", "Grep"} or ( + tool_name == "Bash" and command.startswith("dhee ") ) if recoverable: warning = ( "Contract supervisor unavailable; deny mode allowed this " - "read-only/remediation call." + "native read-only/remediation call." ) _record_enforcement_warning( repo_root, diff --git a/dhee/core/scene.py b/dhee/core/scene.py index 0ed45be..78acfd3 100644 --- a/dhee/core/scene.py +++ b/dhee/core/scene.py @@ -229,16 +229,10 @@ def close_scene(self, scene_id: str, timestamp: Optional[str] = None) -> None: # Generate summary (LLM when enabled, otherwise deterministic extractive fallback). memories = self.db.get_scene_memories(scene_id) - summary = None - if self.use_llm_summarization and self.llm: - summary = self._summarize_scene(scene, memories) - if not summary: - summary = self._extractive_scene_summary(memories) + summary = self._generate_scene_summary(scene, memories) if summary: updates["summary"] = summary - # Derive title from summary - title = summary.split(".")[0][:120] - updates["title"] = title + updates["title"] = self._title_from_summary(summary) if updates: self.db.update_scene(scene_id, updates) @@ -270,6 +264,77 @@ def auto_close_stale(self, user_id: str) -> List[str]: # Summarization # ------------------------------------------------------------------ + def summarize_scene(self, scene_id: str, *, force: bool = False) -> Optional[str]: + """Generate and persist a summary for an existing scene.""" + scene = self.db.get_scene(scene_id) + if not scene: + return None + existing = str(scene.get("summary") or "").strip() + if existing and not force: + return existing + + memories = self.db.get_scene_memories(scene_id) + summary = self._generate_scene_summary(scene, memories) + if not summary: + return None + self.db.update_scene( + scene_id, + { + "summary": summary, + "title": self._title_from_summary(summary), + }, + ) + return summary + + def summarize_unsummarized( + self, + *, + user_id: Optional[str] = None, + limit: int = 100, + ) -> Dict[str, Any]: + """Fill missing scene summaries for a bounded maintenance slice.""" + bounded_limit = max(1, min(int(limit), 2_000)) + scenes = self.db.get_scenes(user_id=user_id, limit=bounded_limit) + summarized: List[Dict[str, Any]] = [] + skipped_existing = 0 + skipped_empty = 0 + + for scene in scenes: + if str(scene.get("summary") or "").strip(): + skipped_existing += 1 + continue + summary = self.summarize_scene(str(scene["id"]), force=True) + if not summary: + skipped_empty += 1 + continue + summarized.append( + { + "id": scene.get("id"), + "title": self._title_from_summary(summary), + } + ) + + return { + "scanned_count": len(scenes), + "summarized_count": len(summarized), + "skipped_existing": skipped_existing, + "skipped_empty": skipped_empty, + "summaries": summarized[:50], + "truncated_summaries": max(0, len(summarized) - 50), + } + + def _generate_scene_summary( + self, + scene: Dict[str, Any], + memories: List[Dict[str, Any]], + ) -> Optional[str]: + summary = None + if self.use_llm_summarization and self.llm: + summary = self._summarize_scene(scene, memories) + if not summary: + summary = self._extractive_scene_summary(memories) + return summary + def _summarize_scene( self, scene: Dict[str, Any], memories: List[Dict[str, Any]] ) -> Optional[str]: @@ -319,6 +384,14 @@ def _extractive_scene_summary(memories: List[Dict[str, Any]], max_lines: int = 4 return None return " | ".join(snippets) + @staticmethod + def _title_from_summary(summary: str) -> str: + cleaned = " ".join(str(summary or "").split()) + if not cleaned: + return "Untitled scene" + first_sentence = cleaned.split(".")[0].strip() + return (first_sentence or cleaned)[:120] + # ------------------------------------------------------------------ # Search # ------------------------------------------------------------------ diff --git a/dhee/doctor.py b/dhee/doctor.py index 780b875..d53319a 100644 --- a/dhee/doctor.py +++ b/dhee/doctor.py @@ -49,6 +49,7 @@ class DoctorReport: generated_at: float = 0.0 core: dict[str, Any] = field(default_factory=dict) runtime: dict[str, Any] = field(default_factory=dict) + provider_health: dict[str, Any] = field(default_factory=dict) router: dict[str, Any] = field(default_factory=dict) context: dict[str, Any] = field(default_factory=dict) cognition: dict[str, Any] = field(default_factory=dict) @@ -62,6 +63,7 @@ def to_dict(self) -> dict[str, Any]: "generated_at": self.generated_at, "core": self.core, "runtime": self.runtime, + "provider_health": self.provider_health, "router": self.router, "context": self.context, "cognition": self.cognition, @@ -123,6 +125,61 @@ def _runtime_section() -> dict[str, Any]: return {"error": f"{type(exc).__name__}: {exc}"} +def _provider_health_section() -> dict[str, Any]: + try: + from dhee.cli_config import get_api_key, load_config + from dhee.provider_defaults import provider_defaults + except Exception as exc: + return {"ok": False, "status": "config_error", "error": f"{type(exc).__name__}: {exc}"} + + config = load_config() + provider = str(config.get("provider") or "nvidia").strip().lower() + defaults = provider_defaults(provider) + model = str(config.get("llm_model") or defaults.get("llm_model") or "").strip() + if provider != "nvidia": + return { + "ok": None, + "status": "not_implemented", + "provider": provider, + "model": model, + } + + api_key = get_api_key(provider) + if not api_key: + return { + "ok": False, + "status": "missing_api_key", + "provider": provider, + "model": model, + "required_env": defaults.get("env_var"), + } + + try: + from dhee.llms.nvidia import NvidiaLLM + + llm = NvidiaLLM( + { + "api_key": api_key, + "model": model, + "temperature": 0, + "max_tokens": 2, + "timeout": 8, + "max_retries": 0, + "app_retries": 1, + } + ) + return llm.ping() + except Exception as exc: + return { + "ok": False, + "status": "unavailable", + "provider": provider, + "model": model, + "error_type": type(exc).__name__, + "error": str(exc), + } + + def _context_section() -> dict[str, Any]: out: dict[str, Any] = {} try: @@ -597,6 +654,7 @@ def build_report() -> DoctorReport: core = _core_section() runtime = _runtime_section() + provider_health = _provider_health_section() router = _router_section() context = _context_section() cognition = _cognition_section() @@ -609,6 +667,7 @@ def build_report() -> DoctorReport: generated_at=time.time(), core=core, runtime=runtime, + provider_health=provider_health, router=router, context=context, cognition=cognition, @@ -627,6 +686,7 @@ def format_human(report: DoctorReport) -> str: lines: list[str] = [] core = report.core runtime = report.runtime + provider_health = report.provider_health router = report.router context = report.context cog = report.cognition @@ -674,6 +734,17 @@ def format_human(report: DoctorReport) -> str: lines.append(f" runtime dir: {paths.get('runtime_dir')}") lines.append("") + # Provider health + lines.append("[ provider health ]") + lines.append(f" provider: {provider_health.get('provider', '?')}") + lines.append(f" model: {provider_health.get('model', '?')}") + lines.append(f" status: {provider_health.get('status', '?')}") + if provider_health.get("latency_ms") is not None: + lines.append(f" latency: {provider_health.get('latency_ms')} ms") + if provider_health.get("error_type"): + lines.append(f" error: {provider_health.get('error_type')}: {provider_health.get('error')}") + lines.append("") + # Router lines.append("[ router ]") if "error" in router: diff --git a/dhee/llms/nvidia.py b/dhee/llms/nvidia.py index 28acc81..52d3329 100644 --- a/dhee/llms/nvidia.py +++ b/dhee/llms/nvidia.py @@ -1,7 +1,7 @@ import logging import os import time -from typing import Optional +from typing import Any, Dict, Optional from dhee.llms.base import BaseLLM from dhee.provider_defaults import DEFAULT_NVIDIA_LLM_MODEL @@ -59,6 +59,40 @@ def __init__(self, config: Optional[dict] = None): self.top_p = self.config.get("top_p", 0.7) self.enable_thinking = self.config.get("enable_thinking", False) + def ping(self) -> Dict[str, Any]: + """Check that the configured NVIDIA model accepts a minimal chat call.""" + started = time.perf_counter() + try: + response = self.client.chat.completions.create( + model=self.model, + messages=[{"role": "user", "content": "Reply with OK."}], + temperature=0, + top_p=1, + max_tokens=2, + stream=False, + ) + content = "" + if getattr(response, "choices", None): + content = str(response.choices[0].message.content or "").strip() + return { + "ok": True, + "status": "ready", + "provider": "nvidia", + "model": self.model, + "latency_ms": round((time.perf_counter() - started) * 1000, 2), + "response_preview": content[:20], + } + except Exception as exc: + return { + "ok": False, + "status": "unavailable", + "provider": "nvidia", + "model": self.model, + "latency_ms": round((time.perf_counter() - started) * 1000, 2), + "error_type": type(exc).__name__, + "error": str(exc), + } + def generate(self, prompt: str) -> str: from openai import APITimeoutError, APIConnectionError diff --git a/dhee/memory/main.py b/dhee/memory/main.py index 7d8a27c..3c5911d 100644 --- a/dhee/memory/main.py +++ b/dhee/memory/main.py @@ -6,11 +6,14 @@ import math import os import re +import shutil +import sqlite3 import time import uuid from dataclasses import dataclass from datetime import datetime, date, timezone from enum import Enum +from pathlib import Path from typing import Any, Dict, List, Optional, Tuple, Union from dhee.configs.base import MemoryConfig @@ -77,6 +80,19 @@ logger = logging.getLogger(__name__) +_SCENE_BENCHMARK_NOISE_RE = re.compile( + r"\b(?:you are fixing a real bug|benchmark prompt|swe-bench|fail[- ]to[- ]pass)\b", + re.IGNORECASE, +) +_SCENE_OPERATIONAL_VERB_RE = re.compile( + r"^\s*(?:edited|wrote|created|updated|modified|touched)\s+(.{1,240})\s*$", + re.IGNORECASE, +) +_SCENE_PATH_HINT_RE = re.compile( + r"(?:^|\s)(?:/|~/|\./|[\w.-]+\.(?:py|js|ts|tsx|jsx|go|rs|java|rb|php|sh|sql|toml|yaml|yml|json|md)\b)", + re.IGNORECASE, +) + # --------------------------------------------------------------------------- # Inline helpers (formerly in deleted core/acceptance and core/policy modules) # --------------------------------------------------------------------------- @@ -1159,11 +1175,17 @@ def enrich_pending( max_batches: int = 5, ) -> Dict[str, Any]: """Batch-enrich memories that were stored with deferred enrichment.""" - return self._write_pipeline.enrich_pending( + result = self._write_pipeline.enrich_pending( user_id=user_id, batch_size=batch_size, max_batches=max_batches, ) + if self.scene_processor: + result["scene_summaries"] = self.scene_processor.summarize_unsummarized( + user_id=user_id, + limit=max(1, min(int(batch_size) * int(max_batches), 500)), + ) + return result def reextract( self, @@ -1953,6 +1975,146 @@ def _get_quality_health_memories( break return memories + @staticmethod + def _scene_noise_kind(scene: Dict[str, Any]) -> Optional[str]: + title = str(scene.get("title") or "").strip() + summary = str(scene.get("summary") or "").strip() + topic = str(scene.get("topic") or "").strip() + text = "\n".join(part for part in (title, summary, topic) if part) + if not text: + return None + if _SCENE_BENCHMARK_NOISE_RE.search(text): + return "benchmark_prompt_scene" + operational = _SCENE_OPERATIONAL_VERB_RE.match(title) + if operational and _SCENE_PATH_HINT_RE.search(operational.group(1)): + return "operational_scene" + return None + + def _scene_noise_candidates( + self, + *, + user_id: Optional[str], + limit: int, + ) -> List[Dict[str, Any]]: + bounded_limit = max(1, min(int(limit), 50_000)) + query = """ + SELECT id, user_id, title, summary, topic, start_time, end_time + FROM scenes + WHERE tombstone = 0 + """ + params: List[Any] = [] + if user_id: + query += " AND user_id = ?" + params.append(user_id) + query += " ORDER BY start_time DESC LIMIT ?" + params.append(bounded_limit) + with self.db._get_connection() as conn: + rows = [dict(row) for row in conn.execute(query, params).fetchall()] + + candidates: List[Dict[str, Any]] = [] + for row in rows: + kind = self._scene_noise_kind(row) + if not kind: + continue + row["noise_kind"] = kind + candidates.append(row) + return candidates + + def _backup_history_db(self, reason: str) -> Optional[str]: + raw_path = getattr(self.config, "history_db_path", None) + if not raw_path: + return None + db_path = Path(str(raw_path)).expanduser().resolve() + if not db_path.exists(): + return None + timestamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") + backup_dir = db_path.parent / "backups" + backup_dir.mkdir(parents=True, exist_ok=True) + backup_path = backup_dir / f"{db_path.stem}-{reason}-{timestamp}{db_path.suffix}" + shutil.copy2(db_path, backup_path) + return str(backup_path) + + def _tombstone_scene_noise( + self, + candidates: List[Dict[str, Any]], + *, + dry_run: bool, + ) -> Dict[str, Any]: + counts = { + "scene_noise_tombstoned": 0, + "operational_scenes_tombstoned": 0, + "benchmark_scenes_tombstoned": 0, + "scene_noise_events_written": 0, + } + if not candidates: + return {**counts, "scene_noise_backup": None} + + for scene in candidates: + if scene["noise_kind"] == "operational_scene": + counts["operational_scenes_tombstoned"] += 1 + elif scene["noise_kind"] == "benchmark_prompt_scene": + counts["benchmark_scenes_tombstoned"] += 1 + counts["scene_noise_tombstoned"] = len(candidates) + if dry_run: + return {**counts, "scene_noise_backup": None} + + backup_path = self._backup_history_db("scene-noise") + now = datetime.now(timezone.utc).isoformat() + events: List[Dict[str, Any]] = [] + for scene in candidates: + scene_id = str(scene["id"]) + event_time = str(scene.get("start_time") or scene.get("end_time") or now) + event_type = ( + "operational_event" + if scene["noise_kind"] == "operational_scene" + else "test_fixture" + ) + value_text = str(scene.get("title") or scene.get("summary") or scene_id) + events.append( + { + "id": f"scene-noise:{scene_id}", + "memory_id": f"scene:{scene_id}", + "user_id": scene.get("user_id") or "default", + "conversation_id": None, + "session_id": None, + "turn_id": 0, + "actor_id": "dhee_scene_repair", + "actor_role": "maintenance", + "event_time": event_time, + "event_type": event_type, + "canonical_key": f"scene_noise:{scene['noise_kind']}:{scene_id}", + "value_text": value_text, + "value_num": None, + "value_unit": None, + "currency": None, + "normalized_time_start": event_time, + "normalized_time_end": event_time, + "time_granularity": "instant", + "entity_key": str(scene.get("topic") or scene.get("title") or scene_id)[:240], + "value_norm": " ".join(value_text.lower().split())[:500], + "confidence": 1.0, + "superseded_by": None, + } + ) + counts["scene_noise_events_written"] = self.db.add_episodic_events(events) + scene_ids = [(str(scene["id"]),) for scene in candidates] + with self.db._get_connection() as conn: + conn.executemany( + "UPDATE scenes SET tombstone = 1, strength = 0.0 WHERE id = ?", + scene_ids, + ) + try: + conn.executemany( + "UPDATE memories SET scene_id = NULL WHERE scene_id = ?", + scene_ids, + ) + except sqlite3.OperationalError as exc: + if "no such column" not in str(exc).lower(): + raise + with self.db._get_connection() as conn: + conn.execute("VACUUM") + return {**counts, "scene_noise_backup": backup_path} + def get_stats(self, user_id: Optional[str] = None, agent_id: Optional[str] = None) -> Dict[str, Any]: memories = self._get_quality_health_memories(user_id=user_id, agent_id=agent_id) sml_count = sum(1 for m in memories if m.get("layer") == "sml") @@ -2094,6 +2256,9 @@ def audit_memory_quality( "evidence_without_distillation": 0, "contract_repairs_available": 0, "profile_contamination": 0, + "scene_noise": 0, + "operational_scene_noise": 0, + "benchmark_scene_noise": 0, } samples: Dict[str, List[Dict[str, Any]]] = { "unresolved_test_noise": [], @@ -2104,6 +2269,7 @@ def audit_memory_quality( "damaged_canonical": [], "evidence_without_distillation": [], "profile_contamination": [], + "scene_noise": [], } def _sample(memory: Dict[str, Any], issue: str, updates: List[str]) -> None: @@ -2242,6 +2408,26 @@ def _sample(memory: Dict[str, Any], issue: str, updates: List[str]) -> None: } ) + scene_noise = self._scene_noise_candidates(user_id=user_id, limit=limit) + counts["scene_noise"] = len(scene_noise) + for scene in scene_noise: + if scene["noise_kind"] == "operational_scene": + counts["operational_scene_noise"] += 1 + elif scene["noise_kind"] == "benchmark_prompt_scene": + counts["benchmark_scene_noise"] += 1 + bucket = samples.setdefault("scene_noise", []) + if len(bucket) < 8: + bucket.append( + { + "id": scene.get("id"), + "user_id": scene.get("user_id"), + "noise_kind": scene.get("noise_kind"), + "title": scene.get("title"), + } + ) + if scene_noise: + counts["contract_repairs_available"] += len(scene_noise) + checks: List[Dict[str, Any]] = [] def _check(name: str, passed: bool, severity: str, detail: str) -> None: @@ -2323,6 +2509,12 @@ def _check(name: str, passed: bool, severity: str, detail: str) -> None: "critical", f"{counts['profile_contamination']} derived profile anchors contain tool/test/doc noise.", ) + _check( + "no_scene_noise", + counts["scene_noise"] == 0, + "critical", + f"{counts['scene_noise']} operational/test prompt scenes remain active.", + ) failed_critical = [check for check in checks if check["severity"] == "critical" and not check["passed"]] failed_warnings = [check for check in checks if check["severity"] == "warning" and not check["passed"]] @@ -2335,6 +2527,8 @@ def _check(name: str, passed: bool, severity: str, detail: str) -> None: recommended_actions.append("Run repair_memory_quality with dry_run=false, then rerun this audit.") if counts["degraded_or_queued"]: recommended_actions.append("Run enrich_pending after repair to reconcile queued/degraded rows.") + if counts["scene_noise"]: + recommended_actions.append("Run repair_memory_quality with dry_run=false to tombstone operational/test prompt scenes.") if require_personal_model and counts["canonical_personal"] == 0: recommended_actions.append("Ingest explicit goals, preferences, decisions, style, and product philosophy as canonical personal memories.") if profile_l and counts["canonical_profile_matches"] == 0: @@ -2404,6 +2598,10 @@ def repair_memory_quality( "queued_reopened": 0, "malformed_metadata_normalized": 0, "profiles_cleaned": 0, + "scene_noise_tombstoned": 0, + "operational_scenes_tombstoned": 0, + "benchmark_scenes_tombstoned": 0, + "scene_noise_events_written": 0, } for memory in memories: raw_metadata = memory.get("metadata") @@ -2520,11 +2718,31 @@ def repair_memory_quality( if not dry_run: self.db.update_profile(str(profile["id"]), updates) + scene_noise = self._scene_noise_candidates(user_id=user_id, limit=limit) + for scene in scene_noise: + repaired.append( + { + "id": scene.get("id"), + "memory_class": scene.get("noise_kind"), + "updates": ["tombstone", "episodic_event"], + "title": scene.get("title"), + } + ) + scene_repair = self._tombstone_scene_noise(scene_noise, dry_run=dry_run) + for key in ( + "scene_noise_tombstoned", + "operational_scenes_tombstoned", + "benchmark_scenes_tombstoned", + "scene_noise_events_written", + ): + counts[key] = int(scene_repair.get(key, 0) or 0) + result = { "dry_run": dry_run, "scanned_count": len(memories), "repair_count": len(repaired), **counts, + "scene_noise_backup": scene_repair.get("scene_noise_backup"), "repairs": repaired[:200], "truncated_repairs": max(0, len(repaired) - 200), } diff --git a/dhee/provider_defaults.py b/dhee/provider_defaults.py index 8a74176..48733a2 100644 --- a/dhee/provider_defaults.py +++ b/dhee/provider_defaults.py @@ -8,7 +8,7 @@ DEFAULT_PROVIDER = "nvidia" DEFAULT_COLLECTION = "dhee" -DEFAULT_NVIDIA_LLM_MODEL = "moonshotai/kimi-k2.5" +DEFAULT_NVIDIA_LLM_MODEL = "moonshotai/kimi-k2.6" DEFAULT_NVIDIA_EMBEDDER_MODEL = "nvidia/llama-nemotron-embed-vl-1b-v2" DEFAULT_NVIDIA_RERANK_MODEL = "nvidia/llama-nemotron-rerank-vl-1b-v2" DEFAULT_NVIDIA_EMBEDDING_DIMS = 2048 diff --git a/dhee/router/handlers.py b/dhee/router/handlers.py index 8490b12..2f3ebda 100644 --- a/dhee/router/handlers.py +++ b/dhee/router/handlers.py @@ -8,7 +8,9 @@ from __future__ import annotations import os +import shlex import subprocess +import sys import time import hashlib from pathlib import Path @@ -650,6 +652,16 @@ def handle_dhee_bash(arguments: Dict[str, Any]) -> Dict[str, Any]: cmd = str(arguments.get("command", "")).strip() if not cmd: return {"error": "command is required"} + for prefix in ("python3 -m ", "python -m "): + if cmd.startswith(prefix): + cmd = f"{shlex.quote(sys.executable)} -m {cmd[len(prefix):]}" + break + else: + if cmd == "pytest" or cmd.startswith("pytest "): + suffix = cmd[len("pytest"):].lstrip() + cmd = f"{shlex.quote(sys.executable)} -m pytest" + if suffix: + cmd = f"{cmd} {suffix}" from dhee.contract_runtime import command_preview, guard_router_call, record_router_observation, router_refusal, router_result_runtime contract_guard = guard_router_call("dhee_bash", arguments) @@ -687,10 +699,25 @@ def handle_dhee_bash(arguments: Dict[str, Any]) -> Dict[str, Any]: started = time.perf_counter() timed_out = False + env = os.environ.copy() + python_bin = str(Path(sys.executable).resolve().parent) + env["PATH"] = ( + python_bin + if not env.get("PATH") + else os.pathsep.join([python_bin, str(env["PATH"])]) + ) + if cwd: + existing_pythonpath = env.get("PYTHONPATH") + env["PYTHONPATH"] = ( + str(cwd) + if not existing_pythonpath + else os.pathsep.join([str(cwd), existing_pythonpath]) + ) try: proc = subprocess.run( [os.environ.get("SHELL") or "/bin/sh", "-lc", cmd], cwd=cwd, + env=env, capture_output=True, timeout=timeout, text=False, diff --git a/dhee/simple.py b/dhee/simple.py index e58b042..655cc4a 100644 --- a/dhee/simple.py +++ b/dhee/simple.py @@ -36,6 +36,7 @@ LLMConfig, MemoryConfig, VectorStoreConfig, + resolve_dhee_data_dir, ) from dhee.memory.main import FullMemory from dhee.provider_defaults import ( @@ -187,8 +188,8 @@ def _get_data_dir() -> Path: """Get the data directory for Dhee storage.""" data_dir = os.environ.get("DHEE_DATA_DIR") if data_dir: - return Path(data_dir) - return Path.home() / ".dhee" + return Path(resolve_dhee_data_dir(data_dir)) + return Path(resolve_dhee_data_dir(Path.home() / ".dhee")) class Engram: @@ -229,7 +230,7 @@ def __init__( self._provider = "mock" if in_memory and data_dir is None: data_dir = tempfile.mkdtemp(prefix="dhee_") - self._data_dir = Path(data_dir) if data_dir else _get_data_dir() + self._data_dir = Path(resolve_dhee_data_dir(data_dir)) if data_dir else _get_data_dir() self._data_dir.mkdir(parents=True, exist_ok=True) # Build configuration. The persistent default is zvec. If a live memory @@ -588,6 +589,19 @@ def stats( """ return self._memory.get_stats(user_id=user_id, agent_id=agent_id) + def provider_health(self) -> Dict[str, Any]: + """Return a live health check for the configured model provider.""" + llm = getattr(self._memory, "llm", None) + ping = getattr(llm, "ping", None) + if callable(ping): + return ping() + return { + "ok": self._provider == "mock", + "status": "ready" if self._provider == "mock" else "not_supported", + "provider": self._provider, + "model": getattr(llm, "model", None), + } + def repair_memory_quality( self, user_id: Optional[str] = None, diff --git a/dhee/verification_runner.py b/dhee/verification_runner.py index f93fd1a..54686e5 100644 --- a/dhee/verification_runner.py +++ b/dhee/verification_runner.py @@ -268,10 +268,29 @@ def _execute_command(repo_root: Path, command: str, timeout_sec: int) -> Dict[st "stderr_tail": reason, "observation": {"blocked_reason": reason}, } + normalized_parts = list(parts or []) + if normalized_parts[:2] in (["python", "-m"], ["python3", "-m"]): + normalized_parts[0] = sys.executable + elif normalized_parts and normalized_parts[0] == "pytest": + normalized_parts = [sys.executable, "-m", "pytest", *normalized_parts[1:]] + env = os.environ.copy() + python_bin = str(Path(sys.executable).resolve().parent) + env["PATH"] = ( + python_bin + if not env.get("PATH") + else os.pathsep.join([python_bin, str(env["PATH"])]) + ) + existing_pythonpath = env.get("PYTHONPATH") + env["PYTHONPATH"] = ( + str(repo_root) + if not existing_pythonpath + else os.pathsep.join([str(repo_root), existing_pythonpath]) + ) try: proc = subprocess.run( - list(parts or []), + normalized_parts, cwd=str(repo_root), + env=env, text=True, capture_output=True, timeout=max(1, int(timeout_sec or DEFAULT_TIMEOUT_SEC)), diff --git a/install.sh b/install.sh index 42869a3..155da12 100755 --- a/install.sh +++ b/install.sh @@ -24,7 +24,7 @@ DHEE_HOME="$HOME/.dhee" VENV_DIR="$DHEE_HOME/.venv" BIN_DIR="$HOME/.local/bin" MIN_PYTHON="3.9" -DEFAULT_PACKAGE="dhee>=7.2.4" +DEFAULT_PACKAGE="dhee>=7.2.5" PACKAGE="${DHEE_INSTALL_PACKAGE:-$DEFAULT_PACKAGE}" FALLBACK_PACKAGE="${DHEE_FALLBACK_PACKAGE:-git+https://github.com/Sankhya-AI/Dhee.git@main}" diff --git a/pyproject.toml b/pyproject.toml index 32f2a22..191c2ad 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "dhee" -version = "7.2.4" +version = "7.2.5" description = "Dhee - world memory layer and context compiler for AI agents" readme = "README.md" requires-python = ">=3.9" diff --git a/tests/conftest.py b/tests/conftest.py index 59461f5..63b261e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -7,7 +7,9 @@ def pytest_configure(config): - if os.environ.get("DHEE_TEST_LIVE_DATA"): + if os.environ.get("DHEE_TEST_LIVE_DATA") and os.environ.get("DHEE_TEST_ALLOW_LIVE_FIXTURES"): return temp_home = tempfile.mkdtemp(prefix="dhee-pytest-home-") + os.environ["HOME"] = temp_home os.environ["DHEE_DATA_DIR"] = os.path.join(temp_home, ".dhee") + os.environ["DHEE_PYTEST_ISOLATED_DATA_DIR"] = "1" diff --git a/tests/test_data_dir_paths.py b/tests/test_data_dir_paths.py new file mode 100644 index 0000000..66db528 --- /dev/null +++ b/tests/test_data_dir_paths.py @@ -0,0 +1,29 @@ +import os + +from dhee.configs.base import _dhee_data_dir +from dhee.simple import Engram, _get_data_dir + + +def test_dhee_data_dir_expands_tilde(monkeypatch, tmp_path): + home = tmp_path / "home" + home.mkdir() + monkeypatch.setenv("HOME", str(home)) + monkeypatch.setenv("DHEE_DATA_DIR", "~/clear-water") + + expected = home / "clear-water" + + assert _dhee_data_dir() == os.path.abspath(str(expected)) + assert _get_data_dir() == expected.absolute() + + +def test_engram_expands_explicit_data_dir_tilde(monkeypatch, tmp_path): + home = tmp_path / "home" + home.mkdir() + monkeypatch.setenv("HOME", str(home)) + + memory = Engram(provider="mock", data_dir="~/explicit-dhee", in_memory=True) + + try: + assert memory.data_dir == (home / "explicit-dhee").absolute() + finally: + memory.close() diff --git a/tests/test_nvidia_llm.py b/tests/test_nvidia_llm.py new file mode 100644 index 0000000..c2270c4 --- /dev/null +++ b/tests/test_nvidia_llm.py @@ -0,0 +1,59 @@ +import sys +from types import SimpleNamespace + +from dhee.llms.nvidia import NvidiaLLM + + +class _FakeCompletions: + def __init__(self, response=None, error=None): + self.response = response + self.error = error + self.calls = [] + + def create(self, **kwargs): + self.calls.append(kwargs) + if self.error: + raise self.error + return self.response + + +class _FakeOpenAI: + completions = None + + def __init__(self, **_kwargs): + self.chat = SimpleNamespace(completions=self.completions) + + +def test_nvidia_llm_ping_uses_configured_model(monkeypatch): + response = SimpleNamespace( + choices=[SimpleNamespace(message=SimpleNamespace(content="OK"))] + ) + completions = _FakeCompletions(response=response) + _FakeOpenAI.completions = completions + monkeypatch.setitem(sys.modules, "openai", SimpleNamespace(OpenAI=_FakeOpenAI)) + monkeypatch.setenv("NVIDIA_API_KEY", "test-key") + + llm = NvidiaLLM({"model": "moonshotai/kimi-k2.6"}) + health = llm.ping() + + assert health["ok"] is True + assert health["status"] == "ready" + assert health["model"] == "moonshotai/kimi-k2.6" + assert completions.calls[0]["model"] == "moonshotai/kimi-k2.6" + assert completions.calls[0]["max_tokens"] == 2 + + +def test_nvidia_llm_ping_reports_model_failure(monkeypatch): + completions = _FakeCompletions(error=RuntimeError("404 model not found")) + _FakeOpenAI.completions = completions + monkeypatch.setitem(sys.modules, "openai", SimpleNamespace(OpenAI=_FakeOpenAI)) + monkeypatch.setenv("NVIDIA_API_KEY", "test-key") + + llm = NvidiaLLM({"model": "moonshotai/kimi-k2.5"}) + health = llm.ping() + + assert health["ok"] is False + assert health["status"] == "unavailable" + assert health["model"] == "moonshotai/kimi-k2.5" + assert health["error_type"] == "RuntimeError" + assert "404" in health["error"] diff --git a/tests/test_packaging.py b/tests/test_packaging.py index 0df03ab..2058d90 100644 --- a/tests/test_packaging.py +++ b/tests/test_packaging.py @@ -62,7 +62,7 @@ def test_curl_installer_verifies_handoff_bus(): assert "Cross-agent handoff bus ready" in installer assert "from dhee.core.kernel import _get_bus" in installer assert "for bin_name in dhee dhee-mcp engram-bus" in installer - assert 'DEFAULT_PACKAGE="dhee>=7.2.4"' in installer + assert 'DEFAULT_PACKAGE="dhee>=7.2.5"' in installer assert "DHEE_INSTALL_PACKAGE" in installer assert "FALLBACK_PACKAGE" in installer assert "DHEE_INIT_REPO" in installer diff --git a/tests/test_scene_maintenance.py b/tests/test_scene_maintenance.py new file mode 100644 index 0000000..0f1d1e2 --- /dev/null +++ b/tests/test_scene_maintenance.py @@ -0,0 +1,73 @@ +from dhee.simple import Engram + + +def test_enrich_pending_fills_missing_scene_summary(tmp_path): + memory = Engram(provider="mock", in_memory=True, data_dir=str(tmp_path)) + try: + memory.add( + "Dhee should summarize scenes during lifecycle maintenance.", + user_id="default", + infer=False, + ) + scenes = memory.memory.db.get_scenes(user_id="default", limit=10) + assert len(scenes) == 1 + assert not scenes[0].get("summary") + + result = memory.enrich_pending(user_id="default", batch_size=5, max_batches=1) + + assert result["scene_summaries"]["summarized_count"] == 1 + summarized = memory.memory.db.get_scene(scenes[0]["id"]) + assert "Dhee should summarize scenes" in summarized["summary"] + assert summarized["title"].startswith("Dhee should summarize scenes") + finally: + memory.close() + + +def test_repair_memory_quality_tombstones_operational_scene_noise(tmp_path): + memory = Engram(provider="mock", in_memory=True, data_dir=str(tmp_path)) + try: + memory.memory.db.add_scene( + { + "id": "scene-edited-file", + "user_id": "default", + "title": "Edited /tmp/project/app.py", + "summary": None, + "topic": "Edited /tmp/project/app.py", + "start_time": "2026-01-01T00:00:00+00:00", + "end_time": "2026-01-01T00:01:00+00:00", + "memory_ids": [], + "participants": [], + "strength": 1.0, + } + ) + + before = memory.audit_memory_quality( + user_id="default", + require_personal_model=False, + ) + assert before["ready"] is False + assert before["counts"]["scene_noise"] == 1 + assert before["counts"]["operational_scene_noise"] == 1 + + repair = memory.repair_memory_quality(user_id="default", dry_run=False) + + assert repair["scene_noise_tombstoned"] == 1 + assert repair["operational_scenes_tombstoned"] == 1 + assert repair["scene_noise_events_written"] == 1 + assert repair["scene_noise_backup"] + assert memory.memory.db.get_scene("scene-edited-file") is None + + events = memory.memory.db.get_episodic_events(user_id="default", limit=10) + assert any( + event["memory_id"] == "scene:scene-edited-file" + and event["event_type"] == "operational_event" + for event in events + ) + after = memory.audit_memory_quality( + user_id="default", + require_personal_model=False, + ) + assert after["ready"] is True + assert after["counts"]["scene_noise"] == 0 + finally: + memory.close() From 9720a6a5de30bf07949eae3494255caee3f7900c Mon Sep 17 00:00:00 2001 From: Ashish-dwi99 Date: Fri, 3 Jul 2026 08:38:32 +0530 Subject: [PATCH 3/3] Ignore local env backups --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 9a8eadf..b82c16d 100644 --- a/.gitignore +++ b/.gitignore @@ -48,6 +48,7 @@ venv.bak/ # Environment variables .env +.env.bak-* .env.local .env.*.local