From e2f3b2def2d7056580ab056e91a2d66f033efcd3 Mon Sep 17 00:00:00 2001 From: Jimisola Laursen Date: Sun, 16 Aug 2026 00:03:31 +0200 Subject: [PATCH] fix(mcp): reload the served snapshot when project files change MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An MCP server parsed its project once at startup and served that snapshot for the lifetime of the process. Harnesses keep the server up for days while builds regenerate annotations.yml and JUnit XML underneath it, so it answered from a tree that no longer existed — and the stale answer was well-formed, indistinguishable from a real one (#437). Record which local files each parse read: the four data YAMLs and reqstool_config.yml, stamped whether or not they exist, plus every test_results glob pattern with the concrete files it matched. Absent files are tracked deliberately — an annotations.yml the build has yet to generate is the common trigger. ProjectSession.ensure_fresh() re-stats that fingerprint and rebuilds only on a mismatch; the LSP keeps rebuilding from its client's file-change notifications. The MCP tools now resolve the repository per call rather than closing over it at startup, without which reloading would be invisible to them. A reload that fails raises instead of falling back to the superseded snapshot, and the failed build re-stamps its inputs so a tree that does not parse is parsed once, not once per request. Adds a refresh tool for unconditional reloads and a snapshot field on get_status reporting built_at, tracked_files and warnings — where a test_results pattern matching no files is reported as such rather than counted as zero tests. Only local sources are watched; remote ones are version-pinned downloads. Requirements: MCP_0006, MCP_0007, MCP_0008 Signed-off-by: Jimisola Laursen --- CLAUDE.md | 1 + docs/modules/ROOT/pages/mcp.adoc | 37 +++- docs/reqstool/requirements.yml | 18 ++ docs/reqstool/software_verification_cases.yml | 18 ++ .../mcp-snapshot-freshness/.openspec.yaml | 2 + .../changes/mcp-snapshot-freshness/design.md | 56 ++++++ .../mcp-snapshot-freshness/proposal.md | 66 +++++++ .../mcp-snapshot-freshness/specs/mcp/spec.md | 19 ++ .../changes/mcp-snapshot-freshness/tasks.md | 49 +++++ src/reqstool/common/exceptions.py | 11 ++ src/reqstool/common/project_session.py | 144 ++++++++++---- src/reqstool/common/snapshot_fingerprint.py | 179 ++++++++++++++++++ src/reqstool/mcp/server.py | 71 +++++-- .../combined_raw_datasets_generator.py | 67 ++++++- src/reqstool/models/raw_datasets.py | 8 + .../reqstool/mcp/test_mcp_integration.py | 3 +- .../mcp/test_mcp_reload_integration.py | 126 ++++++++++++ .../common/test_project_session_freshness.py | 162 ++++++++++++++++ .../common/test_snapshot_fingerprint.py | 172 +++++++++++++++++ .../reqstool/mcp/test_server_freshness.py | 147 ++++++++++++++ 20 files changed, 1297 insertions(+), 59 deletions(-) create mode 100644 openspec/changes/mcp-snapshot-freshness/.openspec.yaml create mode 100644 openspec/changes/mcp-snapshot-freshness/design.md create mode 100644 openspec/changes/mcp-snapshot-freshness/proposal.md create mode 100644 openspec/changes/mcp-snapshot-freshness/specs/mcp/spec.md create mode 100644 openspec/changes/mcp-snapshot-freshness/tasks.md create mode 100644 src/reqstool/common/snapshot_fingerprint.py create mode 100644 tests/integration/reqstool/mcp/test_mcp_reload_integration.py create mode 100644 tests/unit/reqstool/common/test_project_session_freshness.py create mode 100644 tests/unit/reqstool/common/test_snapshot_fingerprint.py create mode 100644 tests/unit/reqstool/mcp/test_server_freshness.py diff --git a/CLAUDE.md b/CLAUDE.md index d1d4e93a..16f86f4d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -161,6 +161,7 @@ Full rationale in `docs/DESIGN.md`. - **Cycle detection covers both chains**: `CircularImportError` for the import chain, `CircularImplementationError` for the implementation chain. - **FK constraints scope evidence from implementation children**: SVCs/MVRs/annotations referencing out-of-scope requirements are rejected by SQLite FK checks on insert — no explicit filtering needed. - **Test results need explicit scoping**: no FK (keyed by FQN), so a scope check is required when inserting test results from implementation children. +- **Long-lived servers must not bind the repository at startup**: `ProjectSession` reloads itself (`ensure_fresh()`, driven by the input fingerprint in `common/snapshot_fingerprint.py`), so an MCP tool that captured `session.repo` in its closure would keep serving the pre-reload database. Resolve the repository per request via the server's `_repo()` helper. A snapshot that cannot be reloaded is an error, never a fallback to the superseded data — that silent staleness was bug #437. - **The requirement "complete" verdict has ONE source of truth**: `StatisticsService`'s per-requirement computation (producing `RequirementStatus`). Every consumer — `status`/`report`/`export`, the MCP tools (`get_status`, `get_requirement_status`, `get_requirements_status`), and LSP — MUST derive completeness from that same computation. Do NOT re-implement "is this requirement met / are its automated tests satisfied" anywhere else (e.g. a parallel helper in `common/queries/details.py`). Two parallel verdict paths silently drifted and caused bug #411 (consolidation tracked in #412); a private re-derivation is the regression to guard against. When the verdict logic must be reused, extract/call the shared predicate — never copy its traversal. ## Key Conventions diff --git a/docs/modules/ROOT/pages/mcp.adoc b/docs/modules/ROOT/pages/mcp.adoc index e401cf1e..7efeae31 100644 --- a/docs/modules/ROOT/pages/mcp.adoc +++ b/docs/modules/ROOT/pages/mcp.adoc @@ -29,7 +29,7 @@ reqstool mcp local -p /path/to/reqstool ---- The server starts and blocks, serving MCP requests over stdio. The project at the given path is -loaded once at startup. +loaded at startup and reloaded whenever its files change — see <>. === Other sources @@ -164,7 +164,17 @@ Overall traceability status across all requirements — completion counts, test *Parameters:* none -*Returns:* status summary dict +*Returns:* status summary dict, plus a `snapshot` field describing the data it was computed +from: `{ built_at, reload, tracked_files, warnings }`. See <>. + +==== `refresh` + +Forces an immediate reload of the project from disk. Reloading is automatic when input files +change, so this is only needed to reload unconditionally or to confirm what is being served. + +*Parameters:* none + +*Returns:* `{ built_at, reload, tracked_files, warnings }` ==== `get_requirement_status` @@ -210,6 +220,29 @@ in_progress = [ ] ---- +[[snapshot-freshness]] +== Snapshot Freshness + +An MCP server is typically spawned by an AI harness and left running for days, while builds +regenerate `annotations.yml` and JUnit XML underneath it. The server therefore records which +local files it parsed — including files it looked for and did not find, and the concrete files +each `test_results` pattern matched — and re-checks them before answering each request. If +anything changed, the project is reloaded first. + +Consequences worth knowing: + +* *Files that appear later count as changes.* A server started before the first build picks up + the `annotations.yml` that build produces; it does not keep reporting zero implementations. +* *A project that no longer parses is an error, not a stale answer.* If the sources changed but + the new state cannot be loaded — a half-written YAML file, say — tools fail with the parse + error rather than answering from the superseded snapshot. Fixing the file restores service on + the next call. +* *Absent build artifacts are reported, not counted as zero.* When a configured `test_results` + pattern matches no files, `get_status` says so under `snapshot.warnings` instead of silently + reporting a project with no tests. +* *Only local sources are watched.* Remote sources (git, maven, npm, pypi) are version-pinned + downloads, so they are never treated as stale. Use `refresh` to reload those. + == reqstool-ai https://github.com/reqstool/reqstool-ai[reqstool-ai] provides a marketplace of AI agents, diff --git a/docs/reqstool/requirements.yml b/docs/reqstool/requirements.yml index 3c93f05d..4e69b766 100644 --- a/docs/reqstool/requirements.yml +++ b/docs/reqstool/requirements.yml @@ -248,6 +248,24 @@ requirements: description: The system shall report an identical completion verdict and output structure for a given requirement across the status CLI command and the MCP get_requirement_status and get_requirements_status tools, in both build-only and post-build scoping modes, derived from a single shared verdict computation and a single shared serializer. categories: ["functional-suitability"] revision: "0.11.0" + - id: MCP_0006 + title: MCP snapshot freshness + significance: shall + description: The system shall answer MCP queries from a snapshot that reflects the current state of the served project's local input files, reloading them automatically when they change and on explicit request, so that a long-lived server does not answer from the snapshot captured when it started. + categories: ["functional-suitability"] + revision: "0.11.0" + - id: MCP_0007 + title: MCP snapshot reload failure reporting + significance: shall + description: The system shall report an error when the served project's input files have changed but the new state cannot be loaded, and shall not answer from the superseded snapshot. + categories: ["reliability"] + revision: "0.11.0" + - id: MCP_0008 + title: MCP snapshot provenance and missing artifact reporting + significance: shall + description: The system shall report when the served snapshot was parsed, and shall explicitly report configured test result patterns that match no files, so that absent build artifacts are distinguishable from an absence of tests. + categories: ["interaction-capability"] + revision: "0.11.0" # --- data-sources capability (derived from openspec/specs/data-sources) --- - id: SOURCE_0001 diff --git a/docs/reqstool/software_verification_cases.yml b/docs/reqstool/software_verification_cases.yml index 126ca68c..433eec31 100644 --- a/docs/reqstool/software_verification_cases.yml +++ b/docs/reqstool/software_verification_cases.yml @@ -242,6 +242,24 @@ cases: description: "GIVEN the same dataset WHEN get_requirement_status and get_requirements_status are called THEN they report the same completed verdict and output structure as the status command for the same requirement, in both build-only and post-build modes" verification: automated-test revision: "0.11.0" + - id: SVC_MCP_0006 + requirement_ids: ["MCP_0006"] + title: "MCP server follows changes to the served project" + description: "GIVEN a running MCP server WHEN the served project's requirements, annotations or test results are changed, added or removed THEN subsequent tool calls answer from the changed files without a restart, and WHEN nothing changed THEN the project is not reparsed" + verification: automated-test + revision: "0.11.0" + - id: SVC_MCP_0007 + requirement_ids: ["MCP_0007"] + title: "MCP server reports a failed reload" + description: "GIVEN a running MCP server WHEN the served project has changed but no longer parses THEN tool calls report the load error instead of answering from the previous snapshot" + verification: automated-test + revision: "0.11.0" + - id: SVC_MCP_0008 + requirement_ids: ["MCP_0008"] + title: "MCP server reports snapshot provenance and absent artifacts" + description: "GIVEN a running MCP server WHEN get_status is called THEN it reports when its data was parsed, and WHEN a configured test results pattern matches no files THEN that is reported as a warning rather than as zero tests" + verification: automated-test + revision: "0.11.0" # --- data-sources --- - id: SVC_SOURCE_0001 diff --git a/openspec/changes/mcp-snapshot-freshness/.openspec.yaml b/openspec/changes/mcp-snapshot-freshness/.openspec.yaml new file mode 100644 index 00000000..0c73c8f5 --- /dev/null +++ b/openspec/changes/mcp-snapshot-freshness/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-08-15 diff --git a/openspec/changes/mcp-snapshot-freshness/design.md b/openspec/changes/mcp-snapshot-freshness/design.md new file mode 100644 index 00000000..c445c2f3 --- /dev/null +++ b/openspec/changes/mcp-snapshot-freshness/design.md @@ -0,0 +1,56 @@ +# Design + +## Why not re-parse per request + +Issue #437 suggests re-reading sources and artifacts on every request. That is correct but +expensive in the wrong place: a rebuild walks the whole import chain, including remote imports, +which means network fetches for git/maven/pypi sources on every tool call. + +Stat-then-rebuild gives the same freshness guarantee for local sources at the cost of a few dozen +`stat()` calls and one `rglob` per configured test-result pattern, and remote imports are +version-pinned anyway. The check runs unconditionally per request — no debounce interval — because +a debounce window is exactly the "answered from just before the build" case this change exists to +eliminate. + +## What is fingerprinted + +Per parsed source, when its location is local: + +| Tracked | Why | +|---|---| +| `requirements.yml`, `software_verification_cases.yml`, `manual_verification_results.yml`, `annotations.yml` | The parsed inputs. Stamped whether or not they exist, so a file the build generates later registers as a change. | +| `reqstool_config.yml` | Decides which files and patterns are read at all. | +| each `test_results` pattern + the files it matched | Test results are resolved from globs; a new JUnit XML under a matched pattern has to count. Each matched file is stamped, so a rewritten report counts too. | + +Stamps compare `(exists, st_mtime_ns, st_size)`. Content hashing was not used: it buys protection +against a rewrite that preserves both mtime and size, which no build produces, at the cost of +reading every file on every request. + +Paths are recorded under the **real project directory**, not the temp tree. A local location is +materialized as a symlink into a `TemporaryDirectory` owned by the generator, and that directory is +gone by the time staleness is checked. + +## Failure handling + +`build()` failing leaves the session not ready and the database closed — deliberately, since the +alternative is serving data known to be superseded. Two details make that survivable: + +- The failed build re-stamps the previous fingerprint against current disk. A tree that does not + parse is therefore parsed once, not once per request, and the next edit makes it stale again. +- `ensure_fresh()` distinguishes the two states in its message: *sources changed but reloading them + failed* (this call triggered the rebuild) versus *project is not loaded* (an earlier reload failed + and nothing has changed since). + +## Concurrency + +`ensure_fresh()` can replace the database underneath a request handler, so `ProjectSession` guards +build/close/ensure_fresh with an `RLock`. MCP tools are async and run on the event loop thread, +which is also the thread the SQLite connection is created on, so the rebuild happens on the +connection's own thread — a rebuild blocks the loop for its duration, as the startup build already +does. + +## Where the verdict comes from + +`get_status` continues to derive every number from `StatisticsService`. The `snapshot` field +describes the *inputs*, never the verdict, so this change adds no second opinion about whether a +requirement is complete (see MCP_0005 and CLAUDE.md). diff --git a/openspec/changes/mcp-snapshot-freshness/proposal.md b/openspec/changes/mcp-snapshot-freshness/proposal.md new file mode 100644 index 00000000..4fd78981 --- /dev/null +++ b/openspec/changes/mcp-snapshot-freshness/proposal.md @@ -0,0 +1,66 @@ +## Why + +`reqstool mcp` parses the project once at startup and then serves that snapshot for as long as +the process lives. AI harnesses spawn the server and keep it up for days, while builds regenerate +`annotations.yml` and JUnit XML underneath it. Issue #437 reports the consequence: a server +spawned days earlier answered `get_status` with 14 requirements, 0 implementations, 0 tests, while +the CLI against the same tree reported 55/55 complete. The stale answer is well-formed, so neither +an agent nor a human can tell it apart from a real result. + +Two things cause it. The server binds the repository into its tool closures at startup, so nothing +short of a restart can change what the tools read. And nothing records which files were parsed, so +there is no way to notice that the tree moved on — in particular "the `test_results` pattern +matched no files" is indistinguishable from "this project has no tests". + +## What Changes + +- Capture a **fingerprint** of the local input files each parse read: the four data YAMLs plus + `reqstool_config.yml`, stamped whether or not they exist, and each `test_results` glob pattern + together with the concrete files it matched. Files absent at parse time are tracked deliberately + — an `annotations.yml` the build has yet to generate is the common staleness trigger. Remote + sources (git/maven/npm/pypi) are version-pinned downloads materialized into a temp directory that + is removed after parsing, so they are not fingerprinted and never go stale. +- Add `ProjectSession.ensure_fresh()`: re-stat the fingerprint, rebuild only if it no longer + matches. The LSP keeps calling `rebuild()` directly from its client file-change notifications; + MCP has no such channel, which is why it checks per request. +- **MCP tools resolve the repository per call** instead of closing over it at startup. Without + this, reloading would be invisible to the tools. +- **A failed reload is an error, not a fallback.** If the inputs changed but the new state does not + parse, tools raise `SnapshotReloadError` rather than answering from the superseded snapshot. The + failed build re-stamps the inputs it knew about, so a broken tree is not re-parsed on every + request until it is fixed. +- Add a `refresh` tool that reloads unconditionally, and a `snapshot` field on `get_status` + reporting `built_at`, `tracked_files`, and `warnings` — where a `test_results` pattern matching + no files is reported explicitly instead of being counted as zero tests. + +## Capabilities + +### New Capabilities + + +### Modified Capabilities +- `mcp`: adds requirements that the server answers from a snapshot reflecting the current state of + the served project's local files (reloading automatically and on request), that a reload which + fails is reported as an error rather than answered from the superseded snapshot, and that the + snapshot's parse time and any test-result patterns matching no files are reported. + +## Impact + +- `src/reqstool/common/snapshot_fingerprint.py` — new; the fingerprint and its staleness check. +- `src/reqstool/model_generators/combined_raw_datasets_generator.py` — records the fingerprint per + parsed source; `__parse_source_other` now also returns the test-result files each pattern matched. +- `src/reqstool/models/raw_datasets.py` — `fingerprint` on `RawDataset` and `CombinedRawDataset`. +- `src/reqstool/common/project_session.py` — fingerprint, `built_at`, `initial_urn`, + `ensure_fresh()`, and a lock around rebuilds. +- `src/reqstool/mcp/server.py` — per-call repository resolution, `refresh` tool, `snapshot` field. +- MCP clients — `get_status` gains a `snapshot` field (additive); tool calls can now fail with a + load error where they previously returned stale numbers (intentional). +- Not changed: the CLI, which parses per invocation and was never affected. + +## Non-goals + +- A `verify` tool with CLI-gate semantics (issue #437, point 3). It belongs with the `status`/ + `export` redesign in #311 and must derive from the shared verdict computation (MCP_0005), so it + is deliberately left out of this change. +- Watching remote sources. A `GitLocation` pinned to a moving branch ref is technically mutable; + `refresh` covers it. diff --git a/openspec/changes/mcp-snapshot-freshness/specs/mcp/spec.md b/openspec/changes/mcp-snapshot-freshness/specs/mcp/spec.md new file mode 100644 index 00000000..2cef5754 --- /dev/null +++ b/openspec/changes/mcp-snapshot-freshness/specs/mcp/spec.md @@ -0,0 +1,19 @@ +## ADDED Requirements + +### Requirement: MCP_0006 +The system SHALL implement MCP_0006. + +#### Scenario: SVC_MCP_0006 +The system SHALL pass SVC_MCP_0006. + +### Requirement: MCP_0007 +The system SHALL implement MCP_0007. + +#### Scenario: SVC_MCP_0007 +The system SHALL pass SVC_MCP_0007. + +### Requirement: MCP_0008 +The system SHALL implement MCP_0008. + +#### Scenario: SVC_MCP_0008 +The system SHALL pass SVC_MCP_0008. diff --git a/openspec/changes/mcp-snapshot-freshness/tasks.md b/openspec/changes/mcp-snapshot-freshness/tasks.md new file mode 100644 index 00000000..b77fc9d1 --- /dev/null +++ b/openspec/changes/mcp-snapshot-freshness/tasks.md @@ -0,0 +1,49 @@ +## 1. reqstool SSOT + +- [x] 1.1 Add `MCP_0006` (snapshot freshness), `MCP_0007` (reload failure reporting) and `MCP_0008` (snapshot provenance and missing artifact reporting) to `docs/reqstool/requirements.yml` under the mcp capability block +- [x] 1.2 Add `SVC_MCP_0006`, `SVC_MCP_0007` and `SVC_MCP_0008` to `docs/reqstool/software_verification_cases.yml` +- [x] 1.3 Run `openspec validate mcp-snapshot-freshness --type change --strict` and confirm it passes + +## 2. Fingerprint the parsed inputs + +- [x] 2.1 Add `SnapshotFingerprint` (`FileStamp`, `GlobSpec`) with `stale_reasons()`, `is_stale()`, `restamped()` and `warnings()` +- [x] 2.2 Capture it per parsed source in `CombinedRawDatasetsGenerator.__parse_source`, for `LocalLocation` only, recording paths under the real project directory rather than the temp symlink tree +- [x] 2.3 Stamp the four data YAMLs and `reqstool_config.yml` whether or not they exist, so a file the build generates later registers as a change +- [x] 2.4 Record each `test_results` pattern with the concrete files it matched — `__parse_source_other` returns them instead of discarding them +- [x] 2.5 Aggregate onto `CombinedRawDataset`, mirroring `urn_source_paths` + +## 3. Reload the session when its inputs change + +- [x] 3.1 Track `fingerprint`, `built_at` and `initial_urn` on `ProjectSession` +- [x] 3.2 Add `ensure_fresh()` — rebuild only when the fingerprint no longer matches disk; raise `SnapshotReloadError` when the session cannot serve a snapshot that matches +- [x] 3.3 Re-stamp the known inputs after a failed build, so a tree that does not parse is not re-parsed on every request +- [x] 3.4 Guard build/close/ensure_fresh with a lock — a reload replaces the database underneath request handlers +- [x] 3.5 Leave the LSP path alone: it rebuilds from client file-change notifications and does not call `ensure_fresh()` + +## 4. MCP server + +- [x] 4.1 Resolve the repository per tool call via a `_repo()` helper instead of binding it at startup — reloading is invisible to the tools otherwise +- [x] 4.2 Add a `refresh` tool that reloads unconditionally and reports the resulting snapshot +- [x] 4.3 Add a `snapshot` field to `get_status` with `built_at`, `reload`, `tracked_files` and `warnings` +- [x] 4.4 Report `test_results` patterns matching no files as warnings; report a missing annotations file only for the served URN, since imported sources are not expected to carry annotations +- [x] 4.5 Add `@Requirements` annotations for `MCP_0006`, `MCP_0007` and `MCP_0008` + +## 5. Tests + +- [x] 5.1 Unit-test the fingerprint: modified, removed and later-created files; new and rewritten test results; zero-match patterns; `restamped()` +- [x] 5.2 Unit-test `ProjectSession.ensure_fresh()` against a writable copy of a fixture: unchanged tree is not rebuilt, changed annotations and test results are picked up, a broken tree errors and recovers once fixed +- [x] 5.3 Unit-test the MCP tools driven while the project changes underneath a live server +- [x] 5.4 Integration-test a real spawned server (`tests/integration/reqstool/mcp/test_mcp_reload_integration.py`) against a writable project copy: annotations added after startup are served, `refresh` reloads, a project that no longer parses errors rather than answering stale +- [x] 5.5 Add `@SVCs` annotations to the verifying tests + +## 6. Documentation + +- [x] 6.1 Document the `refresh` tool and the `snapshot` field in `docs/modules/ROOT/pages/mcp.adoc` +- [x] 6.2 Add a "Snapshot Freshness" section covering what is watched, what happens when a reload fails, and that remote sources are not watched + +## 7. Verification + +- [x] 7.1 Run `hatch run dev:pytest --cov=reqstool` (unit and integration) and `hatch run dev:flake8` +- [x] 7.2 Run the CLAUDE.md regression smoke diffs against `main` — CLI output must be byte-identical +- [x] 7.3 Run `reqstool status local -p docs/reqstool` and confirm the new SVCs are covered +- [x] 7.4 Run `openspec validate --all --strict` diff --git a/src/reqstool/common/exceptions.py b/src/reqstool/common/exceptions.py index 17615545..12d165f3 100644 --- a/src/reqstool/common/exceptions.py +++ b/src/reqstool/common/exceptions.py @@ -48,6 +48,17 @@ def __init__(self, ref: str, url: str): super().__init__(f"ref '{ref}' not found in {url}") +class SnapshotReloadError(Exception): + """Raised when a session's input files changed but the new state cannot be parsed. + + Answering from the superseded snapshot would be a well-formed but wrong answer, so + callers get this instead. + """ + + def __init__(self, message: str): + super().__init__(message) + + class EnvVarInterpolationError(Exception): """Raised when environment variable interpolation of YAML input fails. diff --git a/src/reqstool/common/project_session.py b/src/reqstool/common/project_session.py index 40289eb3..bd1830e9 100644 --- a/src/reqstool/common/project_session.py +++ b/src/reqstool/common/project_session.py @@ -2,7 +2,11 @@ import logging +import threading +from datetime import datetime, timezone +from reqstool.common.exceptions import SnapshotReloadError +from reqstool.common.snapshot_fingerprint import SnapshotFingerprint from reqstool.common.validators.lifecycle_validator import LifecycleValidator from reqstool.common.validators.semantic_validator import SemanticValidator from reqstool.common.validator_error_holder import ValidationErrorHolder @@ -22,6 +26,10 @@ class ProjectSession: Keeps the SQLite database open for the lifetime of the session (unlike the build_database() context manager which closes on exit). Suitable for servers (MCP, LSP) that need persistent read access after a one-time build. + + A session records a fingerprint of the local files it parsed. Servers without an + external change signal call `ensure_fresh()` before serving a request; servers driven + by client file-change notifications (LSP) call `rebuild()` directly. """ def __init__(self, location: LocationInterface, parsing_config: ParsingConfig = ParsingConfig()): @@ -32,6 +40,11 @@ def __init__(self, location: LocationInterface, parsing_config: ParsingConfig = self._urn_source_paths: dict[str, dict[str, str]] = {} self._ready: bool = False self._error: str | None = None + self._fingerprint: SnapshotFingerprint | None = None + self._built_at: str | None = None + self._initial_urn: str | None = None + # ensure_fresh() may rebuild the database underneath concurrent request handlers. + self._lock = threading.RLock() @property def ready(self) -> bool: @@ -49,46 +62,103 @@ def repo(self) -> RequirementsRepository | None: def urn_source_paths(self) -> dict[str, dict[str, str]]: return self._urn_source_paths + @property + def fingerprint(self) -> SnapshotFingerprint | None: + return self._fingerprint + + @property + def built_at(self) -> str | None: + """When the current snapshot was parsed (ISO 8601, UTC), or None if never built.""" + return self._built_at + + @property + def initial_urn(self) -> str | None: + """URN of the source this session was opened on (imports and implementations excluded).""" + return self._initial_urn + def build(self) -> None: - self.close() - self._error = None - db = RequirementsDatabase() - try: - holder = ValidationErrorHolder() - semantic_validator = SemanticValidator(validation_error_holder=holder) - - crdg = CombinedRawDatasetsGenerator( - initial_location=self._location, - semantic_validator=semantic_validator, - database=db, - parsing_config=self._parsing_config, - ) - crd = crdg.combined_raw_datasets - - DatabaseFilterProcessor(db, crd.raw_datasets).apply_filters() - LifecycleValidator(RequirementsRepository(db)) - - self._db = db - self._repo = RequirementsRepository(db) - self._urn_source_paths = dict(crd.urn_source_paths) - self._ready = True - logger.info("Built project session for %s", self._location) - except SystemExit as e: - logger.warning("build() called sys.exit(%s) for %s", e.code, self._location) - self._error = f"Pipeline error (exit code {e.code})" - db.close() - except Exception as e: - logger.error("Failed to build project session for %s: %s", self._location, e) - self._error = str(e) - db.close() + with self._lock: + previous_fingerprint = self._fingerprint + self.close() + self._error = None + db = RequirementsDatabase() + try: + holder = ValidationErrorHolder() + semantic_validator = SemanticValidator(validation_error_holder=holder) + + crdg = CombinedRawDatasetsGenerator( + initial_location=self._location, + semantic_validator=semantic_validator, + database=db, + parsing_config=self._parsing_config, + ) + crd = crdg.combined_raw_datasets + + DatabaseFilterProcessor(db, crd.raw_datasets).apply_filters() + LifecycleValidator(RequirementsRepository(db)) + + self._db = db + self._repo = RequirementsRepository(db) + self._urn_source_paths = dict(crd.urn_source_paths) + self._fingerprint = crd.fingerprint + self._initial_urn = crd.initial_model_urn + self._built_at = datetime.now(timezone.utc).isoformat() + self._ready = True + logger.info("Built project session for %s", self._location) + except SystemExit as e: + logger.warning("build() called sys.exit(%s) for %s", e.code, self._location) + self._error = f"Pipeline error (exit code {e.code})" + db.close() + self._fingerprint = self.__fingerprint_after_failure(previous_fingerprint) + except Exception as e: + logger.error("Failed to build project session for %s: %s", self._location, e) + self._error = str(e) + db.close() + self._fingerprint = self.__fingerprint_after_failure(previous_fingerprint) + + @staticmethod + def __fingerprint_after_failure(previous: SnapshotFingerprint | None) -> SnapshotFingerprint | None: + """Keep watching the inputs we knew about, stamped as they are now. + + Without this a working tree that fails to parse — a half-written YAML file, say — + would be re-parsed on every single request until it is fixed. + """ + return previous.restamped() if previous is not None else None def rebuild(self) -> None: self.build() + def ensure_fresh(self) -> bool: + """Rebuild if the local input files no longer match the loaded snapshot. + + Returns True if a rebuild happened. Raises SnapshotReloadError if the session + cannot serve a snapshot that matches disk — answering from a snapshot known to be + superseded is what this whole mechanism exists to prevent. + """ + with self._lock: + if self._fingerprint is not None: + stale_reasons = self._fingerprint.stale_reasons(limit=5) + if not stale_reasons: + if self._ready: + return False + raise SnapshotReloadError(f"reqstool project is not loaded: {self._error}") + logger.info("Reloading snapshot for %s: %s", self._location, "; ".join(stale_reasons)) + + self.build() + + if not self._ready: + raise SnapshotReloadError(f"reqstool project sources changed but reloading them failed: {self._error}") + + return True + def close(self) -> None: - if self._db is not None: - self._db.close() - self._db = None - self._repo = None - self._urn_source_paths = {} - self._ready = False + with self._lock: + if self._db is not None: + self._db.close() + self._db = None + self._repo = None + self._urn_source_paths = {} + self._fingerprint = None + self._built_at = None + self._initial_urn = None + self._ready = False diff --git a/src/reqstool/common/snapshot_fingerprint.py b/src/reqstool/common/snapshot_fingerprint.py new file mode 100644 index 00000000..a97f4c75 --- /dev/null +++ b/src/reqstool/common/snapshot_fingerprint.py @@ -0,0 +1,179 @@ +# Copyright © LFV + +"""Fingerprint of the local input files a parsed snapshot was built from. + +A long-lived server (MCP) parses once and then serves that snapshot. The fingerprint +records what was read — and what was looked for but absent — so the server can tell, +cheaply and per request, whether the snapshot still matches the working tree. + +Only local sources are fingerprinted. Remote sources (git/maven/npm/pypi) are +version-pinned downloads materialized into a temp directory that is removed once +parsing finishes, so there is nothing stable to stat. +""" + +import logging +import os +from dataclasses import dataclass +from pathlib import Path +from typing import List, Optional, Tuple + +logger = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class FileStamp: + """Identity of a single input file at capture time. + + Absent files are stamped too (``exists=False``): an ``annotations.yml`` that the build + has not generated yet is the common staleness trigger, and it can only be detected by + having recorded that we looked for it. + """ + + path: str + kind: str + urn: str + exists: bool = True + mtime_ns: int = 0 + size: int = 0 + + @staticmethod + def capture(path: str, kind: str, urn: str) -> "FileStamp": + try: + st = os.stat(path) + except OSError: + return FileStamp(path=path, kind=kind, urn=urn, exists=False) + return FileStamp(path=path, kind=kind, urn=urn, exists=True, mtime_ns=st.st_mtime_ns, size=st.st_size) + + def changed_on_disk(self) -> Optional[str]: + """Return a human-readable reason if disk no longer matches this stamp, else None.""" + current = FileStamp.capture(self.path, self.kind, self.urn) + + if current.exists is not self.exists: + return f"{'removed' if self.exists else 'added'}: {self.path}" + + if current.exists and (current.mtime_ns != self.mtime_ns or current.size != self.size): + return f"modified: {self.path}" + + return None + + +@dataclass(frozen=True) +class GlobSpec: + """A test-results glob pattern and the files it matched at capture time. + + Test results are resolved from patterns, not fixed paths, so freshness means + "the pattern still matches the same files, unchanged" — a new JUnit XML dropped + into a report directory has to count as a change. + """ + + root: str + pattern: str + urn: str + matched: Tuple[FileStamp, ...] = () + + @staticmethod + def capture(root: str, pattern: str, urn: str, matched_paths: List[str]) -> "GlobSpec": + stamps = tuple(sorted((FileStamp.capture(p, "test_results", urn) for p in matched_paths), key=lambda s: s.path)) + return GlobSpec(root=root, pattern=pattern, urn=urn, matched=stamps) + + @property + def matched_paths(self) -> Tuple[str, ...]: + return tuple(stamp.path for stamp in self.matched) + + def changed_on_disk(self) -> Optional[str]: + """Return a human-readable reason if the pattern no longer resolves as captured, else None.""" + try: + current = tuple(sorted(str(p) for p in Path(self.root).rglob(self.pattern))) + except OSError as e: + return f"test results unreadable for pattern {self.pattern!r} under {self.root}: {e}" + + if current != self.matched_paths: + added = len(set(current) - set(self.matched_paths)) + removed = len(set(self.matched_paths) - set(current)) + return f"test results changed for pattern {self.pattern!r} under {self.root} (+{added}/-{removed} files)" + + for stamp in self.matched: + reason = stamp.changed_on_disk() + if reason is not None: + return f"test result {reason}" + + return None + + +@dataclass(frozen=True) +class SnapshotFingerprint: + """The full set of local inputs a snapshot was built from.""" + + stamps: Tuple[FileStamp, ...] = () + globs: Tuple[GlobSpec, ...] = () + + @staticmethod + def merge(parts: List["SnapshotFingerprint"]) -> "SnapshotFingerprint": + stamps: List[FileStamp] = [] + globs: List[GlobSpec] = [] + for part in parts: + stamps.extend(part.stamps) + globs.extend(part.globs) + return SnapshotFingerprint(stamps=tuple(stamps), globs=tuple(globs)) + + @property + def tracked_file_count(self) -> int: + return len(self.stamps) + sum(len(g.matched) for g in self.globs) + + def stale_reasons(self, limit: Optional[int] = None) -> List[str]: + """Reasons this snapshot no longer matches disk, at most ``limit`` of them.""" + reasons: List[str] = [] + + for checkable in (*self.stamps, *self.globs): + reason = checkable.changed_on_disk() + if reason is not None: + reasons.append(reason) + if limit is not None and len(reasons) >= limit: + break + + return reasons + + def is_stale(self) -> bool: + return bool(self.stale_reasons(limit=1)) + + def restamped(self) -> "SnapshotFingerprint": + """Same tracked inputs, re-read from disk. + + Used after a failed reload: the paths we know about are still the right ones to + watch, but re-stamping them stops a broken working tree from being re-parsed on + every single request — the next edit flips it stale again. + """ + stamps = tuple(FileStamp.capture(s.path, s.kind, s.urn) for s in self.stamps) + globs = [] + for g in self.globs: + try: + matched_paths = [str(p) for p in Path(g.root).rglob(g.pattern)] + except OSError: + matched_paths = list(g.matched_paths) + globs.append(GlobSpec.capture(g.root, g.pattern, g.urn, matched_paths)) + return SnapshotFingerprint(stamps=stamps, globs=tuple(globs)) + + def warnings(self, primary_urn: Optional[str] = None) -> List[str]: + """Input conditions that make a well-formed answer misleading. + + A test-results pattern matching nothing is reported as such rather than silently + counted as zero tests — that is the incremental-build trap this exists for. + + A missing annotations file is only worth reporting for ``primary_urn``, the source + the session was opened on: imported sources supply requirements, and are not + expected to carry implementation annotations of their own. + """ + warnings: List[str] = [] + + for g in self.globs: + if not g.matched: + warnings.append(f"[{g.urn}] test_results pattern {g.pattern!r} matched no files under {g.root}") + + for s in self.stamps: + if s.kind == "annotations" and not s.exists and s.urn == primary_urn: + warnings.append( + f"[{s.urn}] no annotations file at {s.path} — " + "implementation and test annotations are reported as absent" + ) + + return warnings diff --git a/src/reqstool/mcp/server.py b/src/reqstool/mcp/server.py index 2b41a657..42dd9f4f 100644 --- a/src/reqstool/mcp/server.py +++ b/src/reqstool/mcp/server.py @@ -4,6 +4,8 @@ import logging from typing import Literal +from reqstool_python_decorators.decorators.decorators import Requirements + from reqstool.common.project_session import ProjectSession from reqstool.common.enrichment.enricher import BUILT_IN_PRESETS, enrich_text from reqstool.common.queries.details import ( @@ -41,8 +43,29 @@ def start_server( # noqa: C901 if session.repo is None: raise RuntimeError("Project session repo is None after successful build") - repo: RequirementsRepository = session.repo - urn_source_paths = session.urn_source_paths + + @Requirements("MCP_0006", "MCP_0007") + def _repo() -> RequirementsRepository: + """Reload the snapshot if the project's input files changed, then return the repository. + + Every tool must resolve the repository through this. Binding it once at startup is + what let long-lived servers serve a snapshot from before the last build (#437). + """ + session.ensure_fresh() + repo = session.repo + if repo is None: + raise RuntimeError(f"reqstool project is not loaded: {session.error}") + return repo + + @Requirements("MCP_0008") + def _snapshot_info() -> dict: + fingerprint = session.fingerprint + return { + "built_at": session.built_at, + "reload": "automatic on input change", + "tracked_files": fingerprint.tracked_file_count if fingerprint is not None else 0, + "warnings": fingerprint.warnings(primary_urn=session.initial_urn) if fingerprint is not None else [], + } mcp = MCPServer(name="reqstool") @@ -57,12 +80,12 @@ def start_server( # noqa: C901 async def list_requirements(urn: str | None = None, lifecycle_state: str | None = None) -> list[dict]: """List requirements with id, title, and lifecycle state. Filter by urn and/or lifecycle_state (draft|effective|deprecated|obsolete).""" - return get_requirements_list(repo, urn=urn, lifecycle_state=lifecycle_state) + return get_requirements_list(_repo(), urn=urn, lifecycle_state=lifecycle_state) @mcp.tool() async def get_requirement(id: str) -> dict: """Get full details for a requirement by ID (e.g. REQ_010).""" - result = get_requirement_details(id, repo, urn_source_paths) + result = get_requirement_details(id, _repo(), session.urn_source_paths) if result is None: raise ValueError(f"Requirement {id!r} not found") return result @@ -73,18 +96,18 @@ async def get_requirements_status(urn: str | None = None, include_post_build: bo automated_tests, manual_tests. Use this to find requirements that are incomplete, partially tested, or not yet implemented. Optionally filter by URN. Set include_post_build=True for parity with `status --with-post-tests` (scopes to post-build-phase SVCs too).""" - return _get_requirements_status_all(repo, urn=urn, include_post_build=include_post_build) + return _get_requirements_status_all(_repo(), urn=urn, include_post_build=include_post_build) @mcp.tool() async def list_svcs(urn: str | None = None, lifecycle_state: str | None = None) -> list[dict]: """List SVCs with id, title, lifecycle state, and verification type. Filter by urn and/or lifecycle_state (draft|effective|deprecated|obsolete).""" - return get_svcs_list(repo, urn=urn, lifecycle_state=lifecycle_state) + return get_svcs_list(_repo(), urn=urn, lifecycle_state=lifecycle_state) @mcp.tool() async def get_svc(id: str) -> dict: """Get full details for an SVC by ID (e.g. SVC_010).""" - result = get_svc_details(id, repo, urn_source_paths) + result = get_svc_details(id, _repo(), session.urn_source_paths) if result is None: raise ValueError(f"SVC {id!r} not found") return result @@ -92,27 +115,44 @@ async def get_svc(id: str) -> dict: @mcp.tool() async def list_mvrs(urn: str | None = None, passed: bool | None = None) -> list[dict]: """List MVRs with id and passed status. Filter by urn and/or passed (True|False).""" - return get_mvrs_list(repo, urn=urn, passed=passed) + return get_mvrs_list(_repo(), urn=urn, passed=passed) @mcp.tool() async def get_mvr(id: str) -> dict: """Get full details for an MVR by ID.""" - result = get_mvr_details(id, repo, urn_source_paths) + result = get_mvr_details(id, _repo(), session.urn_source_paths) if result is None: raise ValueError(f"MVR {id!r} not found") return result @mcp.tool() async def get_status() -> dict: - """Get overall traceability status — completion per requirement, test totals.""" - return StatisticsService(repo).to_status_dict() + """Get overall traceability status — completion per requirement, test totals. + + The `snapshot` field reports when the served data was parsed and warns about + configured test-result patterns that matched no files (an unbuilt or partially + built project reports zero tests, which is not the same as having no tests).""" + status = StatisticsService(_repo()).to_status_dict() + status["snapshot"] = _snapshot_info() + return status + + @mcp.tool() + async def refresh() -> dict: + """Force an immediate reload of the project from disk. + + Reloading is automatic when input files change, so this is only needed to reload + unconditionally — after a build, for instance — or to confirm what is being served.""" + session.build() + if not session.ready: + raise RuntimeError(f"Failed to reload reqstool project: {session.error}") + return _snapshot_info() @mcp.tool() async def get_requirement_status(id: str, include_post_build: bool = False) -> dict: """Status check for one requirement: lifecycle_state, completed, implementation_type, automated_tests, manual_tests. Set include_post_build=True for parity with `status --with-post-tests` (scopes to post-build-phase SVCs too).""" - result = _get_requirement_status(id, repo, include_post_build=include_post_build) + result = _get_requirement_status(id, _repo(), include_post_build=include_post_build) if result is None: raise ValueError(f"Requirement {id!r} not found") return result @@ -120,7 +160,7 @@ async def get_requirement_status(id: str, include_post_build: bool = False) -> d @mcp.tool() async def list_annotations(urn: str | None = None) -> list[dict]: """List implementation annotations (@Requirements) found in source code. Optionally filter by URN.""" - impl_annotations = repo.get_annotations_impls(urn=urn) + impl_annotations = _repo().get_annotations_impls(urn=urn) result = [] for urn_id, ann_list in impl_annotations.items(): for ann in ann_list: @@ -137,12 +177,12 @@ async def list_annotations(urn: str | None = None) -> list[dict]: @mcp.tool() async def list_urns() -> list[dict]: """List all URNs in the project graph with variant, title, url, location, and file paths.""" - return get_urns_list(repo, urn_source_paths) + return get_urns_list(_repo(), session.urn_source_paths) @mcp.tool() async def get_urn_details(urn: str) -> dict: """Get details for a URN: variant, title, location, file paths, and entity counts.""" - result = _get_urn_details(urn, repo, urn_source_paths) + result = _get_urn_details(urn, _repo(), session.urn_source_paths) if result is None: raise ValueError(f"URN {urn!r} not found") return result @@ -160,6 +200,7 @@ async def enrich_document(content: str, preset: str) -> str: if preset not in BUILT_IN_PRESETS: raise ValueError(f"Unknown preset {preset!r}. Valid: {sorted(BUILT_IN_PRESETS)}") config = BUILT_IN_PRESETS[preset] + repo = _repo() return enrich_text(content, repo.get_all_requirements(), repo.get_all_svcs(), repo.get_all_mvrs(), config) try: diff --git a/src/reqstool/model_generators/combined_raw_datasets_generator.py b/src/reqstool/model_generators/combined_raw_datasets_generator.py index 437cbe8c..6ffa2e62 100644 --- a/src/reqstool/model_generators/combined_raw_datasets_generator.py +++ b/src/reqstool/model_generators/combined_raw_datasets_generator.py @@ -3,11 +3,13 @@ import logging import os from collections import defaultdict +from pathlib import Path from typing import Dict, List, Optional, Set, Tuple from reqstool_python_decorators.decorators.decorators import Requirements from reqstool.common.exceptions import CircularImplementationError, CircularImportError, MissingRequirementsFileError +from reqstool.common.snapshot_fingerprint import FileStamp, GlobSpec, SnapshotFingerprint from reqstool.common.utils import TempDirectoryManager, Utils from reqstool.common.validators.semantic_validator import SemanticValidator from reqstool.location_resolver.location_resolver import LocationResolver @@ -81,6 +83,7 @@ def __generate(self) -> CombinedRawDataset: urn_parsing_order=self._parsing_order, parsing_graph=self._parsing_graph, urn_source_paths=urn_source_paths, + fingerprint=SnapshotFingerprint.merge([rd.fingerprint for rd in raw_datasets.values()]), ) self.semantic_validator.validate_post_parsing(combined_raw_dataset=combined_raw_datasets) @@ -277,7 +280,7 @@ def __parse_source(self, current_location_handler: LocationResolver) -> RawDatas logging.info(f"{requirements_indata.dst_path}") # parse file sources other than requirements.yml - annotations_data, svcs_data, automated_tests, mvrs_data = self.__parse_source_other( + annotations_data, svcs_data, automated_tests, mvrs_data, test_result_files = self.__parse_source_other( actual_tmp_path, requirements_indata, rmg ) @@ -286,6 +289,14 @@ def __parse_source(self, current_location_handler: LocationResolver) -> RawDatas # Capture resolved file paths for LocalLocation only source_paths = self.__extract_source_paths(current_location_handler.current, requirements_indata) + fingerprint = self.__extract_fingerprint( + location=current_location_handler.current, + requirements_indata=requirements_indata, + actual_tmp_path=actual_tmp_path, + test_result_files=test_result_files, + urn=rmg.requirements_data.metadata.urn, + ) + raw_dataset = RawDataset( requirements_data=rmg.requirements_data, annotations_data=annotations_data, @@ -295,6 +306,7 @@ def __parse_source(self, current_location_handler: LocationResolver) -> RawDatas location_type=location_type, location_uri=location_uri, source_paths=source_paths, + fingerprint=fingerprint, ) return raw_dataset @@ -347,6 +359,51 @@ def __extract_source_paths(location: LocationInterface, requirements_indata: Req source_paths["annotations"] = paths.annotations_yml.path return source_paths + @staticmethod + def __extract_fingerprint( + location: LocationInterface, + requirements_indata: RequirementsIndata, + actual_tmp_path: str, + test_result_files: Dict[str, List[Path]], + urn: str, + ) -> SnapshotFingerprint: + """Stamp the local input files this source was parsed from. + + Only LocalLocation is fingerprinted: everything else is a version-pinned download + materialized under a temp directory that is removed once parsing finishes. + """ + if not isinstance(location, LocalLocation): + return SnapshotFingerprint() + + # actual_tmp_path is a symlink into the temp tree; the temp tree is gone by the time + # staleness is checked, so record paths under the real project directory instead. + real_root = os.readlink(actual_tmp_path) + + paths = requirements_indata.requirements_indata_paths + stamps = [ + FileStamp.capture(os.path.join(real_root, "reqstool_config.yml"), "config", urn), + # Stamped whether or not they exist — an annotations.yml the build has yet to + # generate is exactly the change a long-lived server must notice. + FileStamp.capture(paths.requirements_yml.path, "requirements", urn), + FileStamp.capture(paths.svcs_yml.path, "svcs", urn), + FileStamp.capture(paths.mvrs_yml.path, "mvrs", urn), + FileStamp.capture(paths.annotations_yml.path, "annotations", urn), + ] + + globs = [ + GlobSpec.capture( + root=real_root, + pattern=pattern, + urn=urn, + matched_paths=[ + os.path.join(real_root, os.path.relpath(str(f), actual_tmp_path)) for f in matched_files + ], + ) + for pattern, matched_files in test_result_files.items() + ] + + return SnapshotFingerprint(stamps=tuple(stamps), globs=tuple(globs)) + @Requirements("INGEST_0002", "INGEST_0003", "INGEST_0004") def __parse_source_other( self, actual_tmp_path: str, requirements_indata: RequirementsIndata, rmg: RequirementsModelGenerator @@ -356,6 +413,7 @@ def __parse_source_other( mvrs_data: MVRsData = None automated_tests: TestsData = None tests = {} + test_result_files: Dict[str, List[Path]] = {} # get current urn current_urn = rmg.requirements_data.metadata.urn @@ -371,9 +429,10 @@ def __parse_source_other( for test_result_pattern in requirements_indata.test_results_patterns: - test_result_files = Utils.get_matching_files(path=actual_tmp_path, patterns=[test_result_pattern]) + matching_files = Utils.get_matching_files(path=actual_tmp_path, patterns=[test_result_pattern]) + test_result_files[test_result_pattern] = matching_files - automated_tests_results = TestDataModelGenerator(test_result_files, urn=current_urn).model + automated_tests_results = TestDataModelGenerator(matching_files, urn=current_urn).model tests |= automated_tests_results.tests @@ -394,4 +453,4 @@ def __parse_source_other( uri=requirements_indata.requirements_indata_paths.annotations_yml.path, urn=current_urn ).model - return annotations_data, svcs_data, automated_tests, mvrs_data + return annotations_data, svcs_data, automated_tests, mvrs_data, test_result_files diff --git a/src/reqstool/models/raw_datasets.py b/src/reqstool/models/raw_datasets.py index 948265c7..da8efa37 100644 --- a/src/reqstool/models/raw_datasets.py +++ b/src/reqstool/models/raw_datasets.py @@ -4,6 +4,7 @@ from pydantic import BaseModel, ConfigDict, Field +from reqstool.common.snapshot_fingerprint import SnapshotFingerprint from reqstool.models.annotations import AnnotationsData from reqstool.models.mvrs import MVRsData from reqstool.models.requirements import RequirementsData @@ -31,6 +32,10 @@ class RawDataset(BaseModel): # Resolved file paths (file_type → absolute path), only populated for LocalLocation source_paths: Dict[str, str] = Field(default_factory=dict) + # Identity of the local input files this dataset was parsed from, for staleness detection + # by long-lived servers. Empty for non-local locations (version-pinned, materialized to tmp). + fingerprint: SnapshotFingerprint = Field(default_factory=SnapshotFingerprint) + class CombinedRawDataset(BaseModel): model_config = ConfigDict(arbitrary_types_allowed=True) @@ -42,3 +47,6 @@ class CombinedRawDataset(BaseModel): # Aggregated resolved file paths: urn → file_type → absolute path (LSP only, LocalLocation only) urn_source_paths: Dict[str, Dict[str, str]] = Field(default_factory=dict) + + # Merged fingerprint of every local input file parsed into this dataset + fingerprint: SnapshotFingerprint = Field(default_factory=SnapshotFingerprint) diff --git a/tests/integration/reqstool/mcp/test_mcp_integration.py b/tests/integration/reqstool/mcp/test_mcp_integration.py index a3b4de3d..44035299 100644 --- a/tests/integration/reqstool/mcp/test_mcp_integration.py +++ b/tests/integration/reqstool/mcp/test_mcp_integration.py @@ -24,7 +24,7 @@ def _parse_result(result) -> list | dict: async def test_list_tools(mcp_session): - """Server advertises all 9 expected tools.""" + """Server advertises all 10 expected tools.""" result = await mcp_session.list_tools() tool_names = {t.name for t in result.tools} expected = { @@ -37,6 +37,7 @@ async def test_list_tools(mcp_session): "get_status", "get_requirement_status", "list_annotations", + "refresh", } assert expected.issubset(tool_names), f"Missing tools: {expected - tool_names}" diff --git a/tests/integration/reqstool/mcp/test_mcp_reload_integration.py b/tests/integration/reqstool/mcp/test_mcp_reload_integration.py new file mode 100644 index 00000000..759bfc57 --- /dev/null +++ b/tests/integration/reqstool/mcp/test_mcp_reload_integration.py @@ -0,0 +1,126 @@ +# Copyright © LFV + +"""End-to-end proof that a running MCP server follows the project it was pointed at. + +The shared `mcp_session` fixture serves the pristine fixture directory, so these tests run +their own server against a writable copy — the point is to change files underneath a live +server, which is exactly what a build does while an AI harness keeps the server spawned. +""" + +import asyncio +import json +import shutil +import sys +from pathlib import Path + +import pytest +import pytest_asyncio +from mcp.client.session import ClientSession +from mcp.client.stdio import StdioServerParameters, stdio_client +from reqstool_python_decorators.decorators.decorators import SVCs + +pytestmark = [pytest.mark.integration, pytest.mark.asyncio(loop_scope="session")] + +FIXTURE_DIR = Path(__file__).resolve().parents[3] / "fixtures" / "reqstool-regression-python" + +ADDED_IMPLEMENTATION_FQN = "reqstool_regression.added.AfterServerStartup" + + +def _parse_result(result) -> list | dict: + blocks = [json.loads(b.text) for b in result.content if hasattr(b, "text")] + return blocks if len(blocks) != 1 else blocks[0] + + +@pytest.fixture(scope="module") +def mutable_project(tmp_path_factory) -> Path: + dst = tmp_path_factory.mktemp("reqstool-reload") / "project" + shutil.copytree(FIXTURE_DIR, dst) + return dst + + +@pytest_asyncio.fixture(loop_scope="session", scope="module") +async def reload_session(mutable_project): + """A server spawned against the writable copy, kept alive across the tests below.""" + ready: asyncio.Queue = asyncio.Queue() + done = asyncio.Event() + + async def _lifecycle(): + params = StdioServerParameters( + command=sys.executable, + args=["-m", "reqstool.command", "mcp", "local", "-p", str(mutable_project)], + ) + try: + async with stdio_client(params) as (read, write): + async with ClientSession(read, write) as session: + await session.initialize() + await ready.put(session) + await done.wait() + except Exception as exc: + await ready.put(exc) + + task = asyncio.create_task(_lifecycle()) + result = await ready.get() + if isinstance(result, Exception): + raise result + + yield result + + done.set() + await task + + +@SVCs("SVC_MCP_0006") +async def test_annotations_written_after_startup_are_served(reload_session, mutable_project): + """#437: the server answered from its spawn-time snapshot for as long as it stayed up.""" + before = _parse_result(await reload_session.call_tool("list_annotations", {})) + assert not any(a["fqn"] == ADDED_IMPLEMENTATION_FQN for a in before) + + annotations = mutable_project / "annotations.yml" + original = annotations.read_text() + existing = ' fullyQualifiedName: "requirements_example.RequirementsExample"\n' + assert existing in original + annotations.write_text( + original.replace( + existing, + f'{existing} - elementKind: "CLASS"\n fullyQualifiedName: "{ADDED_IMPLEMENTATION_FQN}"\n', + 1, + ) + ) + + after = _parse_result(await reload_session.call_tool("list_annotations", {})) + + assert any(a["fqn"] == ADDED_IMPLEMENTATION_FQN for a in after) + assert len(after) > len(before) + + +@SVCs("SVC_MCP_0008") +async def test_get_status_reports_the_snapshot_it_answered_from(reload_session): + status = _parse_result(await reload_session.call_tool("get_status", {})) + + assert status["snapshot"]["built_at"] is not None + assert status["snapshot"]["tracked_files"] > 0 + + +@SVCs("SVC_MCP_0006") +async def test_refresh_tool_reloads_on_demand(reload_session): + before = _parse_result(await reload_session.call_tool("get_status", {}))["snapshot"] + + refreshed = _parse_result(await reload_session.call_tool("refresh", {})) + + assert refreshed["built_at"] != before["built_at"] + + +@SVCs("SVC_MCP_0007") +async def test_a_project_that_no_longer_parses_is_an_error_not_a_stale_answer(reload_session, mutable_project): + requirements = mutable_project / "requirements.yml" + original = requirements.read_text() + requirements.write_text(": this is not: [ valid yaml") + + try: + result = await reload_session.call_tool("get_status", {}) + assert result.is_error + assert "reloading them failed" in str(result.content) + finally: + requirements.write_text(original) + + assert not (await reload_session.call_tool("get_status", {})).is_error diff --git a/tests/unit/reqstool/common/test_project_session_freshness.py b/tests/unit/reqstool/common/test_project_session_freshness.py new file mode 100644 index 00000000..15cf6671 --- /dev/null +++ b/tests/unit/reqstool/common/test_project_session_freshness.py @@ -0,0 +1,162 @@ +# Copyright © LFV + +import os +import shutil + +import pytest +from reqstool_python_decorators.decorators.decorators import SVCs + +from reqstool.common.exceptions import SnapshotReloadError +from reqstool.common.project_session import ProjectSession +from reqstool.locations.local_location import LocalLocation +from reqstool.services.statistics_service import StatisticsService + +ANNOTATIONS_WITH_EXTRA_IMPL = """\ +--- +requirement_annotations: + implementations: + REQ_101: + - elementKind: "CLASS" + fullyQualifiedName: "com.example.RequirementsExample" + - elementKind: "METHOD" + fullyQualifiedName: "com.example.RequirementsExample.addedAfterStartup" + REQ_201: + - elementKind: "METHOD" + fullyQualifiedName: "com.example.RequirementsExample.someMethod" +""" + + +@pytest.fixture +def project_copy(tmp_path, local_testdata_resources_rootdir_w_path): + """A writable copy of the ms-101 fixture, so a test can simulate a build changing it.""" + dst = tmp_path / "ms-101" + shutil.copytree(local_testdata_resources_rootdir_w_path("test_basic/baseline/ms-101"), dst) + return dst + + +def _implementation_count(session: ProjectSession) -> int: + return sum(len(impls) for impls in session.repo.get_annotations_impls().values()) + + +def _automated_tests(session: ProjectSession) -> dict: + """Automated test totals as the status command computes them.""" + return StatisticsService(session.repo).to_status_dict()["totals"]["automated_tests"] + + +@pytest.fixture +def session(project_copy): + session = ProjectSession(LocalLocation(path=str(project_copy))) + session.build() + assert session.ready + yield session + session.close() + + +@SVCs("SVC_MCP_0006") +def test_unchanged_project_is_not_rebuilt(session): + built_at = session.built_at + + assert session.ensure_fresh() is False + assert session.built_at == built_at + + +@SVCs("SVC_MCP_0006") +def test_edited_annotations_are_picked_up_without_a_restart(session, project_copy): + """The reported failure: a build regenerates annotations while the server keeps serving.""" + before = _implementation_count(session) + + (project_copy / "annotations.yml").write_text(ANNOTATIONS_WITH_EXTRA_IMPL) + + assert session.ensure_fresh() is True + assert _implementation_count(session) == before + 1 + + +@SVCs("SVC_MCP_0006") +def test_test_results_written_after_startup_are_picked_up( + session, project_copy, local_testdata_resources_rootdir_w_path +): + """The reported scenario: the server was started before the build produced any JUnit XML. + + Test results are resolved from a glob, so files appearing under a matched pattern have + to count as a change — not stay reported as zero passing tests. + """ + results = project_copy / "test_results" + shutil.rmtree(results) + session.build() + assert _automated_tests(session)["passed"] == 0 + + shutil.copytree(local_testdata_resources_rootdir_w_path("test_basic/baseline/ms-101/test_results"), results) + + assert session.ensure_fresh() is True + assert _automated_tests(session)["passed"] > 0 + + +@SVCs("SVC_MCP_0006") +def test_an_annotations_file_created_after_startup_is_picked_up(session, project_copy): + """Files absent at build time are tracked too — that is the unbuilt-project case.""" + os.remove(project_copy / "annotations.yml") + session.build() + assert session.repo.get_annotations_impls() == {} + + (project_copy / "annotations.yml").write_text(ANNOTATIONS_WITH_EXTRA_IMPL) + + assert session.ensure_fresh() is True + assert len(session.repo.get_annotations_impls()) > 0 + + +@SVCs("SVC_MCP_0007") +def test_a_broken_reload_reports_an_error_rather_than_the_superseded_snapshot(session, project_copy): + (project_copy / "requirements.yml").write_text(": this is not: [ valid yaml") + + with pytest.raises(SnapshotReloadError, match="sources changed but reloading them failed"): + session.ensure_fresh() + + assert not session.ready + assert session.repo is None + + +@SVCs("SVC_MCP_0007") +def test_an_unfixed_project_keeps_reporting_the_error_without_reparsing(session, project_copy): + requirements = project_copy / "requirements.yml" + original = requirements.read_text() + requirements.write_text(": this is not: [ valid yaml") + + with pytest.raises(SnapshotReloadError, match="sources changed but reloading them failed"): + session.ensure_fresh() + # Re-stamped on failure: the broken tree is not re-parsed until it changes again. + with pytest.raises(SnapshotReloadError, match="project is not loaded"): + session.ensure_fresh() + + requirements.write_text(original) + + assert session.ensure_fresh() is True + assert session.ready + + +@SVCs("SVC_MCP_0008") +def test_snapshot_records_when_it_was_built_and_what_it_tracks(session, project_copy): + assert session.built_at is not None + assert session.initial_urn == "ms-101" + + fingerprint = session.fingerprint + tracked = {stamp.path for stamp in fingerprint.stamps} + assert str(project_copy / "requirements.yml") in tracked + assert str(project_copy / "annotations.yml") in tracked + # The two JUnit XML files of the fixture, matched by the configured pattern. + assert fingerprint.tracked_file_count == len(fingerprint.stamps) + 2 + + built_at = session.built_at + (project_copy / "annotations.yml").write_text(ANNOTATIONS_WITH_EXTRA_IMPL) + session.ensure_fresh() + + assert session.built_at != built_at + + +@SVCs("SVC_MCP_0008") +def test_absent_test_results_are_warned_about_rather_than_counted_as_zero(session, project_copy): + shutil.rmtree(project_copy / "test_results") + session.build() + + assert _automated_tests(session)["passed"] == 0 + warnings = session.fingerprint.warnings(primary_urn=session.initial_urn) + assert warnings == [f"[ms-101] test_results pattern 'test_results/**/*.xml' matched no files under {project_copy}"] diff --git a/tests/unit/reqstool/common/test_snapshot_fingerprint.py b/tests/unit/reqstool/common/test_snapshot_fingerprint.py new file mode 100644 index 00000000..a124905f --- /dev/null +++ b/tests/unit/reqstool/common/test_snapshot_fingerprint.py @@ -0,0 +1,172 @@ +# Copyright © LFV + +import os + +from reqstool.common.snapshot_fingerprint import FileStamp, GlobSpec, SnapshotFingerprint + +URN = "ms-101" + + +def _write(path, content: str = "x"): + with open(path, "w") as f: + f.write(content) + return str(path) + + +def _fingerprint_of(paths, kind: str = "requirements") -> SnapshotFingerprint: + return SnapshotFingerprint(stamps=tuple(FileStamp.capture(str(p), kind, URN) for p in paths)) + + +# --------------------------------------------------------------------------- +# FileStamp +# --------------------------------------------------------------------------- + + +def test_unchanged_file_is_not_stale(tmp_path): + fingerprint = _fingerprint_of([_write(tmp_path / "requirements.yml")]) + + assert not fingerprint.is_stale() + assert fingerprint.stale_reasons() == [] + + +def test_modified_file_is_stale(tmp_path): + path = _write(tmp_path / "requirements.yml", "before") + fingerprint = _fingerprint_of([path]) + + _write(path, "after (a different size, so mtime granularity cannot hide the change)") + + assert fingerprint.is_stale() + assert fingerprint.stale_reasons() == [f"modified: {path}"] + + +def test_removed_file_is_stale(tmp_path): + path = _write(tmp_path / "requirements.yml") + fingerprint = _fingerprint_of([path]) + + os.remove(path) + + assert fingerprint.stale_reasons() == [f"removed: {path}"] + + +def test_file_absent_at_capture_that_appears_is_stale(tmp_path): + """The reported failure mode: a build generates annotations.yml after the server started.""" + path = str(tmp_path / "annotations.yml") + fingerprint = _fingerprint_of([path], kind="annotations") + + assert not fingerprint.is_stale() + + _write(path, "annotations: {}") + + assert fingerprint.stale_reasons() == [f"added: {path}"] + + +def test_stale_reasons_honours_limit(tmp_path): + paths = [_write(tmp_path / f"f{i}.yml") for i in range(4)] + fingerprint = _fingerprint_of(paths) + for path in paths: + os.remove(path) + + assert len(fingerprint.stale_reasons(limit=2)) == 2 + assert len(fingerprint.stale_reasons()) == 4 + + +# --------------------------------------------------------------------------- +# GlobSpec — test results are resolved from patterns, not fixed paths +# --------------------------------------------------------------------------- + + +def _test_results_fingerprint(root, pattern: str = "**/*.xml") -> SnapshotFingerprint: + matched = [str(p) for p in root.rglob(pattern)] + return SnapshotFingerprint(globs=(GlobSpec.capture(str(root), pattern, URN, matched),)) + + +def test_new_test_result_file_is_stale(tmp_path): + reports = tmp_path / "target" / "surefire-reports" + reports.mkdir(parents=True) + _write(reports / "TEST-a.xml", "") + + fingerprint = _test_results_fingerprint(tmp_path) + assert not fingerprint.is_stale() + + _write(reports / "TEST-b.xml", "") + + assert fingerprint.stale_reasons() == [ + f"test results changed for pattern '**/*.xml' under {tmp_path} (+1/-0 files)" + ] + + +def test_rewritten_test_result_file_is_stale(tmp_path): + reports = tmp_path / "target" / "surefire-reports" + reports.mkdir(parents=True) + xml = reports / "TEST-a.xml" + _write(xml, "") + + fingerprint = _test_results_fingerprint(tmp_path) + + _write(xml, "") + + assert fingerprint.stale_reasons() == [f"test result modified: {xml}"] + + +def test_pattern_matching_no_files_is_reported_as_a_warning(tmp_path): + fingerprint = _test_results_fingerprint(tmp_path, pattern="target/surefire-reports/*.xml") + + assert not fingerprint.is_stale() + assert fingerprint.warnings() == [ + f"[{URN}] test_results pattern 'target/surefire-reports/*.xml' matched no files under {tmp_path}" + ] + + +# --------------------------------------------------------------------------- +# SnapshotFingerprint +# --------------------------------------------------------------------------- + + +def test_missing_annotations_warns_only_for_the_primary_urn(tmp_path): + primary = FileStamp.capture(str(tmp_path / "annotations.yml"), "annotations", "ms-101") + imported = FileStamp.capture(str(tmp_path / "sys" / "annotations.yml"), "annotations", "sys-101") + fingerprint = SnapshotFingerprint(stamps=(primary, imported)) + + warnings = fingerprint.warnings(primary_urn="ms-101") + + assert len(warnings) == 1 + assert "[ms-101]" in warnings[0] + + +def test_missing_annotations_of_imported_sources_is_not_a_warning(tmp_path): + imported = FileStamp.capture(str(tmp_path / "sys" / "annotations.yml"), "annotations", "sys-101") + + assert SnapshotFingerprint(stamps=(imported,)).warnings(primary_urn="ms-101") == [] + + +def test_restamped_fingerprint_tracks_the_same_inputs_as_they_are_now(tmp_path): + path = _write(tmp_path / "requirements.yml", "before") + fingerprint = _fingerprint_of([path]) + _write(path, "after — a broken edit that fails to parse") + + assert fingerprint.is_stale() + + restamped = fingerprint.restamped() + + assert not restamped.is_stale() + assert [s.path for s in restamped.stamps] == [path] + + _write(path, "the next edit, which must be noticed again") + assert restamped.is_stale() + + +def test_merge_combines_every_source(tmp_path): + a = _fingerprint_of([_write(tmp_path / "a.yml")]) + b = _test_results_fingerprint(tmp_path) + + merged = SnapshotFingerprint.merge([a, b]) + + assert len(merged.stamps) == 1 + assert len(merged.globs) == 1 + assert merged.tracked_file_count == 1 + len(b.globs[0].matched) + + +def test_empty_fingerprint_is_never_stale(): + """Remote sources are version-pinned downloads: nothing local to watch.""" + assert not SnapshotFingerprint().is_stale() + assert SnapshotFingerprint().tracked_file_count == 0 diff --git a/tests/unit/reqstool/mcp/test_server_freshness.py b/tests/unit/reqstool/mcp/test_server_freshness.py new file mode 100644 index 00000000..833d421f --- /dev/null +++ b/tests/unit/reqstool/mcp/test_server_freshness.py @@ -0,0 +1,147 @@ +# Copyright © LFV + +"""The MCP server serves a long-lived snapshot; these tests drive it while the project changes. + +Tools are exercised from inside the fake server's run() because start_server() closes the +session as soon as run() returns — the same window a real client operates in. +""" + +import asyncio +import shutil +from unittest.mock import patch + +import mcp.server.mcpserver +import pytest +from reqstool_python_decorators.decorators.decorators import SVCs + +from reqstool.common.exceptions import SnapshotReloadError +from reqstool.locations.local_location import LocalLocation +from reqstool.mcp import server as mcp_server + +ANNOTATIONS_WITH_EXTRA_IMPL = """\ +--- +requirement_annotations: + implementations: + REQ_101: + - elementKind: "CLASS" + fullyQualifiedName: "com.example.RequirementsExample" + - elementKind: "METHOD" + fullyQualifiedName: "com.example.RequirementsExample.addedAfterStartup" + REQ_201: + - elementKind: "METHOD" + fullyQualifiedName: "com.example.RequirementsExample.someMethod" +""" + + +class _DrivenMCPServer: + """Stand-in for MCPServer that hands the registered tools to a test-supplied scenario.""" + + instances: list["_DrivenMCPServer"] = [] + scenario = None + + def __init__(self, name=None, **kwargs): + self.tools = {} + self.result = None + _DrivenMCPServer.instances.append(self) + + def tool(self): + def decorator(fn): + self.tools[fn.__name__] = fn + return fn + + return decorator + + def run(self, transport, **kwargs): + # Tools are async so they execute on the event loop thread (SQLite affinity). + self.result = asyncio.run(_DrivenMCPServer.scenario(self.tools)) + + +def _serve(project_path, scenario): + """Start the server against project_path, run scenario(tools) against it, return its value.""" + _DrivenMCPServer.scenario = scenario + _DrivenMCPServer.instances.clear() + with patch.object(mcp.server.mcpserver, "MCPServer", _DrivenMCPServer): + mcp_server.start_server(location=LocalLocation(path=str(project_path)), transport="stdio") + return _DrivenMCPServer.instances[-1].result + + +@pytest.fixture +def project_copy(tmp_path, local_testdata_resources_rootdir_w_path): + dst = tmp_path / "ms-101" + shutil.copytree(local_testdata_resources_rootdir_w_path("test_basic/baseline/ms-101"), dst) + return dst + + +@SVCs("SVC_MCP_0006") +def test_tools_serve_the_current_project_not_the_startup_snapshot(project_copy): + """#437: a server spawned before a build kept answering from its spawn-time snapshot.""" + + async def scenario(tools): + before = await tools["list_annotations"]() + + (project_copy / "annotations.yml").write_text(ANNOTATIONS_WITH_EXTRA_IMPL) + + after = await tools["list_annotations"]() + return before, after + + before, after = _serve(project_copy, scenario) + + assert len(after) > len(before) + assert any(a["fqn"].endswith("addedAfterStartup") for a in after) + assert not any(a["fqn"].endswith("addedAfterStartup") for a in before) + + +@SVCs("SVC_MCP_0006") +def test_refresh_tool_reloads_unconditionally(project_copy): + async def scenario(tools): + first = await tools["get_status"]() + refreshed = await tools["refresh"]() + return first["snapshot"], refreshed + + first_snapshot, refreshed = _serve(project_copy, scenario) + + assert refreshed["built_at"] != first_snapshot["built_at"] + assert refreshed["tracked_files"] == first_snapshot["tracked_files"] + + +@SVCs("SVC_MCP_0007") +def test_a_tool_errors_when_the_changed_project_cannot_be_reloaded(project_copy): + """A well-formed answer from a snapshot known to be superseded is the dangerous case.""" + + async def scenario(tools): + (project_copy / "requirements.yml").write_text(": this is not: [ valid yaml") + + with pytest.raises(SnapshotReloadError, match="sources changed but reloading them failed"): + await tools["get_status"]() + with pytest.raises(SnapshotReloadError): + await tools["list_requirements"]() + return True + + assert _serve(project_copy, scenario) is True + + +@SVCs("SVC_MCP_0008") +def test_get_status_reports_when_its_data_was_parsed(project_copy): + async def scenario(tools): + return await tools["get_status"]() + + status = _serve(project_copy, scenario) + + assert status["snapshot"]["built_at"] is not None + assert status["snapshot"]["tracked_files"] > 0 + assert status["snapshot"]["warnings"] == [] + + +@SVCs("SVC_MCP_0008") +def test_get_status_warns_when_no_test_results_were_found(project_copy): + """Zero tests because nothing was built must be distinguishable from zero tests.""" + shutil.rmtree(project_copy / "test_results") + + async def scenario(tools): + return await tools["get_status"]() + + status = _serve(project_copy, scenario) + + assert status["totals"]["automated_tests"]["passed"] == 0 + assert len(status["snapshot"]["warnings"]) == 1 + assert "matched no files" in status["snapshot"]["warnings"][0]