diff --git a/src/validation/golden_corpus.py b/src/validation/golden_corpus.py index 998c442..34f2295 100644 --- a/src/validation/golden_corpus.py +++ b/src/validation/golden_corpus.py @@ -51,6 +51,21 @@ def corpus_records() -> dict[str, list[dict[str, object]]]: } +def canonical_content_hash(content: str | bytes) -> str: + """Compute the SHA-256 hash of a UTF-8 string with normalized LF line endings. + + Newline Rule: All occurrences of CRLF (\\r\\n) are replaced with LF (\\n). + Encoding: UTF-8. + Invalid UTF-8: Input bytes that are not valid UTF-8 will raise a UnicodeDecodeError. + """ + if isinstance(content, bytes): + text = content.decode("utf-8") + else: + text = content + normalized = text.replace("\r\n", "\n") + return hashlib.sha256(normalized.encode("utf-8")).hexdigest() + + def write_golden_corpus(root: Path = GOLDEN_ROOT) -> dict[str, str]: root.mkdir(parents=True, exist_ok=True) hashes: dict[str, str] = {} @@ -58,10 +73,12 @@ def write_golden_corpus(root: Path = GOLDEN_ROOT) -> dict[str, str]: path = root / filename lines = [json.dumps(record, sort_keys=True, separators=(",", ":"), ensure_ascii=False) for record in records] content = "\n".join(lines) + "\n" - if path.exists() and path.read_text(encoding="utf-8") != content: - raise RuntimeError(f"golden corpus mutation detected: {path}") - path.write_text(content, encoding="utf-8") - hashes[filename] = hashlib.sha256(content.encode()).hexdigest() + if path.exists(): + existing = path.read_text(encoding="utf-8").replace("\r\n", "\n") + if existing != content: + raise RuntimeError(f"golden corpus mutation detected: {path}") + path.write_text(content, encoding="utf-8", newline="\n") + hashes[filename] = canonical_content_hash(content) return hashes diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..41b859d --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1 @@ +"""Test package for CompTextv7.""" diff --git a/tests/test_compression_signals_ts.py b/tests/test_compression_signals_ts.py index 7c9a1b3..70c8a5f 100644 --- a/tests/test_compression_signals_ts.py +++ b/tests/test_compression_signals_ts.py @@ -3,9 +3,11 @@ import textwrap from pathlib import Path +from tests.utils import resolve_tsc_executable + REPO_ROOT = Path(__file__).resolve().parents[1] DASHBOARD_APP = REPO_ROOT / "dashboard" / "app" -TSC = DASHBOARD_APP / "node_modules" / ".bin" / "tsc" +TSC = resolve_tsc_executable() def run_compression_script(tmp_path: Path, script: str): diff --git a/tests/test_core_foundation_ts.py b/tests/test_core_foundation_ts.py index dc53c9f..6b605c6 100644 --- a/tests/test_core_foundation_ts.py +++ b/tests/test_core_foundation_ts.py @@ -3,9 +3,11 @@ import textwrap from pathlib import Path +from tests.utils import resolve_tsc_executable + REPO_ROOT = Path(__file__).resolve().parents[1] DASHBOARD_APP = REPO_ROOT / "dashboard" / "app" -TSC = DASHBOARD_APP / "node_modules" / ".bin" / "tsc" +TSC = resolve_tsc_executable() def run_foundation_script(tmp_path: Path, script: str): diff --git a/tests/test_golden_corpus_canonical.py b/tests/test_golden_corpus_canonical.py new file mode 100644 index 0000000..1d02809 --- /dev/null +++ b/tests/test_golden_corpus_canonical.py @@ -0,0 +1,40 @@ +from __future__ import annotations + +import pytest +from src.validation.golden_corpus import canonical_content_hash + +def test_canonical_content_hash_line_endings() -> None: + # 1. LF line endings + content_lf = "line1\nline2\nline3\n" + hash_lf = canonical_content_hash(content_lf) + + # 2. CRLF line endings + content_crlf = "line1\r\nline2\r\nline3\r\n" + hash_crlf = canonical_content_hash(content_crlf) + + # 3. Mixed line endings + content_mixed = "line1\r\nline2\nline3\r\n" + hash_mixed = canonical_content_hash(content_mixed) + + # All three must yield the identical hash + assert hash_lf == hash_crlf + assert hash_lf == hash_mixed + +def test_canonical_content_hash_detects_mutations() -> None: + # Ensure byte changes other than line endings are detected + content_orig = "line1\nline2\nline3\n" + content_mutated = "line1\nline2_altered\nline3\n" + + assert canonical_content_hash(content_orig) != canonical_content_hash(content_mutated) + +def test_canonical_content_hash_bytes_support() -> None: + content_str = "line1\nline2\r\n" + content_bytes = b"line1\nline2\r\n" + + assert canonical_content_hash(content_str) == canonical_content_hash(content_bytes) + +def test_canonical_content_hash_invalid_utf8_raises_error() -> None: + # Invalid UTF-8 bytes (e.g. 0xff) must raise a UnicodeDecodeError + invalid_bytes = b"line1\n\xff\n" + with pytest.raises(UnicodeDecodeError): + canonical_content_hash(invalid_bytes) diff --git a/tests/test_reference_index_event_fingerprints_ts.py b/tests/test_reference_index_event_fingerprints_ts.py index 5578b37..d9b8a30 100644 --- a/tests/test_reference_index_event_fingerprints_ts.py +++ b/tests/test_reference_index_event_fingerprints_ts.py @@ -3,9 +3,11 @@ import textwrap from pathlib import Path +from tests.utils import resolve_tsc_executable + REPO_ROOT = Path(__file__).resolve().parents[1] DASHBOARD_APP = REPO_ROOT / "dashboard" / "app" -TSC = DASHBOARD_APP / "node_modules" / ".bin" / "tsc" +TSC = resolve_tsc_executable() def run_foundation_script(tmp_path: Path, script: str): out_dir = tmp_path / "compiled" diff --git a/tests/test_replay_artifact_writer_ts.py b/tests/test_replay_artifact_writer_ts.py index 8256201..c4ae777 100644 --- a/tests/test_replay_artifact_writer_ts.py +++ b/tests/test_replay_artifact_writer_ts.py @@ -3,9 +3,11 @@ import textwrap from pathlib import Path +from tests.utils import resolve_tsc_executable + REPO_ROOT = Path(__file__).resolve().parents[1] DASHBOARD_APP = REPO_ROOT / "dashboard" / "app" -TSC = DASHBOARD_APP / "node_modules" / ".bin" / "tsc" +TSC = resolve_tsc_executable() def run_foundation_script(tmp_path: Path, script: str): out_dir = tmp_path / "compiled" diff --git a/tests/test_shared_stable_hashing_ts.py b/tests/test_shared_stable_hashing_ts.py index 66adc6c..0eca726 100644 --- a/tests/test_shared_stable_hashing_ts.py +++ b/tests/test_shared_stable_hashing_ts.py @@ -3,9 +3,11 @@ import textwrap from pathlib import Path +from tests.utils import resolve_tsc_executable + REPO_ROOT = Path(__file__).resolve().parents[1] DASHBOARD_APP = REPO_ROOT / "dashboard" / "app" -TSC = DASHBOARD_APP / "node_modules" / ".bin" / "tsc" +TSC = resolve_tsc_executable() def run_foundation_script(tmp_path: Path, script: str): diff --git a/tests/test_validation_hardening.py b/tests/test_validation_hardening.py index 9f0cabc..bfe8726 100644 --- a/tests/test_validation_hardening.py +++ b/tests/test_validation_hardening.py @@ -9,7 +9,9 @@ from src.validation.token_telemetry import SUPPORTED_ENCODINGS, count_tokens, drift_fingerprint -def test_golden_corpus_hashes_are_stable() -> None: +from src.validation.golden_corpus import canonical_content_hash + +def test_golden_corpus_canonical_content_hashes_are_stable() -> None: hashes = write_golden_corpus() assert set(hashes) == { @@ -19,8 +21,8 @@ def test_golden_corpus_hashes_are_stable() -> None: "mixed_incident_reference.jsonl", } for filename, digest in hashes.items(): - content = Path("datasets/golden", filename).read_bytes() - assert hashlib.sha256(content).hexdigest() == digest + content_bytes = Path("datasets/golden", filename).read_bytes() + assert canonical_content_hash(content_bytes) == digest def test_token_telemetry_supports_required_encodings_deterministically() -> None: diff --git a/tests/utils/__init__.py b/tests/utils/__init__.py index e69de29..a243166 100644 --- a/tests/utils/__init__.py +++ b/tests/utils/__init__.py @@ -0,0 +1,27 @@ +import os +import shutil +from pathlib import Path + +def resolve_tsc_executable() -> Path: + """Resolve the path to the TypeScript compiler (tsc). + + Prefers the local Node dependency in the dashboard application, + falling back to system PATH. Raises FileNotFoundError if missing. + """ + repo_root = Path(__file__).resolve().parents[2] + dashboard_app = repo_root / "dashboard" / "app" + + tsc_bin = "tsc.cmd" if os.name == "nt" else "tsc" + local_tsc = dashboard_app / "node_modules" / ".bin" / tsc_bin + + if local_tsc.exists(): + return local_tsc + + system_tsc = shutil.which(tsc_bin) + if system_tsc: + return Path(system_tsc) + + raise FileNotFoundError( + "TypeScript compiler (tsc) not found. Please run 'npm install' " + "in dashboard/app or install TypeScript globally." + )