From 98d9cb3f967a193ecadb943596988905ed8686c5 Mon Sep 17 00:00:00 2001 From: Josh Riesenbach <15370822+jedijashwa@users.noreply.github.com> Date: Tue, 4 Aug 2026 08:06:12 +0200 Subject: [PATCH 1/3] feat: Add clear orphans command --- src/semble/cache.py | 12 ++++-- src/semble/cli.py | 91 ++++++++++++++++++++++++++++++++------------- tests/test_cli.py | 41 +++++++++++++++++++- 3 files changed, 113 insertions(+), 31 deletions(-) diff --git a/src/semble/cache.py b/src/semble/cache.py index d53ca666..bdf09b53 100644 --- a/src/semble/cache.py +++ b/src/semble/cache.py @@ -24,15 +24,19 @@ from semble.index import SembleIndex -def find_index_from_cache_folder(path: str) -> Path: - """Finds an index from a cache folder and a project path.""" +def cache_key(path: str) -> str: + """Compute the sha256 cache key for a local path or git URL.""" if is_git_url(path): data = path.encode("utf-8") else: normalized = Path(path).expanduser().resolve() data = str(normalized).encode("utf-8") - subdir_path = hashlib.new("sha256", data).hexdigest() - cache_dir = resolve_cache_folder() / subdir_path + return hashlib.new("sha256", data).hexdigest() + + +def find_index_from_cache_folder(path: str) -> Path: + """Finds an index from a cache folder and a project path.""" + cache_dir = resolve_cache_folder() / cache_key(path) return cache_dir / "index" diff --git a/src/semble/cli.py b/src/semble/cli.py index c89c225d..eb15dd09 100644 --- a/src/semble/cli.py +++ b/src/semble/cli.py @@ -5,12 +5,13 @@ import sys import warnings from importlib.util import find_spec +from pathlib import Path from shutil import rmtree from typing import Literal from model2vec.utils import get_package_extras -from semble.cache import find_index_from_cache_folder, resolve_cache_folder +from semble.cache import cache_key, find_index_from_cache_folder, resolve_cache_folder from semble.index import SembleIndex from semble.index.types import PersistencePath from semble.installer.agents import AGENTS, IntegrationType @@ -22,7 +23,7 @@ _CLI_DISPATCH_ARGS = frozenset( {"search", "find-related", "install", "uninstall", "savings", "-h", "--help", "clear", "--version", "-V"} ) -_CLEAR_CHOICE = Literal["all", "index", "savings"] +_CLEAR_CHOICE = Literal["all", "index", "savings", "orphans"] _SHA_256_REGEX = re.compile(r"^[a-f0-9]{64}$") @@ -138,33 +139,69 @@ def _run_find_related( _maybe_save_index(index, path) +def _clear_indexes(cache_folder: Path) -> None: + """Remove all valid index entries from the cache folder.""" + indexes = [] + for path in cache_folder.glob("*/index"): + if not _SHA_256_REGEX.match(path.parent.name): + continue + if PersistencePath.from_path(path).non_existing(): + continue + indexes.append(path) + + if not indexes: + print(f"No indexes found to clear in `{cache_folder}`") + else: + for path in indexes: + index_folder = path.parent + rmtree(index_folder) + print(f"Cleared index at `{index_folder}`") + + +def _clear_savings(cache_folder: Path) -> None: + """Remove the savings file from the cache folder.""" + path = cache_folder / "savings.jsonl" + if not path.exists(): + print(f"No savings file found at `{path}`") + else: + path.unlink() + print(f"Cleared savings at `{path}`") + + +def _clear_orphans(cache_folder: Path) -> None: + """Remove index entries whose local root_path no longer exists.""" + orphans = [] + for path in cache_folder.glob("*/index"): + if not _SHA_256_REGEX.match(path.parent.name): + continue + try: + with open(path / "metadata.json", encoding="utf-8") as f: + root_path = json.load(f).get("root_path") + except (OSError, json.JSONDecodeError): + continue + # Git-URL entries store their temp clone dir as root_path, so only trust entries whose key matches. + if not root_path or cache_key(root_path) != path.parent.name: + continue + if not Path(root_path).exists(): + orphans.append((path.parent, root_path)) + + if not orphans: + print("No orphaned indexes found") + else: + for index_folder, root_path in orphans: + rmtree(index_folder) + print(f"Cleared orphaned index for `{root_path}`") + + def _run_clear(clear_type: _CLEAR_CHOICE) -> None: """Run the `clear` subcommand.""" cache_folder = resolve_cache_folder() if clear_type == "index" or clear_type == "all": - indexes = [] - for path in cache_folder.glob("*/index"): - if not _SHA_256_REGEX.match(path.parent.name): - continue - if PersistencePath.from_path(path).non_existing(): - continue - indexes.append(path) - - if not indexes: - print(f"No indexes found to clear in `{cache_folder}`") - else: - for path in indexes: - index_folder = path.parent - rmtree(index_folder) - print(f"Cleared index at `{index_folder}`") - + _clear_indexes(cache_folder) if clear_type == "savings" or clear_type == "all": - path = cache_folder / "savings.jsonl" - if not path.exists(): - print(f"No savings file found at `{path}`") - else: - path.unlink() - print(f"Cleared savings at `{path}`") + _clear_savings(cache_folder) + if clear_type == "orphans": + _clear_orphans(cache_folder) def _cli_main() -> None: @@ -186,7 +223,11 @@ def _cli_main() -> None: _add_content_args(search_p) clear_p = sub.add_parser("clear", help="Clear the index cache.") - clear_p.add_argument("type", choices=["all", "index", "savings"], help="Type of cache to clear.") + clear_p.add_argument( + "type", + choices=["all", "index", "savings", "orphans"], + help="Type of cache to clear. `orphans` removes indexes whose source path no longer exists.", + ) related_p = sub.add_parser("find-related", help="Find code similar to a specific location.") related_p.add_argument("file_path", help="File path as shown in search results.") diff --git a/tests/test_cli.py b/tests/test_cli.py index 539d71c1..eb0f4887 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,3 +1,5 @@ +import hashlib +import json import sys import warnings from importlib.resources import files @@ -240,7 +242,7 @@ def test_agent_file_tools_are_bash_only() -> None: assert not any("mcp__" in t for t in tools) -def _make_valid_index_dir(cache_folder: Path, sha: str = "a" * 64) -> Path: +def _make_valid_index_dir(cache_folder: Path, sha: str = "a" * 64, metadata: str = "{}") -> Path: """Create a fake valid index directory with the expected structure.""" index_dir = cache_folder / sha / "index" index_dir.mkdir(parents=True) @@ -248,7 +250,7 @@ def _make_valid_index_dir(cache_folder: Path, sha: str = "a" * 64) -> Path: (index_dir / "chunks.json").write_text("[]") (index_dir / "bm25_index").write_text("") (index_dir / "semantic_index").write_text("") - (index_dir / "metadata.json").write_text("{}") + (index_dir / "metadata.json").write_text(metadata) return index_dir @@ -291,6 +293,40 @@ def test_run_clear_index( assert not (tmp_path / ("b" * 64)).exists() +@pytest.mark.parametrize( + "scenario", + ["orphan", "live", "mismatched_key", "no_root_path"], +) +def test_run_clear_orphans(scenario: str, tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + """_run_clear('orphans') removes entries whose local root is gone, and keeps everything else.""" + cache_folder = tmp_path / "cache" + cache_folder.mkdir() + root = tmp_path / "repo" + root.mkdir() + sha = hashlib.sha256(str(root.resolve()).encode("utf-8")).hexdigest() + if scenario == "orphan": + _make_valid_index_dir(cache_folder, sha, metadata=json.dumps({"root_path": str(root)})) + root.rmdir() + elif scenario == "live": + _make_valid_index_dir(cache_folder, sha, metadata=json.dumps({"root_path": str(root)})) + elif scenario == "mismatched_key": + # A git-URL entry: the dir name hashes the URL, not the (missing) root_path + _make_valid_index_dir(cache_folder, "a" * 64, metadata=json.dumps({"root_path": str(root / "clone")})) + elif scenario == "no_root_path": + _make_valid_index_dir(cache_folder, "b" * 64) + + with patch("semble.cli.resolve_cache_folder", return_value=cache_folder): + _run_clear("orphans") + + out = capsys.readouterr().out + if scenario == "orphan": + assert str(root) in out + assert not (cache_folder / sha).exists() + else: + assert "No orphaned indexes found" in out + assert len(list(cache_folder.iterdir())) == 1 + + @pytest.mark.parametrize( ("create_file", "expected"), [ @@ -348,6 +384,7 @@ def test_run_clear_all( ("index", True, False, ["Cleared index", "e" * 64]), ("savings", False, True, ["Cleared savings"]), ("all", True, True, ["Cleared index", "Cleared savings"]), + ("orphans", False, False, ["No orphaned indexes found"]), ], ) def test_cli_clear_command( From 50d50f21d3b75556adb271a461a7940907afeac5 Mon Sep 17 00:00:00 2001 From: Josh Riesenbach <15370822+jedijashwa@users.noreply.github.com> Date: Tue, 4 Aug 2026 10:48:26 +0200 Subject: [PATCH 2/3] fix: Validate root_path type when clearing orphans --- src/semble/cli.py | 2 +- tests/test_cli.py | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/src/semble/cli.py b/src/semble/cli.py index eb15dd09..8c786fb8 100644 --- a/src/semble/cli.py +++ b/src/semble/cli.py @@ -180,7 +180,7 @@ def _clear_orphans(cache_folder: Path) -> None: except (OSError, json.JSONDecodeError): continue # Git-URL entries store their temp clone dir as root_path, so only trust entries whose key matches. - if not root_path or cache_key(root_path) != path.parent.name: + if not isinstance(root_path, str) or not root_path or cache_key(root_path) != path.parent.name: continue if not Path(root_path).exists(): orphans.append((path.parent, root_path)) diff --git a/tests/test_cli.py b/tests/test_cli.py index eb0f4887..863d5b38 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -327,6 +327,26 @@ def test_run_clear_orphans(scenario: str, tmp_path: Path, capsys: pytest.Capture assert len(list(cache_folder.iterdir())) == 1 +def test_run_clear_orphans_skips_invalid_metadata(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + """A non-string root_path is skipped without aborting the rest of the cleanup.""" + cache_folder = tmp_path / "cache" + cache_folder.mkdir() + root = tmp_path / "repo" + root.mkdir() + sha = hashlib.sha256(str(root.resolve()).encode("utf-8")).hexdigest() + _make_valid_index_dir(cache_folder, sha, metadata=json.dumps({"root_path": str(root)})) + root.rmdir() + _make_valid_index_dir(cache_folder, "f" * 64, metadata=json.dumps({"root_path": 123})) + + with patch("semble.cli.resolve_cache_folder", return_value=cache_folder): + _run_clear("orphans") + + out = capsys.readouterr().out + assert str(root) in out + assert not (cache_folder / sha).exists() + assert (cache_folder / ("f" * 64)).exists() + + @pytest.mark.parametrize( ("create_file", "expected"), [ From 24c5582d8acb710327c4826f9e11ac80f52d4e55 Mon Sep 17 00:00:00 2001 From: Josh Riesenbach <15370822+jedijashwa@users.noreply.github.com> Date: Tue, 4 Aug 2026 16:33:22 +0200 Subject: [PATCH 3/3] fix: Handle non-object metadata when clearing orphans --- src/semble/cli.py | 3 ++- tests/test_cli.py | 8 ++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/semble/cli.py b/src/semble/cli.py index 8c786fb8..dad4770d 100644 --- a/src/semble/cli.py +++ b/src/semble/cli.py @@ -176,7 +176,8 @@ def _clear_orphans(cache_folder: Path) -> None: continue try: with open(path / "metadata.json", encoding="utf-8") as f: - root_path = json.load(f).get("root_path") + metadata = json.load(f) + root_path = metadata.get("root_path") if isinstance(metadata, dict) else None except (OSError, json.JSONDecodeError): continue # Git-URL entries store their temp clone dir as root_path, so only trust entries whose key matches. diff --git a/tests/test_cli.py b/tests/test_cli.py index 863d5b38..e4245f60 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -328,7 +328,7 @@ def test_run_clear_orphans(scenario: str, tmp_path: Path, capsys: pytest.Capture def test_run_clear_orphans_skips_invalid_metadata(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: - """A non-string root_path is skipped without aborting the rest of the cleanup.""" + """Malformed cache entries are skipped without aborting the rest of the cleanup.""" cache_folder = tmp_path / "cache" cache_folder.mkdir() root = tmp_path / "repo" @@ -337,6 +337,9 @@ def test_run_clear_orphans_skips_invalid_metadata(tmp_path: Path, capsys: pytest _make_valid_index_dir(cache_folder, sha, metadata=json.dumps({"root_path": str(root)})) root.rmdir() _make_valid_index_dir(cache_folder, "f" * 64, metadata=json.dumps({"root_path": 123})) + _make_valid_index_dir(cache_folder, "0" * 64, metadata="not json") + _make_valid_index_dir(cache_folder, "1" * 64, metadata="[]") + (cache_folder / "not-a-sha" / "index").mkdir(parents=True) with patch("semble.cli.resolve_cache_folder", return_value=cache_folder): _run_clear("orphans") @@ -344,7 +347,8 @@ def test_run_clear_orphans_skips_invalid_metadata(tmp_path: Path, capsys: pytest out = capsys.readouterr().out assert str(root) in out assert not (cache_folder / sha).exists() - assert (cache_folder / ("f" * 64)).exists() + for kept in ("f" * 64, "0" * 64, "1" * 64, "not-a-sha"): + assert (cache_folder / kept).exists() @pytest.mark.parametrize(