feat: Switch to semble-grammars

pyproject.toml

@@ -32,9 +32,9 @@ dependencies = [
     "bm25s>=0.2.0",
     "pathspec>=0.12",
     "tree-sitter>=0.25,<0.26",
-    "tree-sitter-language-pack>=1.0,<1.8.0,!=1.6.3",
     "orjson",
     "questionary>=2.0,<3.0",
+    "semble-grammars>=0.1.2",
 ]
 
 [project.optional-dependencies]
@@ -146,3 +146,6 @@ exclude_also = ["if __name__ == .__main__.:", "@abstractmethod"]
 
 [tool.uv]
 exclude-newer = "1 week"
+
+[tool.uv.exclude-newer-package]
+semble-grammars = false

src/semble/chunking/chunking.py

@@ -1,6 +1,6 @@
 import logging
 
-from semble.chunking.core import chunk, chunk_lines, is_supported_language
+from semble.chunking.core import chunk, chunk_lines
 from semble.types import Chunk
 
 logger = logging.getLogger(__name__)
@@ -15,7 +15,7 @@ def chunk_source(source: str, file_path: str, language: str | None) -> list[Chun
     if not source.strip():
         return []
     chunk_boundaries = None
-    if language is not None and is_supported_language(language):
+    if language is not None:
         chunk_boundaries = chunk(source, language, _DESIRED_CHUNK_LENGTH_CHARS)
     # This is an if because the error state of the parser above
     # is a None.

src/semble/chunking/core.py

@@ -4,22 +4,15 @@
 from functools import cache
 from logging import getLogger
 
+from semble_grammars import LanguageNotFoundError, UnsupportedPlatformError, get_parser
 from tree_sitter import Node, Parser
-from tree_sitter_language_pack import DownloadError, LanguageNotFoundError, SupportedLanguage, get_parser
-
-from semble.index.files import ALL_LANGUAGES
 
 logger = getLogger(__name__)
 
 _RECURSION_DEPTH = 500
 _MIN_CHUNK_SIZE = 50
 
 
-def is_supported_language(language: str) -> bool:
-    """Check if the language is supported by tree-sitter."""
-    return language in ALL_LANGUAGES
-
-
 @dataclass
 class ChunkBoundary:
     """The output of the internal chunking algorithm."""
@@ -29,14 +22,14 @@ class ChunkBoundary:
 
 
 @cache
-def _cached_get_parser(language: SupportedLanguage) -> Parser | None:
+def _cached_get_parser(language: str) -> Parser | None:
     """Gets a parser from tree_sitter."""
     try:
         return get_parser(language)
     except LanguageNotFoundError:
         logger.warning("Language %s not found, falling back to line chunking", language)
-    except DownloadError:
-        logger.warning("Failed to download language %s, falling back to line chunking", language)
+    except UnsupportedPlatformError:
+        logger.warning("No bundled grammars for this platform, falling back to line chunking")
     except Exception:
         logger.error("Uncaught exception in _cached_get_parser", exc_info=True)
     return None

src/semble/installer/config.py

@@ -1,11 +1,12 @@
 from __future__ import annotations
 
 import json
+from functools import cache
 from pathlib import Path
-from typing import Literal, TypeVar, cast
+from typing import Literal, TypeVar
 
+from semble_grammars import get_parser
 from tree_sitter import Node, Parser
-from tree_sitter_language_pack import SupportedLanguage, download, get_parser
 
 from semble.installer.agents import SEMBLE_END, SEMBLE_START, Action
 
@@ -15,23 +16,14 @@
 _CODEX_MCP_HEADER = "[mcp_servers.semble]"
 _CODEX_MCP_BLOCK = '[mcp_servers.semble]\ncommand = "uvx"\nargs = ["--from", "semble[mcp]", "semble"]\n'
 
-_json5_parser_cache: Parser | None | bool = False  # False = not yet attempted
-
 
+@cache
 def _json5_parser() -> Parser | None:
-    """Return a tree-sitter JSON5 parser, downloading the grammar if needed.
-
-    "json5" ships in tree-sitter-language-pack but isn't in its typed language list, hence the cast.
-    """
-    global _json5_parser_cache
-    if _json5_parser_cache is not False:
-        return _json5_parser_cache  # type: ignore[return-value]
+    """Return a tree-sitter JSON5 parser, or None if unavailable."""
     try:
-        download(["json5"])
-        _json5_parser_cache = get_parser(cast(SupportedLanguage, "json5"))
+        return get_parser("json5")
     except Exception:
-        _json5_parser_cache = None
-    return _json5_parser_cache  # type: ignore[return-value]
+        return None
 
 
 def _json5_object(text: str) -> JsonObjectResult:

src/semble/version.py

@@ -1,2 +1,2 @@
-__version_triple__ = (0, 5, 1)
+__version_triple__ = (0, 5, 2)
 __version__ = ".".join(map(str, __version_triple__))

tests/test_chunker.py

@@ -2,7 +2,7 @@
 from unittest.mock import patch
 
 import pytest
-from tree_sitter_language_pack import DownloadError
+from semble_grammars import UnsupportedPlatformError
 
 from semble.chunking.chunking import Chunk, chunk_lines, chunk_source
 from semble.chunking.core import ChunkBoundary, _cached_get_parser, chunk
@@ -102,12 +102,11 @@ def test_get_parser(caplog: pytest.LogCaptureFixture) -> None:
         _cached_get_parser("hello")
         assert len(caplog.records) == 0
 
-    with patch("semble.chunking.core.get_parser", side_effect=DownloadError):
+    with patch("semble.chunking.core.get_parser", side_effect=UnsupportedPlatformError):
         with caplog.at_level(logging.WARNING, logger="semble.chunking.core"):
             _cached_get_parser("Python")
             assert len(caplog.records) == 1
-            assert "Failed to download" in caplog.records[0].message
-            assert "Python" in caplog.records[0].message
+            assert "No bundled grammars for this platform" in caplog.records[0].message
 
             caplog.clear()
             _cached_get_parser("Python")
@@ -131,9 +130,9 @@ def test_chunks_is_none() -> None:
         assert chunks is None
 
 
-def test_download_error() -> None:
-    """Test that chunk returns None when parser is not available."""
-    with patch("semble.chunking.core.get_parser", side_effect=DownloadError):
+def test_unsupported_platform_error() -> None:
+    """Test that chunk returns None when the current platform has no bundled grammars."""
+    with patch("semble.chunking.core.get_parser", side_effect=UnsupportedPlatformError):
         chunks = chunk("x = 1", "python", 10)
         assert chunks is None
 

tests/test_installer.py

@@ -17,6 +17,7 @@
 )
 from semble.installer.config import (
     _CODEX_MCP_HEADER,
+    _json5_parser,
     merge_json_member,
     merge_toml_block,
     remove_json_member,
@@ -167,12 +168,13 @@ def test_merge_mcp_writes_under_agent_key(tmp_path, agent_id, key):
 
 
 def test_mcp_skipped_when_grammar_unavailable(claude_agent, monkeypatch):
-    """When the JSON5 grammar cannot be downloaded, merge/remove return 'skipped'."""
+    """When the JSON5 grammar is unavailable, merge/remove return 'skipped'."""
     claude_agent.mcp.path.write_text('{ "mcpServers": {} }')
-    monkeypatch.setattr("semble.installer.config.download", lambda _: 1 / 0)
-    monkeypatch.setattr("semble.installer.config._json5_parser_cache", False)
+    monkeypatch.setattr("semble.installer.config.get_parser", lambda _: 1 / 0)
+    _json5_parser.cache_clear()
     assert merge_mcp(claude_agent).action == "skipped"
     assert remove_mcp(claude_agent).action == "skipped"
+    _json5_parser.cache_clear()
 
 
 def test_merge_mcp_reparse_guard(claude_agent, monkeypatch):