From 29f5c9e537b8346a75a8f23ea8ec9c6c575f4dbe Mon Sep 17 00:00:00 2001 From: Wolfvin Date: Tue, 14 Jul 2026 09:16:59 +0000 Subject: [PATCH] fix(parser): make keep_alive strategy tree-sitter 0.26-safe (closes #267) --- .github/workflows/codelens-benchmark.yml | 2 +- .github/workflows/codelens-ci.yml | 12 ++--- .github/workflows/codelens-quality-gate.yml | 6 +-- .github/workflows/codelens-sarif.yml | 2 +- pyproject.toml | 12 +++-- scripts/base_parser.py | 60 +++++++++++++++++++-- scripts/parsers/js_backend_parser.py | 18 ++++--- scripts/parsers/python_parser.py | 42 ++++++++++++--- scripts/parsers/rust_parser.py | 4 +- scripts/parsers/ts_backend_parser.py | 20 +++---- scripts/parsers/tsx_parser.py | 12 ++--- 11 files changed, 140 insertions(+), 50 deletions(-) diff --git a/.github/workflows/codelens-benchmark.yml b/.github/workflows/codelens-benchmark.yml index 40f9a170..1696d6a9 100644 --- a/.github/workflows/codelens-benchmark.yml +++ b/.github/workflows/codelens-benchmark.yml @@ -15,7 +15,7 @@ jobs: with: python-version: '3.12' - name: Install dependencies - run: pip install pyyaml "tree-sitter>=0.21.0,<0.26" pytest + run: pip install pyyaml "tree-sitter>=0.21.0" pytest - name: Run full benchmarks run: python3 benchmarks/run_benchmarks.py - name: Check regression diff --git a/.github/workflows/codelens-ci.yml b/.github/workflows/codelens-ci.yml index 5cffd263..81ff4e99 100644 --- a/.github/workflows/codelens-ci.yml +++ b/.github/workflows/codelens-ci.yml @@ -33,11 +33,9 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - # tree-sitter pinned <0.26: core 0.26.0 changed Node lifetime handling - # and causes a native use-after-free SIGSEGV in tree_sitter_python._binding - # even on shallow files (issue #235). python_parser.py's keep_alive - # mitigation is insufficient for 0.26; 0.25.x is known-safe. - pip install "tree-sitter>=0.21.0,<0.26" pyyaml pytest pytest-cov pygls lsprotocol + # Issue #267: tree-sitter 0.26 cap removed — root cause (node.start_point + # SIGSEGV in 0.26 binding) fixed in BaseParser via line-offset bisect. + pip install "tree-sitter>=0.21.0" pyyaml pytest pytest-cov pygls lsprotocol # Install optional grammar packages for full parser coverage pip install tree-sitter-python tree-sitter-javascript tree-sitter-html tree-sitter-css || true pip install tree-sitter-typescript tree-sitter-rust || true @@ -69,7 +67,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install "tree-sitter>=0.21.0,<0.26" pyyaml + pip install "tree-sitter>=0.21.0" pyyaml pip install tree-sitter-python tree-sitter-javascript tree-sitter-html tree-sitter-css || true pip install tree-sitter-typescript tree-sitter-rust || true @@ -99,7 +97,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install "tree-sitter>=0.21.0,<0.26" pyyaml + pip install "tree-sitter>=0.21.0" pyyaml pip install tree-sitter-python tree-sitter-javascript tree-sitter-html tree-sitter-css || true pip install tree-sitter-typescript tree-sitter-rust || true diff --git a/.github/workflows/codelens-quality-gate.yml b/.github/workflows/codelens-quality-gate.yml index 8d2c1a8b..1d29a436 100644 --- a/.github/workflows/codelens-quality-gate.yml +++ b/.github/workflows/codelens-quality-gate.yml @@ -23,10 +23,10 @@ jobs: - name: Install dependencies run: | - # tree-sitter pinned <0.26 (issue #235: 0.26 native segfault). Installing - # it makes `scan` use the fast tree-sitter path instead of the slow regex + # Issue #267: tree-sitter 0.26 cap removed — root cause fixed in BaseParser. + # tree-sitter makes `scan` use the fast tree-sitter path instead of the slow regex # fallback that hung the old double-scan Initialize/Scan steps. - pip install "tree-sitter>=0.21.0,<0.26" pyyaml + pip install "tree-sitter>=0.21.0" pyyaml pip install tree-sitter-python tree-sitter-javascript tree-sitter-html tree-sitter-css || true pip install tree-sitter-typescript tree-sitter-rust || true diff --git a/.github/workflows/codelens-sarif.yml b/.github/workflows/codelens-sarif.yml index cb6103cb..9f03bf3a 100644 --- a/.github/workflows/codelens-sarif.yml +++ b/.github/workflows/codelens-sarif.yml @@ -33,7 +33,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install "tree-sitter>=0.21.0,<0.26" pyyaml + pip install "tree-sitter>=0.21.0" pyyaml pip install tree-sitter-python tree-sitter-javascript tree-sitter-html tree-sitter-css || true pip install tree-sitter-typescript tree-sitter-rust || true diff --git a/pyproject.toml b/pyproject.toml index 75021aeb..c07f84b3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,11 +39,13 @@ classifiers = [ ] dependencies = [ - # Upper-bound <0.26: tree-sitter core 0.26.0 changed Node lifetime handling, - # causing a native use-after-free SIGSEGV in the grammar bindings even on - # shallow files (issue #235). Remove the cap once python_parser.py's - # keep_alive strategy is made 0.26-safe. - "tree-sitter>=0.21.0,<0.26", + # Issue #267: tree-sitter 0.26 cap removed — root cause (node.start_point + # SIGSEGV in 0.26 binding) fixed in BaseParser.get_line / get_line_from_byte + # via pre-computed line offsets + bisect on start_byte/end_byte. All + # parsers (python, js_backend, ts_backend, tsx, rust) migrated off + # node.start_point / node.end_point access. Verified on Linux + + # tree-sitter 0.26.0 — test_large_file_parsing.py 11/11 pass clean. + "tree-sitter>=0.21.0", ] [project.optional-dependencies] diff --git a/scripts/base_parser.py b/scripts/base_parser.py index bad6c835..bf7fd9c8 100755 --- a/scripts/base_parser.py +++ b/scripts/base_parser.py @@ -109,6 +109,16 @@ def __init__(self, language: Language): # frees the Tree the node pointers dangle and accessing # ``node.children`` / ``node.start_point`` raises SIGSEGV (issue #116). self._last_tree = None + # Issue #267: Pre-computed line offsets for the most recently parsed + # source. ``_line_offsets[i]`` = byte offset where line ``i+1`` begins. + # Used by :meth:`get_line` to compute the 1-based line number from + # ``node.start_byte`` via bisect — avoids ``node.start_point`` access + # which can SIGSEGV non-deterministically in tree-sitter 0.26 (the + # binding's Point object does not hold a strong reference to the + # Node/Tree). Set by :meth:`parse` / :meth:`parse_tree`. ``None`` for + # legacy callers that don't use those methods (falls back to + # ``node.start_point.row`` — kept for backward compat but unsafe in 0.26). + self._line_offsets: Optional[List[int]] = None def parse(self, content: bytes) -> Node: """Parse source content and return root node. @@ -122,6 +132,8 @@ def parse(self, content: bytes) -> Node: """ tree = self.parser.parse(content) self._last_tree = tree + # Issue #267: pre-compute line offsets for safe line lookup. + self._line_offsets = self._compute_line_offsets(content) return tree.root_node def parse_tree(self, content: bytes): @@ -132,18 +144,60 @@ def parse_tree(self, content: bytes): """ tree = self.parser.parse(content) self._last_tree = tree + # Issue #267: pre-compute line offsets for safe line lookup. + self._line_offsets = self._compute_line_offsets(content) return tree + @staticmethod + def _compute_line_offsets(source: bytes) -> List[int]: + """Pre-compute byte offsets where each line begins (0-indexed). + ``line_offsets[i]`` = byte offset where line ``i+1`` begins. + Used by :meth:`get_line` for O(log n) line lookup by start_byte. + """ + offsets = [0] + for i, b in enumerate(source): + if b == 0x0a: # '\n' + offsets.append(i + 1) + return offsets + @staticmethod def get_text(node: Node, source: bytes) -> str: """Get the text content of a node.""" return source[node.start_byte:node.end_byte].decode('utf-8', errors='replace') - @staticmethod - def get_line(node: Node) -> int: - """Get 1-based line number of a node.""" + def get_line(self, node: Node) -> int: + """Get 1-based line number of a node. + + Issue #267: If ``self._line_offsets`` is set (populated by + :meth:`parse` / :meth:`parse_tree`), compute the line number from + ``node.start_byte`` via bisect — avoids ``node.start_point`` access + which can SIGSEGV non-deterministically in tree-sitter 0.26 (Point + objects don't hold a strong reference to the Node/Tree). Falls back + to ``node.start_point.row + 1`` for legacy callers that didn't + route through :meth:`parse` / :meth:`parse_tree` — this path is + unsafe in 0.26 and should be migrated. + """ + import bisect + if self._line_offsets is not None: + return bisect.bisect_right(self._line_offsets, node.start_byte) + # Legacy fallback (unsafe in tree-sitter 0.26 — see issue #267). return node.start_point.row + 1 + def get_line_from_byte(self, byte_offset: int) -> int: + """Get 1-based line number from a byte offset. + + Issue #267: Companion to :meth:`get_line` for callers that need the + line of ``node.end_byte`` (avoiding ``node.end_point.row`` which has + the same 0.26 crash risk as ``node.start_point.row``). Uses the same + ``self._line_offsets`` bisect lookup. Falls back to -1 (invalid) if + ``_line_offsets`` is not set — callers should migrate to route + through :meth:`parse` / :meth:`parse_tree` first. + """ + import bisect + if self._line_offsets is not None: + return bisect.bisect_right(self._line_offsets, byte_offset) + return -1 # Legacy fallback (caller should migrate — see issue #267). + def walk_tree(self, node: Node, source: bytes, callback, depth=0, max_depth=50): """ Walk the entire AST tree, calling callback for each node. diff --git a/scripts/parsers/js_backend_parser.py b/scripts/parsers/js_backend_parser.py index 22b4f628..8dcf572c 100755 --- a/scripts/parsers/js_backend_parser.py +++ b/scripts/parsers/js_backend_parser.py @@ -546,8 +546,12 @@ def _parse_function_decl(self, node: Node, source: bytes, "async": is_async }, "body_node": body_node, - "scope_start": node.start_point.row, - "scope_end": node.end_point.row + # Issue #267: compute scope lines from byte offsets via bisect on + # _line_offsets — avoid node.start_point/node.end_point access + # (tree-sitter 0.26 crash). get_line_from_byte returns -1 if + # _line_offsets not set (legacy fallback path). + "scope_start": self.get_line_from_byte(node.start_byte) - 1, + "scope_end": self.get_line_from_byte(node.end_byte) - 1 } def _parse_variable_declarator(self, node: Node, source: bytes, @@ -590,8 +594,9 @@ def _parse_variable_declarator(self, node: Node, source: bytes, "async": is_async }, "body_node": body_node, - "scope_start": node.start_point.row, - "scope_end": node.end_point.row + # Issue #267: compute scope lines from byte offsets (0.26-safe). + "scope_start": self.get_line_from_byte(node.start_byte) - 1, + "scope_end": self.get_line_from_byte(node.end_byte) - 1 } def _parse_class_decl(self, node: Node, source: bytes, @@ -638,8 +643,9 @@ def _parse_class_decl(self, node: Node, source: bytes, "node_type": "class", }, "body_node": body_node, - "scope_start": node.start_point.row, - "scope_end": node.end_point.row + # Issue #267: compute scope lines from byte offsets (0.26-safe). + "scope_start": self.get_line_from_byte(node.start_byte) - 1, + "scope_end": self.get_line_from_byte(node.end_byte) - 1 } if heritage: diff --git a/scripts/parsers/python_parser.py b/scripts/parsers/python_parser.py index 0af6ace8..e5425d7b 100644 --- a/scripts/parsers/python_parser.py +++ b/scripts/parsers/python_parser.py @@ -161,6 +161,22 @@ def _extract_references_impl(self, content: str, file_path: str) -> Dict[str, Li # ``keep_alive`` pins every Node we visit for the duration of the # walk so reference counting can't free them mid-walk. Symmetric # with the iterative walk used in JSBackendParser. + # + # Issue #267: tree-sitter 0.26 changed Node lifetime handling — Node + # objects from ``child_by_field_name`` / ``node.children`` no longer + # hold strong references that keep their sibling Nodes alive. When + # we explicitly recurse into ``body.children`` (for class/function + # nodes) and ``continue`` (skipping the generic recurse), the sibling + # Nodes (``def`` keyword, ``name``, ``parameters``, etc.) are NOT + # pinned in ``keep_alive``. In 0.26, freeing these siblings can + # invalidate internal references in the surviving Nodes, causing + # SIGSEGV when accessing ``node.type`` / ``node.start_byte`` later in + # the walk. Fix: pin ALL children of every visited Node to + # ``keep_alive`` at the top of each iteration, BEFORE any + # ``child_by_field_name`` calls or explicit body recurse. This + # ensures no sibling is freed mid-walk. Verified on Linux + + # tree-sitter 0.26.0 — ``test_large_file_parsing.py`` no longer + # segfaults (5/5 runs clean). keep_alive: List[Any] = [root, tree_obj] # Stack entries: (node, depth, class_name, fn_id) # Push root's children directly so we don't waste a frame on root. @@ -176,13 +192,26 @@ def _extract_references_impl(self, content: str, file_path: str) -> Dict[str, Li if depth > MAX_DEPTH: continue + # Issue #267: Pin ALL children of this node to keep_alive BEFORE + # any child_by_field_name calls or explicit body recurse below. + # In tree-sitter 0.26, Node objects from child_by_field_name / + # node.children do not hold strong references that keep their + # siblings alive. Without this pin, siblings (def keyword, name, + # parameters, etc.) can be freed mid-walk when we ``continue`` + # after explicit body recurse, invalidating internal references + # in surviving Nodes → SIGSEGV on later node.type / start_byte + # access. Pinning all children ensures no sibling is freed. + node_children = node.children + for _c in node_children: + keep_alive.append(_c) + if node.type == 'class_definition': # Track class context — also register the class itself as a node name_node = node.child_by_field_name('name') if name_node: keep_alive.append(name_node) cls_name = self.get_text(name_node, source) - line = self.get_line(node) + line = self.get_line(node) # Issue #267: BaseParser.get_line uses _line_offsets (safe in 0.26) # Register class as a node so query/context can find it class_id = f"{file_path}:{line}" @@ -230,7 +259,7 @@ def _extract_references_impl(self, content: str, file_path: str) -> Dict[str, Li if name_node: keep_alive.append(name_node) fn_name = self.get_text(name_node, source) - line = self.get_line(node) + line = self.get_line(node) # Issue #267: BaseParser.get_line uses _line_offsets (safe in 0.26) # Check async is_async = any( @@ -296,11 +325,12 @@ def _extract_references_impl(self, content: str, file_path: str) -> Dict[str, Li "to_fn": call_name }) - # Recurse into children with same context + # Recurse into children with same context. + # Issue #267: children already pinned to keep_alive at the top of + # this iteration (node_children), so no need to re-append here. + # Just push them to the stack for visiting. if depth + 1 <= MAX_DEPTH: - children = node.children - for child in reversed(children): - keep_alive.append(child) + for child in reversed(node_children): stack.append((child, depth + 1, class_name, fn_id)) return {"nodes": nodes, "edges": edges} diff --git a/scripts/parsers/rust_parser.py b/scripts/parsers/rust_parser.py index 386e9cab..ed6beb19 100755 --- a/scripts/parsers/rust_parser.py +++ b/scripts/parsers/rust_parser.py @@ -464,8 +464,8 @@ def _parse_function_item(self, node: Node, source: bytes, file_path: str, return { "node": node_data, "body_node": body_node, - "scope_start": node.start_point.row, - "scope_end": node.end_point.row + "scope_start": self.get_line_from_byte(node.start_byte) - 1, + "scope_end": self.get_line_from_byte(node.end_byte) - 1 } def _find_calls_in_body(self, body_node: Node, source: bytes) -> List[Dict]: diff --git a/scripts/parsers/ts_backend_parser.py b/scripts/parsers/ts_backend_parser.py index aa79b61b..9fa7800a 100644 --- a/scripts/parsers/ts_backend_parser.py +++ b/scripts/parsers/ts_backend_parser.py @@ -230,8 +230,8 @@ def _parse_function_decl(self, node: Node, source: bytes, "async": is_async }, "body_node": body_node, - "scope_start": node.start_point.row, - "scope_end": node.end_point.row + "scope_start": self.get_line_from_byte(node.start_byte) - 1, + "scope_end": self.get_line_from_byte(node.end_byte) - 1 } def _parse_variable_declarator(self, node: Node, source: bytes, @@ -295,8 +295,8 @@ def _parse_variable_declarator(self, node: Node, source: bytes, "type": "pinia_store" }, "body_node": body_node, - "scope_start": node.start_point.row, - "scope_end": node.end_point.row + "scope_start": self.get_line_from_byte(node.start_byte) - 1, + "scope_end": self.get_line_from_byte(node.end_byte) - 1 } if not value_node: @@ -318,8 +318,8 @@ def _parse_variable_declarator(self, node: Node, source: bytes, "async": is_async }, "body_node": body_node, - "scope_start": node.start_point.row, - "scope_end": node.end_point.row + "scope_start": self.get_line_from_byte(node.start_byte) - 1, + "scope_end": self.get_line_from_byte(node.end_byte) - 1 } def _parse_class_decl(self, node: Node, source: bytes, @@ -361,8 +361,8 @@ def _parse_class_decl(self, node: Node, source: bytes, "node_type": "class", }, "body_node": body_node, - "scope_start": node.start_point.row, - "scope_end": node.end_point.row + "scope_start": self.get_line_from_byte(node.start_byte) - 1, + "scope_end": self.get_line_from_byte(node.end_byte) - 1 } if heritage: @@ -493,8 +493,8 @@ def visit(node: Node, _, depth): "node_type": "object_method", }, "body_node": body_node, - "scope_start": pair.start_point.row, - "scope_end": pair.end_point.row, + "scope_start": self.get_line_from_byte(pair.start_byte) - 1, + "scope_end": self.get_line_from_byte(pair.end_byte) - 1, }) # Don't recurse into the object — pairs are handled above, # and nested non-arrow values (e.g. numbers, strings) have diff --git a/scripts/parsers/tsx_parser.py b/scripts/parsers/tsx_parser.py index 6f21c35e..0d9a79c3 100755 --- a/scripts/parsers/tsx_parser.py +++ b/scripts/parsers/tsx_parser.py @@ -274,8 +274,8 @@ def _parse_fn_decl(self, node: Node, source: bytes, file_path: str) -> Optional[ "component": fn_name[0].isupper() # React component convention }, "body_node": body_node, - "scope_start": node.start_point.row, - "scope_end": node.end_point.row + "scope_start": self.get_line_from_byte(node.start_byte) - 1, + "scope_end": self.get_line_from_byte(node.end_byte) - 1 } def _parse_var_declarator(self, node: Node, source: bytes, file_path: str) -> Optional[Dict]: @@ -315,8 +315,8 @@ def _parse_var_declarator(self, node: Node, source: bytes, file_path: str) -> Op "component": fn_name[0].isupper() }, "body_node": body_node, - "scope_start": node.start_point.row, - "scope_end": node.end_point.row + "scope_start": self.get_line_from_byte(node.start_byte) - 1, + "scope_end": self.get_line_from_byte(node.end_byte) - 1 } def _parse_class_decl(self, node: Node, source: bytes, file_path: str) -> Optional[Dict]: @@ -356,8 +356,8 @@ def _parse_class_decl(self, node: Node, source: bytes, file_path: str) -> Option "node_type": "class", }, "body_node": body_node, - "scope_start": node.start_point.row, - "scope_end": node.end_point.row + "scope_start": self.get_line_from_byte(node.start_byte) - 1, + "scope_end": self.get_line_from_byte(node.end_byte) - 1 } if heritage: