From 788d4835aa45ee162c48b0461ea7e765b76d4d63 Mon Sep 17 00:00:00 2001 From: squid-protocol Date: Wed, 16 Sep 2026 20:57:28 -0400 Subject: [PATCH 1/6] feat(rexx): add REXX language profile + registration surfaces (#2504) - languages/rexx.py: full baseline schema; z/OS TSO/E + SAA + ooRexx directives - recursive_block_rexx prism dialect: nested /* */ peel without recursive_block's // line token (// is REXX's remainder operator); REXX quote morphology (doubled-quote escape, line-bounded) in both nested-comment strippers - .cmd registered as a batch/rexx collision; batch gains an internal_discriminator (its empty rules dict scores 0 in Tier 3, so without one every batch .cmd would lose the lexical scan to rexx) - detector: rexx in Mode A label slicing + _CLASS_START_NAMED_EXTRACTION_LANGS - analysis_lens: strictness row (False,False,True,False) + infra ecosystem Co-Authored-By: Claude Opus 4.8 --- gitgalaxy/core/detector.py | 11 + gitgalaxy/core/prism.py | 70 ++- gitgalaxy/standards/analysis_lens.py | 8 + gitgalaxy/standards/gitgalaxy_config.py | 12 + .../standards/language_standards/__init__.py | 2 + .../language_standards/_lens_config.py | 6 +- .../language_standards/languages/batch.py | 22 + .../language_standards/languages/rexx.py | 431 ++++++++++++++++++ 8 files changed, 539 insertions(+), 23 deletions(-) create mode 100644 gitgalaxy/standards/language_standards/languages/rexx.py diff --git a/gitgalaxy/core/detector.py b/gitgalaxy/core/detector.py index 1cc093a5d..9f89a8673 100644 --- a/gitgalaxy/core/detector.py +++ b/gitgalaxy/core/detector.py @@ -603,6 +603,11 @@ def get_mode(cls, lang_id: str) -> Optional[str]: "pli", "powershell", "python", + # #2504: rexx's class_start is ooRexx's `::CLASS name` with the name in + # group 1; the generic fallback (`class|struct|...`, lowercase-only, no + # `::` anchor) can never match the directive form. Tree-sitter-blind; + # verified against planted corpus programs. + "rexx", "ruby", "rust", "scala", @@ -3410,6 +3415,12 @@ def _function_slice( # body early; that is the same approximation ada's nested # subprograms already take. "pli", + # #2504: rexx is COBOL's own shape -- a subroutine runs from + # its `label:` to its RETURN/EXIT (both already in the shared + # assembly_returns terminator vocabulary) or the next label, + # never a brace. Routines don't nest, so Mode A's greedy + # label-to-label body is the real boundary. + "rexx", ) or family in ("column_sensitive"): mode_name = "Mode_A_Labels" sats, impact = self._slice_by_labels(code, rules, offset, spatial_map) diff --git a/gitgalaxy/core/prism.py b/gitgalaxy/core/prism.py index 9b5279cca..1328e7269 100644 --- a/gitgalaxy/core/prism.py +++ b/gitgalaxy/core/prism.py @@ -423,10 +423,11 @@ def _positional_comment_segment(self, text: str, lang_id: str, family: str) -> s ) return positional_comments - if family in ("recursive_block", "recursive_block_haskell", "recursive_block_lisp"): + if family in ("recursive_block", "recursive_block_haskell", "recursive_block_lisp", "recursive_block_rexx"): # #2908 Phase 2 follow-up: rust/scala/swift (recursive_block), - # haskell (recursive_block_haskell) and scheme - # (recursive_block_lisp) -- see _positional_nested_comments. + # haskell (recursive_block_haskell), scheme + # (recursive_block_lisp) and rexx (recursive_block_rexx, #2504) + # -- see _positional_nested_comments. return self._positional_nested_comments(text, family) if lang_id == "perl": @@ -724,17 +725,25 @@ def _positional_nested_comments(self, text: str, family: str) -> str: s_line, b_start, b_end = delims[0], delims[1], delims[2] # Same construction as _strip_nested_comments -- see that method's - # own comment for why each alternative exists and why lisp's char - # literal must be tried first. + # own comment for why each alternative exists, why lisp's char + # literal must be tried first, and why rexx swaps the quote branches. lisp_char_literal = r"#\\(?:[a-zA-Z0-9][a-zA-Z0-9-]{0,31}|[^\s])|" if family == "recursive_block_lisp" else "" - combined_pattern = re.compile( - lisp_char_literal - + r'(?(),&|\]\s])(?:\\.|[^'\\]){0,10}'" - + r"|(?(),&|\]\s])(?:\\.|[^'\\]){0,10}'" + + r"|(? str: return "".join(ch if ch == "\n" else "\x00" for ch in s) @@ -808,7 +817,7 @@ def _strip_segment_comments(self, text: str, lang_id: str, family: str) -> tuple # line_exclusive/recursive_block/positional_anchored/block_exclusive/ # non_lexical), so none of these branches, nor the generic REGEX_MATRIX # stripper below, ever actually ran for any language. - if family in ("recursive_block", "recursive_block_haskell", "recursive_block_lisp"): + if family in ("recursive_block", "recursive_block_haskell", "recursive_block_lisp", "recursive_block_rexx"): # #621: recursive_block_haskell added because Haskell's {- -} # blocks genuinely nest (unlike the standard_block family's flat # delimiters) but use -- for line comments and {- -} rather than @@ -1494,14 +1503,31 @@ def _strip_nested_comments(self, text: str, family: str = "recursive_block") -> # it goes through the same mask/unmask path as a string so the code stream # keeps it verbatim. Bounded exactly like detector.py's _LISP_SCOPE_TOKEN. lisp_char_literal = r"#\\(?:[a-zA-Z0-9][a-zA-Z0-9-]{0,31}|[^\s])|" if family == "recursive_block_lisp" else "" - combined_pattern = re.compile( - lisp_char_literal - + r'(?(),&|\]\s])(?:\\.|[^'\\]){0,10}'" - + r"|(?(),&|\]\s])(?:\\.|[^'\\]){0,10}'" + + r"|(? str: diff --git a/gitgalaxy/standards/analysis_lens.py b/gitgalaxy/standards/analysis_lens.py index 3c92fc719..61f82f418 100644 --- a/gitgalaxy/standards/analysis_lens.py +++ b/gitgalaxy/standards/analysis_lens.py @@ -198,6 +198,11 @@ def get_policy(mode="baseline"): "powershell": (False, False, True, True), # non-terminating errors continue by default "proto": None, "python": (False, False, True, True), + # #2504: dynamically typed (everything is a string); conditions (SYNTAX/ + # ERROR/NOVALUE) are ignored unless SIGNAL ON installs a handler; no raw + # memory model of its own; a routine without PROCEDURE shares the caller's + # whole variable pool -- implicit globals by default. + "rexx": (False, False, True, False), "ruby": (False, False, True, True), "rust": (True, True, True, True), "scala": (True, False, True, True), @@ -991,6 +996,9 @@ def strictness_constants(lang_id: str) -> tuple[int, float]: "infra": { "shell", "powershell", + # #2504: z/OS automation scripting -- ADDRESS TSO/ISPEXEC host + # commands are the language's purpose, shell's own profile. + "rexx", "dockerfile", "yaml", "makefile", diff --git a/gitgalaxy/standards/gitgalaxy_config.py b/gitgalaxy/standards/gitgalaxy_config.py index 76c13f5fb..e442f0381 100644 --- a/gitgalaxy/standards/gitgalaxy_config.py +++ b/gitgalaxy/standards/gitgalaxy_config.py @@ -538,6 +538,18 @@ # a stateless per-line stripper is exactly what caused the bug. # Examples: scheme. "recursive_block_lisp": {"delimiters": [";", "#|", "|#"]}, + # 2d. Recursive Block, REXX dialect (#2504) + # Same nested-block-peeling algorithm as recursive_block ("comments + # may be nested within other comments", TSO/E REXX Reference), but + # WITHOUT recursive_block's `//` line token: `//` is REXX's + # integer-remainder operator (`a // b`), so the shared family would + # truncate real arithmetic lines. The line token is ooRexx/Regina/ + # NetRexx's `--` (classic z/OS REXX has no line comment at all; + # adjacent `--` double-negation is legal but vanishingly rare in real + # source, the accepted trade). prism.py also swaps the quote-masking + # branches for this family: REXX strings double their quote to escape, + # never backslash, and cannot span lines. + "recursive_block_rexx": {"delimiters": ["--", "/*", "*/"]}, # 3. Line Exclusive # The language possesses no native multi-line block syntax. The engine ignores closing tags. # Examples: Python, Shell, Makefile, Ruby, Perl, Assembly. diff --git a/gitgalaxy/standards/language_standards/__init__.py b/gitgalaxy/standards/language_standards/__init__.py index 4b0153044..c73f91802 100644 --- a/gitgalaxy/standards/language_standards/__init__.py +++ b/gitgalaxy/standards/language_standards/__init__.py @@ -118,6 +118,7 @@ from .languages import powershell as _powershell from .languages import proto as _proto from .languages import python as _python +from .languages import rexx as _rexx from .languages import ruby as _ruby from .languages import rust as _rust from .languages import scala as _scala @@ -209,4 +210,5 @@ "bms": _bms.DEFINITION, "db2_sql": _db2_sql.DEFINITION, "hlasm": _hlasm.DEFINITION, + "rexx": _rexx.DEFINITION, } diff --git a/gitgalaxy/standards/language_standards/_lens_config.py b/gitgalaxy/standards/language_standards/_lens_config.py index 2e2723d42..f382941d8 100644 --- a/gitgalaxy/standards/language_standards/_lens_config.py +++ b/gitgalaxy/standards/language_standards/_lens_config.py @@ -58,7 +58,11 @@ class LensConfig(TypedDict): # internal_discriminator (Tier 2: CSECT/DSECT/USING/... in operation-field # position), mainframe-sibling ecosystem gravity (Tier 1.5) or the lexical # scan (Tier 3). ".mac" and ".hlasm" are uncontested and stay Tier 1. - "COLLISION_FREQUENCIES": {".inc", ".h", ".py", ".cshtml", ".c", ".y", ".m", ".map", ".sql", ".ddl", ".dml", ".asm"}, + # #2504: ".cmd" is claimed by BOTH batch (Windows/OS2) and rexx (z/OS, + # OS/2), same mechanism -- rexx's internal_discriminator (a .cmd opening + # with `/*` is REXX, the platform loaders' own dispatch rule) resolves it; + # ".rexx" and ".exec" are uncontested and stay Tier 1. + "COLLISION_FREQUENCIES": {".inc", ".h", ".py", ".cshtml", ".c", ".y", ".m", ".map", ".sql", ".ddl", ".dml", ".asm", ".cmd"}, "PROSE_ANCHORS": { "README", "LICENSE", diff --git a/gitgalaxy/standards/language_standards/languages/batch.py b/gitgalaxy/standards/language_standards/languages/batch.py index a72f0ec73..47df43736 100644 --- a/gitgalaxy/standards/language_standards/languages/batch.py +++ b/gitgalaxy/standards/language_standards/languages/batch.py @@ -8,6 +8,7 @@ # of this project, or at https://polyformproject.org/licenses/noncommercial/1.0.0/ # ============================================================================== +import re from typing import Any DEFINITION: dict[str, Any] = { @@ -17,5 +18,26 @@ "discriminators": [], "shebangs": [], "lexical_family": "line_exclusive", + # Collision resolution for `.cmd` (#2504: rexx claims it too, and batch's + # empty rules dict scores 0 in the Tier 3 lexical scan, so without a + # Tier 2 anchor every real batch file would lose the scan to any language + # with rules). These are batch-only line shapes REXX cannot carry: + # `@echo off/on`, SETLOCAL/ENDLOCAL, `goto :label`, the `%~dp0`/`%%~x` + # argument modifiers and `%ERRORLEVEL%`/`errorlevel N` tests. Bare `rem` + # and `set X=1` are deliberately left out -- `rem = n // 7` and + # `set = 1` are legal REXX assignments (registry order checks batch's + # discriminator FIRST, so a false batch hit on real REXX would lock the + # wrong language, the costlier direction). + "internal_discriminator": re.compile( + r"^[ \t]*@?ECHO[ \t]+(?:OFF|ON)\b" + r"|^[ \t]*(?:SETLOCAL|ENDLOCAL)\b" + r"|^[ \t]*GOTO[ \t]+:?[A-Za-z_]" + r"|^[ \t]*@?SET[ \t]+\"?[A-Za-z_][\w]{0,63}=" + r"|^[ \t]*IF[ \t]+(?:NOT[ \t]+)?(?:EXIST|DEFINED)\b" + r"|%%?~[A-Za-z]{0,10}[0-9]" + r"|%ERRORLEVEL%" + r"|\bERRORLEVEL[ \t]+[0-9]", + re.M | re.I, + ), "rules": {}, } diff --git a/gitgalaxy/standards/language_standards/languages/rexx.py b/gitgalaxy/standards/language_standards/languages/rexx.py new file mode 100644 index 000000000..15594fa2b --- /dev/null +++ b/gitgalaxy/standards/language_standards/languages/rexx.py @@ -0,0 +1,431 @@ +# ============================================================================== +# GitGalaxy +# Copyright (c) 2026 Joe Esquibel +# +# This source code is licensed under the PolyForm Noncommercial License 1.0.0. +# You may not use this file except in compliance with the License. +# A copy of the license can be found in the LICENSE file in the root directory +# of this project, or at https://polyformproject.org/licenses/noncommercial/1.0.0/ +# ============================================================================== + +import re +from typing import Any + +from .._shared_patterns import GLOBAL_FRAGILE_DEBT, GLOBAL_PLANNED_DEBT + +# #2504: REXX -- z/OS TSO/E REXX and classic (SAA) REXX, plus the ooRexx +# directive surface (`::requires` / `::routine` / `::class` / `::method`), +# since `.rexx` files in the wild host ooRexx code too. +# +# A REXX symbol is letters, digits and `. ! ? _` plus the national characters +# `@ # $` (TSO/E REXX Reference, "Tokens"). `\w` covers letters/digits/`_`; +# `! ? @ # $` are regex NON-word characters, so a plain `\b` cannot guard a +# keyword against them (`?EXIT` would satisfy `\bEXIT`): bare keywords use +# these explicit guards, PL/I's discipline. `.` joins both guards -- a +# compound variable's tail (`rc.exit`) must not read as the keyword -- and +# stays OUT of the label-name class below (a label is a simple symbol). +_ID = r"[\w.!?@#$]" +_NAME = r"[\w!?@#$]" +_L = r"(?`, statement-position `ARG