Add in-repository Tree-sitter parser and grammar for Bash++

.github/workflows/editor-support.yml

@@ -0,0 +1,46 @@
+name: Editor Support
+
+on:
+  pull_request:
+    paths:
+      - ".github/workflows/editor-support.yml"
+      - "makefile"
+      - "src/**"
+      - "test-suite/lsp-semantic-tokens.bpp"
+      - "test-suite/lsp-fixtures/**"
+      - "zed/**"
+  push:
+    branches: ["main"]
+    paths:
+      - ".github/workflows/editor-support.yml"
+      - "makefile"
+      - "src/**"
+      - "test-suite/lsp-semantic-tokens.bpp"
+      - "test-suite/lsp-fixtures/**"
+      - "zed/**"
+  workflow_dispatch:
+
+jobs:
+  semantic-tokens:
+    runs-on: ubuntu-latest
+    container:
+      image: debian:stable
+      options: --user root
+    steps:
+      - name: Install dependencies
+        run: |
+          apt-get update
+          apt-get install -y build-essential flex bison git jq libutfcpp-dev nlohmann-json3-dev
+          git config --global --add safe.directory "$GITHUB_WORKSPACE"
+      - uses: actions/checkout@v4
+      - name: Test language server
+        run: make test-lsp
+
+  zed-extension:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install WebAssembly target
+        run: rustup target add wasm32-wasip1
+      - name: Test Zed extension
+        run: make test-zed

.gitignore

@@ -11,3 +11,5 @@ debian/*.7
 vscode/node_modules
 vscode/*.vsix
 vscode/dist
+zed/target
+zed/extension.wasm

README.md

@@ -70,6 +70,9 @@ Language server-specific prerequisites:
  - `nlohmann-json3-dev`
 
 Optional:
+ - `jq` for running the language server integration tests
+ - `cargo` and the `wasm32-wasip1` Rust target for validating the Zed extension
+ - the `wasm32-wasip2` Rust target for installing the extension in current Zed releases
  - `pandoc` and `perl` for building the documentation
  - `debhelper` for building the Debian package and keeping version numbers up-to-date via `dpkg-parsechangelog`
 
@@ -86,6 +89,8 @@ $ sudo apt install build-essential flex bison libutfcpp-dev pandoc perl debhelpe
 $ make          # Build the Bash++ compiler and language server, bin/bpp and bin/bpp-lsp
 $ make manpages # Build the manpages, which can then be found under debian/
 $ make test     # Run the test suite to verify the compiler works correctly
+$ make test-lsp # Run the language server integration tests
+$ make test-zed # Validate the Zed extension
 ```
 
 ## Using the compiler
@@ -120,3 +125,10 @@ $ shellwatch compiled-program.sh
 ## VSCode extension
 
 The [Bash++ extension for Visual Studio Code](https://marketplace.visualstudio.com/items?itemName=rail5.bashpp) provides IDE language support for Bash++, including syntax highlighting and (optionally) language server features such as code completion, go-to definition, etc.
+
+## Zed extension
+
+The development extension under [`zed/`](zed/) recognizes `.bpp` files, reuses
+Zed's built-in Bash grammar for baseline highlighting, and starts an installed
+`bpp-lsp` from `PATH`. Enable Zed's `combined` semantic-token mode to layer
+Bash++ identifier highlighting from the compiler AST over the Bash syntax.

makefile

@@ -14,16 +14,28 @@ include mk/docs.mk
 test:
 	bin/bpp -Istdlib/ test-suite/run.bpp
 
+test-lsp: bin/bpp bin/bpp-lsp
+	bin/bpp -Istdlib/ test-suite/lsp-semantic-tokens.bpp bin/bpp-lsp
+
 vscode:
 	@cd vscode && $(MAKE) --no-print-directory
 
 clean-vscode:
 	@cd vscode && $(MAKE) --no-print-directory clean
 	@echo "Cleaned up VSCode extension files."
 
-clean: clean-flexbison clean-lsp clean-meta clean-objects clean-bin clean-std clean-manpages clean-technical-docs clean-vscode
+zed:
+	cargo build --manifest-path zed/Cargo.toml --target wasm32-wasip1
+
+test-zed:
+	cargo check --manifest-path zed/Cargo.toml --target wasm32-wasip1
+
+clean-zed:
+	rm -rf zed/target zed/extension.wasm
+
+clean: clean-flexbison clean-lsp clean-meta clean-objects clean-bin clean-std clean-manpages clean-technical-docs clean-vscode clean-zed
 
-.PHONY: all test vscode clean-vscode
+.PHONY: all test test-lsp vscode clean-vscode zed test-zed clean-zed
 
 ifeq ($(filter clean%,$(MAKECMDGOALS)),)
 -include $(shell find bin -name '*.d' 2>/dev/null)

src/AST/BashppParser.cpp

@@ -18,8 +18,10 @@ extern void yyset_in(FILE* in_str, yyscan_t scanner);
 
 extern void initLexer(yyscan_t yyscanner);
 extern void destroyLexer(yyscan_t yyscanner);
+extern std::vector<AST::LexerToken> get_lexer_tokens(yyscan_t yyscanner);
 
 extern bool set_display_lexer_output(bool enable, yyscan_t yyscanner);
+extern void set_collect_lexer_tokens(bool enable, yyscan_t yyscanner);
 extern void set_utf16_mode(bool enable, yyscan_t yyscanner);
 
 #include <flexbison/generated/parser.tab.hpp>
@@ -78,6 +80,7 @@ void AST::BashppParser::_initialize_lexer() {
 	initLexer(lexer);
 	set_utf16_mode(utf16_mode, lexer);
 	set_display_lexer_output(display_lexer_output, lexer);
+	set_collect_lexer_tokens(collect_lexer_tokens, lexer);
 }
 
 void AST::BashppParser::_destroy_lexer() {
@@ -97,6 +100,7 @@ void AST::BashppParser::_parse() {
 			errors,
 			lexer);
 		parser.parse(); // Returns an int, not needed by us
+		lexer_tokens = ::get_lexer_tokens(lexer);
 	} catch (...) {
 		_destroy_lexer();
 		throw;
@@ -113,6 +117,10 @@ void AST::BashppParser::setDisplayLexerOutput(bool enabled) {
 	display_lexer_output = enabled;
 }
 
+void AST::BashppParser::setCollectLexerTokens(bool enabled) {
+	collect_lexer_tokens = enabled;
+}
+
 void AST::BashppParser::setInputFromFilePath(const std::string& file_path) {
 	input_type = InputType::FILEPATH;
 	input_source = file_path;
@@ -144,3 +152,7 @@ std::shared_ptr<AST::Program> AST::BashppParser::program() {
 const std::vector<AST::ParserError>& AST::BashppParser::get_errors() const {
 	return errors;
 }
+
+const std::vector<AST::LexerToken>& AST::BashppParser::get_lexer_tokens() const {
+	return lexer_tokens;
+}

src/AST/BashppParser.h

@@ -11,6 +11,7 @@
 #include <variant>
 #include <vector>
 #include <AST/ASTNode.h>
+#include <AST/LexerToken.h>
 #include <AST/Nodes/Nodes.h>
 #include <error/ParserError.h>
 
@@ -32,8 +33,10 @@ class BashppParser {
 
 		bool utf16_mode = false; // Whether to use UTF-16 mode for character counting
 		bool display_lexer_output = false;
+		bool collect_lexer_tokens = false;
 
 		std::vector<ParserError> errors;
+		std::vector<LexerToken> lexer_tokens;
 
 		std::string input_file_path = "<stdin>";
 		std::vector<std::string> include_chain;
@@ -56,6 +59,7 @@ class BashppParser {
 	public:
 		void setUTF16Mode(bool enabled);
 		void setDisplayLexerOutput(bool enabled);
+		void setCollectLexerTokens(bool enabled);
 
 		void setInputFromFilePath(const std::string& file_path);
 		void setInputFromFilePtr(FILE* file_ptr, const std::string& file_path);
@@ -66,6 +70,7 @@ class BashppParser {
 		std::shared_ptr<AST::Program> program();
 
 		const std::vector<ParserError>& get_errors() const;
+		const std::vector<LexerToken>& get_lexer_tokens() const;
 };
 
 } // namespace AST

src/AST/LexerToken.h

@@ -0,0 +1,24 @@
+/*
+ * Copyright (C) 2026 Andrew S. Rightenburg
+ * Bash++: Bash with classes
+ * SPDX-License-Identifier: GPL-3.0-or-later
+ */
+
+#pragma once
+
+#include <string>
+
+#include <include/ParserPosition.h>
+
+namespace AST {
+
+/**
+ * @brief A lexer symbol and its exact source range for editor features.
+ */
+struct LexerToken {
+	std::string kind;
+	std::string text;
+	ParserLocation location;
+};
+
+} // namespace AST

src/flexbison/lexer.l

@@ -7,11 +7,13 @@
  * SPDX-License-Identifier: GPL-3.0-or-later
  */
 #include "parser.tab.hpp" // Bison header for token types
+#include <AST/LexerToken.h>
 #include <AST/Token.h> // For AST::Token<T>
 #include <flexbison/ModeStack.h>
 #include <cstdint>
 #include <stack>
 #include <deque>
+#include <vector>
 
 /**
 GNU Bison Docs, 10.1.7.2: Complete Symbols
@@ -103,7 +105,9 @@ struct LexerState {
 struct LexerExtra {
 	LexerState lexerState;
 	ModeStack modeStack;
+	std::vector<AST::LexerToken> lexerTokens;
 	bool display_lexer_output = false;
+	bool collect_lexer_tokens = false;
 };
 
 extern LexerExtra* yyget_extra(yyscan_t yyscanner);
@@ -137,6 +141,10 @@ void set_display_lexer_output(bool enable, yyscan_t yyscanner) {
 	get_lexer_extra(yyscanner)->display_lexer_output = enable;
 }
 
+void set_collect_lexer_tokens(bool enable, yyscan_t yyscanner) {
+	get_lexer_extra(yyscanner)->collect_lexer_tokens = enable;
+}
+
 size_t utf8_char_count(const std::string& s) {
 	size_t count = 0;
 	for (unsigned char c : s) {
@@ -291,6 +299,47 @@ void set_received_local_keyword(bool received, yyscan_t yyscanner) {
 */
 yy::parser::symbol_type maybe_get_lvalue_token(yy::parser::symbol_type token, yyscan_t yyscanner);
 
+void recordLexerToken(
+	const yy::parser::symbol_type& token,
+	const std::string& text,
+	yyscan_t yyscanner
+) {
+	auto* extra = get_lexer_extra(yyscanner);
+	if (!extra->collect_lexer_tokens) return;
+
+	ParserLocation location = token.location;
+	std::string recorded_text = text;
+	if (token.kind() == yy::parser::symbol_kind::S_SINGLEQUOTED_STRING) {
+		// The lexer combines several matches while assembling this token, so the
+		// semantic value retains a more accurate range than the final match.
+		const auto& value = token.value.as<AST::Token<std::string>>();
+		location.begin.line = value.getLine();
+		location.begin.column = value.getCharPositionInLine();
+		recorded_text = value.getValue();
+	}
+
+	extra->lexerTokens.push_back(AST::LexerToken{
+		.kind = std::string(yy::parser::symbol_name(token.kind())),
+		.text = std::move(recorded_text),
+		.location = location
+	});
+}
+
+void recordComment(
+	const std::string& text,
+	const ParserLocation& location,
+	yyscan_t yyscanner
+) {
+	auto* extra = get_lexer_extra(yyscanner);
+	if (!extra->collect_lexer_tokens) return;
+
+	extra->lexerTokens.push_back(AST::LexerToken{
+		.kind = "COMMENT",
+		.text = text,
+		.location = location
+	});
+}
+
 void updateLexerState(yyscan_t yyscanner) {
 	thisLexerState.expecting_assignment_operator = false;
 	thisLexerState.parsed_assignment_operator = false;
@@ -448,10 +497,14 @@ ANGLEBRACKET_INCLUDE_PATH <([^>])+>
 	*  return yy::parser::make_AT();
 	*/
 	#define emit(tokenType, ...) \
+	do { \
 		updateLexerState(yyscanner); \
-		return maybe_get_lvalue_token(yy::parser::make_##tokenType( \
+		auto token = maybe_get_lvalue_token(yy::parser::make_##tokenType( \
 			__VA_ARGS__ __VA_OPT__(,) current_match_location(yyscanner) \
-		), yyscanner);
+		), yyscanner); \
+		recordLexerToken(token, std::string(yytext, yyleng), yyscanner); \
+		return token; \
+	} while (false)
 
 	/**
 	* Helper to get the text of the current token as a std::string.
@@ -465,7 +518,7 @@ ANGLEBRACKET_INCLUDE_PATH <([^>])+>
 
 <SKIP_AFTER_DELIM_MODE>{
 	[ \t\n]+    { /* Ignore whitespace and newlines after a DELIM */ }
-	[\#][^\n]*  { /* Ignore comments */ }
+	[\#][^\n]*  { recordComment(std::string(yytext, yyleng), current_match_location(yyscanner), yyscanner); }
 	.           {
 		// Any other character means we're done skipping
 		thisModeStack.pop(); // Exit SKIP_AFTER_DELIM_MODE
@@ -989,7 +1042,7 @@ function/[ \t]+[a-zA-Z_][a-zA-Z_0-9]* {
 
 {INTEGER} { emit(INTEGER, tokenText); }
 
-[ \t]*[\#][^\n]*  { /* Ignore comments */ }
+[ \t]*[\#][^\n]*  { recordComment(std::string(yytext, yyleng), current_match_location(yyscanner), yyscanner); }
 
 <BASH_IF_MODE>{
 	"then"/[ \t\n] {
@@ -1560,6 +1613,11 @@ extern void initLexer(yyscan_t yyscanner) {
 	thisModeStack.bind(yyscanner);
 	thisModeStack.push(SKIP_AFTER_DELIM_MODE); // Initial mode
 	thisLexerState.reset();
+	get_lexer_extra(yyscanner)->lexerTokens.clear();
+}
+
+extern std::vector<AST::LexerToken> get_lexer_tokens(yyscan_t yyscanner) {
+	return get_lexer_extra(yyscanner)->lexerTokens;
 }
 
 extern void destroyLexer(yyscan_t yyscanner) {

src/lsp/BashppServer.h

@@ -80,6 +80,7 @@ class BashppServer {
 		GenericResponseMessage handleRename(const GenericRequestMessage& request);
 		GenericResponseMessage handleReferences(const GenericRequestMessage& request);
 		GenericResponseMessage handleCompletion(const GenericRequestMessage& request);
+		GenericResponseMessage handleSemanticTokens(const GenericRequestMessage& request);
 
 		CompletionList handleATCompletion(const CompletionParams& params);
 		CompletionList handleDOTCompletion(const CompletionParams& params);
@@ -286,14 +287,15 @@ class BashppServer {
 		 * @brief Maps request types to the functions that handle them.
 		 * 
 		 */
-		static constexpr std::array<RequestHandlerEntry, 8> request_handlers = {{
+		static constexpr std::array<RequestHandlerEntry, 9> request_handlers = {{
 			{"initialize",                  &BashppServer::handleInitialize},
 			{"textDocument/definition",     &BashppServer::handleDefinition},
 			{"textDocument/completion",     &BashppServer::handleCompletion},
 			{"textDocument/hover",          &BashppServer::handleHover},
 			{"textDocument/documentSymbol", &BashppServer::handleDocumentSymbol},
 			{"textDocument/rename",         &BashppServer::handleRename},
 			{"textDocument/references",     &BashppServer::handleReferences},
+			{"textDocument/semanticTokens/full", &BashppServer::handleSemanticTokens},
 			{"shutdown",                    &BashppServer::shutdown}
 		}};