feat(discovery) Add fast discovery

package.json

@@ -19,7 +19,7 @@
   },
   "qna": "https://github.com/robotcodedev/robotcode/discussions/categories/q-a",
   "engines": {
-    "vscode": "^1.108.0"
+    "vscode": "^1.105.0"
   },
   "categories": [
     "Programming Languages",
@@ -883,6 +883,53 @@
             "default": false,
             "scope": "resource"
           },
+          "robotcode.debug.logCommandArgs": {
+            "type": "boolean",
+            "description": "Log full command arguments for debug launcher and robot run commands.",
+            "default": false,
+            "scope": "resource"
+          },
+          "robotcode.debug.listenerLogLevel": {
+            "type": "string",
+            "enum": [
+              "TRACE",
+              "DEBUG",
+              "INFO",
+              "WARN",
+              "ERROR",
+              "FAIL",
+              "OFF"
+            ],
+            "enumDescriptions": [
+              "Process listener log/message callbacks at TRACE and above.",
+              "Process listener log/message callbacks at DEBUG and above.",
+              "Process listener log/message callbacks at INFO and above.",
+              "Process listener log/message callbacks at WARN and above.",
+              "Process listener log/message callbacks at ERROR and above.",
+              "Process listener log/message callbacks at FAIL and above.",
+              "Disable listener log/message callback processing."
+            ],
+            "description": "Minimum Robot Framework message level processed by RobotCode listener log/message hooks during test/debug runs.",
+            "default": "TRACE",
+            "scope": "resource"
+          },
+          "robotcode.debug.listenerLogTraffic": {
+            "type": "string",
+            "enum": [
+              "all",
+              "warnAndAbove",
+              "off"
+            ],
+            "enumDescriptions": [
+              "Process all listener log/message callbacks.",
+              "Process only WARN/ERROR/FAIL listener log/message callbacks.",
+              "Disable listener log/message callback processing."
+            ],
+            "description": "Controls how much Robot Framework log/message traffic RobotCode listeners process during test/debug runs.",
+            "default": "all",
+            "scope": "resource",
+            "markdownDeprecationMessage": "Deprecated in favor of `robotcode.debug.listenerLogLevel`."
+          },
           "robotcode.debug.useExternalDebugpy": {
             "type": "boolean",
             "description": "Use the debugpy in python environment, not from the python extension.",
@@ -1286,6 +1333,44 @@
             "default": true,
             "description": "Enable/disable whether Robot Framework tests and tasks are integrated into the VSCode Test/Test Explorer view.",
             "scope": "resource"
+          },
+          "robotcode.testExplorer.fastDiscovery.enabled": {
+            "type": "boolean",
+            "default": false,
+            "description": "Experimental optimization for very large workspaces. Prefilters files containing Test Cases/Tasks before full discovery.",
+            "scope": "resource"
+          },
+          "robotcode.testExplorer.fastDiscovery.prefilterCommand": {
+            "type": "string",
+            "enum": [
+              "auto",
+              "gitGrep",
+              "ripGrep",
+              "grep",
+              "none"
+            ],
+            "default": "auto",
+            "markdownDescription": "Selects prefilter command for fast discovery. `auto` tries `git grep` first, then `ripgrep`, then `grep`, and falls back to normal discovery if unavailable. `none` disables prefiltering.",
+            "scope": "resource"
+          },
+          "robotcode.testExplorer.fastDiscovery.command.enabled": {
+            "type": "boolean",
+            "default": false,
+            "description": "Use `robotcode discover fast` for workspace discovery when fast discovery prefiltering is enabled. Automatically falls back to full discovery if unsupported options are detected.",
+            "scope": "resource"
+          },
+          "robotcode.testExplorer.discovery.runEmptySuite": {
+            "type": "boolean",
+            "default": true,
+            "markdownDescription": "Keep empty suites in Test Explorer discovery results. Disable to show only suites with executable tests/tasks.",
+            "scope": "resource"
+          },
+          "robotcode.testExplorer.discovery.fastTimeoutMs": {
+            "type": "integer",
+            "default": 0,
+            "minimum": 0,
+            "markdownDescription": "Override fast discovery timeout in milliseconds. `0` uses adaptive timeout based on candidate count.",
+            "scope": "resource"
           }
         }
       },
@@ -2158,7 +2243,7 @@
     "@jgoz/esbuild-plugin-typecheck": "^4.0.4",
     "@types/fs-extra": "^11.0.4",
     "@types/node": "^22.17.0",
-    "@types/vscode": "^1.108.0",
+    "@types/vscode": "^1.105.0",
     "@types/vscode-notebook-renderer": "^1.72.4",
     "@vscode/python-extension": "^1.0.6",
     "@vscode/vsce": "^3.9.1",

packages/core/src/robotcode/core/ignore_spec.py

@@ -313,6 +313,8 @@ def _iter_files(
     verbose_callback: Optional[Callable[[str], None]] = None,
     verbose_trace: bool = False,
 ) -> Iterator[Path]:
+    ignore_file_names = tuple(ignore_files)
+
     if verbose_callback is not None and verbose_trace:
         verbose_callback(f"iter_files: {path}")
 
@@ -334,27 +336,29 @@ def _iter_files(
                 parents.insert(0, p)
 
             for p in parents:
-                ignore_file = next((p / f for f in ignore_files if (p / f).is_file()), None)
-
-                if ignore_file is not None:
+                for ignore_file in (p / f for f in ignore_file_names if (p / f).is_file()):
                     if verbose_callback is not None:
                         verbose_callback(f"using ignore file: '{ignore_file}'")
                     parent_spec = parent_spec + IgnoreSpec.from_gitignore(ignore_file)
-                    ignore_files = [ignore_file.name]
-
-    ignore_file = next((path / f for f in ignore_files if (path / f).is_file()), None)
 
-    if ignore_file is not None:
+    spec = parent_spec
+    for ignore_file in (path / f for f in ignore_file_names if (path / f).is_file()):
         if verbose_callback is not None:
             verbose_callback(f"using ignore file: '{ignore_file}'")
-        spec = parent_spec + IgnoreSpec.from_gitignore(ignore_file)
-        ignore_files = [ignore_file.name]
-    else:
-        spec = parent_spec
+        spec = spec + IgnoreSpec.from_gitignore(ignore_file)
 
     if not path.is_dir():
         if spec is not None and spec.matches(path):
             return
+
+        parent = path.parent
+        while True:
+            if spec is not None and spec.matches(parent):
+                return
+            if parent == parent.parent:
+                break
+            parent = parent.parent
+
         yield path
         return
 
@@ -368,7 +372,7 @@ def _iter_files(
         if p.is_dir():
             yield from _iter_files(
                 p,
-                ignore_files=ignore_files,
+                ignore_files=ignore_file_names,
                 include_hidden=include_hidden,
                 parent_spec=spec,
                 verbose_callback=verbose_callback,

packages/core/src/robotcode/core/text_document.py

@@ -344,7 +344,7 @@ def get_cache(
 
         e = self._cache[reference]
 
-        with e.lock:
+        with e.lock(timeout=-1):
             if not e.has_data:
                 e.data = entry(self, *args, **kwargs)
                 e.has_data = True

packages/debugger/src/robotcode/debugger/launcher/server.py

@@ -125,6 +125,9 @@ async def _launch(
         outputLog: Optional[bool] = False,  # noqa: N803
         outputTimestamps: Optional[bool] = False,  # noqa: N803
         groupOutput: Optional[bool] = False,  # noqa: N803
+        logCommandArgs: Optional[bool] = False,  # noqa: N803
+        listenerLogLevel: Optional[str] = "TRACE",  # noqa: N803
+        listenerLogTraffic: Optional[str] = "all",  # noqa: N803
         stopOnEntry: Optional[bool] = False,  # noqa: N803
         dryRun: Optional[bool] = None,  # noqa: N803
         mode: Optional[str] = None,
@@ -239,6 +242,9 @@ async def _launch(
             run_args.insert(0, "--")
 
         env = {k: ("" if v is None else str(v)) for k, v in env.items()} if env else {}
+        env["ROBOTCODE_DEBUG_LOG_COMMAND_ARGS"] = "1" if logCommandArgs else "0"
+        env["ROBOTCODE_DEBUG_LISTENER_LOG_LEVEL"] = (listenerLogLevel or "TRACE").strip()
+        env["ROBOTCODE_DEBUG_LISTENER_LOG_TRAFFIC"] = (listenerLogTraffic or "all").strip()
 
         if console in ["integratedTerminal", "externalTerminal"]:
             await self.send_request_async(

packages/debugger/src/robotcode/debugger/listeners.py

@@ -1,5 +1,7 @@
+import os
 import re
 from dataclasses import dataclass
+from functools import lru_cache
 from pathlib import Path
 from typing import Any, Dict, Iterator, List, Optional, Union, cast
 
@@ -56,6 +58,57 @@ def source_from_attributes(attributes: Dict[str, Any]) -> str:
     return s or ""
 
 
+_LISTENER_LOG_LEVEL_VALUES = {
+    "TRACE": 10,
+    "DEBUG": 20,
+    "INFO": 30,
+    "WARN": 40,
+    "ERROR": 50,
+    "FAIL": 60,
+    "OFF": 100,
+}
+
+
+def _normalize_log_level(value: Optional[str], *, default: str) -> str:
+    normalized = (value or "").strip().upper()
+
+    if normalized in ["WARNING", "WARN"]:
+        return "WARN"
+
+    if normalized in _LISTENER_LOG_LEVEL_VALUES:
+        return normalized
+
+    return default
+
+
+def _legacy_listener_log_traffic_to_level() -> str:
+    value = (os.getenv("ROBOTCODE_DEBUG_LISTENER_LOG_TRAFFIC") or "all").strip().lower()
+
+    if value in ["warnandabove", "warn_and_above", "warn-and-above"]:
+        return "WARN"
+    if value == "off":
+        return "OFF"
+
+    return "TRACE"
+
+
+@lru_cache(maxsize=1)
+def _get_listener_log_level_threshold() -> int:
+    configured_level = os.getenv("ROBOTCODE_DEBUG_LISTENER_LOG_LEVEL")
+    level_name = _normalize_log_level(configured_level, default=_legacy_listener_log_traffic_to_level())
+    return _LISTENER_LOG_LEVEL_VALUES[level_name]
+
+
+def _should_process_listener_log(level: Optional[str]) -> bool:
+    threshold = _get_listener_log_level_threshold()
+    if threshold >= _LISTENER_LOG_LEVEL_VALUES["OFF"]:
+        return False
+
+    message_level = _normalize_log_level(level, default="INFO")
+
+    return _LISTENER_LOG_LEVEL_VALUES[message_level] >= threshold
+
+
 class ListenerV2:
     ROBOT_LISTENER_API_VERSION = "2"
 
@@ -221,6 +274,9 @@ def end_keyword(self, name: str, attributes: Dict[str, Any]) -> None:
     RE_FILE_LINE_MATCHER = re.compile(r".+\sin\sfile\s'(?P<file>.*)'\son\sline\s(?P<line>\d+):(?P<message>.*)")
 
     def log_message(self, message: LogMessage) -> None:
+        if not _should_process_listener_log(message.get("level")):
+            return
+
         if message["level"] in ["FAIL", "ERROR", "WARN"]:
             current_frame = Debugger.instance.full_stack_frames[0] if Debugger.instance.full_stack_frames else None
 
@@ -274,6 +330,9 @@ def log_message(self, message: LogMessage) -> None:
         Debugger.instance.log_message(message)
 
     def message(self, message: LogMessage) -> None:
+        if not _should_process_listener_log(message.get("level")):
+            return
+
         if message["level"] in ["FAIL", "ERROR", "WARN"]:
             current_frame = Debugger.instance.full_stack_frames[0] if Debugger.instance.full_stack_frames else None
 

packages/debugger/src/robotcode/debugger/run.py

@@ -102,6 +102,14 @@ def _debug_adapter_server(
 debugpy_connected = threading.Event()
 
 
+def _should_log_command_args() -> bool:
+    value = os.getenv("ROBOTCODE_DEBUG_LOG_COMMAND_ARGS")
+    if value is None:
+        return True
+
+    return value.lower() in ["on", "1", "yes", "true"]
+
+
 @_logger.call
 def start_debugpy(
     app: Application,
@@ -167,6 +175,8 @@ def run_debugger(
     output_timestamps: bool = False,
     group_output: bool = False,
 ) -> int:
+    app.verbose(lambda: f"debug run: initial robot args={args}")
+
     if debug and debugpy and not is_debugpy_installed():
         app.warning("Debugpy not installed")
 
@@ -222,6 +232,7 @@ def run_debugger(
             "robotcode.debugger.listeners.ListenerV2",
             *args,
         ]
+        app.verbose(lambda: f"debug run: robot args with listeners={args}")
 
         Debugger.instance.stop_on_entry = stop_on_entry
         Debugger.instance.output_messages = output_messages
@@ -241,6 +252,8 @@ def run_debugger(
 
             app.verbose("Start robot")
             try:
+                if _should_log_command_args():
+                    app.echo(f"robot python api argv: {args}")
                 app.verbose(f"Create robot context with args: {args}")
                 robot_ctx = robot.make_context("robot", args, parent=ctx)
                 robot.invoke(robot_ctx)

packages/language_server/src/robotcode/language_server/common/parts/diagnostics.py

@@ -349,7 +349,12 @@ def run_workspace_diagnostics(self) -> None:
                 self._break_diagnostics_loop_event.clear()
 
                 documents = sorted(
-                    [doc for doc in self.parent.documents.documents if self._doc_need_update(doc)],
+                    [
+                        doc
+                        for doc in self.parent.documents.documents
+                        if self._doc_need_update(doc)
+                        and (doc.opened_in_editor or self.get_diagnostics_mode(doc.uri) == DiagnosticsMode.WORKSPACE)
+                    ],
                     key=lambda d: not d.opened_in_editor,
                 )
 

packages/language_server/src/robotcode/language_server/robotframework/configuration.py

@@ -60,6 +60,10 @@ class AnalysisConfig(ConfigBase):
     diagnostic_mode: DiagnosticsMode = DiagnosticsMode.OPENFILESONLY
     progress_mode: AnalysisProgressMode = AnalysisProgressMode.OFF
     references_code_lens: bool = False
+    # Controls whether the language server should eagerly load all workspace
+    # Robot Framework documents for analysis. This is independent from
+    # diagnostic_mode to avoid coupling references behavior to diagnostics.
+    load_workspace_documents: bool = True
     find_unused_references: bool = False
     cache: CacheConfig = field(default_factory=CacheConfig)
     robot: AnalysisRobotConfig = field(default_factory=AnalysisRobotConfig)