Fix pytest CI matrix compatibility and prevent hangs

.github/workflows/build.yml

@@ -197,6 +197,9 @@ jobs:
       - name: Install test requirements
         run: python -m pip install --upgrade -r build/test-requirements.txt
 
+      - name: Install pytest
+        run: python -m pip install --upgrade pytest
+
       - name: Run Python unit tests
         run: python python_files/tests/run_all.py
 

.github/workflows/pr-check.yml

@@ -156,9 +156,9 @@ jobs:
         # We're not running CI on macOS for now because it's one less matrix entry to lower the number of runners used,
         # macOS runners are expensive, and we assume that Ubuntu is enough to cover the Unix case.
         os: [ubuntu-latest, windows-latest]
-        # Run the tests on the oldest and most recent versions of Python.
-        python: ['3.10', '3.x', '3.13'] # run for 3 pytest versions, most recent stable, oldest version supported and pre-release
-        pytest-version: ['pytest', 'pytest@pre-release', 'pytest==6.2.0']
+        python: ['3.10', '3.11', '3.12', '3.13', '3.14']
+        # Test approximately one year of pytest releases and upcoming pytest changes.
+        pytest-version: ['pytest==8.4.*', 'pytest@pre-release']
 
     steps:
       - name: Checkout
@@ -172,18 +172,6 @@ jobs:
         with:
           python-version: ${{ matrix.python }}
 
-      - name: Install specific pytest version
-        if: matrix.pytest-version == 'pytest@pre-release'
-        run: |
-          python -m pip install --pre pytest
-
-      - name: Install specific pytest version
-        if: matrix.pytest-version != 'pytest@pre-release'
-        run: |
-          python -m pip install "${{ matrix.pytest-version }}"
-
-      - name: Install specific pytest version
-        run: python -m pytest --version
       - name: Install base Python requirements
         uses: brettcannon/pip-secure-install@92f400e3191171c1858cc0e0d9ac6320173fdb0c # v1.0.0
         with:
@@ -193,6 +181,17 @@ jobs:
       - name: Install test requirements
         run: python -m pip install --upgrade -r build/test-requirements.txt
 
+      - name: Install specific pytest version (pre-release)
+        if: matrix.pytest-version == 'pytest@pre-release'
+        run: python -m pip install --upgrade --pre pytest
+
+      - name: Install specific pytest version
+        if: matrix.pytest-version != 'pytest@pre-release'
+        run: python -m pip install --upgrade "${{ matrix.pytest-version }}"
+
+      - name: Print pytest version
+        run: python -m pytest --version
+
       - name: Run Python unit tests
         run: python python_files/tests/run_all.py
 

build/test-requirements.txt

@@ -7,9 +7,6 @@ pylint
 pycodestyle
 pydocstyle
 prospector
-# pytest-black 0.6.0 uses the deprecated `path` arg in pytest_collect_file,
-# which was removed in pytest 8.1. Pin to <8.1 to maintain compatibility.
-pytest<8.1
 flask
 fastapi
 uvicorn
@@ -41,4 +38,3 @@ pytest-describe
 
 # for pytest-ruff related tests
 pytest-ruff
-pytest-black

python_files/tests/pytestadapter/helpers.py

@@ -12,7 +12,7 @@
 import tempfile
 import threading
 import uuid
-from typing import Any, Dict, List, Optional, Tuple
+from typing import Any, Dict, List, Optional, Tuple, Union
 
 if sys.platform == "win32":
     from namedpipe import NPopen  # pylint: disable=import-error # cspell: disable-line
@@ -28,6 +28,8 @@
 TEST_DATA_PATH = pathlib.Path(__file__).parent / ".data"
 CONTENT_LENGTH: str = "Content-Length:"
 CONTENT_TYPE: str = "Content-Type:"
+PIPE_RESULT_TIMEOUT_SECONDS = 10
+TEST_SUBPROCESS_TIMEOUT_SECONDS = 300
 
 
 @contextlib.contextmanager
@@ -242,10 +244,36 @@ def _listen_on_pipe_new(listener, result: List[str], completed: threading.Event)
         result.append("".join(all_data))
 
 
-def _run_test_code(proc_args: List[str], proc_env, proc_cwd: str, completed: threading.Event):
-    result = subprocess.run(proc_args, env=proc_env, cwd=proc_cwd, check=False)
-    completed.set()
-    return result
+def _run_test_code(
+    proc_args: List[str],
+    proc_env,
+    proc_cwd: Union[str, os.PathLike[str]],
+    completed: threading.Event,
+) -> subprocess.CompletedProcess:
+    try:
+        return subprocess.run(
+            proc_args,
+            env=proc_env,
+            cwd=proc_cwd,
+            check=False,
+            timeout=TEST_SUBPROCESS_TIMEOUT_SECONDS,
+        )
+    finally:
+        completed.set()
+
+
+def _wait_for_pipe_result(
+    listener_thread: threading.Thread,
+    process_result: subprocess.CompletedProcess,
+    result: List[str],
+) -> None:
+    listener_thread.join(timeout=PIPE_RESULT_TIMEOUT_SECONDS)
+    if listener_thread.is_alive():
+        if process_result.returncode:
+            raise subprocess.CalledProcessError(process_result.returncode, process_result.args)
+        raise TimeoutError("Timed out waiting for the test subprocess pipe result")
+    if process_result.returncode and not result:
+        raise subprocess.CalledProcessError(process_result.returncode, process_result.args)
 
 
 def runner(args: List[str]) -> Optional[List[Dict[str, Any]]]:
@@ -359,18 +387,12 @@ def runner_with_cwd_env(
 
             result = []  # result is a string array to store the data during threading
             t1: threading.Thread = threading.Thread(
-                target=_listen_on_pipe_new, args=(pipe, result, completed)
+                target=_listen_on_pipe_new, args=(pipe, result, completed), daemon=True
             )
             t1.start()
 
-            t2 = threading.Thread(
-                target=_run_test_code,
-                args=(process_args, env, path, completed),
-            )
-            t2.start()
-
-            t1.join()
-            t2.join()
+            process_result = _run_test_code(process_args, env, path, completed)
+            _wait_for_pipe_result(t1, process_result, result)
 
             return process_data_received(result[0]) if result else None
     else:  # Unix design
@@ -397,19 +419,12 @@ def runner_with_cwd_env(
 
         result = []  # result is a string array to store the data during threading
         t1: threading.Thread = threading.Thread(
-            target=_listen_on_fifo, args=(pipe_name, result, completed)
+            target=_listen_on_fifo, args=(pipe_name, result, completed), daemon=True
         )
         t1.start()
 
-        t2: threading.Thread = threading.Thread(
-            target=_run_test_code,
-            args=(process_args, env, path, completed),
-        )
-
-        t2.start()
-
-        t1.join()
-        t2.join()
+        process_result = _run_test_code(process_args, env, path, completed)
+        _wait_for_pipe_result(t1, process_result, result)
 
         return process_data_received(result[0]) if result else None
 

python_files/tests/pytestadapter/test_discovery.py

@@ -476,10 +476,11 @@ def test_config_sub_folder():
             expected_discovery_test_output.ruff_test_expected_output,
             "--ruff",
         ),
-        (
+        pytest.param(
             "2496-black-formatter",
             expected_discovery_test_output.black_formatter_expected_output,
             "--black",
+            marks=pytest.mark.skip(reason="pytest-black does not support pytest 8.1 or newer"),
         ),
     ],
 )

python_files/tests/pytestadapter/test_helpers.py

@@ -0,0 +1,54 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+import os
+import subprocess
+import sys
+import threading
+
+import pytest
+
+from . import helpers
+
+
+def test_run_test_code_times_out(tmp_path, monkeypatch):
+    monkeypatch.setattr(helpers, "TEST_SUBPROCESS_TIMEOUT_SECONDS", 0.01)
+    completed = threading.Event()
+
+    with pytest.raises(subprocess.TimeoutExpired):
+        helpers._run_test_code(  # noqa: SLF001
+            [sys.executable, "-c", "import time; time.sleep(1)"],
+            os.environ.copy(),
+            str(tmp_path),
+            completed,
+        )
+
+    assert completed.is_set()
+
+
+def test_wait_for_pipe_result_surfaces_subprocess_failure():
+    listener_thread = threading.Thread(target=lambda: None)
+    listener_thread.start()
+    process_result = subprocess.CompletedProcess(["pytest"], returncode=2)
+
+    with pytest.raises(subprocess.CalledProcessError):
+        helpers._wait_for_pipe_result(  # noqa: SLF001
+            listener_thread, process_result, []
+        )
+
+
+def test_wait_for_pipe_result_times_out(monkeypatch):
+    monkeypatch.setattr(helpers, "PIPE_RESULT_TIMEOUT_SECONDS", 0.01)
+    release_listener = threading.Event()
+    listener_thread = threading.Thread(target=release_listener.wait, daemon=True)
+    listener_thread.start()
+    process_result = subprocess.CompletedProcess(["pytest"], returncode=0)
+
+    try:
+        with pytest.raises(TimeoutError, match="Timed out waiting"):
+            helpers._wait_for_pipe_result(  # noqa: SLF001
+                listener_thread, process_result, []
+            )
+    finally:
+        release_listener.set()
+        listener_thread.join()