diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 78b19331116d..515d84416eb9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -197,6 +197,9 @@ jobs: - name: Install test requirements run: python -m pip install --upgrade -r build/test-requirements.txt + - name: Install pytest + run: python -m pip install --upgrade pytest + - name: Run Python unit tests run: python python_files/tests/run_all.py diff --git a/.github/workflows/pr-check.yml b/.github/workflows/pr-check.yml index c58844bbf110..655e0de23ee5 100644 --- a/.github/workflows/pr-check.yml +++ b/.github/workflows/pr-check.yml @@ -156,9 +156,9 @@ jobs: # We're not running CI on macOS for now because it's one less matrix entry to lower the number of runners used, # macOS runners are expensive, and we assume that Ubuntu is enough to cover the Unix case. os: [ubuntu-latest, windows-latest] - # Run the tests on the oldest and most recent versions of Python. - python: ['3.10', '3.x', '3.13'] # run for 3 pytest versions, most recent stable, oldest version supported and pre-release - pytest-version: ['pytest', 'pytest@pre-release', 'pytest==6.2.0'] + python: ['3.10', '3.11', '3.12', '3.13', '3.14'] + # Test approximately one year of pytest releases and upcoming pytest changes. + pytest-version: ['pytest==8.4.*', 'pytest@pre-release'] steps: - name: Checkout @@ -172,18 +172,6 @@ jobs: with: python-version: ${{ matrix.python }} - - name: Install specific pytest version - if: matrix.pytest-version == 'pytest@pre-release' - run: | - python -m pip install --pre pytest - - - name: Install specific pytest version - if: matrix.pytest-version != 'pytest@pre-release' - run: | - python -m pip install "${{ matrix.pytest-version }}" - - - name: Install specific pytest version - run: python -m pytest --version - name: Install base Python requirements uses: brettcannon/pip-secure-install@92f400e3191171c1858cc0e0d9ac6320173fdb0c # v1.0.0 with: @@ -193,6 +181,17 @@ jobs: - name: Install test requirements run: python -m pip install --upgrade -r build/test-requirements.txt + - name: Install specific pytest version (pre-release) + if: matrix.pytest-version == 'pytest@pre-release' + run: python -m pip install --upgrade --pre pytest + + - name: Install specific pytest version + if: matrix.pytest-version != 'pytest@pre-release' + run: python -m pip install --upgrade "${{ matrix.pytest-version }}" + + - name: Print pytest version + run: python -m pytest --version + - name: Run Python unit tests run: python python_files/tests/run_all.py diff --git a/build/test-requirements.txt b/build/test-requirements.txt index d1acb9a33d84..f19194c2c772 100644 --- a/build/test-requirements.txt +++ b/build/test-requirements.txt @@ -7,9 +7,6 @@ pylint pycodestyle pydocstyle prospector -# pytest-black 0.6.0 uses the deprecated `path` arg in pytest_collect_file, -# which was removed in pytest 8.1. Pin to <8.1 to maintain compatibility. -pytest<8.1 flask fastapi uvicorn @@ -41,4 +38,3 @@ pytest-describe # for pytest-ruff related tests pytest-ruff -pytest-black diff --git a/python_files/tests/pytestadapter/helpers.py b/python_files/tests/pytestadapter/helpers.py index 7cc5e678bd61..86f3b127b5fb 100644 --- a/python_files/tests/pytestadapter/helpers.py +++ b/python_files/tests/pytestadapter/helpers.py @@ -12,7 +12,7 @@ import tempfile import threading import uuid -from typing import Any, Dict, List, Optional, Tuple +from typing import Any, Dict, List, Optional, Tuple, Union if sys.platform == "win32": from namedpipe import NPopen # pylint: disable=import-error # cspell: disable-line @@ -28,6 +28,8 @@ TEST_DATA_PATH = pathlib.Path(__file__).parent / ".data" CONTENT_LENGTH: str = "Content-Length:" CONTENT_TYPE: str = "Content-Type:" +PIPE_RESULT_TIMEOUT_SECONDS = 10 +TEST_SUBPROCESS_TIMEOUT_SECONDS = 300 @contextlib.contextmanager @@ -242,10 +244,36 @@ def _listen_on_pipe_new(listener, result: List[str], completed: threading.Event) result.append("".join(all_data)) -def _run_test_code(proc_args: List[str], proc_env, proc_cwd: str, completed: threading.Event): - result = subprocess.run(proc_args, env=proc_env, cwd=proc_cwd, check=False) - completed.set() - return result +def _run_test_code( + proc_args: List[str], + proc_env, + proc_cwd: Union[str, os.PathLike[str]], + completed: threading.Event, +) -> subprocess.CompletedProcess: + try: + return subprocess.run( + proc_args, + env=proc_env, + cwd=proc_cwd, + check=False, + timeout=TEST_SUBPROCESS_TIMEOUT_SECONDS, + ) + finally: + completed.set() + + +def _wait_for_pipe_result( + listener_thread: threading.Thread, + process_result: subprocess.CompletedProcess, + result: List[str], +) -> None: + listener_thread.join(timeout=PIPE_RESULT_TIMEOUT_SECONDS) + if listener_thread.is_alive(): + if process_result.returncode: + raise subprocess.CalledProcessError(process_result.returncode, process_result.args) + raise TimeoutError("Timed out waiting for the test subprocess pipe result") + if process_result.returncode and not result: + raise subprocess.CalledProcessError(process_result.returncode, process_result.args) def runner(args: List[str]) -> Optional[List[Dict[str, Any]]]: @@ -359,18 +387,12 @@ def runner_with_cwd_env( result = [] # result is a string array to store the data during threading t1: threading.Thread = threading.Thread( - target=_listen_on_pipe_new, args=(pipe, result, completed) + target=_listen_on_pipe_new, args=(pipe, result, completed), daemon=True ) t1.start() - t2 = threading.Thread( - target=_run_test_code, - args=(process_args, env, path, completed), - ) - t2.start() - - t1.join() - t2.join() + process_result = _run_test_code(process_args, env, path, completed) + _wait_for_pipe_result(t1, process_result, result) return process_data_received(result[0]) if result else None else: # Unix design @@ -397,19 +419,12 @@ def runner_with_cwd_env( result = [] # result is a string array to store the data during threading t1: threading.Thread = threading.Thread( - target=_listen_on_fifo, args=(pipe_name, result, completed) + target=_listen_on_fifo, args=(pipe_name, result, completed), daemon=True ) t1.start() - t2: threading.Thread = threading.Thread( - target=_run_test_code, - args=(process_args, env, path, completed), - ) - - t2.start() - - t1.join() - t2.join() + process_result = _run_test_code(process_args, env, path, completed) + _wait_for_pipe_result(t1, process_result, result) return process_data_received(result[0]) if result else None diff --git a/python_files/tests/pytestadapter/test_discovery.py b/python_files/tests/pytestadapter/test_discovery.py index c2bb5a039850..7e7336b87c65 100644 --- a/python_files/tests/pytestadapter/test_discovery.py +++ b/python_files/tests/pytestadapter/test_discovery.py @@ -476,10 +476,11 @@ def test_config_sub_folder(): expected_discovery_test_output.ruff_test_expected_output, "--ruff", ), - ( + pytest.param( "2496-black-formatter", expected_discovery_test_output.black_formatter_expected_output, "--black", + marks=pytest.mark.skip(reason="pytest-black does not support pytest 8.1 or newer"), ), ], ) diff --git a/python_files/tests/pytestadapter/test_helpers.py b/python_files/tests/pytestadapter/test_helpers.py new file mode 100644 index 000000000000..552c571d51f8 --- /dev/null +++ b/python_files/tests/pytestadapter/test_helpers.py @@ -0,0 +1,54 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import os +import subprocess +import sys +import threading + +import pytest + +from . import helpers + + +def test_run_test_code_times_out(tmp_path, monkeypatch): + monkeypatch.setattr(helpers, "TEST_SUBPROCESS_TIMEOUT_SECONDS", 0.01) + completed = threading.Event() + + with pytest.raises(subprocess.TimeoutExpired): + helpers._run_test_code( # noqa: SLF001 + [sys.executable, "-c", "import time; time.sleep(1)"], + os.environ.copy(), + str(tmp_path), + completed, + ) + + assert completed.is_set() + + +def test_wait_for_pipe_result_surfaces_subprocess_failure(): + listener_thread = threading.Thread(target=lambda: None) + listener_thread.start() + process_result = subprocess.CompletedProcess(["pytest"], returncode=2) + + with pytest.raises(subprocess.CalledProcessError): + helpers._wait_for_pipe_result( # noqa: SLF001 + listener_thread, process_result, [] + ) + + +def test_wait_for_pipe_result_times_out(monkeypatch): + monkeypatch.setattr(helpers, "PIPE_RESULT_TIMEOUT_SECONDS", 0.01) + release_listener = threading.Event() + listener_thread = threading.Thread(target=release_listener.wait, daemon=True) + listener_thread.start() + process_result = subprocess.CompletedProcess(["pytest"], returncode=0) + + try: + with pytest.raises(TimeoutError, match="Timed out waiting"): + helpers._wait_for_pipe_result( # noqa: SLF001 + listener_thread, process_result, [] + ) + finally: + release_listener.set() + listener_thread.join()