diff --git a/sdk/ml/azure-ai-ml/tests/component/e2etests/test_component.py b/sdk/ml/azure-ai-ml/tests/component/e2etests/test_component.py index c9b96e292eb1..75b383f007a0 100644 --- a/sdk/ml/azure-ai-ml/tests/component/e2etests/test_component.py +++ b/sdk/ml/azure-ai-ml/tests/component/e2etests/test_component.py @@ -1223,7 +1223,6 @@ def test_load_component_from_flow(self, client: MLClient, randstr, params_overri assert component._get_origin_code_value() == created_component._get_origin_code_value() - @pytest.mark.live_test_only("Needs re-recording to work with new test proxy sanitizers") def test_load_component_from_flow_in_registry(self, registry_client: MLClient, randstr): target_path: str = "./tests/test_configs/flows/runs/with_environment.yml" component = load_component( diff --git a/sdk/ml/azure-ai-ml/tests/connection/e2etests/test_connections.py b/sdk/ml/azure-ai-ml/tests/connection/e2etests/test_connections.py index 294fc52047f6..54fde4480bba 100644 --- a/sdk/ml/azure-ai-ml/tests/connection/e2etests/test_connections.py +++ b/sdk/ml/azure-ai-ml/tests/connection/e2etests/test_connections.py @@ -40,7 +40,6 @@ @pytest.mark.core_sdk_test @pytest.mark.usefixtures("recorded_test") class TestWorkspaceConnections(AzureRecordedTestCase): - @pytest.mark.live_test_only("Needs re-recording to work with new common sanitizers") def test_secret_population( self, client: MLClient, @@ -122,7 +121,6 @@ def test_workspace_connections_create_update_and_delete_git_pat( with pytest.raises(Exception): client.connections.get(name=wps_connection_name) - @pytest.mark.live_test_only("Needs re-recording to work with new common sanitizers") def test_workspace_connections_create_update_and_delete_snowflake_user_pwd( self, client: MLClient, @@ -898,7 +896,6 @@ def test_python_feed_crud( with pytest.raises(Exception): client.connections.get(name=wps_connection_name) - @pytest.mark.live_test_only("Needs re-recording to work with new common sanitizers") def test_container_registry_managed_id_crud( self, client: MLClient, diff --git a/sdk/ml/azure-ai-ml/tests/datastore/e2etests/test_datastore.py b/sdk/ml/azure-ai-ml/tests/datastore/e2etests/test_datastore.py index 80f78c5fbf5f..812d92f9aaf0 100644 --- a/sdk/ml/azure-ai-ml/tests/datastore/e2etests/test_datastore.py +++ b/sdk/ml/azure-ai-ml/tests/datastore/e2etests/test_datastore.py @@ -79,7 +79,6 @@ def test_hdfs_keytab( with pytest.raises(Exception): client.datastores.get(random_name) - @pytest.mark.live_test_only("Needs re-recording to work with new common sanitizers") def test_blob_store( self, client: MLClient, @@ -106,7 +105,6 @@ def test_blob_store( with pytest.raises(Exception): client.datastores.get(random_name) - @pytest.mark.live_test_only("Needs re-recording to work with new common sanitizers") def test_blob_store_credential_less( self, client: MLClient, @@ -129,7 +127,6 @@ def test_blob_store_credential_less( with pytest.raises(Exception): client.datastores.get(random_name) - @pytest.mark.live_test_only("Needs re-recording to work with new common sanitizers") def test_file_store( self, client: MLClient, @@ -217,7 +214,6 @@ def test_adls_gen2_store( with pytest.raises(Exception): client.datastores.get(random_name) - @pytest.mark.live_test_only("Needs re-recording to work with new common sanitizers") def test_credential_less_adls_gen2_store( self, client: MLClient, diff --git a/sdk/ml/azure-ai-ml/tests/environment/e2etests/test_environment.py b/sdk/ml/azure-ai-ml/tests/environment/e2etests/test_environment.py index 011db149a929..5059c2903069 100644 --- a/sdk/ml/azure-ai-ml/tests/environment/e2etests/test_environment.py +++ b/sdk/ml/azure-ai-ml/tests/environment/e2etests/test_environment.py @@ -89,7 +89,6 @@ def test_environment_create_or_update_docker(self, client: MLClient, env_name: C assert env_dump["id"] == ARM_ID_PREFIX + environment_id assert env_dump["image"] == environment.image - @pytest.mark.live_test_only("Needs re-recording to work with new test proxy sanitizers") def test_environment_create_or_update_docker_context( self, client: MLClient, env_name: Callable[[str], str] ) -> None: @@ -218,7 +217,6 @@ def test_environment_get_latest_label(self, client: MLClient, randstr: Callable[ sleep_if_live(2) assert client.environments.get(name, label="latest").version == version - @pytest.mark.live_test_only("Needs re-recording to work with new test proxy sanitizers") def test_registry_environment_create_conda_and_get( self, only_registry_client: MLClient, env_name: Callable[[str], str] ) -> None: diff --git a/sdk/ml/azure-ai-ml/tests/internal/e2etests/test_pipeline_job.py b/sdk/ml/azure-ai-ml/tests/internal/e2etests/test_pipeline_job.py index 06cc1091bb62..0408de1f674c 100644 --- a/sdk/ml/azure-ai-ml/tests/internal/e2etests/test_pipeline_job.py +++ b/sdk/ml/azure-ai-ml/tests/internal/e2etests/test_pipeline_job.py @@ -287,7 +287,6 @@ def pipeline_with_command_components(tsv_file, content): assert_job_cancel(pipeline_job, client, experiment_name="v15_v2_interop") - @pytest.mark.live_test_only("Needs re-recording to work with new common sanitizers") def test_pipeline_with_setting_node_output_mode(self, client: MLClient): # get dataset training_data = Input(type=AssetTypes.URI_FILE, path="https://dprepdata.blob.core.windows.net/demo/Titanic.csv") diff --git a/sdk/ml/azure-ai-ml/tests/test_e2e_recording_marker_audit.py b/sdk/ml/azure-ai-ml/tests/test_e2e_recording_marker_audit.py new file mode 100644 index 000000000000..0862418072d3 --- /dev/null +++ b/sdk/ml/azure-ai-ml/tests/test_e2e_recording_marker_audit.py @@ -0,0 +1,53 @@ +import ast +from pathlib import Path + + +LIVE_ONLY_RECORDING_REASONS = ( + "needs re-recording", + "new common sanitizers", + "new test proxy sanitizers", +) + + +def _marker_name(decorator: ast.expr) -> str: + if isinstance(decorator, ast.Call): + decorator = decorator.func + parts = [] + while isinstance(decorator, ast.Attribute): + parts.append(decorator.attr) + decorator = decorator.value + if isinstance(decorator, ast.Name): + parts.append(decorator.id) + return ".".join(reversed(parts)) + + +def _marker_reason(decorator: ast.expr) -> str: + if isinstance(decorator, ast.Call): + if decorator.args and isinstance(decorator.args[0], ast.Constant) and isinstance(decorator.args[0].value, str): + return decorator.args[0].value + for keyword in decorator.keywords: + if keyword.arg == "reason" and isinstance(keyword.value, ast.Constant) and isinstance(keyword.value.value, str): + return keyword.value.value + return "" + + +def test_e2e_recording_staleness_markers_are_removed() -> None: + tests_root = Path(__file__).parent + offenders = [] + + for path in tests_root.glob("**/e2etests/*.py"): + module = ast.parse(path.read_text(encoding="utf-8")) + for node in ast.walk(module): + if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + continue + for decorator in node.decorator_list: + if _marker_name(decorator) != "pytest.mark.live_test_only": + continue + reason = _marker_reason(decorator).lower() + if any(marker in reason for marker in LIVE_ONLY_RECORDING_REASONS): + offenders.append(f"{path.relative_to(tests_root)}::{node.name}") + + assert offenders == [], ( + "Playback-gating tests must not stay live-only solely because recordings need re-recording or sanitizer updates: " + + ", ".join(offenders) + )