Fix importlib mode shadowing an installed package with a namespace package (#13257)

AUTHORS

@@ -498,6 +498,7 @@ Vidar T. Fauske
 Vijay Arora
 Virendra Patil
 Virgil Dupras
+Vismay Tiwari
 Vitaly Lashmanov
 Vivaan Verma
 Vlad Dragos

changelog/13257.bugfix.rst

@@ -0,0 +1 @@
+Fixed a regression in ``--import-mode=importlib`` where a test file whose rootdir-relative path reused the name of an installed package (for example ``mypkg/tests/test_foo.py`` next to the real ``mypkg/mypkg`` package) would shadow that package in ``sys.modules`` with a namespace package, breaking imports such as ``from mypkg import version`` in the tests.

src/_pytest/pathlib.py

@@ -661,6 +661,30 @@ def _import_module_using_spec(
             parent_module_name:         "a.b"
             parent_module_path:         Path("a/b/")
     """
+    # If ``module_path`` is a directory (a namespace-package candidate) but a real,
+    # importable package with the same fully-qualified name already lives *inside* it,
+    # importing it as a namespace package here would shadow the real one in
+    # ``sys.modules`` and break imports from it in the test modules -- for example an
+    # installed ``mypkg`` (at ``mypkg/mypkg/``) while the rootdir-relative path of the
+    # tests is ``mypkg/tests/...`` (#13257). Defer to the real package in that case.
+    #
+    # The "inside this directory" check is important: a test directory named after a
+    # standard-library module (``math``, ``time``, ...) must still be shadowed by a
+    # namespace package, since the real module lives elsewhere (see
+    # ``test_importlib_same_name_as_stl``).
+    if module_path.is_dir():
+        try:
+            existing_spec = importlib.util.find_spec(module_name)
+        except (ImportError, ValueError, ImportWarning):
+            existing_spec = None
+        if (
+            existing_spec is not None
+            and existing_spec.origin is not None
+            and not spec_matches_module_path(existing_spec, module_path)
+            and module_path in Path(existing_spec.origin).parents
+        ):
+            return importlib.import_module(module_name)
+
     # Attempt to import the parent module, seems is our responsibility:
     # https://github.com/python/cpython/blob/73906d5c908c1e0b73c5436faeff7d93698fc074/Lib/importlib/_bootstrap.py#L1308-L1311
     parent_module_name, _, name = module_name.rpartition(".")

testing/test_pathlib.py

@@ -1696,6 +1696,39 @@ def test_ns_import_same_name_directory_12592(
     assert result.ret == ExitCode.OK
 
 
+def test_ns_import_shadowed_by_installed_package_13257(
+    pytester: Pytester, monkeypatch: MonkeyPatch
+) -> None:
+    """Regression for #13257: with ``--import-mode=importlib``, a test path that reuses
+    the name of an installed package (``mypkg/tests/...`` next to the real ``mypkg/mypkg``
+    package) must not shadow the real package with a namespace package in ``sys.modules``.
+    """
+    pkg_root = pytester.path / "mypkg"
+    (pkg_root / "mypkg").mkdir(parents=True)
+    (pkg_root / "mypkg" / "__init__.py").write_text(
+        "version = '1.2.3'\n", encoding="UTF-8"
+    )
+    (pkg_root / "tests").mkdir()
+    (pkg_root / "tests" / "version_test.py").write_text(
+        dedent(
+            """
+            from mypkg import version
+
+
+            def test_version():
+                assert version == "1.2.3"
+            """
+        ),
+        encoding="UTF-8",
+    )
+    # Make `mypkg` importable as the real package, as a regular/editable install would.
+    monkeypatch.syspath_prepend(str(pkg_root))
+
+    result = pytester.runpytest("--import-mode=importlib", "mypkg/tests")
+    assert result.ret == ExitCode.OK
+    result.assert_outcomes(passed=1)
+
+
 def test_is_importable(pytester: Pytester) -> None:
     pytester.syspathinsert()