Fix importlib mode shadowing stdlib modules from same-named test dirs (#12303)

AUTHORS

@@ -170,6 +170,7 @@ Fabien Zarifian
 Fabio Zadrozny
 Farbod Ahmadian
 faph
+Fazeel Usmani
 Felix Hofstätter
 Felix Nieuwenhuizen
 Feng Ma

changelog/12303.bugfix.rst

@@ -0,0 +1,7 @@
+Fixed ``--import-mode=importlib`` shadowing stdlib and installed packages when test directories share their name (e.g. ``test/``).
+
+Test modules whose top-level directory collides with an external package are now
+registered in ``sys.modules`` under a ``_pytest_shadow_<name>`` prefix (e.g.
+``_pytest_shadow_test.test_demo`` instead of ``test.test_demo``).  This is an
+internal detail and should not affect test behaviour, but it will appear in
+tracebacks and ``__name__``.

src/_pytest/pathlib.py

@@ -11,6 +11,7 @@
 from errno import ENOENT
 from errno import ENOTDIR
 import fnmatch
+from functools import cache
 from functools import partial
 from importlib.machinery import ModuleSpec
 from importlib.machinery import PathFinder
@@ -542,19 +543,29 @@ def import_path(
         except CouldNotResolvePathError:
             pass
         else:
-            # If the given module name is already in sys.modules, do not import it again.
-            with contextlib.suppress(KeyError):
-                return sys.modules[module_name]
-
-            mod = _import_module_using_spec(
-                module_name, path, pkg_root, insert_modules=False
-            )
-            if mod is not None:
-                return mod
+            # Skip dotted names that would shadow stdlib/installed packages (#12303).
+            if "." not in module_name or not _top_level_shadows_external(
+                module_name, pkg_root
+            ):
+                # If the given module name is already in sys.modules, do not import it again.
+                with contextlib.suppress(KeyError):
+                    return sys.modules[module_name]
+
+                mod = _import_module_using_spec(
+                    module_name, path, pkg_root, insert_modules=False
+                )
+                if mod is not None:
+                    return mod
 
         # Could not import the module with the current sys.path, so we fall back
         # to importing the file as a single module, not being a part of a package.
         module_name = module_name_from_path(path, root)
+
+        # Prefix to avoid shadowing stdlib/installed packages (#12303).
+        if "." in module_name and _top_level_shadows_external(module_name, root):
+            top, _, rest = module_name.partition(".")
+            module_name = f"_pytest_shadow_{top}.{rest}"
+
         with contextlib.suppress(KeyError):
             return sys.modules[module_name]
 
@@ -754,6 +765,112 @@ def spec_matches_module_path(module_spec: ModuleSpec | None, module_path: Path)
     return False
 
 
+def _top_level_shadows_external(module_name: str, local_root: Path) -> bool:
+    """Return True if the top-level component of *module_name* would collide
+    with a stdlib or installed package that lives outside *local_root*.
+    See #12303."""
+    top = module_name.partition(".")[0]
+    return _top_shadows_external_cached(top, local_root)
+
+
+@cache
+def _top_shadows_external_cached(top: str, local_root: Path) -> bool:
+    """Cached core of :func:`_top_level_shadows_external`.
+
+    Keyed on the top-level name and root so that every test file under the
+    same directory reuses a single ``find_spec`` + ``iterdir`` result."""
+    try:
+        with warnings.catch_warnings():
+            warnings.simplefilter("ignore", ImportWarning)
+            existing_spec = importlib.util.find_spec(top)
+    except (ImportError, ValueError, AttributeError):
+        return False
+    if existing_spec is None:
+        return False
+
+    # Built-in or frozen modules are always external.
+    if existing_spec.origin in ("built-in", "frozen"):
+        return True
+
+    # Also pick up dirs whose name normalizes to `top` (e.g. ".tests" → "_tests").
+    # Resolve everything so symlinks like /var → /private/var don't trip us up.
+    local_candidates: list[Path] = [local_root / top]
+    try:
+        for child in local_root.iterdir():
+            if (
+                child.is_dir()
+                and child.name.replace(".", "_") == top
+                and child not in local_candidates
+            ):
+                local_candidates.append(child)
+    except OSError:
+        pass
+    resolved_candidates = [c.resolve() for c in local_candidates if c.exists()]
+
+    def _is_local(p: Path) -> bool:
+        rp = p.resolve()
+        for candidate in resolved_candidates:
+            try:
+                rp.relative_to(candidate)
+                return True
+            except ValueError:
+                pass
+        return False
+
+    # Check whether the spec found by find_spec points into local_root.
+    spec_is_local = False
+    if existing_spec.origin is not None:
+        if _is_local(Path(existing_spec.origin)):
+            spec_is_local = True
+    if not spec_is_local and existing_spec.submodule_search_locations:
+        for loc in existing_spec.submodule_search_locations:
+            if _is_local(Path(loc)):
+                spec_is_local = True
+                break
+
+    if not spec_is_local:
+        # The module found by find_spec lives outside local_root → shadow.
+        return True
+
+    # find_spec returned the local package (e.g. because the project root is
+    # reachable via '' or an explicit sys.path entry).  An external module
+    # with the same name may still exist behind it.  Search only non-local
+    # sys.path entries via PathFinder so we never modify sys.path.
+    local_root_resolved = local_root.resolve()
+
+    def _is_local_path_entry(entry: str) -> bool:
+        p = Path.cwd() if entry == "" else Path(entry)
+        try:
+            p.resolve().relative_to(local_root_resolved)
+            return True
+        except ValueError:
+            return False
+
+    non_local = [p for p in sys.path if not _is_local_path_entry(p)]
+    try:
+        behind = PathFinder.find_spec(top, path=non_local)
+    except (ImportError, ValueError, AttributeError):
+        behind = None
+
+    if behind is None:
+        return False
+
+    # Guard against namespace packages that span multiple project directories
+    # (e.g. dist1/com + dist2/com).  If the "behind" spec's locations are all
+    # already present in the original spec, it is the same package — not a
+    # genuinely external shadow.
+    def _spec_locations(spec: ModuleSpec) -> set[str]:
+        locs: set[str] = set()
+        if spec.origin is not None:
+            locs.add(str(Path(spec.origin).resolve()))
+        if spec.submodule_search_locations:
+            for loc in spec.submodule_search_locations:
+                locs.add(str(Path(loc).resolve()))
+        return locs
+
+    return bool(_spec_locations(behind) - _spec_locations(existing_spec))
+
+
 # Implement a special _is_same function on Windows which returns True if the two filenames
 # compare equal, to circumvent os.path.samefile returning False for mounts in UNC (#7678).
 if sys.platform.startswith("win"):
@@ -817,8 +934,10 @@ def insert_missing_modules(modules: dict[str, ModuleType], module_name: str) ->
                     # a warning and raise ModuleNotFoundError. To avoid the
                     # warning, we check sys.meta_path explicitly and raise the error
                     # ourselves to fall back to creating a dummy module.
-                    if not sys.meta_path:
+                    if not sys.meta_path:  # pragma: no cover
                         raise ModuleNotFoundError
+                    # May import an unrelated module on name collision;
+                    # callers use the _pytest_shadow_ prefix to avoid this (#12303).
                     parent_module = importlib.import_module(parent_module_name)
                 except ModuleNotFoundError:
                     parent_module = ModuleType(