Merge branch 'main' into fix-149738

Author: sepehr-rs

Doc/library/site.rst

@@ -356,7 +356,79 @@ Module contents
       This function used to be called unconditionally.
 
 
-.. function:: addsitedir(sitedir, known_paths=None, *, defer_processing_start_files=False)
+.. function:: makepath(*paths)
+
+   Join *paths* with :func:`os.path.join`, attempt to make the result
+   absolute with :func:`os.path.abspath`, and return a 2-tuple containing
+   the absolute path and its case-normalized form as produced by
+   :func:`os.path.normcase`.  If :func:`os.path.abspath` raises
+   :exc:`OSError`, the joined path is used unchanged for the
+   case-normalization step.
+
+   The second element of the returned tuple is the form used throughout the
+   :mod:`!site` module to compare paths on case-insensitive file systems, and
+   is what populates the ``known_paths`` sets that prevent duplicate
+   :data:`sys.path` entries in various APIs within this module.
+
+
+.. class:: StartupState(known_paths=None)
+
+   Instances of this class accumulate interpreter startup configuration data
+   from one or more site directories.  They are the preferred interface for
+   batching the processing of :file:`.pth` and :file:`.start` files across
+   multiple site directories, so that every :data:`sys.path` extension is
+   visible before any startup code runs.
+
+   The optional *known_paths* argument is a set of case-normalized paths
+   (which can be produced by :func:`makepath`) used to prevent duplicate
+   :data:`sys.path` entries.  When ``None`` (the default), the set is built
+   from the current :data:`sys.path`.  :func:`main` implicitly uses an
+   instance of this class.
+
+   Typical use:
+
+   .. code-block:: python
+
+      state = site.StartupState()
+      for sitedir in site_dirs:
+          state.addsitedir(sitedir)
+      state.process()
+
+   .. versionadded:: 3.15
+
+   .. method:: addsitedir(sitedir)
+
+      Read the :file:`.pth` and :file:`.start` files in *sitedir* and
+      record their :data:`sys.path` extensions, deprecated :file:`.pth`
+      ``import`` lines, and :file:`.start` entry points on this state.
+      The recorded data is not applied until :meth:`process` is called.
+
+   .. method:: addusersitepackages()
+
+      Add the per-user site-packages directory, if enabled and if it exists.
+      The directory's startup data is accumulated for later processing by
+      :meth:`process`.
+
+   .. method:: addsitepackages(prefixes=None)
+
+      Add global site-packages directories, computed from *prefixes* or from
+      the global :data:`PREFIXES` when *prefixes* is ``None``.  Each
+      directory's startup data is accumulated for later processing by
+      :meth:`process`.
+
+   .. method:: process()
+
+      Apply the accumulated state by first adding the path extensions to
+      :data:`sys.path`, then executing the :file:`.start` file entry points
+      and :file:`.pth` file ``import`` lines (:ref:`deprecated
+      <site-pth-files>`).
+
+      This method is not idempotent and must not be called more than once
+      on the same instance.  Doing so will apply the accumulated state
+      more than once, re-running entry points and ``import`` lines.
+
+
+.. function:: addsitedir(sitedir, known_paths=None)
 
    Add a directory to sys.path and parse the :file:`.pth` and :file:`.start`
    files found in that directory.  Typically used in :mod:`sitecustomize` or
@@ -366,17 +438,15 @@ Module contents
    used to prevent duplicate :data:`sys.path` entries.  When ``None`` (the
    default), the set is built from the current :data:`sys.path`.
 
-   While :file:`.pth` and :file:`.start` files are always parsed, set
-   *defer_processing_start_files* to ``True`` to prevent processing the
-   startup data found in those files, so that you can process them explicitly
-   (this is typically used by the :func:`main` function).
+   For batched processing across multiple site directories, build a
+   :class:`StartupState` explicitly and call :meth:`StartupState.addsitedir`
+   on it; that defers :file:`.pth` and :file:`.start` processing until a
+   single :meth:`StartupState.process` call, ensuring every :data:`sys.path`
+   extension is visible before any startup code runs.
 
    .. versionchanged:: 3.15
 
       Also processes :file:`.start` files.  See :ref:`site-start-files`.
-      All :file:`.pth` and :file:`.start` files are now read and
-      accumulated before any path extensions, ``import`` line execution,
-      or entry point invocations take place.
 
 
 .. function:: getsitepackages()
@@ -447,4 +517,3 @@ value greater than 2 if there is an error.
    * :pep:`370` -- Per user site-packages directory
    * :pep:`829` -- Startup entry points and the deprecation of import lines in ``.pth`` files
    * :ref:`sys-path-init` -- The initialization of :data:`sys.path`.
-

Doc/reference/import.rst

@@ -122,6 +122,12 @@ Importing ``parent.one`` will implicitly execute ``parent/__init__.py`` and
 ``parent.three`` will execute ``parent/two/__init__.py`` and
 ``parent/three/__init__.py`` respectively.
 
+A subdirectory inside a regular package that does not contain an
+``__init__.py`` file is treated as an implicit
+:ref:`namespace package <reference-namespace-package>` (a "namespace
+subpackage") rooted in that parent.  See :pep:`420` for the underlying
+specification.
+
 
 .. _reference-namespace-package:
 
@@ -153,6 +159,12 @@ physically located next to ``parent/two``.  In this case, Python will create a
 namespace package for the top-level ``parent`` package whenever it or one of
 its subpackages is imported.
 
+Namespace packages may also be nested inside a regular package.  When the
+import system searches a regular package's ``__path__`` and encounters a
+subdirectory that does not contain an ``__init__.py`` file, that
+subdirectory becomes a :term:`portion` contributing to a namespace
+subpackage of the enclosing regular package.
+
 See also :pep:`420` for the namespace package specification.
 
 

Doc/whatsnew/3.15.rst

@@ -478,7 +478,18 @@ matching :file:`.start` file is found, ``import`` lines in :file:`.pth` files
 are ignored.  There is no change to :data:`sys.path` extension lines in
 :file:`.pth` files.
 
-(Contributed by Barry Warsaw in :gh:`148641`.)
+The :mod:`site` module also provides :class:`site.StartupState` to batch
+startup processing for multiple site directories, ensuring all static path
+extensions are applied before any startup code is executed.  :func:`site.main`
+uses an instance of this class implicitly to batch process all startup
+configuration files during normal interpreter startup.  Callers needing the
+same batching behavior can build a :class:`~site.StartupState` directly and
+drive it with :meth:`~site.StartupState.addsitedir`,
+:meth:`~site.StartupState.addusersitepackages`, and
+:meth:`~site.StartupState.addsitepackages`, then call
+:meth:`~site.StartupState.process` once at the end of the batch.
+
+(Contributed by Barry Warsaw in :gh:`148641` and :gh:`150228`.)
 
 
 .. _whatsnew315-abi3t:

Include/cpython/pystats.h

@@ -163,6 +163,7 @@ typedef struct _optimization_stats {
     uint64_t jit_code_size;
     uint64_t jit_trampoline_size;
     uint64_t jit_data_size;
+    uint64_t jit_got_size;
     uint64_t jit_padding_size;
     uint64_t jit_freed_memory_size;
     uint64_t trace_total_memory_hist[_Py_UOP_HIST_SIZE];

Include/internal/pycore_interpframe.h

@@ -230,7 +230,9 @@ _PyFrame_GetLocalsArray(_PyInterpreterFrame *frame)
 static inline _PyStackRef*
 _PyFrame_GetStackPointer(_PyInterpreterFrame *frame)
 {
+#ifndef _Py_JIT
     assert(frame->stackpointer != NULL);
+#endif
     _PyStackRef *sp = frame->stackpointer;
 #ifndef NDEBUG
     frame->stackpointer = NULL;
@@ -241,7 +243,10 @@ _PyFrame_GetStackPointer(_PyInterpreterFrame *frame)
 static inline void
 _PyFrame_SetStackPointer(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer)
 {
+/* Avoid bloating the JIT code */
+#ifndef _Py_JIT
     assert(frame->stackpointer == NULL);
+#endif
     frame->stackpointer = stack_pointer;
 }
 

Include/internal/pycore_jit.h

@@ -31,6 +31,8 @@ _Py_CODEUNIT *_PyJIT_Entry(
 int _PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction *trace, size_t length);
 void _PyJIT_Free(_PyExecutorObject *executor);
 PyAPI_FUNC(int) _PyJIT_AddressInJitCode(PyInterpreterState *interp, uintptr_t addr);
+PyAPI_FUNC(void) _Py_jit_assert_within_stack_bounds(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, int lineno);
+PyAPI_FUNC(int) _Py_jit_assertion_failure(int line);
 
 #endif  // _Py_JIT
 

Include/internal/pycore_uop.h

@@ -38,12 +38,7 @@ typedef struct _PyUOpInstruction{
 
 // Fitness is the target length of the trace we translate initially. The uop
 // buffer has a small amount of extra space for entry/loop-closing overhead.
-#if defined(Py_DEBUG) && defined(_Py_JIT)
-    // With asserts, the stencils are a lot larger
-#define FITNESS_INITIAL 1000
-#else
 #define FITNESS_INITIAL 2500
-#endif
 
 #define UOP_TRACE_BUFFER_OVERHEAD 10
 #define UOP_MAX_TRACE_LENGTH (FITNESS_INITIAL + UOP_TRACE_BUFFER_OVERHEAD)

Lib/dis.py

@@ -84,7 +84,7 @@ def _try_compile(source, name):
     return compile(source, name, 'exec')
 
 def dis(x=None, *, file=None, depth=None, show_caches=False, adaptive=False,
-        show_offsets=False, show_positions=False):
+        show_offsets=False, show_positions=False, show_jit=False):
     """Disassemble classes, methods, functions, and other compiled objects.
 
     With no argument, disassemble the last traceback.
@@ -95,7 +95,8 @@ def dis(x=None, *, file=None, depth=None, show_caches=False, adaptive=False,
     """
     if x is None:
         distb(file=file, show_caches=show_caches, adaptive=adaptive,
-              show_offsets=show_offsets, show_positions=show_positions)
+              show_offsets=show_offsets, show_positions=show_positions,
+              show_jit=show_jit)
         return
     # Extract functions from methods.
     if hasattr(x, '__func__'):
@@ -116,12 +117,14 @@ def dis(x=None, *, file=None, depth=None, show_caches=False, adaptive=False,
             if isinstance(x1, _have_code):
                 print("Disassembly of %s:" % name, file=file)
                 try:
-                    dis(x1, file=file, depth=depth, show_caches=show_caches, adaptive=adaptive, show_offsets=show_offsets, show_positions=show_positions)
+                    dis(x1, file=file, depth=depth, show_caches=show_caches, adaptive=adaptive,
+                        show_offsets=show_offsets, show_positions=show_positions, show_jit=show_jit)
                 except TypeError as msg:
                     print("Sorry:", msg, file=file)
                 print(file=file)
     elif hasattr(x, 'co_code'): # Code object
-        _disassemble_recursive(x, file=file, depth=depth, show_caches=show_caches, adaptive=adaptive, show_offsets=show_offsets, show_positions=show_positions)
+        _disassemble_recursive(x, file=file, depth=depth, show_caches=show_caches, adaptive=adaptive,
+                               show_offsets=show_offsets, show_positions=show_positions, show_jit=show_jit)
     elif isinstance(x, (bytes, bytearray)): # Raw bytecode
         labels_map = _make_labels_map(x)
         label_width = 4 + len(str(len(labels_map)))
@@ -132,12 +135,13 @@ def dis(x=None, *, file=None, depth=None, show_caches=False, adaptive=False,
         arg_resolver = ArgResolver(labels_map=labels_map)
         _disassemble_bytes(x, arg_resolver=arg_resolver, formatter=formatter)
     elif isinstance(x, str):    # Source code
-        _disassemble_str(x, file=file, depth=depth, show_caches=show_caches, adaptive=adaptive, show_offsets=show_offsets, show_positions=show_positions)
+        _disassemble_str(x, file=file, depth=depth, show_caches=show_caches, adaptive=adaptive,
+                         show_offsets=show_offsets, show_positions=show_positions, show_jit=show_jit)
     else:
         raise TypeError("don't know how to disassemble %s objects" %
                         type(x).__name__)
 
-def distb(tb=None, *, file=None, show_caches=False, adaptive=False, show_offsets=False, show_positions=False):
+def distb(tb=None, *, file=None, show_caches=False, adaptive=False, show_offsets=False, show_positions=False, show_jit=False):
     """Disassemble a traceback (default: last traceback)."""
     if tb is None:
         try:
@@ -148,7 +152,8 @@ def distb(tb=None, *, file=None, show_caches=False, adaptive=False, show_offsets
         except AttributeError:
             raise RuntimeError("no last traceback to disassemble") from None
         while tb.tb_next: tb = tb.tb_next
-    disassemble(tb.tb_frame.f_code, tb.tb_lasti, file=file, show_caches=show_caches, adaptive=adaptive, show_offsets=show_offsets, show_positions=show_positions)
+    disassemble(tb.tb_frame.f_code, tb.tb_lasti, file=file, show_caches=show_caches, adaptive=adaptive,
+                show_offsets=show_offsets, show_positions=show_positions, show_jit=show_jit)
 
 # The inspect module interrogates this dictionary to build its
 # list of CO_* constants. It is also used by pretty_flags to
@@ -216,14 +221,14 @@ def _deoptop(op):
     name = _all_opname[op]
     return _all_opmap[deoptmap[name]] if name in deoptmap else op
 
-def _get_code_array(co, adaptive):
+def _get_code_array(co, adaptive, show_jit):
     if adaptive:
         code = co._co_code_adaptive
         res = []
         found = False
         for i in range(0, len(code), 2):
             op, arg = code[i], code[i+1]
-            if op == ENTER_EXECUTOR:
+            if op == ENTER_EXECUTOR and not show_jit:
                 try:
                     ex = get_executor(co, i)
                 except (ValueError, RuntimeError):
@@ -656,7 +661,7 @@ def get_argval_argrepr(self, op, arg, offset):
                 argrepr = 'not in' if argval else 'in'
         return argval, argrepr
 
-def get_instructions(x, *, first_line=None, show_caches=None, adaptive=False):
+def get_instructions(x, *, first_line=None, show_caches=None, adaptive=False, show_jit=False):
     """Iterator for the opcodes in methods, functions or code
 
     Generates a series of Instruction named tuples giving the details of
@@ -679,7 +684,7 @@ def get_instructions(x, *, first_line=None, show_caches=None, adaptive=False):
                                names=co.co_names,
                                varname_from_oparg=co._varname_from_oparg,
                                labels_map=_make_labels_map(original_code))
-    return _get_instructions_bytes(_get_code_array(co, adaptive),
+    return _get_instructions_bytes(_get_code_array(co, adaptive, show_jit),
                                    linestarts=linestarts,
                                    line_offset=line_offset,
                                    co_positions=co.co_positions(),
@@ -792,6 +797,8 @@ def _get_instructions_bytes(code, linestarts=None, line_offset=0, co_positions=N
         positions = Positions(*next(co_positions, ()))
         deop = _deoptop(op)
         op = code[offset]
+        if op == ENTER_EXECUTOR:
+            arg = code[offset+1]
 
         if arg_resolver:
             argval, argrepr = arg_resolver.get_argval_argrepr(op, arg, offset)
@@ -820,7 +827,7 @@ def _get_instructions_bytes(code, linestarts=None, line_offset=0, co_positions=N
 
 
 def disassemble(co, lasti=-1, *, file=None, show_caches=False, adaptive=False,
-                show_offsets=False, show_positions=False):
+                show_offsets=False, show_positions=False, show_jit=False):
     """Disassemble a code object."""
     linestarts = dict(findlinestarts(co))
     exception_entries = _parse_exception_table(co)
@@ -840,12 +847,12 @@ def disassemble(co, lasti=-1, *, file=None, show_caches=False, adaptive=False,
                                names=co.co_names,
                                varname_from_oparg=co._varname_from_oparg,
                                labels_map=labels_map)
-    _disassemble_bytes(_get_code_array(co, adaptive), lasti, linestarts,
+    _disassemble_bytes(_get_code_array(co, adaptive, show_jit), lasti, linestarts,
                        exception_entries=exception_entries, co_positions=co.co_positions(),
                        original_code=co.co_code, arg_resolver=arg_resolver, formatter=formatter)
 
-def _disassemble_recursive(co, *, file=None, depth=None, show_caches=False, adaptive=False, show_offsets=False, show_positions=False):
-    disassemble(co, file=file, show_caches=show_caches, adaptive=adaptive, show_offsets=show_offsets, show_positions=show_positions)
+def _disassemble_recursive(co, *, file=None, depth=None, show_caches=False, adaptive=False, show_offsets=False, show_positions=False, show_jit=False):
+    disassemble(co, file=file, show_caches=show_caches, adaptive=adaptive, show_offsets=show_offsets, show_positions=show_positions, show_jit=show_jit)
     if depth is None or depth > 0:
         if depth is not None:
             depth = depth - 1
@@ -855,7 +862,8 @@ def _disassemble_recursive(co, *, file=None, depth=None, show_caches=False, adap
                 print("Disassembly of %r:" % (x,), file=file)
                 _disassemble_recursive(
                     x, file=file, depth=depth, show_caches=show_caches,
-                    adaptive=adaptive, show_offsets=show_offsets, show_positions=show_positions
+                    adaptive=adaptive, show_offsets=show_offsets,
+                    show_positions=show_positions, show_jit=show_jit
                 )
 
 
@@ -1054,7 +1062,7 @@ class Bytecode:
 
     Iterating over this yields the bytecode operations as Instruction instances.
     """
-    def __init__(self, x, *, first_line=None, current_offset=None, show_caches=False, adaptive=False, show_offsets=False, show_positions=False):
+    def __init__(self, x, *, first_line=None, current_offset=None, show_caches=False, adaptive=False, show_offsets=False, show_positions=False, show_jit=False):
         self.codeobj = co = _get_code_object(x)
         if first_line is None:
             self.first_line = co.co_firstlineno
@@ -1070,6 +1078,7 @@ def __init__(self, x, *, first_line=None, current_offset=None, show_caches=False
         self.adaptive = adaptive
         self.show_offsets = show_offsets
         self.show_positions = show_positions
+        self.show_jit = show_jit
 
     def __iter__(self):
         co = self.codeobj
@@ -1079,7 +1088,7 @@ def __iter__(self):
                                    names=co.co_names,
                                    varname_from_oparg=co._varname_from_oparg,
                                    labels_map=labels_map)
-        return _get_instructions_bytes(_get_code_array(co, self.adaptive),
+        return _get_instructions_bytes(_get_code_array(co, self.adaptive, self.show_jit),
                                        linestarts=self._linestarts,
                                        line_offset=self._line_offset,
                                        co_positions=co.co_positions(),
@@ -1111,7 +1120,7 @@ def dis(self):
         else:
             offset = -1
         with io.StringIO() as output:
-            code = _get_code_array(co, self.adaptive)
+            code = _get_code_array(co, self.adaptive, self.show_jit)
             offset_width = len(str(max(len(code) - 2, 9999))) if self.show_offsets else 0
             if self.show_positions:
                 lineno_width = _get_positions_width(co)