Merge branch 'main' into main

Author: dplusj

.github/workflows/mypy.yml

@@ -19,6 +19,7 @@ on:
       - "Tools/build/consts_getter.py"
       - "Tools/build/deepfreeze.py"
       - "Tools/build/generate-build-details.py"
+      - "Tools/build/generate_levenshtein_examples.py"
       - "Tools/build/generate_sbom.py"
       - "Tools/build/generate_stdlib_module_names.py"
       - "Tools/build/mypy.ini"

Include/internal/pycore_genobject.h

@@ -33,6 +33,9 @@ PyAPI_FUNC(int) _PyGen_FetchStopIterationValue(PyObject **);
 PyAPI_FUNC(PyObject *)_PyCoro_GetAwaitableIter(PyObject *o);
 PyAPI_FUNC(PyObject *)_PyAsyncGenValueWrapperNew(PyThreadState *state, PyObject *);
 
+// Exported for external JIT support
+PyAPI_FUNC(PyObject *) _PyCoro_ComputeOrigin(int origin_depth, _PyInterpreterFrame *current_frame);
+
 extern PyTypeObject _PyCoroWrapper_Type;
 extern PyTypeObject _PyAsyncGenWrappedValue_Type;
 extern PyTypeObject _PyAsyncGenAThrow_Type;

Include/internal/pycore_instruments.h

@@ -122,6 +122,15 @@ typedef struct _PyCoMonitoringData {
 extern int
 _Py_Instrumentation_GetLine(PyCodeObject *code, _PyCoLineInstrumentationData *line_data, int index);
 
+static inline uint8_t
+_PyCode_GetOriginalOpcode(_PyCoLineInstrumentationData *line_data, int index)
+{
+    return line_data->data[index*line_data->bytes_per_entry];
+}
+
+// Exported for external JIT support
+PyAPI_FUNC(uint8_t) _PyCode_Deinstrument(uint8_t opcode);
+
 #ifdef __cplusplus
 }
 #endif

Lib/_colorize.py

@@ -1,3 +1,4 @@
+import builtins
 import os
 import sys
 
@@ -202,25 +203,25 @@ class Difflib(ThemeSection):
 @dataclass(frozen=True, kw_only=True)
 class FancyCompleter(ThemeSection):
     # functions and methods
-    function: str = ANSIColors.BOLD_BLUE
-    builtin_function_or_method: str = ANSIColors.BOLD_BLUE
-    method: str = ANSIColors.BOLD_CYAN
-    method_wrapper: str = ANSIColors.BOLD_CYAN
-    wrapper_descriptor: str = ANSIColors.BOLD_CYAN
-    method_descriptor: str = ANSIColors.BOLD_CYAN
+    function: builtins.str = ANSIColors.BOLD_BLUE
+    builtin_function_or_method: builtins.str = ANSIColors.BOLD_BLUE
+    method: builtins.str = ANSIColors.BOLD_CYAN
+    method_wrapper: builtins.str = ANSIColors.BOLD_CYAN
+    wrapper_descriptor: builtins.str = ANSIColors.BOLD_CYAN
+    method_descriptor: builtins.str = ANSIColors.BOLD_CYAN
 
     # numbers
-    int: str = ANSIColors.BOLD_YELLOW
-    float: str = ANSIColors.BOLD_YELLOW
-    complex: str = ANSIColors.BOLD_YELLOW
-    bool: str = ANSIColors.BOLD_YELLOW
+    int: builtins.str = ANSIColors.BOLD_YELLOW
+    float: builtins.str = ANSIColors.BOLD_YELLOW
+    complex: builtins.str = ANSIColors.BOLD_YELLOW
+    bool: builtins.str = ANSIColors.BOLD_YELLOW
 
     # others
-    type: str = ANSIColors.BOLD_MAGENTA
-    module: str = ANSIColors.CYAN
-    NoneType: str = ANSIColors.GREY
-    bytes: str = ANSIColors.BOLD_GREEN
-    str: str = ANSIColors.BOLD_GREEN
+    type: builtins.str = ANSIColors.BOLD_MAGENTA
+    module: builtins.str = ANSIColors.CYAN
+    NoneType: builtins.str = ANSIColors.GREY
+    bytes: builtins.str = ANSIColors.BOLD_GREEN
+    str: builtins.str = ANSIColors.BOLD_GREEN
 
 
 @dataclass(frozen=True, kw_only=True)

Lib/test/test__colorize.py

@@ -5,6 +5,7 @@
 import unittest
 import unittest.mock
 import _colorize
+from test.support import cpython_only, import_helper
 from test.support.os_helper import EnvironmentVarGuard
 
 
@@ -22,6 +23,15 @@ def supports_virtual_terminal():
         return contextlib.nullcontext()
 
 
+class TestImportTime(unittest.TestCase):
+
+    @cpython_only
+    def test_lazy_import(self):
+        import_helper.ensure_lazy_imports(
+            "_colorize", {"copy", "re"}
+        )
+
+
 class TestTheme(unittest.TestCase):
 
     def test_attributes(self):

Objects/genobject.c

@@ -1110,9 +1110,6 @@ make_gen(PyTypeObject *type, PyFunctionObject *func)
     return (PyObject *)gen;
 }
 
-static PyObject *
-compute_cr_origin(int origin_depth, _PyInterpreterFrame *current_frame);
-
 PyObject *
 _Py_MakeCoro(PyFunctionObject *func)
 {
@@ -1150,7 +1147,7 @@ _Py_MakeCoro(PyFunctionObject *func)
         assert(frame);
         assert(_PyFrame_IsIncomplete(frame));
         frame = _PyFrame_GetFirstComplete(frame->previous);
-        PyObject *cr_origin = compute_cr_origin(origin_depth, frame);
+        PyObject *cr_origin = _PyCoro_ComputeOrigin(origin_depth, frame);
         ((PyCoroObject *)coro)->cr_origin_or_finalizer = cr_origin;
         if (!cr_origin) {
             Py_DECREF(coro);
@@ -1535,8 +1532,8 @@ PyTypeObject _PyCoroWrapper_Type = {
     0,                                          /* tp_free */
 };
 
-static PyObject *
-compute_cr_origin(int origin_depth, _PyInterpreterFrame *current_frame)
+PyObject *
+_PyCoro_ComputeOrigin(int origin_depth, _PyInterpreterFrame *current_frame)
 {
     _PyInterpreterFrame *frame = current_frame;
     /* First count how many frames we have */
@@ -1581,7 +1578,7 @@ PyCoro_New(PyFrameObject *f, PyObject *name, PyObject *qualname)
     if (origin_depth == 0) {
         ((PyCoroObject *)coro)->cr_origin_or_finalizer = NULL;
     } else {
-        PyObject *cr_origin = compute_cr_origin(origin_depth, _PyEval_GetFrame());
+        PyObject *cr_origin = _PyCoro_ComputeOrigin(origin_depth, _PyEval_GetFrame());
         ((PyCoroObject *)coro)->cr_origin_or_finalizer = cr_origin;
         if (!cr_origin) {
             Py_DECREF(coro);

Python/instrumentation.c

@@ -185,6 +185,12 @@ opcode_has_event(int opcode)
     );
 }
 
+uint8_t
+_PyCode_Deinstrument(uint8_t opcode)
+{
+    return DE_INSTRUMENT[opcode];
+}
+
 static inline bool
 is_instrumented(int opcode)
 {
@@ -330,12 +336,6 @@ _PyInstruction_GetLength(PyCodeObject *code, int offset)
     return 1 + _PyOpcode_Caches[inst.op.code];
 }
 
-static inline uint8_t
-get_original_opcode(_PyCoLineInstrumentationData *line_data, int index)
-{
-    return line_data->data[index*line_data->bytes_per_entry];
-}
-
 static inline uint8_t *
 get_original_opcode_ptr(_PyCoLineInstrumentationData *line_data, int index)
 {
@@ -401,7 +401,7 @@ dump_instrumentation_data_lines(PyCodeObject *code, _PyCoLineInstrumentationData
         fprintf(out, ", lines = NULL");
     }
     else {
-        int opcode = get_original_opcode(lines, i);
+        int opcode = _PyCode_GetOriginalOpcode(lines, i);
         int line_delta = get_line_delta(lines, i);
         if (opcode == 0) {
             fprintf(out, ", lines = {original_opcode = No LINE (0), line_delta = %d)", line_delta);
@@ -571,7 +571,7 @@ sanity_check_instrumentation(PyCodeObject *code)
         }
         if (opcode == INSTRUMENTED_LINE) {
             CHECK(data->lines);
-            opcode = get_original_opcode(data->lines, i);
+            opcode = _PyCode_GetOriginalOpcode(data->lines, i);
             CHECK(valid_opcode(opcode));
             CHECK(opcode != END_FOR);
             CHECK(opcode != RESUME);
@@ -588,7 +588,7 @@ sanity_check_instrumentation(PyCodeObject *code)
              * *and* we are executing a INSTRUMENTED_LINE instruction
              * that has de-instrumented itself, then we will execute
              * an invalid INSTRUMENTED_INSTRUCTION */
-            CHECK(get_original_opcode(data->lines, i) != INSTRUMENTED_INSTRUCTION);
+            CHECK(_PyCode_GetOriginalOpcode(data->lines, i) != INSTRUMENTED_INSTRUCTION);
         }
         if (opcode == INSTRUMENTED_INSTRUCTION) {
             CHECK(data->per_instruction_opcodes[i] != 0);
@@ -603,7 +603,7 @@ sanity_check_instrumentation(PyCodeObject *code)
             }
             CHECK(active_monitors.tools[event] != 0);
         }
-        if (data->lines && get_original_opcode(data->lines, i)) {
+        if (data->lines && _PyCode_GetOriginalOpcode(data->lines, i)) {
             int line1 = compute_line(code, get_line_delta(data->lines, i));
             int line2 = _PyCode_CheckLineNumber(i*sizeof(_Py_CODEUNIT), &range);
             CHECK(line1 == line2);
@@ -655,7 +655,7 @@ _Py_GetBaseCodeUnit(PyCodeObject *code, int i)
         return inst;
     }
     if (opcode == INSTRUMENTED_LINE) {
-        opcode = get_original_opcode(code->_co_monitoring->lines, i);
+        opcode = _PyCode_GetOriginalOpcode(code->_co_monitoring->lines, i);
     }
     if (opcode == INSTRUMENTED_INSTRUCTION) {
         opcode = code->_co_monitoring->per_instruction_opcodes[i];
@@ -714,7 +714,7 @@ de_instrument_line(PyCodeObject *code, _Py_CODEUNIT *bytecode, _PyCoMonitoringDa
         return;
     }
     _PyCoLineInstrumentationData *lines = monitoring->lines;
-    int original_opcode = get_original_opcode(lines, i);
+    int original_opcode = _PyCode_GetOriginalOpcode(lines, i);
     if (original_opcode == INSTRUMENTED_INSTRUCTION) {
         set_original_opcode(lines, i, monitoring->per_instruction_opcodes[i]);
     }
@@ -1391,7 +1391,7 @@ _Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame,
     Py_DECREF(line_obj);
     uint8_t original_opcode;
 done:
-    original_opcode = get_original_opcode(line_data, i);
+    original_opcode = _PyCode_GetOriginalOpcode(line_data, i);
     assert(original_opcode != 0);
     assert(original_opcode != INSTRUMENTED_LINE);
     assert(_PyOpcode_Deopt[original_opcode] == original_opcode);
@@ -1464,7 +1464,7 @@ initialize_tools(PyCodeObject *code)
         int opcode = instr->op.code;
         assert(opcode != ENTER_EXECUTOR);
         if (opcode == INSTRUMENTED_LINE) {
-            opcode = get_original_opcode(code->_co_monitoring->lines, i);
+            opcode = _PyCode_GetOriginalOpcode(code->_co_monitoring->lines, i);
         }
         if (opcode == INSTRUMENTED_INSTRUCTION) {
             opcode = code->_co_monitoring->per_instruction_opcodes[i];
@@ -1849,7 +1849,7 @@ force_instrument_lock_held(PyCodeObject *code, PyInterpreterState *interp)
     if (removed_line_tools) {
         _PyCoLineInstrumentationData *line_data = code->_co_monitoring->lines;
         for (int i = code->_co_firsttraceable; i < code_len;) {
-            if (get_original_opcode(line_data, i)) {
+            if (_PyCode_GetOriginalOpcode(line_data, i)) {
                 remove_line_tools(code, i, removed_line_tools);
             }
             i += _PyInstruction_GetLength(code, i);
@@ -1876,7 +1876,7 @@ force_instrument_lock_held(PyCodeObject *code, PyInterpreterState *interp)
     if (new_line_tools) {
         _PyCoLineInstrumentationData *line_data = code->_co_monitoring->lines;
         for (int i = code->_co_firsttraceable; i < code_len;) {
-            if (get_original_opcode(line_data, i)) {
+            if (_PyCode_GetOriginalOpcode(line_data, i)) {
                 add_line_tools(code, i, new_line_tools);
             }
             i += _PyInstruction_GetLength(code, i);

Tools/build/generate_levenshtein_examples.py

@@ -13,7 +13,7 @@
 _CASE_COST = 1
 
 
-def _substitution_cost(ch_a, ch_b):
+def _substitution_cost(ch_a: str, ch_b: str) -> int:
     if ch_a == ch_b:
         return 0
     if ch_a.lower() == ch_b.lower():
@@ -22,7 +22,7 @@ def _substitution_cost(ch_a, ch_b):
 
 
 @lru_cache(None)
-def levenshtein(a, b):
+def levenshtein(a: str, b: str) -> int:
     if not a or not b:
         return (len(a) + len(b)) * _MOVE_COST
     option1 = levenshtein(a[:-1], b[:-1]) + _substitution_cost(a[-1], b[-1])
@@ -31,7 +31,7 @@ def levenshtein(a, b):
     return min(option1, option2, option3)
 
 
-def main():
+def main() -> None:
     parser = argparse.ArgumentParser(description=__doc__)
     parser.add_argument('output_path', metavar='FILE', type=str)
     parser.add_argument('--overwrite', dest='overwrite', action='store_const',
@@ -48,7 +48,7 @@ def main():
         )
         return
 
-    examples = set()
+    examples: set[tuple[str, str, int]] = set()
     # Create a lot of non-empty examples, which should end up with a Gauss-like
     # distribution for even costs (moves) and odd costs (case substitutions).
     while len(examples) < 9990:

Tools/build/mypy.ini

@@ -9,6 +9,7 @@ files =
     Tools/build/consts_getter.py,
     Tools/build/deepfreeze.py,
     Tools/build/generate-build-details.py,
+    Tools/build/generate_levenshtein_examples.py,
     Tools/build/generate_sbom.py,
     Tools/build/generate_stdlib_module_names.py,
     Tools/build/verify_ensurepip_wheels.py,