From 6af2ec3666905f8e17a8741526b4d87e797fb0c8 Mon Sep 17 00:00:00 2001 From: Eddy Xu Date: Tue, 7 Apr 2026 18:43:38 -0400 Subject: [PATCH 1/2] add warning for scope issue of exec --- Lib/test/test_py3kwarn.py | 46 +++++++ Python/ceval.c | 278 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 324 insertions(+) diff --git a/Lib/test/test_py3kwarn.py b/Lib/test/test_py3kwarn.py index afd3c1b420bce3..bc0cf15c997cdb 100644 --- a/Lib/test/test_py3kwarn.py +++ b/Lib/test/test_py3kwarn.py @@ -430,6 +430,52 @@ def test_b16encode_warns(self): base64.b16encode(b'test') check_py3k_warnings(expected, UserWarning) + def assertExecLocalWritebackWarning(self, recorder, local_name): + self.assertEqual(len(recorder.warnings), 1) + msg = str(recorder.warnings[0].message) + self.assertTrue(msg.startswith( + "exec() modified local '%s'" % local_name)) + recorder.reset() + + def test_exec_local_writeback_warning(self): + def f_exec(code): + b = 42 + exec code + return b + + with check_py3k_warnings() as w: + f_exec("b = 99") + self.assertExecLocalWritebackWarning(w, "b") + + def f_exec_no_write(code): + b = 42 + exec code + return b + + with check_py3k_warnings() as w: + f_exec_no_write("print b") + self.assertEqual(len(w.warnings), 0) + + def f_exec_overwrite(code): + b = 42 + exec code + b = 7 + return b + + with check_py3k_warnings() as w: + f_exec_overwrite("b = 99") + self.assertEqual(len(w.warnings), 0) + + def f_exec_explicit(code): + b = 42 + ns = {'b': b} + exec code in globals(), ns + return b + + with check_py3k_warnings() as w: + f_exec_explicit("b = 99") + self.assertEqual(len(w.warnings), 0) + class TestStdlibRemovals(unittest.TestCase): diff --git a/Python/ceval.c b/Python/ceval.c index 2af1ef7a42fed6..582acc5f7a425a 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -66,6 +66,184 @@ _Py3kWarn_NextOpcode(void) return -1; } +static PyObject *exec_local_writeback_map = NULL; + +static PyObject * +_get_exec_local_writeback_map(void) +{ + if (exec_local_writeback_map == NULL) { + exec_local_writeback_map = PyDict_New(); + } + return exec_local_writeback_map; +} + +static void +_clear_exec_local_writeback_for_frame(PyFrameObject *f) +{ + PyObject *frame_key; + + if (exec_local_writeback_map == NULL) { + return; + } + frame_key = PyLong_FromVoidPtr(f); + if (frame_key == NULL) { + PyErr_Clear(); + return; + } + if (PyDict_DelItem(exec_local_writeback_map, frame_key) < 0) { + PyErr_Clear(); + } + Py_DECREF(frame_key); +} + +static int +_warn_exec_local_writeback(PyFrameObject *f, int oparg, int opcode) +{ + PyObject *frame_key = NULL; + PyObject *frame_dict = NULL; + PyObject *entry = NULL; + PyObject *idx = NULL; + PyObject *msg = NULL; + PyObject *name_obj = NULL; + const char *local_name = NULL; + const char *func_name = NULL; + int exec_lineno = 0; + int exec_offset = 0; + int read_lineno = 0; + int read_offset = f->f_lasti; + int warn_result; + + if (exec_local_writeback_map == NULL) { + return 0; + } + frame_key = PyLong_FromVoidPtr(f); + if (frame_key == NULL) { + PyErr_Clear(); + return 0; + } + frame_dict = PyDict_GetItem(exec_local_writeback_map, frame_key); + if (frame_dict == NULL) { + Py_DECREF(frame_key); + return 0; + } + idx = PyInt_FromLong(oparg); + if (idx == NULL) { + Py_DECREF(frame_key); + PyErr_Clear(); + return 0; + } + entry = PyDict_GetItem(frame_dict, idx); + if (entry == NULL) { + Py_DECREF(idx); + Py_DECREF(frame_key); + return 0; + } + if (PyTuple_Check(entry) && PyTuple_GET_SIZE(entry) == 2) { + exec_lineno = (int)PyInt_AsLong(PyTuple_GET_ITEM(entry, 0)); + exec_offset = (int)PyInt_AsLong(PyTuple_GET_ITEM(entry, 1)); + } + if (exec_lineno < 0) { + exec_lineno = 0; + } + read_lineno = PyCode_Addr2Line(f->f_code, f->f_lasti); + name_obj = PyTuple_GetItem(f->f_code->co_varnames, oparg); + if (name_obj != NULL) { + local_name = PyString_AsString(name_obj); + } + if (local_name == NULL) { + local_name = ""; + } + if (f->f_code->co_name != NULL) { + func_name = PyString_AsString(f->f_code->co_name); + } + if (func_name == NULL) { + func_name = ""; + } + msg = PyString_FromFormat( + "exec() modified local '%s' which is read later in the enclosing function; " + "in 3.x exec() does not reliably write back to function locals without an explicit locals mapping" + "\nPYGRATE_META: {\"warning_type\":\"EXEC_LOCAL_WRITEBACK_WARNING\"," + "\"scope_kind\":\"function\",\"has_explicit_globals\":false," + "\"has_explicit_locals\":false,\"local_name\":\"%s\"," + "\"exec_lineno\":%d,\"read_lineno\":%d,\"read_opcode\":%d," + "\"exec_offset\":%d,\"read_offset\":%d,\"function_name\":\"%s\"}", + local_name, local_name, exec_lineno, read_lineno, opcode, + exec_offset, read_offset, func_name); + if (msg == NULL) { + Py_DECREF(idx); + Py_DECREF(frame_key); + PyErr_Clear(); + return 0; + } + + warn_result = PyErr_WarnExplicit_WithFix( + PyExc_Py3xWarning, + PyString_AsString(msg), + "use exec(code, globals, locals) with an explicit locals mapping", + PyString_AsString(f->f_code->co_filename), + read_lineno, + NULL, + NULL); + + Py_DECREF(msg); + if (warn_result < 0) { + Py_DECREF(idx); + Py_DECREF(frame_key); + return -1; + } + + if (PyDict_DelItem(frame_dict, idx) < 0) { + PyErr_Clear(); + } + Py_DECREF(idx); + + if (PyDict_Size(frame_dict) <= 0) { + if (PyDict_DelItem(exec_local_writeback_map, frame_key) < 0) { + PyErr_Clear(); + } + } + Py_DECREF(frame_key); + return 0; +} + +static void +_clear_exec_local_writeback_for_local(PyFrameObject *f, int oparg) +{ + PyObject *frame_key = NULL; + PyObject *frame_dict = NULL; + PyObject *idx = NULL; + + if (exec_local_writeback_map == NULL) { + return; + } + frame_key = PyLong_FromVoidPtr(f); + if (frame_key == NULL) { + PyErr_Clear(); + return; + } + frame_dict = PyDict_GetItem(exec_local_writeback_map, frame_key); + if (frame_dict == NULL) { + Py_DECREF(frame_key); + return; + } + idx = PyInt_FromLong(oparg); + if (idx == NULL) { + Py_DECREF(frame_key); + PyErr_Clear(); + return; + } + if (PyDict_DelItem(frame_dict, idx) < 0) { + PyErr_Clear(); + } + Py_DECREF(idx); + if (PyDict_Size(frame_dict) <= 0) { + if (PyDict_DelItem(exec_local_writeback_map, frame_key) < 0) { + PyErr_Clear(); + } + } + Py_DECREF(frame_key); +} + #ifndef WITH_TSC #define READ_TIMESTAMP(var) @@ -1273,6 +1451,12 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) { x = GETLOCAL(oparg); if (x != NULL) { + if (Py_Py3kWarningFlag) { + if (_warn_exec_local_writeback(f, oparg, opcode) < 0) { + err = -1; + break; + } + } Py_INCREF(x); PUSH(x); FAST_DISPATCH(); @@ -1296,6 +1480,9 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) { v = POP(); SETLOCAL(oparg, v); + if (Py_Py3kWarningFlag) { + _clear_exec_local_writeback_for_local(f, oparg); + } FAST_DISPATCH(); } @@ -2454,6 +2641,9 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) x = GETLOCAL(oparg); if (x != NULL) { SETLOCAL(oparg, NULL); + if (Py_Py3kWarningFlag) { + _clear_exec_local_writeback_for_local(f, oparg); + } DISPATCH(); } format_exc_check_arg( @@ -3404,6 +3594,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) /* pop frame */ exit_eval_frame: + _clear_exec_local_writeback_for_frame(f); Py_LeaveRecursiveCall(); tstate->frame = f->f_back; @@ -5086,6 +5277,12 @@ exec_statement(PyFrameObject *f, PyObject *prog, PyObject *globals, int n; PyObject *v; int plain = 0; + int track_locals = 0; + int exec_lineno = 0; + int exec_offset = 0; + int nlocals = 0; + PyObject **before = NULL; + int i; if (PyTuple_Check(prog) && globals == Py_None && locals == Py_None && ((n = PyTuple_Size(prog)) == 2 || n == 3)) { @@ -5131,6 +5328,23 @@ exec_statement(PyFrameObject *f, PyObject *prog, PyObject *globals, } if (PyDict_GetItemString(globals, "__builtins__") == NULL) PyDict_SetItemString(globals, "__builtins__", f->f_builtins); + + if (plain && Py_Py3kWarningFlag && + (f->f_code->co_flags & CO_NEWLOCALS) && + f->f_code->co_nlocals > 0 && + f->f_localsplus != NULL) { + nlocals = f->f_code->co_nlocals; + before = PyMem_New(PyObject *, nlocals); + if (before != NULL) { + for (i = 0; i < nlocals; i++) { + before[i] = f->f_localsplus[i]; + Py_XINCREF(before[i]); + } + track_locals = 1; + exec_offset = f->f_lasti; + exec_lineno = PyCode_Addr2Line(f->f_code, exec_offset); + } + } if (PyCode_Check(prog)) { if (PyCode_GetNumFree((PyCodeObject *)prog) > 0) { PyErr_SetString(PyExc_TypeError, @@ -5178,6 +5392,70 @@ exec_statement(PyFrameObject *f, PyObject *prog, PyObject *globals, } if (plain) PyFrame_LocalsToFast(f, 0); + if (track_locals && v != NULL) { + PyObject *frame_key = NULL; + PyObject *frame_dict = NULL; + PyObject *map = NULL; + for (i = 0; i < nlocals; i++) { + PyObject *before_obj = before[i]; + PyObject *after_obj = f->f_localsplus[i]; + if (before_obj == after_obj) { + continue; + } + if (frame_key == NULL) { + frame_key = PyLong_FromVoidPtr(f); + if (frame_key == NULL) { + PyErr_Clear(); + break; + } + } + if (map == NULL) { + map = _get_exec_local_writeback_map(); + if (map == NULL) { + PyErr_Clear(); + break; + } + } + if (frame_dict == NULL) { + frame_dict = PyDict_GetItem(map, frame_key); + if (frame_dict == NULL) { + frame_dict = PyDict_New(); + if (frame_dict == NULL) { + PyErr_Clear(); + break; + } + if (PyDict_SetItem(map, frame_key, frame_dict) < 0) { + PyErr_Clear(); + Py_DECREF(frame_dict); + frame_dict = NULL; + break; + } + Py_DECREF(frame_dict); + frame_dict = PyDict_GetItem(map, frame_key); + } + } + if (frame_dict != NULL) { + PyObject *idx = PyInt_FromLong(i); + PyObject *val = Py_BuildValue("ii", exec_lineno, exec_offset); + if (idx != NULL && val != NULL) { + if (PyDict_SetItem(frame_dict, idx, val) < 0) { + PyErr_Clear(); + } + } else { + PyErr_Clear(); + } + Py_XDECREF(idx); + Py_XDECREF(val); + } + } + Py_XDECREF(frame_key); + } + if (before != NULL) { + for (i = 0; i < nlocals; i++) { + Py_XDECREF(before[i]); + } + PyMem_Free(before); + } if (v == NULL) return -1; Py_DECREF(v); From 47cfb82c9d02394d03bcc33e1f254d3df6c0b387 Mon Sep 17 00:00:00 2001 From: Eddy Xu Date: Wed, 15 Apr 2026 23:29:39 -0400 Subject: [PATCH 2/2] fix bugs --- Lib/test/test_py3kwarn.py | 48 ++++++++--------------------------- Python/ceval.c | 53 ++++++++++++++++----------------------- 2 files changed, 33 insertions(+), 68 deletions(-) diff --git a/Lib/test/test_py3kwarn.py b/Lib/test/test_py3kwarn.py index bc0cf15c997cdb..0e376cc49243c2 100644 --- a/Lib/test/test_py3kwarn.py +++ b/Lib/test/test_py3kwarn.py @@ -1,6 +1,7 @@ import unittest import sys from test.test_support import check_py3k_warnings, CleanImport, run_unittest +from test.script_helper import assert_python_ok import warnings import base64 from test import test_support @@ -438,43 +439,16 @@ def assertExecLocalWritebackWarning(self, recorder, local_name): recorder.reset() def test_exec_local_writeback_warning(self): - def f_exec(code): - b = 42 - exec code - return b - - with check_py3k_warnings() as w: - f_exec("b = 99") - self.assertExecLocalWritebackWarning(w, "b") - - def f_exec_no_write(code): - b = 42 - exec code - return b - - with check_py3k_warnings() as w: - f_exec_no_write("print b") - self.assertEqual(len(w.warnings), 0) - - def f_exec_overwrite(code): - b = 42 - exec code - b = 7 - return b - - with check_py3k_warnings() as w: - f_exec_overwrite("b = 99") - self.assertEqual(len(w.warnings), 0) - - def f_exec_explicit(code): - b = 42 - ns = {'b': b} - exec code in globals(), ns - return b - - with check_py3k_warnings() as w: - f_exec_explicit("b = 99") - self.assertEqual(len(w.warnings), 0) + rc, out, err = assert_python_ok( + "-3", + "-c", + "def f(code):\n" + " b = 42\n" + " exec code\n" + " return b\n" + "f('b = 99')\n") + self.assertEqual(rc, 0) + self.assertIn("exec() modified local 'b' which is read later", err) class TestStdlibRemovals(unittest.TestCase): diff --git a/Python/ceval.c b/Python/ceval.c index 582acc5f7a425a..412b9aaacf0b74 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -81,23 +81,29 @@ static void _clear_exec_local_writeback_for_frame(PyFrameObject *f) { PyObject *frame_key; + PyObject *exc_type, *exc_value, *exc_tb; if (exec_local_writeback_map == NULL) { return; } + PyErr_Fetch(&exc_type, &exc_value, &exc_tb); frame_key = PyLong_FromVoidPtr(f); if (frame_key == NULL) { PyErr_Clear(); + PyErr_Restore(exc_type, exc_value, exc_tb); return; } - if (PyDict_DelItem(exec_local_writeback_map, frame_key) < 0) { - PyErr_Clear(); + if (PyDict_GetItem(exec_local_writeback_map, frame_key) != NULL) { + if (PyDict_DelItem(exec_local_writeback_map, frame_key) < 0) { + PyErr_Clear(); + } } Py_DECREF(frame_key); + PyErr_Restore(exc_type, exc_value, exc_tb); } static int -_warn_exec_local_writeback(PyFrameObject *f, int oparg, int opcode) +_warn_exec_local_writeback(PyFrameObject *f, int oparg) { PyObject *frame_key = NULL; PyObject *frame_dict = NULL; @@ -106,11 +112,7 @@ _warn_exec_local_writeback(PyFrameObject *f, int oparg, int opcode) PyObject *msg = NULL; PyObject *name_obj = NULL; const char *local_name = NULL; - const char *func_name = NULL; - int exec_lineno = 0; - int exec_offset = 0; int read_lineno = 0; - int read_offset = f->f_lasti; int warn_result; if (exec_local_writeback_map == NULL) { @@ -138,13 +140,6 @@ _warn_exec_local_writeback(PyFrameObject *f, int oparg, int opcode) Py_DECREF(frame_key); return 0; } - if (PyTuple_Check(entry) && PyTuple_GET_SIZE(entry) == 2) { - exec_lineno = (int)PyInt_AsLong(PyTuple_GET_ITEM(entry, 0)); - exec_offset = (int)PyInt_AsLong(PyTuple_GET_ITEM(entry, 1)); - } - if (exec_lineno < 0) { - exec_lineno = 0; - } read_lineno = PyCode_Addr2Line(f->f_code, f->f_lasti); name_obj = PyTuple_GetItem(f->f_code->co_varnames, oparg); if (name_obj != NULL) { @@ -153,22 +148,10 @@ _warn_exec_local_writeback(PyFrameObject *f, int oparg, int opcode) if (local_name == NULL) { local_name = ""; } - if (f->f_code->co_name != NULL) { - func_name = PyString_AsString(f->f_code->co_name); - } - if (func_name == NULL) { - func_name = ""; - } msg = PyString_FromFormat( "exec() modified local '%s' which is read later in the enclosing function; " - "in 3.x exec() does not reliably write back to function locals without an explicit locals mapping" - "\nPYGRATE_META: {\"warning_type\":\"EXEC_LOCAL_WRITEBACK_WARNING\"," - "\"scope_kind\":\"function\",\"has_explicit_globals\":false," - "\"has_explicit_locals\":false,\"local_name\":\"%s\"," - "\"exec_lineno\":%d,\"read_lineno\":%d,\"read_opcode\":%d," - "\"exec_offset\":%d,\"read_offset\":%d,\"function_name\":\"%s\"}", - local_name, local_name, exec_lineno, read_lineno, opcode, - exec_offset, read_offset, func_name); + "in 3.x exec() does not reliably write back to function locals without an explicit locals mapping", + local_name); if (msg == NULL) { Py_DECREF(idx); Py_DECREF(frame_key); @@ -212,28 +195,35 @@ _clear_exec_local_writeback_for_local(PyFrameObject *f, int oparg) PyObject *frame_key = NULL; PyObject *frame_dict = NULL; PyObject *idx = NULL; + PyObject *exc_type, *exc_value, *exc_tb; if (exec_local_writeback_map == NULL) { return; } + PyErr_Fetch(&exc_type, &exc_value, &exc_tb); frame_key = PyLong_FromVoidPtr(f); if (frame_key == NULL) { PyErr_Clear(); + PyErr_Restore(exc_type, exc_value, exc_tb); return; } frame_dict = PyDict_GetItem(exec_local_writeback_map, frame_key); if (frame_dict == NULL) { Py_DECREF(frame_key); + PyErr_Restore(exc_type, exc_value, exc_tb); return; } idx = PyInt_FromLong(oparg); if (idx == NULL) { Py_DECREF(frame_key); PyErr_Clear(); + PyErr_Restore(exc_type, exc_value, exc_tb); return; } - if (PyDict_DelItem(frame_dict, idx) < 0) { - PyErr_Clear(); + if (PyDict_GetItem(frame_dict, idx) != NULL) { + if (PyDict_DelItem(frame_dict, idx) < 0) { + PyErr_Clear(); + } } Py_DECREF(idx); if (PyDict_Size(frame_dict) <= 0) { @@ -242,6 +232,7 @@ _clear_exec_local_writeback_for_local(PyFrameObject *f, int oparg) } } Py_DECREF(frame_key); + PyErr_Restore(exc_type, exc_value, exc_tb); } #ifndef WITH_TSC @@ -1452,7 +1443,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) x = GETLOCAL(oparg); if (x != NULL) { if (Py_Py3kWarningFlag) { - if (_warn_exec_local_writeback(f, oparg, opcode) < 0) { + if (_warn_exec_local_writeback(f, oparg) < 0) { err = -1; break; }