diff --git a/Doc/library/pickletools.rst b/Doc/library/pickletools.rst index 7a771ea3ab93d4..e753ad3b08b81a 100644 --- a/Doc/library/pickletools.rst +++ b/Doc/library/pickletools.rst @@ -79,6 +79,9 @@ Command-line options A pickle file to read, or ``-`` to indicate reading from standard input. +.. versionadded:: next + Output is in color by default and can be + :ref:`controlled using environment variables `. Programmatic interface diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index eb08f8c4ed69e7..3c2c7a7e399d09 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -1053,6 +1053,15 @@ pickle (Contributed by Zackery Spytz and Serhiy Storchaka in :gh:`77188`.) +pickletools +----------- + +* The output of the :mod:`pickletools` command-line interface is colored by + default. This can be controlled with + :ref:`environment variables `. + (Contributed by Hugo van Kemenade in :gh:`149026`.) + + pprint ------ diff --git a/Lib/_colorize.py b/Lib/_colorize.py index 379ca2529b6585..62806b1d8d7bcf 100644 --- a/Lib/_colorize.py +++ b/Lib/_colorize.py @@ -359,6 +359,23 @@ class LiveProfiler(ThemeSection): ) +@dataclass(frozen=True, kw_only=True) +class Pickletools(ThemeSection): + annotation: str = ANSIColors.GREY + arg_number: str = ANSIColors.YELLOW + arg_string: str = ANSIColors.GREEN + mark: str = ANSIColors.GREY + op_call: str = ANSIColors.GREEN + op_container: str = ANSIColors.INTENSE_BLUE + op_memo: str = ANSIColors.MAGENTA + op_meta: str = ANSIColors.GREY + op_stack: str = ANSIColors.BOLD_RED + opcode_code: str = ANSIColors.CYAN + position: str = ANSIColors.GREY + proto: str = ANSIColors.YELLOW + reset: str = ANSIColors.RESET + + @dataclass(frozen=True, kw_only=True) class Syntax(ThemeSection): prompt: str = ANSIColors.BOLD_MAGENTA @@ -429,6 +446,7 @@ class Theme: fancycompleter: FancyCompleter = field(default_factory=FancyCompleter) http_server: HttpServer = field(default_factory=HttpServer) live_profiler: LiveProfiler = field(default_factory=LiveProfiler) + pickletools: Pickletools = field(default_factory=Pickletools) syntax: Syntax = field(default_factory=Syntax) timeit: Timeit = field(default_factory=Timeit) tokenize: Tokenize = field(default_factory=Tokenize) @@ -444,6 +462,7 @@ def copy_with( fancycompleter: FancyCompleter | None = None, http_server: HttpServer | None = None, live_profiler: LiveProfiler | None = None, + pickletools: Pickletools | None = None, syntax: Syntax | None = None, timeit: Timeit | None = None, tokenize: Tokenize | None = None, @@ -462,6 +481,7 @@ def copy_with( fancycompleter=fancycompleter or self.fancycompleter, http_server=http_server or self.http_server, live_profiler=live_profiler or self.live_profiler, + pickletools=pickletools or self.pickletools, syntax=syntax or self.syntax, timeit=timeit or self.timeit, tokenize=tokenize or self.tokenize, @@ -484,6 +504,7 @@ def no_colors(cls) -> Self: fancycompleter=FancyCompleter.no_colors(), http_server=HttpServer.no_colors(), live_profiler=LiveProfiler.no_colors(), + pickletools=Pickletools.no_colors(), syntax=Syntax.no_colors(), timeit=Timeit.no_colors(), tokenize=Tokenize.no_colors(), diff --git a/Lib/ensurepip/__init__.py b/Lib/ensurepip/__init__.py index 93b4e7a820f3ad..5a55525d6bd235 100644 --- a/Lib/ensurepip/__init__.py +++ b/Lib/ensurepip/__init__.py @@ -10,7 +10,7 @@ __all__ = ["version", "bootstrap"] -_PIP_VERSION = "26.0.1" +_PIP_VERSION = "26.1" # Directory of system wheel packages. Some Linux distribution packaging # policies recommend against bundling dependencies. For example, Fedora diff --git a/Lib/ensurepip/_bundled/pip-26.0.1-py3-none-any.whl b/Lib/ensurepip/_bundled/pip-26.1-py3-none-any.whl similarity index 73% rename from Lib/ensurepip/_bundled/pip-26.0.1-py3-none-any.whl rename to Lib/ensurepip/_bundled/pip-26.1-py3-none-any.whl index 580d09a920422f..b51afa14f7c0ad 100644 Binary files a/Lib/ensurepip/_bundled/pip-26.0.1-py3-none-any.whl and b/Lib/ensurepip/_bundled/pip-26.1-py3-none-any.whl differ diff --git a/Lib/http/cookiejar.py b/Lib/http/cookiejar.py index 68cf16c93cc1c8..13e5b104a81ea2 100644 --- a/Lib/http/cookiejar.py +++ b/Lib/http/cookiejar.py @@ -1032,10 +1032,13 @@ def set_ok_domain(self, cookie, request): if j == 0: # domain like .foo.bar tld = domain[i+1:] sld = domain[j+1:i] - if sld.lower() in ("co", "ac", "com", "edu", "org", "net", - "gov", "mil", "int", "aero", "biz", "cat", "coop", - "info", "jobs", "mobi", "museum", "name", "pro", - "travel", "eu") and len(tld) == 2: + known_slds = ( + "co", "ac", "com", "edu", "org", "net", + "gov", "mil", "int", "aero", "biz", "cat", "coop", + "info", "jobs", "mobi", "museum", "name", "pro", + "travel", "eu", "tv", "or", "nom", "sch", "web", + ) + if sld.lower() in known_slds and len(tld) == 2: # domain like .co.uk _debug(" country-code second level domain %s", domain) return False diff --git a/Lib/pickletools.py b/Lib/pickletools.py index 29baf3be7ebb6e..976e218db19298 100644 --- a/Lib/pickletools.py +++ b/Lib/pickletools.py @@ -16,6 +16,8 @@ import re import sys +lazy from _colorize import decolor, get_theme + __all__ = ['dis', 'genops', 'optimize'] bytes_types = pickle.bytes_types @@ -2209,6 +2211,32 @@ def __init__(self, name, code, arg, name2i[d.name] = i code2i[d.code] = i +# Group opcode names into categories for colourised CLI output. +_opcode_categories = frozendict( + op_call=frozenset({ + "BUILD", "EXT1", "EXT2", "EXT4", "GLOBAL", "INST", "NEWOBJ", + "NEWOBJ_EX", "OBJ", "REDUCE", "STACK_GLOBAL", + }), + op_container=frozenset({ + "ADDITEMS", "APPEND", "APPENDS", "DICT", "EMPTY_DICT", "EMPTY_LIST", + "EMPTY_SET", "EMPTY_TUPLE", "FROZENSET", "LIST", "SETITEM", + "SETITEMS", "TUPLE", "TUPLE1", "TUPLE2", "TUPLE3", + }), + op_memo=frozenset({ + "BINGET", "BINPUT", "GET", "LONG_BINGET", "LONG_BINPUT", "MEMOIZE", + "PUT", + }), + op_meta=frozenset({"BINPERSID", "FRAME", "MARK", "PERSID", "PROTO"}), + op_stack=frozenset({"DUP", "POP", "POP_MARK", "STOP"}), +) +_opcode_color_attr = frozendict({ + name: attr + for attr, names in _opcode_categories.items() + for name in names +}) +assert _opcode_color_attr.keys() <= name2i.keys(), ( + f"unknown opcodes: {_opcode_color_attr.keys() - name2i.keys()}" +) del name2i, code2i, i, d ############################################################################## @@ -2443,13 +2471,19 @@ def dis(pickle, out=None, memo=None, indentlevel=4, annotate=0): indentchunk = ' ' * indentlevel errormsg = None annocol = annotate # column hint for annotations + t = get_theme(tty_file=out).pickletools for opcode, arg, pos in genops(pickle): if pos is not None: - print("%5d:" % pos, end=' ', file=out) + print(f"{t.position}{pos:5d}:{t.reset}", end=' ', file=out) - line = "%-4s %s%s" % (repr(opcode.code)[1:-1], - indentchunk * len(markstack), - opcode.name) + attr = _opcode_color_attr.get(opcode.name) + opcode_color = getattr(t, attr) if attr else "" + opcode_reset = t.reset if attr else "" + line = ( + f"{t.opcode_code}{repr(opcode.code)[1:-1]:<4}{t.reset} " + f"{indentchunk * len(markstack)}" + f"{opcode_color}{opcode.name}{opcode_reset}" + ) maxproto = max(maxproto, opcode.proto) before = opcode.stack_before # don't mutate @@ -2510,18 +2544,26 @@ def dis(pickle, out=None, memo=None, indentlevel=4, annotate=0): line += ' ' * (10 - len(opcode.name)) if arg is not None: if opcode.name in ("STRING", "BINSTRING", "SHORT_BINSTRING"): - line += ' ' + ascii(arg) + arg_text = ascii(arg) else: - line += ' ' + repr(arg) + arg_text = repr(arg) + arg_color = ( + t.arg_number + if isinstance(arg, (int, float)) + else t.arg_string + ) + line += f" {arg_color}{arg_text}{t.reset}" if markmsg: - line += ' ' + markmsg + line += f" {t.mark}{markmsg}{t.reset}" if annotate: - line += ' ' * (annocol - len(line)) + visible_len = len(decolor(line)) + line += ' ' * (annocol - visible_len) # make a mild effort to align annotations - annocol = len(line) + annocol = max(visible_len, annocol) if annocol > 50: annocol = annotate - line += ' ' + opcode.doc.split('\n', 1)[0] + doc = opcode.doc.split('\n', 1)[0] + line += f" {t.annotation}{doc}{t.reset}" print(line, file=out) if errormsg: @@ -2541,7 +2583,11 @@ def dis(pickle, out=None, memo=None, indentlevel=4, annotate=0): stack.extend(after) - print("highest protocol among opcodes =", maxproto, file=out) + print( + "highest protocol among opcodes =", + f"{t.proto}{maxproto}{t.reset}", + file=out, + ) if stack: raise ValueError("stack not empty after STOP: %r" % stack) @@ -2841,10 +2887,7 @@ def __init__(self, value): def _main(args=None): import argparse - parser = argparse.ArgumentParser( - description='disassemble one or more pickle files', - color=True, - ) + parser = argparse.ArgumentParser(description='disassemble one or more pickle files') parser.add_argument( 'pickle_file', nargs='+', help='the pickle file') diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index e323742665234c..81967fb8a83740 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -308,6 +308,27 @@ def f_set(): self.assertEqual(overridden_outputs, ['all', 'any', 'tuple', 'list', 'set']) + def test_builtin_call_async_genexpr_no_crash(self): + async def f_all(): + return all(await 2 for _ in []) + + async def f_any(): + return any(await 2 for _ in []) + + async def f_tuple(): + return tuple(await 2 for _ in []) + + async def f_list(): + return list(await 2 for _ in []) + + async def f_set(): + return set(await 2 for _ in []) + + for f in (f_all, f_any, f_tuple, f_list, f_set): + with self.subTest(func=f.__name__): + with self.assertRaises(TypeError): + run_yielding_async_fn(f) + def test_ascii(self): self.assertEqual(ascii(''), '\'\'') self.assertEqual(ascii(0), '0') diff --git a/Lib/test/test_pickletools.py b/Lib/test/test_pickletools.py index 57285ddf6ebef5..caf2d7ba6bfd8f 100644 --- a/Lib/test/test_pickletools.py +++ b/Lib/test/test_pickletools.py @@ -160,6 +160,7 @@ def test_unknown_opcode_without_pos(self): next(it) +@support.force_not_colorized_test_class class DisTests(unittest.TestCase): maxDiff = None @@ -518,6 +519,7 @@ def test__all__(self): support.check__all__(self, pickletools, not_exported=not_exported) +@support.force_not_colorized_test_class class CommandLineTest(unittest.TestCase): def setUp(self): self.filename = tempfile.mktemp() diff --git a/Lib/test/test_uuid.py b/Lib/test/test_uuid.py index 5f9ab048cdeb6c..055be2994bf435 100755 --- a/Lib/test/test_uuid.py +++ b/Lib/test/test_uuid.py @@ -1182,6 +1182,47 @@ def test_cli_name_required_for_uuid3(self, mock_err): self.assertEqual(cm.exception.code, 2) self.assertIn("error: Incorrect number of arguments", mock_err.getvalue()) + @mock.patch.object(sys, "argv", + ["", "-u", "uuid3", "-n", "@dns", "-N", "python.org"]) + def test_cli_uuid3_outputted_with_valid_namespace_and_name(self): + stdout = io.StringIO() + with contextlib.redirect_stdout(stdout): + self.uuid.main() + + output = stdout.getvalue().strip() + uuid_output = self.uuid.UUID(output) + + # Output should be in the form of uuid3 + self.assertEqual(output, str(uuid_output)) + self.assertEqual(uuid_output.version, 3) + + @mock.patch.object(sys, "argv", + ["", "-u", "uuid3", "-n", + "0d6a16cc-34a7-47d8-b660-214d0ae184d2", + "-N", "some.user"]) + def test_cli_uuid3_outputted_with_custom_namespace_and_name(self): + stdout = io.StringIO() + with contextlib.redirect_stdout(stdout): + self.uuid.main() + + output = stdout.getvalue().strip() + uuid_output = self.uuid.UUID(output) + + # Output should be in the form of uuid3 + self.assertEqual(output, str(uuid_output)) + self.assertEqual(uuid_output.version, 3) + + @mock.patch.object(sys, "argv", + ["", "-u", "uuid3", "-n", "any UUID", "-N", "python.org"]) + @mock.patch('sys.stderr', new_callable=io.StringIO) + def test_cli_uuid3_with_invalid_namespace(self, mock_err): + with self.assertRaises(SystemExit) as cm: + self.uuid.main() + # Check that exception code is the same as argparse.ArgumentParser.error + self.assertEqual(cm.exception.code, 2) + self.assertIn("error: badly formed hexadecimal UUID string", + mock_err.getvalue()) + @mock.patch.object(sys, "argv", [""]) def test_cli_uuid4_outputted_with_no_args(self): stdout = io.StringIO() @@ -1210,8 +1251,8 @@ def test_cli_uuid4_outputted_with_count(self): self.assertEqual(uuid_output.version, 4) @mock.patch.object(sys, "argv", - ["", "-u", "uuid3", "-n", "@dns", "-N", "python.org"]) - def test_cli_uuid3_ouputted_with_valid_namespace_and_name(self): + ["", "-u", "uuid5", "-n", "@dns", "-N", "python.org"]) + def test_cli_uuid5_outputted_with_valid_namespace_and_name(self): stdout = io.StringIO() with contextlib.redirect_stdout(stdout): self.uuid.main() @@ -1221,11 +1262,13 @@ def test_cli_uuid3_ouputted_with_valid_namespace_and_name(self): # Output should be in the form of uuid5 self.assertEqual(output, str(uuid_output)) - self.assertEqual(uuid_output.version, 3) + self.assertEqual(uuid_output.version, 5) @mock.patch.object(sys, "argv", - ["", "-u", "uuid5", "-n", "@dns", "-N", "python.org"]) - def test_cli_uuid5_ouputted_with_valid_namespace_and_name(self): + ["", "-u", "uuid5", "-n", + "0d6a16cc-34a7-47d8-b660-214d0ae184d2", + "-N", "some.user"]) + def test_cli_uuid5_ouputted_with_custom_namespace_and_name(self): stdout = io.StringIO() with contextlib.redirect_stdout(stdout): self.uuid.main() @@ -1237,6 +1280,17 @@ def test_cli_uuid5_ouputted_with_valid_namespace_and_name(self): self.assertEqual(output, str(uuid_output)) self.assertEqual(uuid_output.version, 5) + @mock.patch.object(sys, "argv", + ["", "-u", "uuid5", "-n", "any UUID", "-N", "python.org"]) + @mock.patch('sys.stderr', new_callable=io.StringIO) + def test_cli_uuid5_with_invalid_namespace(self, mock_err): + with self.assertRaises(SystemExit) as cm: + self.uuid.main() + # Check that exception code is the same as argparse.ArgumentParser.error + self.assertEqual(cm.exception.code, 2) + self.assertIn("error: badly formed hexadecimal UUID string", + mock_err.getvalue()) + @mock.patch.object(sys, "argv", ["", "-u", "uuid6"]) def test_cli_uuid6(self): self.do_test_standalone_uuid(6) diff --git a/Lib/uuid.py b/Lib/uuid.py index c0150a59d7cb9a..8c59581464b0d0 100644 --- a/Lib/uuid.py +++ b/Lib/uuid.py @@ -962,7 +962,7 @@ def main(): default="uuid4", help="function to generate the UUID") parser.add_argument("-n", "--namespace", - choices=["any UUID", *namespaces.keys()], + metavar=f"{{any UUID,{','.join(namespaces)}}}", help="uuid3/uuid5 only: " "a UUID, or a well-known predefined UUID addressed " "by namespace name") @@ -984,7 +984,13 @@ def main(): f"{args.uuid} requires a namespace and a name. " "Run 'python -m uuid -h' for more information." ) - namespace = namespaces[namespace] if namespace in namespaces else UUID(namespace) + if namespace in namespaces: + namespace = namespaces[namespace] + else: + try: + namespace = UUID(namespace) + except ValueError as exc: + parser.error(f"{exc}: {args.namespace!r}") for _ in range(args.count): print(uuid_func(namespace, name)) else: diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-04-29-14-06-00.gh-issue-149122.P8k2Lm.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-29-14-06-00.gh-issue-149122.P8k2Lm.rst new file mode 100644 index 00000000000000..f34b6ea857a06c --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-29-14-06-00.gh-issue-149122.P8k2Lm.rst @@ -0,0 +1,4 @@ +Fix a crash in optimized calls to :func:`all`, :func:`any`, :func:`tuple`, +:func:`list`, and :func:`set` with an async generator expression argument +(for example, ``tuple(await x for x in y)``). These calls now correctly raise +``TypeError`` instead of crashing. diff --git a/Misc/NEWS.d/next/Library/2025-06-22-16-29-10.gh-issue-135528.Rt_QhR.rst b/Misc/NEWS.d/next/Library/2025-06-22-16-29-10.gh-issue-135528.Rt_QhR.rst new file mode 100644 index 00000000000000..ab3855582c77ee --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-06-22-16-29-10.gh-issue-135528.Rt_QhR.rst @@ -0,0 +1 @@ +:mod:`http.cookiejar`: add "tv", "or", "nom", "sch", and "web" to the default list of supported country code second-level domains. diff --git a/Misc/NEWS.d/next/Library/2026-04-18-17-37-13.gh-issue-148740.sYnFi0.rst b/Misc/NEWS.d/next/Library/2026-04-18-17-37-13.gh-issue-148740.sYnFi0.rst new file mode 100644 index 00000000000000..7e49cedda7beb2 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-04-18-17-37-13.gh-issue-148740.sYnFi0.rst @@ -0,0 +1,2 @@ +Fix usage for :mod:`uuid` command-line interface to support a custom namespace be +provided for uuid3 and uuid5. diff --git a/Misc/NEWS.d/next/Library/2026-04-26-23-01-50.gh-issue-149026.Akk4Bc.rst b/Misc/NEWS.d/next/Library/2026-04-26-23-01-50.gh-issue-149026.Akk4Bc.rst new file mode 100644 index 00000000000000..d12a92e9f530da --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-04-26-23-01-50.gh-issue-149026.Akk4Bc.rst @@ -0,0 +1 @@ +Add colour to :mod:`pickletools` CLI output. Patch by Hugo van Kemenade. diff --git a/Misc/NEWS.d/next/Library/2026-04-29-14-33-42.gh-issue-149148.EaiYvk.rst b/Misc/NEWS.d/next/Library/2026-04-29-14-33-42.gh-issue-149148.EaiYvk.rst new file mode 100644 index 00000000000000..06186773474fb9 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-04-29-14-33-42.gh-issue-149148.EaiYvk.rst @@ -0,0 +1,2 @@ +:mod:`ensurepip`: Upgrade bundled pip to 26.1. This version fixes +the :cve:`2026-3219` vulnerability. Patch by Victor Stinner. diff --git a/Python/codegen.c b/Python/codegen.c index 389e7cf85d32bb..a371bf332b6d9e 100644 --- a/Python/codegen.c +++ b/Python/codegen.c @@ -3959,6 +3959,12 @@ maybe_optimize_function_call(compiler *c, expr_ty e, jump_target_label end) return 0; } + expr_ty generator_exp = asdl_seq_GET(args, 0); + PySTEntryObject *generator_entry = _PySymtable_Lookup(SYMTABLE(c), (void *)generator_exp); + if (generator_entry->ste_coroutine) { + return 0; + } + location loc = LOC(func); int optimized = 0; @@ -3998,7 +4004,6 @@ maybe_optimize_function_call(compiler *c, expr_ty e, jump_target_label end) } else if (const_oparg == CONSTANT_BUILTIN_SET) { ADDOP_I(c, loc, BUILD_SET, 0); } - expr_ty generator_exp = asdl_seq_GET(args, 0); VISIT(c, expr, generator_exp); NEW_JUMP_TARGET_LABEL(c, loop);