From b98e1a5baa8a721705c27a481e6ecc30babb02e7 Mon Sep 17 00:00:00 2001 From: Maxime David Date: Tue, 11 Aug 2026 00:16:03 -0400 Subject: [PATCH 1/2] gh-154904: Speed up import of shutil by probing compression extensions directly (#154908) --- Lib/shutil.py | 17 +++++++++++------ Lib/test/test_shutil.py | 9 +++++++++ ...26-07-30-02-15-00.gh-issue-154904.Kx9mAq.rst | 4 ++++ 3 files changed, 24 insertions(+), 6 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-07-30-02-15-00.gh-issue-154904.Kx9mAq.rst diff --git a/Lib/shutil.py b/Lib/shutil.py index ce6969d6a4bf5a9..ab75ba9da8894b6 100644 --- a/Lib/shutil.py +++ b/Lib/shutil.py @@ -18,23 +18,28 @@ except ImportError: _ZLIB_SUPPORTED = False +# bz2, lzma and compression.zstd are pure Python wrappers whose only +# importable dependency that may be missing is the extension module they +# wrap. Probe those extensions directly instead: it gives the same answer +# without executing the wrappers, which shutil only needs when an archive +# is actually created or extracted. try: - import bz2 - del bz2 + import _bz2 + del _bz2 _BZ2_SUPPORTED = True except ImportError: _BZ2_SUPPORTED = False try: - import lzma - del lzma + import _lzma + del _lzma _LZMA_SUPPORTED = True except ImportError: _LZMA_SUPPORTED = False try: - from compression import zstd - del zstd + import _zstd + del _zstd _ZSTD_SUPPORTED = True except ImportError: _ZSTD_SUPPORTED = False diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py index d6b3b6a642bee1e..06ebdf9b68f20fc 100644 --- a/Lib/test/test_shutil.py +++ b/Lib/test/test_shutil.py @@ -32,6 +32,7 @@ from test import support from test.support import os_helper, socket_helper from test.support.os_helper import TESTFN, FakePath +from test.support.import_helper import ensure_lazy_imports TESTFN2 = TESTFN + "2" TESTFN_SRC = TESTFN + "_SRC" @@ -2362,6 +2363,14 @@ def _boo(filename, extract_dir, extra): unregister_unpack_format('Boo2') self.assertEqual(get_unpack_formats(), formats) + def test_compression_wrappers_not_imported_by_shutil(self): + # gh-154904: Importing shutil must not pull in the compression + # wrappers: they are only needed once an archive is actually created + # or extracted, and importing them measurably slows down every + # process that uses shutil. + ensure_lazy_imports("shutil", + {"bz2", "lzma", "compression", "compression.zstd"}) + class TestMisc(BaseTest, unittest.TestCase): diff --git a/Misc/NEWS.d/next/Library/2026-07-30-02-15-00.gh-issue-154904.Kx9mAq.rst b/Misc/NEWS.d/next/Library/2026-07-30-02-15-00.gh-issue-154904.Kx9mAq.rst new file mode 100644 index 000000000000000..fa3f930dc21b863 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-30-02-15-00.gh-issue-154904.Kx9mAq.rst @@ -0,0 +1,4 @@ +Speed up :mod:`shutil` import by probing the ``_bz2``, ``_lzma`` and +``_zstd`` extension modules instead of importing the :mod:`bz2`, +:mod:`lzma` and :mod:`compression.zstd` wrappers, which are now only +imported when an archive is actually created or extracted. From 6d2b551399342fc6621195abfde7d79bcb1d7a4b Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Tue, 11 Aug 2026 07:53:55 +0300 Subject: [PATCH 2/2] gh-151365: Temporarily downgrade Git to prevent fatal error (#155495) --- .github/workflows/reusable-check-html-ids.yml | 7 +++++++ .github/workflows/reusable-context.yml | 9 +++++++++ .github/workflows/reusable-docs.yml | 8 ++++++++ 3 files changed, 24 insertions(+) diff --git a/.github/workflows/reusable-check-html-ids.yml b/.github/workflows/reusable-check-html-ids.yml index 41ba1288be1ecf4..bdebacc9eed06bc 100644 --- a/.github/workflows/reusable-check-html-ids.yml +++ b/.github/workflows/reusable-check-html-ids.yml @@ -20,6 +20,13 @@ jobs: with: persist-credentials: false ref: ${{ github.event.pull_request.head.sha }} + - name: 'Downgrade Git' + # Temporarily downgrade to 2.43 until 2.55 is in the runner image, + # to avoid "fatal: shallow file has changed since we read it" bug. + # See https://github.com/python/cpython/issues/151365. + run: | + sudo apt-get install -y --allow-downgrades 'git=1:2.43.*' 'git-man=1:2.43.*' + git --version - name: 'Find merge base' id: merge-base run: | diff --git a/.github/workflows/reusable-context.yml b/.github/workflows/reusable-context.yml index c998cbff181dd12..f307326bae41209 100644 --- a/.github/workflows/reusable-context.yml +++ b/.github/workflows/reusable-context.yml @@ -90,6 +90,15 @@ jobs: || '' }} + - name: 'Downgrade Git' + # Temporarily downgrade to 2.43 until 2.55 is in the runner image, + # to avoid "fatal: shallow file has changed since we read it" bug. + # See https://github.com/python/cpython/issues/151365. + if: github.event_name == 'pull_request' + run: | + sudo apt-get install -y --allow-downgrades 'git=1:2.43.*' 'git-man=1:2.43.*' + git --version + # Adapted from https://github.com/actions/checkout/issues/520#issuecomment-1167205721 - name: Fetch commits to get branch diff if: github.event_name == 'pull_request' diff --git a/.github/workflows/reusable-docs.yml b/.github/workflows/reusable-docs.yml index 199e0fd8d181f0b..6b1a9076c8251b0 100644 --- a/.github/workflows/reusable-docs.yml +++ b/.github/workflows/reusable-docs.yml @@ -47,6 +47,14 @@ jobs: && github.event.pull_request.head.sha || '' }} + - name: 'Downgrade Git' + # Temporarily downgrade to 2.43 until 2.55 is in the runner image, + # to avoid "fatal: shallow file has changed since we read it" bug. + # See https://github.com/python/cpython/issues/151365. + if: github.event_name == 'pull_request' + run: | + sudo apt-get install -y --allow-downgrades 'git=1:2.43.*' 'git-man=1:2.43.*' + git --version # Adapted from https://github.com/actions/checkout/issues/520#issuecomment-1167205721 - name: 'Fetch commits to get branch diff' if: github.event_name == 'pull_request'