diff --git a/cpython-unix/build-cpython.sh b/cpython-unix/build-cpython.sh index 2f2391f10..fdb49c588 100755 --- a/cpython-unix/build-cpython.sh +++ b/cpython-unix/build-cpython.sh @@ -917,74 +917,7 @@ if [ "${PYBUILD_SHARED}" = "1" ]; then LIBPYTHON_SHARED_LIBRARY_BASENAME=libpython${PYTHON_MAJMIN_VERSION}${PYTHON_BINARY_SUFFIX}.so.1.0 LIBPYTHON_SHARED_LIBRARY=${ROOT}/out/python/install/lib/${LIBPYTHON_SHARED_LIBRARY_BASENAME} - # Although we are statically linking libpython, some extension - # modules link against libpython.so even though they are not - # supposed to do that. If you try to import them on an - # interpreter statically linking libpython, all the symbols they - # need are resolved from the main program (because neither glibc - # nor musl has two-level namespaces), so there is hopefully no - # correctness risk, but they need to be able to successfully - # find libpython.so in order to load the module. To allow such - # extensions to load, we set an rpath to point at our lib - # directory, so that if anyone ever tries to find a libpython, - # they successfully find one. See - # https://github.com/astral-sh/python-build-standalone/issues/619 - # for some reports of extensions that need this workaround. - # - # Note that this matches the behavior of Debian/Ubuntu/etc.'s - # interpreter (if package libpython3.x is installed, which it - # usually is thanks to gdb, vim, etc.), because libpython is in - # the system lib directory, as well as the behavior in practice - # on conda-forge miniconda and probably other Conda-family - # Python distributions, which too set an rpath. - # - # There is a downside of making this libpython locatable: some user - # code might do e.g. - # ctypes.CDLL(f"libpython3.{sys.version_info.minor}.so.1.0") - # to get at things in the CPython API not exposed to pure - # Python. This code may _silently misbehave_ on a - # static-libpython interpreter, because you are actually using - # the second copy of libpython. For loading static data or using - # accessors, you might get lucky and things will work, with the - # full set of dangers of C undefined behavior being possible. - # However, there are a few reasons we think this risk is - # tolerable. First, we can't actually fix it by not setting the - # rpath - user code may well find a system libpython3.x.so or - # something which is even more likely to break. Second, this - # exact problem happens with Debian, Conda, etc., so it is very - # unlikely (compared to the extension modules case above) that - # any widely-used code has this problem; the risk is largely - # backwards incompatibility of our own builds. Also, it's quite - # easy for users to fix: simply do - # ctypes.CDLL(None) - # (i.e., dlopen(NULL)), to use symbols already in the process; - # this will work reliably on all interpreters regardless of - # whether they statically or dynamically link libpython. Finally, - # we can (and should, at some point) add a warning, error, or - # silent fix to ctypes for user code that does this, which will - # also cover the case of other libpython3.x.so files on the - # library search path that we cannot suppress. - # - # In the past, when we dynamically linked libpython, we avoided - # using an rpath and instead used a DT_NEEDED entry with - # $ORIGIN/../lib/libpython.so, because LD_LIBRARY_PATH takes - # precedence over DT_RUNPATH, and it's not uncommon to have an - # LD_LIBRARY_PATH that points to some sort of unwanted libpython - # (e.g., actions/setup-python does this as of May 2025). - # Now, though, because we're not actually using code from the - # libpython that's loaded and just need _any_ file of that name - # to satisfy the link, that's not a problem. (This also implies - # another approach to the problem: ensure that libraries find an - # empty dummy libpython.so, which allows the link to succeed but - # ensures they do not use any unwanted symbols. That might be - # worth doing at some point.) - patchelf --force-rpath --set-rpath "\$ORIGIN/../lib" \ - "${ROOT}/out/python/install/bin/python${PYTHON_MAJMIN_VERSION}" - - if [ -n "${PYTHON_BINARY_SUFFIX}" ]; then - patchelf --force-rpath --set-rpath "\$ORIGIN/../lib" \ - "${ROOT}/out/python/install/bin/python${PYTHON_MAJMIN_VERSION}${PYTHON_BINARY_SUFFIX}" - fi + # PYSTANDALONE: do not set RPATH # For libpython3.so (the ABI3 library for embedders), we do # still dynamically link libpython3.x.so.1.0 (the @@ -1250,9 +1183,6 @@ fi # Ideally we'd adjust the build system. But meh. find "${ROOT}/out/python/install" -type d -name __pycache__ -print0 | xargs -0 rm -rf -# PYSTANDALONE: create an empty file to ensure the include directory exists (we remove the contents later) -touch "${ROOT}/out/python/install/include/python${PYTHON_MAJMIN_VERSION}/.empty" - # Ensure lib-dynload exists, or Python complains on startup. LIB_DYNLOAD=${ROOT}/out/python/install/lib/python${PYTHON_MAJMIN_VERSION}${PYTHON_LIB_SUFFIX}/lib-dynload mkdir -p "${LIB_DYNLOAD}" diff --git a/pythonbuild/static.py b/pythonbuild/static.py index 41cf1f761..186d96c89 100644 --- a/pythonbuild/static.py +++ b/pythonbuild/static.py @@ -1,866 +1,866 @@ -#!/usr/bin/env python3 -# PYSTANDALONE: re-added support for static compilation -import pathlib -import re -import sys - -from pythonbuild.cpython import meets_python_minimum_version -from pythonbuild.logging import log -from pythonbuild.utils import NoSearchStringError, static_replace_in_file - - -def add_to_config_c(source_path: pathlib.Path, extension: str, init_fn: str): - """Add an extension to PC/config.c""" - - config_c_path = source_path / "PC" / "config.c" - - lines = [] - - with config_c_path.open("r", encoding="utf8") as fh: - for line in fh: - line = line.rstrip() - - # Insert the init function declaration before the _inittab struct. - if line.startswith("struct _inittab"): - log("adding %s declaration to config.c" % init_fn) - lines.append("extern PyObject* %s(void);" % init_fn) - - # Insert the extension in the _inittab struct. - if line.lstrip().startswith("/* Sentinel */"): - log("marking %s as a built-in extension module" % extension) - lines.append('{"%s", %s},' % (extension, init_fn)) - - lines.append(line) - - with config_c_path.open("w", encoding="utf8") as fh: - fh.write("\n".join(lines)) - - -def remove_from_config_c(source_path: pathlib.Path, extension: str): - """Remove an extension from PC/config.c""" - - config_c_path = source_path / "PC" / "config.c" - - lines: list[str] = [] - - with config_c_path.open("r", encoding="utf8") as fh: - for line in fh: - line = line.rstrip() - - if ('{"%s",' % extension) in line: - log("removing %s as a built-in extension module" % extension) - init_fn = line.strip().strip("{},").partition(", ")[2] - log("removing %s declaration from config.c" % init_fn) - lines = list(filter(lambda line: init_fn not in line, lines)) - continue - - lines.append(line) - - with config_c_path.open("w", encoding="utf8") as fh: - fh.write("\n".join(lines)) - - -def remove_from_extension_modules(source_path: pathlib.Path, extension: str): - """Remove an extension from the set of extension/external modules. - - Call this when an extension will be compiled into libpython instead of - compiled as a standalone extension. - """ - - RE_EXTENSION_MODULES = re.compile('<(Extension|External)Modules Include="([^"]+)"') - - pcbuild_proj_path = source_path / "PCbuild" / "pcbuild.proj" - - lines = [] - - with pcbuild_proj_path.open("r", encoding="utf8") as fh: - for line in fh: - line = line.rstrip() - - m = RE_EXTENSION_MODULES.search(line) - - if m: - modules = [m for m in m.group(2).split(";") if m != extension] - - # Ignore line if new value is empty. - if not modules: - continue - - line = line.replace(m.group(2), ";".join(modules)) - - lines.append(line) - - with pcbuild_proj_path.open("w", encoding="utf8") as fh: - fh.write("\n".join(lines)) - - -def make_project_static_library(source_path: pathlib.Path, project: str): - """Turn a project file into a static library.""" - - proj_path = source_path / "PCbuild" / ("%s.vcxproj" % project) - lines = [] - - found_config_type = False - found_target_ext = False - - with proj_path.open("r", encoding="utf8") as fh: - for line in fh: - line = line.rstrip() - - # Change the project configuration to a static library. - if "DynamicLibrary" in line: - log("changing %s to a static library" % project) - found_config_type = True - line = line.replace("DynamicLibrary", "StaticLibrary") - - elif "StaticLibrary" in line: - log("%s is already a static library" % project) - return - - # Change the output file name from .pyd to .lib because it is no - # longer an extension. - if ".pyd" in line: - log("changing output of %s to a .lib" % project) - found_target_ext = True - line = line.replace(".pyd", ".lib") - # Python 3.13+ uses $(PyStdlibPydExt) instead of literal .pyd. - elif "$(PyStdlibPydExt)" in line: - log("changing output of %s to a .lib (3.13+ style)" % project) - found_target_ext = True - line = line.replace("$(PyStdlibPydExt)", ".lib") - - lines.append(line) - - if not found_config_type: - log("failed to adjust config type for %s" % project) - sys.exit(1) - - if not found_target_ext: - log("failed to adjust target extension for %s" % project) - sys.exit(1) - - with proj_path.open("w", encoding="utf8") as fh: - fh.write("\n".join(lines)) - - -def convert_to_static_library( - source_path: pathlib.Path, - extension: str, - entry: dict, - honor_allow_missing_preprocessor: bool, -): - """Converts an extension to a static library.""" - - proj_path = source_path / "PCbuild" / ("%s.vcxproj" % extension) - - if not proj_path.exists() and entry.get("ignore_missing"): - return False - - # Make the extension's project emit a static library so we can link - # against libpython. - make_project_static_library(source_path, extension) - - # And do the same thing for its dependencies. - for project in entry.get("static_depends", []): - make_project_static_library(source_path, project) - - copy_link_to_lib(proj_path) - - lines: list[str] = [] - - RE_PREPROCESSOR_DEFINITIONS = re.compile( - "]*>([^<]+)" - ) - - found_preprocessor = False - itemgroup_line = None - itemdefinitiongroup_line = None - - with proj_path.open("r", encoding="utf8") as fh: - for i, line in enumerate(fh): - line = line.rstrip() - - # Add Py_BUILD_CORE_BUILTIN to preprocessor definitions so linkage - # data is correct. - m = RE_PREPROCESSOR_DEFINITIONS.search(line) - - # But don't do it if it is an annotation for an individual source file. - if m and " entry. - if "" in line and not itemgroup_line: - itemgroup_line = i - - # Find the first entry. - if "" in line and not itemdefinitiongroup_line: - itemdefinitiongroup_line = i - - lines.append(line) - - if not found_preprocessor: - if honor_allow_missing_preprocessor and entry.get("allow_missing_preprocessor"): - log("not adjusting preprocessor definitions for %s" % extension) - elif itemgroup_line is not None: - log("introducing to %s" % extension) - lines[itemgroup_line:itemgroup_line] = [ - " ", - " ", - " Py_BUILD_CORE_BUILTIN;%(PreprocessorDefinitions)", - " ", - " ", - ] - - itemdefinitiongroup_line = itemgroup_line + 1 - - if "static_depends" in entry: - if not itemdefinitiongroup_line: - log("unable to find for %s" % extension) - sys.exit(1) - - log("changing %s to automatically link library dependencies" % extension) - lines[itemdefinitiongroup_line + 1 : itemdefinitiongroup_line + 1] = [ - " ", - " true", - " ", - ] - - # Ensure the extension project doesn't depend on pythoncore: as a built-in - # extension, pythoncore will depend on it. - - # This logic is a bit hacky. Ideally we'd parse the file as XML and operate - # in the XML domain. But that is more work. The goal here is to strip the - # ... containing the - # {pythoncore ID}. This could leave an item . - # That should be fine. - start_line, end_line = None, None - for i, line in enumerate(lines): - if "{cf7ac3d1-e2df-41d2-bea6-1e2556cdea26}" in line: - for j in range(i, 0, -1): - if "" in lines[j]: - end_line = j - break - - break - - if start_line is not None and end_line is not None: - log("stripping pythoncore dependency from %s" % extension) - for line in lines[start_line : end_line + 1]: - log(line) - - lines = lines[:start_line] + lines[end_line + 1 :] - - with proj_path.open("w", encoding="utf8") as fh: - fh.write("\n".join(lines)) - - # Tell pythoncore to link against the static .lib. - RE_ADDITIONAL_DEPENDENCIES = re.compile( - "([^<]+)" - ) - - pythoncore_path = source_path / "PCbuild" / "pythoncore.vcxproj" - lines = [] - - with pythoncore_path.open("r", encoding="utf8") as fh: - for line in fh: - line = line.rstrip() - - m = RE_ADDITIONAL_DEPENDENCIES.search(line) - - if m: - log("changing pythoncore to link against %s.lib" % extension) - # TODO we shouldn't need this with static linking if the - # project is configured to link library dependencies. - # But removing it results in unresolved external symbols - # when linking the python project. There /might/ be a - # visibility issue with the PyMODINIT_FUNC macro. - line = line.replace( - m.group(1), r"$(OutDir)%s.lib;%s" % (extension, m.group(1)) - ) - - lines.append(line) - - with pythoncore_path.open("w", encoding="utf8") as fh: - fh.write("\n".join(lines)) - - # Change pythoncore to depend on the extension project. - - # pcbuild.proj is the file that matters for msbuild. And order within - # matters. We remove the extension from the "ExtensionModules" set of - # projects. Then we re-add the project to before "pythoncore." - remove_from_extension_modules(source_path, extension) - - pcbuild_proj_path = source_path / "PCbuild" / "pcbuild.proj" - - with pcbuild_proj_path.open("r", encoding="utf8") as fh: - data = fh.read() - - data = data.replace( - '', - ' \n ' - % extension, - ) - - with pcbuild_proj_path.open("w", encoding="utf8") as fh: - fh.write(data) - - # We don't technically need to modify the solution since msbuild doesn't - # use it. But it enables debugging inside Visual Studio, which is - # convenient. - RE_PROJECT = re.compile( - r'Project\("\{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942\}"\) = "([^"]+)", "[^"]+", "{([^\}]+)\}"' - ) - - pcbuild_sln_path = source_path / "PCbuild" / "pcbuild.sln" - lines = [] - - extension_id = None - pythoncore_line = None - - with pcbuild_sln_path.open("r", encoding="utf8") as fh: - # First pass buffers the file, finds the ID of the extension project, - # and finds where the pythoncore project is defined. - for i, line in enumerate(fh): - line = line.rstrip() - - m = RE_PROJECT.search(line) - - if m and m.group(1) == extension: - extension_id = m.group(2) - - if m and m.group(1) == "pythoncore": - pythoncore_line = i - - lines.append(line) - - # Not all projects are in the solution(!!!). Since we don't use the - # solution for building, that's fine to ignore. - if not extension_id: - log("failed to find project %s in solution" % extension) - - if not pythoncore_line: - log("failed to find pythoncore project in solution") - - if extension_id and pythoncore_line: - log("making pythoncore depend on %s" % extension) - - needs_section = ( - not lines[pythoncore_line + 1].lstrip().startswith("ProjectSection") - ) - offset = 1 if needs_section else 2 - - lines.insert( - pythoncore_line + offset, "\t\t{%s} = {%s}" % (extension_id, extension_id) - ) - - if needs_section: - lines.insert( - pythoncore_line + 1, - "\tProjectSection(ProjectDependencies) = postProject", - ) - lines.insert(pythoncore_line + 3, "\tEndProjectSection") - - with pcbuild_sln_path.open("w", encoding="utf8") as fh: - fh.write("\n".join(lines)) - - return True - - -def copy_link_to_lib(p: pathlib.Path): - """Copy the contents of a section to a section.""" - - lines = [] - copy_lines: list[str] = [] - copy_active = False - - with p.open("r", encoding="utf8") as fh: - for line in fh: - line = line.rstrip() - - lines.append(line) - - if "" in line: - copy_active = True - continue - - elif "" in line: - copy_active = False - - log("duplicating section in %s" % p) - lines.append(" ") - lines.extend(copy_lines) - # Ensure the output directory is in the library path for - # lib.exe. The section inherits $(OutDir) from - # property sheets, but does not. Without this, - # dependency libraries referenced by filename only (e.g. - # zlib-ng.lib on 3.14+) cannot be found. - if not any("" in l for l in copy_lines): - lines.append( - " " - "$(OutDir);%(AdditionalLibraryDirectories)" - "" - ) - lines.append(" ") - - if copy_active: - copy_lines.append(line) - - with p.open("w", encoding="utf8") as fh: - fh.write("\n".join(lines)) - - -PYPORT_EXPORT_SEARCH_39 = b""" -#if defined(__CYGWIN__) -# define HAVE_DECLSPEC_DLL -#endif - -#include "exports.h" - -/* only get special linkage if built as shared or platform is Cygwin */ -#if defined(Py_ENABLE_SHARED) || defined(__CYGWIN__) -# if defined(HAVE_DECLSPEC_DLL) -# if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# define PyAPI_FUNC(RTYPE) Py_EXPORTED_SYMBOL RTYPE -# define PyAPI_DATA(RTYPE) extern Py_EXPORTED_SYMBOL RTYPE - /* module init functions inside the core need no external linkage */ - /* except for Cygwin to handle embedding */ -# if defined(__CYGWIN__) -# define PyMODINIT_FUNC Py_EXPORTED_SYMBOL PyObject* -# else /* __CYGWIN__ */ -# define PyMODINIT_FUNC PyObject* -# endif /* __CYGWIN__ */ -# else /* Py_BUILD_CORE */ - /* Building an extension module, or an embedded situation */ - /* public Python functions and data are imported */ - /* Under Cygwin, auto-import functions to prevent compilation */ - /* failures similar to those described at the bottom of 4.1: */ - /* http://docs.python.org/extending/windows.html#a-cookbook-approach */ -# if !defined(__CYGWIN__) -# define PyAPI_FUNC(RTYPE) Py_IMPORTED_SYMBOL RTYPE -# endif /* !__CYGWIN__ */ -# define PyAPI_DATA(RTYPE) extern Py_IMPORTED_SYMBOL RTYPE - /* module init functions outside the core must be exported */ -# if defined(__cplusplus) -# define PyMODINIT_FUNC extern "C" Py_EXPORTED_SYMBOL PyObject* -# else /* __cplusplus */ -# define PyMODINIT_FUNC Py_EXPORTED_SYMBOL PyObject* -# endif /* __cplusplus */ -# endif /* Py_BUILD_CORE */ -# endif /* HAVE_DECLSPEC_DLL */ -#endif /* Py_ENABLE_SHARED */ - -/* If no external linkage macros defined by now, create defaults */ -#ifndef PyAPI_FUNC -# define PyAPI_FUNC(RTYPE) Py_EXPORTED_SYMBOL RTYPE -#endif -#ifndef PyAPI_DATA -# define PyAPI_DATA(RTYPE) extern Py_EXPORTED_SYMBOL RTYPE -#endif -#ifndef PyMODINIT_FUNC -# if defined(__cplusplus) -# define PyMODINIT_FUNC extern "C" Py_EXPORTED_SYMBOL PyObject* -# else /* __cplusplus */ -# define PyMODINIT_FUNC Py_EXPORTED_SYMBOL PyObject* -# endif /* __cplusplus */ -#endif -""" - -PYPORT_EXPORT_SEARCH_38 = b""" -#if defined(__CYGWIN__) -# define HAVE_DECLSPEC_DLL -#endif - -/* only get special linkage if built as shared or platform is Cygwin */ -#if defined(Py_ENABLE_SHARED) || defined(__CYGWIN__) -# if defined(HAVE_DECLSPEC_DLL) -# if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# define PyAPI_FUNC(RTYPE) __declspec(dllexport) RTYPE -# define PyAPI_DATA(RTYPE) extern __declspec(dllexport) RTYPE - /* module init functions inside the core need no external linkage */ - /* except for Cygwin to handle embedding */ -# if defined(__CYGWIN__) -# define PyMODINIT_FUNC __declspec(dllexport) PyObject* -# else /* __CYGWIN__ */ -# define PyMODINIT_FUNC PyObject* -# endif /* __CYGWIN__ */ -# else /* Py_BUILD_CORE */ - /* Building an extension module, or an embedded situation */ - /* public Python functions and data are imported */ - /* Under Cygwin, auto-import functions to prevent compilation */ - /* failures similar to those described at the bottom of 4.1: */ - /* http://docs.python.org/extending/windows.html#a-cookbook-approach */ -# if !defined(__CYGWIN__) -# define PyAPI_FUNC(RTYPE) __declspec(dllimport) RTYPE -# endif /* !__CYGWIN__ */ -# define PyAPI_DATA(RTYPE) extern __declspec(dllimport) RTYPE - /* module init functions outside the core must be exported */ -# if defined(__cplusplus) -# define PyMODINIT_FUNC extern "C" __declspec(dllexport) PyObject* -# else /* __cplusplus */ -# define PyMODINIT_FUNC __declspec(dllexport) PyObject* -# endif /* __cplusplus */ -# endif /* Py_BUILD_CORE */ -# endif /* HAVE_DECLSPEC_DLL */ -#endif /* Py_ENABLE_SHARED */ - -/* If no external linkage macros defined by now, create defaults */ -#ifndef PyAPI_FUNC -# define PyAPI_FUNC(RTYPE) RTYPE -#endif -#ifndef PyAPI_DATA -# define PyAPI_DATA(RTYPE) extern RTYPE -#endif -#ifndef PyMODINIT_FUNC -# if defined(__cplusplus) -# define PyMODINIT_FUNC extern "C" PyObject* -# else /* __cplusplus */ -# define PyMODINIT_FUNC PyObject* -# endif /* __cplusplus */ -#endif -""" - -PYPORT_EXPORT_SEARCH_37 = b""" -#if defined(__CYGWIN__) -# define HAVE_DECLSPEC_DLL -#endif - -/* only get special linkage if built as shared or platform is Cygwin */ -#if defined(Py_ENABLE_SHARED) || defined(__CYGWIN__) -# if defined(HAVE_DECLSPEC_DLL) -# if defined(Py_BUILD_CORE) || defined(Py_BUILD_CORE_BUILTIN) -# define PyAPI_FUNC(RTYPE) __declspec(dllexport) RTYPE -# define PyAPI_DATA(RTYPE) extern __declspec(dllexport) RTYPE - /* module init functions inside the core need no external linkage */ - /* except for Cygwin to handle embedding */ -# if defined(__CYGWIN__) -# define PyMODINIT_FUNC __declspec(dllexport) PyObject* -# else /* __CYGWIN__ */ -# define PyMODINIT_FUNC PyObject* -# endif /* __CYGWIN__ */ -# else /* Py_BUILD_CORE */ - /* Building an extension module, or an embedded situation */ - /* public Python functions and data are imported */ - /* Under Cygwin, auto-import functions to prevent compilation */ - /* failures similar to those described at the bottom of 4.1: */ - /* http://docs.python.org/extending/windows.html#a-cookbook-approach */ -# if !defined(__CYGWIN__) -# define PyAPI_FUNC(RTYPE) __declspec(dllimport) RTYPE -# endif /* !__CYGWIN__ */ -# define PyAPI_DATA(RTYPE) extern __declspec(dllimport) RTYPE - /* module init functions outside the core must be exported */ -# if defined(__cplusplus) -# define PyMODINIT_FUNC extern "C" __declspec(dllexport) PyObject* -# else /* __cplusplus */ -# define PyMODINIT_FUNC __declspec(dllexport) PyObject* -# endif /* __cplusplus */ -# endif /* Py_BUILD_CORE */ -# endif /* HAVE_DECLSPEC_DLL */ -#endif /* Py_ENABLE_SHARED */ - -/* If no external linkage macros defined by now, create defaults */ -#ifndef PyAPI_FUNC -# define PyAPI_FUNC(RTYPE) RTYPE -#endif -#ifndef PyAPI_DATA -# define PyAPI_DATA(RTYPE) extern RTYPE -#endif -#ifndef PyMODINIT_FUNC -# if defined(__cplusplus) -# define PyMODINIT_FUNC extern "C" PyObject* -# else /* __cplusplus */ -# define PyMODINIT_FUNC PyObject* -# endif /* __cplusplus */ -#endif -""" - -PYPORT_EXPORT_REPLACE_NEW = b""" -#include "exports.h" -#define PyAPI_FUNC(RTYPE) __declspec(dllexport) RTYPE -#define PyAPI_DATA(RTYPE) extern __declspec(dllexport) RTYPE -#define PyMODINIT_FUNC __declspec(dllexport) PyObject* -""" - -PYPORT_EXPORT_REPLACE_OLD = b""" -#define PyAPI_FUNC(RTYPE) __declspec(dllexport) RTYPE -#define PyAPI_DATA(RTYPE) extern __declspec(dllexport) RTYPE -#define PyMODINIT_FUNC __declspec(dllexport) PyObject* -""" - -SYSMODULE_WINVER_SEARCH = b""" -#ifdef MS_COREDLL - SET_SYS("dllhandle", PyLong_FromVoidPtr(PyWin_DLLhModule)); - SET_SYS_FROM_STRING("winver", PyWin_DLLVersionString); -#endif -""" - -SYSMODULE_WINVER_REPLACE = b""" -#ifdef MS_COREDLL - SET_SYS("dllhandle", PyLong_FromVoidPtr(PyWin_DLLhModule)); - SET_SYS_FROM_STRING("winver", PyWin_DLLVersionString); -#else - SET_SYS_FROM_STRING("winver", "%s"); -#endif -""" - -SYSMODULE_WINVER_SEARCH_38 = b""" -#ifdef MS_COREDLL - SET_SYS_FROM_STRING("dllhandle", - PyLong_FromVoidPtr(PyWin_DLLhModule)); - SET_SYS_FROM_STRING("winver", - PyUnicode_FromString(PyWin_DLLVersionString)); -#endif -""" - -SYSMODULE_WINVER_REPLACE_38 = b""" -#ifdef MS_COREDLL - SET_SYS_FROM_STRING("dllhandle", - PyLong_FromVoidPtr(PyWin_DLLhModule)); - SET_SYS_FROM_STRING("winver", - PyUnicode_FromString(PyWin_DLLVersionString)); -#else - SET_SYS_FROM_STRING("winver", PyUnicode_FromString("%s")); -#endif -""" - -# In CPython 3.13+, the export macros were moved from pyport.h to exports.h. -# We add Py_NO_ENABLE_SHARED to the conditions so that our static builds -# (which define Py_NO_ENABLE_SHARED) get proper dllexport on BOTH -# Py_EXPORTED_SYMBOL and Py_IMPORTED_SYMBOL. Using dllexport for BOTH -# (instead of dllimport for Py_IMPORTED_SYMBOL) is critical because static -# libraries don't have __imp_ prefixed symbols that dllimport requires. -# This matches the 3.11/3.12 approach where PyAPI_FUNC is unconditionally -# __declspec(dllexport) for all code. -# We also add Py_NO_ENABLE_SHARED to the PyAPI_FUNC/PyMODINIT_FUNC block -# which activates the Py_BUILD_CORE branch for core code. That branch -# defines PyMODINIT_FUNC as plain PyObject* (no dllexport), matching the -# plain "extern" declarations in internal headers like pycore_warnings.h -# (which changed from PyAPI_FUNC() to plain extern in 3.13). -# Without this, _freeze_module gets C2375 "different linkage" errors. -EXPORTS_H_SEARCH_313 = b"""#if defined(_WIN32) || defined(__CYGWIN__) - #if defined(Py_ENABLE_SHARED) - #define Py_IMPORTED_SYMBOL __declspec(dllimport) - #define Py_EXPORTED_SYMBOL __declspec(dllexport) - #define Py_LOCAL_SYMBOL - #else - #define Py_IMPORTED_SYMBOL - #define Py_EXPORTED_SYMBOL - #define Py_LOCAL_SYMBOL - #endif""" - -EXPORTS_H_REPLACE_313 = b"""#if defined(_WIN32) || defined(__CYGWIN__) - #if defined(Py_ENABLE_SHARED) || defined(Py_NO_ENABLE_SHARED) - #define Py_IMPORTED_SYMBOL __declspec(dllexport) - #define Py_EXPORTED_SYMBOL __declspec(dllexport) - #define Py_LOCAL_SYMBOL - #else - #define Py_IMPORTED_SYMBOL - #define Py_EXPORTED_SYMBOL - #define Py_LOCAL_SYMBOL - #endif""" - -# The second block in exports.h: the PyAPI_FUNC/PyMODINIT_FUNC conditional. -EXPORTS_H_LINKAGE_SEARCH_313 = b"/* only get special linkage if built as shared or platform is Cygwin */\n#if defined(Py_ENABLE_SHARED) || defined(__CYGWIN__)" - -EXPORTS_H_LINKAGE_REPLACE_313 = b"/* only get special linkage if built as shared or platform is Cygwin */\n#if defined(Py_ENABLE_SHARED) || defined(Py_NO_ENABLE_SHARED) || defined(__CYGWIN__)" - - -def hack_source_files(source_path: pathlib.Path, python_version: str): - """Apply source modifications to make things work for static builds.""" - - # The PyAPI_FUNC, PyAPI_DATA, and PyMODINIT_FUNC macros define symbol - # visibility. By default, pyport.h looks at Py_ENABLE_SHARED, __CYGWIN__, - # Py_BUILD_CORE, Py_BUILD_CORE_BUILTIN, etc to determine what the macros - # should be. The logic assumes that Python is being built in a certain - # manner - notably that extensions are standalone dynamic libraries. - # - # We force the use of __declspec(dllexport) in all cases to ensure that - # API symbols are exported. This annotation becomes embedded within the - # object file. When that object file is linked, the symbol is exported - # from the final binary. For statically linked binaries, this behavior - # may not be needed. However, by exporting the symbols we allow downstream - # consumers of the object files to produce a binary that can be - # dynamically linked. This is a useful property to have. - - # In CPython 3.13+, exports moved from pyport.h to exports.h. - exports_h = source_path / "Include" / "exports.h" - pyport_h = source_path / "Include" / "pyport.h" - - if meets_python_minimum_version(python_version, "3.13") and exports_h.exists(): - static_replace_in_file(exports_h, EXPORTS_H_SEARCH_313, EXPORTS_H_REPLACE_313) - # Also patch the PyAPI_FUNC/PyMODINIT_FUNC conditional block. - static_replace_in_file( - exports_h, EXPORTS_H_LINKAGE_SEARCH_313, EXPORTS_H_LINKAGE_REPLACE_313 - ) - else: - try: - static_replace_in_file( - pyport_h, PYPORT_EXPORT_SEARCH_39, PYPORT_EXPORT_REPLACE_NEW - ) - except NoSearchStringError: - try: - static_replace_in_file( - pyport_h, PYPORT_EXPORT_SEARCH_38, PYPORT_EXPORT_REPLACE_OLD - ) - except NoSearchStringError: - static_replace_in_file( - pyport_h, PYPORT_EXPORT_SEARCH_37, PYPORT_EXPORT_REPLACE_OLD - ) - - # Modules/getpath.c unconditionally refers to PyWin_DLLhModule, which is - # conditionally defined behind Py_ENABLE_SHARED. Change its usage - # accordingly. This regressed as part of upstream commit - # 99fcf1505218464c489d419d4500f126b6d6dc28. But it was fixed - # in 3.12 by c6858d1e7f4cd3184d5ddea4025ad5dfc7596546. - if meets_python_minimum_version( - python_version, "3.11" - ) and not meets_python_minimum_version(python_version, "3.12"): - try: - static_replace_in_file( - source_path / "Modules" / "getpath.c", - b"#ifdef MS_WINDOWS\n extern HMODULE PyWin_DLLhModule;", - b"#if defined MS_WINDOWS && defined Py_ENABLE_SHARED\n extern HMODULE PyWin_DLLhModule;", - ) - except NoSearchStringError: - pass - - # Similar deal as above. Regression also introduced in upstream commit - # 99fcf1505218464c489d419d4500f126b6d6dc28. - if meets_python_minimum_version(python_version, "3.11"): - try: - static_replace_in_file( - source_path / "Python" / "dynload_win.c", - b"extern HMODULE PyWin_DLLhModule;\n", - b"#ifdef Py_ENABLE_SHARED\nextern HMODULE PyWin_DLLhModule;\n#else\n#define PyWin_DLLhModule NULL\n#endif\n", - ) - except NoSearchStringError: - pass - - # Modules/_winapi.c and Modules/overlapped.c both define an - # ``OverlappedType`` symbol. We rename one to make the symbol conflict - # go away. - try: - overlapped_c = source_path / "Modules" / "overlapped.c" - static_replace_in_file(overlapped_c, b"OverlappedType", b"OOverlappedType") - except NoSearchStringError: - pass - - # Modules/ctypes/callbacks.c has lines like the following: - # #ifndef Py_NO_ENABLE_SHARED - # BOOL WINAPI DllMain(HINSTANCE hinstDLL, DWORD fdwReason, LPVOID lpvRes) - # We currently define Py_ENABLE_SHARED. And I /think/ this check should - # also check against Py_BUILD_CORE_BUILTIN because Py_BUILD_CORE_BUILTIN - # with Py_ENABLE_SHARED is theoretically a valid configuration. - try: - callbacks_c = source_path / "Modules" / "_ctypes" / "callbacks.c" - static_replace_in_file( - callbacks_c, - b"#ifndef Py_NO_ENABLE_SHARED\nBOOL WINAPI DllMain(", - b"#if !defined(Py_NO_ENABLE_SHARED) && !defined(Py_BUILD_CORE_BUILTIN)\nBOOL WINAPI DllMain(", - ) - except NoSearchStringError: - pass - - # Lib/ctypes/__init__.py needs to populate the Python API version. On - # Windows, it assumes a ``pythonXY`` is available. On Cygwin, a - # ``libpythonXY`` DLL. The former assumes that ``sys.dllhandle`` is - # available. And ``sys.dllhandle`` is only populated if ``MS_COREDLL`` - # (a deprecated symbol) is defined. And ``MS_COREDLL`` is not defined - # if ``Py_NO_ENABLE_SHARED`` is defined. The gist of it is that ctypes - # assumes that Python on Windows will use a Python DLL. - # - # The ``pythonapi`` handle obtained in ``ctypes/__init__.py`` needs to - # expose a handle on the Python API. If we have a static library, that - # handle should be the current binary. So all the fancy logic to find - # the DLL can be simplified. - # - # But, ``PyDLL(None)`` doesn't work out of the box because this is - # translated into a call to ``LoadLibrary(NULL)``. Unlike ``dlopen()``, - # ``LoadLibrary()`` won't accept a NULL value. So, we need a way to - # get an ``HMODULE`` for the current executable. Arguably the best way - # to do this is with ``GetModuleHandleEx()`` using the following C code: - # - # HMODULE hModule = NULL; - # GetModuleHandleEx( - # GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS, - # (LPCSTR)SYMBOL_IN_CURRENT_MODULE, - # &hModule); - # - # The ``ctypes`` module has handles on function pointers in the current - # binary. One would think we'd be able to use ``ctypes.cast()`` + - # ``ctypes.addressof()`` to get a pointer to a symbol in the current - # executable. But the addresses appear to be to heap allocated PyObject - # instances, which won't work. - # - # An ideal solution would be to expose the ``HMODULE`` of the current - # module. We /should/ be able to change the behavior of ``sys.dllhandle`` - # to facilitate this. But this is a bit more work. Our hack is to instead - # use ``sys.executable`` with ``LoadLibrary()``. This should hopefully be - # "good enough." - try: - ctypes_init = source_path / "Lib" / "ctypes" / "__init__.py" - static_replace_in_file( - ctypes_init, - b'pythonapi = PyDLL("python dll", None, _sys.dllhandle)', - b"pythonapi = PyDLL(_sys.executable)", - ) - except NoSearchStringError: - pass - - # Python 3.11 made _Py_IDENTIFIER hidden by default. Source files need to - # opt in to unmasking it. Our static build tickles this into not working. - try: - static_replace_in_file( - source_path / "PC" / "_msi.c", - b"#include \n", - b"#define NEEDS_PY_IDENTIFIER\n#include \n", - ) - except (NoSearchStringError, FileNotFoundError): - pass - - # The `sys` module only populates `sys.winver` if MS_COREDLL is defined, - # which it isn't in static builds. We know what the version should be, so - # we go ahead and set it. - majmin = ".".join(python_version.split(".")[0:2]) - # Source changed in 3.10. - try: - static_replace_in_file( - source_path / "Python" / "sysmodule.c", - SYSMODULE_WINVER_SEARCH, - SYSMODULE_WINVER_REPLACE % majmin.encode("ascii"), - ) - except NoSearchStringError: - try: - static_replace_in_file( - source_path / "Python" / "sysmodule.c", - SYSMODULE_WINVER_SEARCH_38, - SYSMODULE_WINVER_REPLACE_38 % majmin.encode("ascii"), - ) - except NoSearchStringError: - pass - - # Producing statically linked binaries invalidates assumptions in the - # layout tool. Update the tool accordingly. - try: - layout_main = source_path / "PC" / "layout" / "main.py" - - # We no longer have a pythonXX.dll file. - try: - # 3.13+ has an if/else block for freethreaded DLL name. - static_replace_in_file( - layout_main, - b" if ns.include_freethreaded:\n yield from in_build(FREETHREADED_PYTHON_DLL_NAME)\n else:\n yield from in_build(PYTHON_DLL_NAME)\n", - b"", - ) - except NoSearchStringError: - static_replace_in_file( - layout_main, b" yield from in_build(PYTHON_DLL_NAME)\n", b"" - ) - except NoSearchStringError: - pass +#!/usr/bin/env python3 +# PYSTANDALONE: re-added support for static compilation +import pathlib +import re +import sys + +from pythonbuild.cpython import meets_python_minimum_version +from pythonbuild.logging import log +from pythonbuild.utils import NoSearchStringError, static_replace_in_file + + +def add_to_config_c(source_path: pathlib.Path, extension: str, init_fn: str): + """Add an extension to PC/config.c""" + + config_c_path = source_path / "PC" / "config.c" + + lines = [] + + with config_c_path.open("r", encoding="utf8") as fh: + for line in fh: + line = line.rstrip() + + # Insert the init function declaration before the _inittab struct. + if line.startswith("struct _inittab"): + log("adding %s declaration to config.c" % init_fn) + lines.append("extern PyObject* %s(void);" % init_fn) + + # Insert the extension in the _inittab struct. + if line.lstrip().startswith("/* Sentinel */"): + log("marking %s as a built-in extension module" % extension) + lines.append('{"%s", %s},' % (extension, init_fn)) + + lines.append(line) + + with config_c_path.open("w", encoding="utf8") as fh: + fh.write("\n".join(lines)) + + +def remove_from_config_c(source_path: pathlib.Path, extension: str): + """Remove an extension from PC/config.c""" + + config_c_path = source_path / "PC" / "config.c" + + lines: list[str] = [] + + with config_c_path.open("r", encoding="utf8") as fh: + for line in fh: + line = line.rstrip() + + if ('{"%s",' % extension) in line: + log("removing %s as a built-in extension module" % extension) + init_fn = line.strip().strip("{},").partition(", ")[2] + log("removing %s declaration from config.c" % init_fn) + lines = list(filter(lambda line: init_fn not in line, lines)) + continue + + lines.append(line) + + with config_c_path.open("w", encoding="utf8") as fh: + fh.write("\n".join(lines)) + + +def remove_from_extension_modules(source_path: pathlib.Path, extension: str): + """Remove an extension from the set of extension/external modules. + + Call this when an extension will be compiled into libpython instead of + compiled as a standalone extension. + """ + + RE_EXTENSION_MODULES = re.compile('<(Extension|External)Modules Include="([^"]+)"') + + pcbuild_proj_path = source_path / "PCbuild" / "pcbuild.proj" + + lines = [] + + with pcbuild_proj_path.open("r", encoding="utf8") as fh: + for line in fh: + line = line.rstrip() + + m = RE_EXTENSION_MODULES.search(line) + + if m: + modules = [m for m in m.group(2).split(";") if m != extension] + + # Ignore line if new value is empty. + if not modules: + continue + + line = line.replace(m.group(2), ";".join(modules)) + + lines.append(line) + + with pcbuild_proj_path.open("w", encoding="utf8") as fh: + fh.write("\n".join(lines)) + + +def make_project_static_library(source_path: pathlib.Path, project: str): + """Turn a project file into a static library.""" + + proj_path = source_path / "PCbuild" / ("%s.vcxproj" % project) + lines = [] + + found_config_type = False + found_target_ext = False + + with proj_path.open("r", encoding="utf8") as fh: + for line in fh: + line = line.rstrip() + + # Change the project configuration to a static library. + if "DynamicLibrary" in line: + log("changing %s to a static library" % project) + found_config_type = True + line = line.replace("DynamicLibrary", "StaticLibrary") + + elif "StaticLibrary" in line: + log("%s is already a static library" % project) + return + + # Change the output file name from .pyd to .lib because it is no + # longer an extension. + if ".pyd" in line: + log("changing output of %s to a .lib" % project) + found_target_ext = True + line = line.replace(".pyd", ".lib") + # Python 3.13+ uses $(PyStdlibPydExt) instead of literal .pyd. + elif "$(PyStdlibPydExt)" in line: + log("changing output of %s to a .lib (3.13+ style)" % project) + found_target_ext = True + line = line.replace("$(PyStdlibPydExt)", ".lib") + + lines.append(line) + + if not found_config_type: + log("failed to adjust config type for %s" % project) + sys.exit(1) + + if not found_target_ext: + log("failed to adjust target extension for %s" % project) + sys.exit(1) + + with proj_path.open("w", encoding="utf8") as fh: + fh.write("\n".join(lines)) + + +def convert_to_static_library( + source_path: pathlib.Path, + extension: str, + entry: dict, + honor_allow_missing_preprocessor: bool, +): + """Converts an extension to a static library.""" + + proj_path = source_path / "PCbuild" / ("%s.vcxproj" % extension) + + if not proj_path.exists() and entry.get("ignore_missing"): + return False + + # Make the extension's project emit a static library so we can link + # against libpython. + make_project_static_library(source_path, extension) + + # And do the same thing for its dependencies. + for project in entry.get("static_depends", []): + make_project_static_library(source_path, project) + + copy_link_to_lib(proj_path) + + lines: list[str] = [] + + RE_PREPROCESSOR_DEFINITIONS = re.compile( + "]*>([^<]+)" + ) + + found_preprocessor = False + itemgroup_line = None + itemdefinitiongroup_line = None + + with proj_path.open("r", encoding="utf8") as fh: + for i, line in enumerate(fh): + line = line.rstrip() + + # Add Py_BUILD_CORE_BUILTIN to preprocessor definitions so linkage + # data is correct. + m = RE_PREPROCESSOR_DEFINITIONS.search(line) + + # But don't do it if it is an annotation for an individual source file. + if m and " entry. + if "" in line and not itemgroup_line: + itemgroup_line = i + + # Find the first entry. + if "" in line and not itemdefinitiongroup_line: + itemdefinitiongroup_line = i + + lines.append(line) + + if not found_preprocessor: + if honor_allow_missing_preprocessor and entry.get("allow_missing_preprocessor"): + log("not adjusting preprocessor definitions for %s" % extension) + elif itemgroup_line is not None: + log("introducing to %s" % extension) + lines[itemgroup_line:itemgroup_line] = [ + " ", + " ", + " Py_BUILD_CORE_BUILTIN;%(PreprocessorDefinitions)", + " ", + " ", + ] + + itemdefinitiongroup_line = itemgroup_line + 1 + + if "static_depends" in entry: + if not itemdefinitiongroup_line: + log("unable to find for %s" % extension) + sys.exit(1) + + log("changing %s to automatically link library dependencies" % extension) + lines[itemdefinitiongroup_line + 1 : itemdefinitiongroup_line + 1] = [ + " ", + " true", + " ", + ] + + # Ensure the extension project doesn't depend on pythoncore: as a built-in + # extension, pythoncore will depend on it. + + # This logic is a bit hacky. Ideally we'd parse the file as XML and operate + # in the XML domain. But that is more work. The goal here is to strip the + # ... containing the + # {pythoncore ID}. This could leave an item . + # That should be fine. + start_line, end_line = None, None + for i, line in enumerate(lines): + if "{cf7ac3d1-e2df-41d2-bea6-1e2556cdea26}" in line: + for j in range(i, 0, -1): + if "" in lines[j]: + end_line = j + break + + break + + if start_line is not None and end_line is not None: + log("stripping pythoncore dependency from %s" % extension) + for line in lines[start_line : end_line + 1]: + log(line) + + lines = lines[:start_line] + lines[end_line + 1 :] + + with proj_path.open("w", encoding="utf8") as fh: + fh.write("\n".join(lines)) + + # Tell pythoncore to link against the static .lib. + RE_ADDITIONAL_DEPENDENCIES = re.compile( + "([^<]+)" + ) + + pythoncore_path = source_path / "PCbuild" / "pythoncore.vcxproj" + lines = [] + + with pythoncore_path.open("r", encoding="utf8") as fh: + for line in fh: + line = line.rstrip() + + m = RE_ADDITIONAL_DEPENDENCIES.search(line) + + if m: + log("changing pythoncore to link against %s.lib" % extension) + # TODO we shouldn't need this with static linking if the + # project is configured to link library dependencies. + # But removing it results in unresolved external symbols + # when linking the python project. There /might/ be a + # visibility issue with the PyMODINIT_FUNC macro. + line = line.replace( + m.group(1), r"$(OutDir)%s.lib;%s" % (extension, m.group(1)) + ) + + lines.append(line) + + with pythoncore_path.open("w", encoding="utf8") as fh: + fh.write("\n".join(lines)) + + # Change pythoncore to depend on the extension project. + + # pcbuild.proj is the file that matters for msbuild. And order within + # matters. We remove the extension from the "ExtensionModules" set of + # projects. Then we re-add the project to before "pythoncore." + remove_from_extension_modules(source_path, extension) + + pcbuild_proj_path = source_path / "PCbuild" / "pcbuild.proj" + + with pcbuild_proj_path.open("r", encoding="utf8") as fh: + data = fh.read() + + data = data.replace( + '', + ' \n ' + % extension, + ) + + with pcbuild_proj_path.open("w", encoding="utf8") as fh: + fh.write(data) + + # We don't technically need to modify the solution since msbuild doesn't + # use it. But it enables debugging inside Visual Studio, which is + # convenient. + RE_PROJECT = re.compile( + r'Project\("\{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942\}"\) = "([^"]+)", "[^"]+", "{([^\}]+)\}"' + ) + + pcbuild_sln_path = source_path / "PCbuild" / "pcbuild.sln" + lines = [] + + extension_id = None + pythoncore_line = None + + with pcbuild_sln_path.open("r", encoding="utf8") as fh: + # First pass buffers the file, finds the ID of the extension project, + # and finds where the pythoncore project is defined. + for i, line in enumerate(fh): + line = line.rstrip() + + m = RE_PROJECT.search(line) + + if m and m.group(1) == extension: + extension_id = m.group(2) + + if m and m.group(1) == "pythoncore": + pythoncore_line = i + + lines.append(line) + + # Not all projects are in the solution(!!!). Since we don't use the + # solution for building, that's fine to ignore. + if not extension_id: + log("failed to find project %s in solution" % extension) + + if not pythoncore_line: + log("failed to find pythoncore project in solution") + + if extension_id and pythoncore_line: + log("making pythoncore depend on %s" % extension) + + needs_section = ( + not lines[pythoncore_line + 1].lstrip().startswith("ProjectSection") + ) + offset = 1 if needs_section else 2 + + lines.insert( + pythoncore_line + offset, "\t\t{%s} = {%s}" % (extension_id, extension_id) + ) + + if needs_section: + lines.insert( + pythoncore_line + 1, + "\tProjectSection(ProjectDependencies) = postProject", + ) + lines.insert(pythoncore_line + 3, "\tEndProjectSection") + + with pcbuild_sln_path.open("w", encoding="utf8") as fh: + fh.write("\n".join(lines)) + + return True + + +def copy_link_to_lib(p: pathlib.Path): + """Copy the contents of a section to a section.""" + + lines = [] + copy_lines: list[str] = [] + copy_active = False + + with p.open("r", encoding="utf8") as fh: + for line in fh: + line = line.rstrip() + + lines.append(line) + + if "" in line: + copy_active = True + continue + + elif "" in line: + copy_active = False + + log("duplicating section in %s" % p) + lines.append(" ") + lines.extend(copy_lines) + # Ensure the output directory is in the library path for + # lib.exe. The section inherits $(OutDir) from + # property sheets, but does not. Without this, + # dependency libraries referenced by filename only (e.g. + # zlib-ng.lib on 3.14+) cannot be found. + if not any("" in l for l in copy_lines): + lines.append( + " " + "$(OutDir);%(AdditionalLibraryDirectories)" + "" + ) + lines.append(" ") + + if copy_active: + copy_lines.append(line) + + with p.open("w", encoding="utf8") as fh: + fh.write("\n".join(lines)) + + +PYPORT_EXPORT_SEARCH_39 = b""" +#if defined(__CYGWIN__) +# define HAVE_DECLSPEC_DLL +#endif + +#include "exports.h" + +/* only get special linkage if built as shared or platform is Cygwin */ +#if defined(Py_ENABLE_SHARED) || defined(__CYGWIN__) +# if defined(HAVE_DECLSPEC_DLL) +# if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) +# define PyAPI_FUNC(RTYPE) Py_EXPORTED_SYMBOL RTYPE +# define PyAPI_DATA(RTYPE) extern Py_EXPORTED_SYMBOL RTYPE + /* module init functions inside the core need no external linkage */ + /* except for Cygwin to handle embedding */ +# if defined(__CYGWIN__) +# define PyMODINIT_FUNC Py_EXPORTED_SYMBOL PyObject* +# else /* __CYGWIN__ */ +# define PyMODINIT_FUNC PyObject* +# endif /* __CYGWIN__ */ +# else /* Py_BUILD_CORE */ + /* Building an extension module, or an embedded situation */ + /* public Python functions and data are imported */ + /* Under Cygwin, auto-import functions to prevent compilation */ + /* failures similar to those described at the bottom of 4.1: */ + /* http://docs.python.org/extending/windows.html#a-cookbook-approach */ +# if !defined(__CYGWIN__) +# define PyAPI_FUNC(RTYPE) Py_IMPORTED_SYMBOL RTYPE +# endif /* !__CYGWIN__ */ +# define PyAPI_DATA(RTYPE) extern Py_IMPORTED_SYMBOL RTYPE + /* module init functions outside the core must be exported */ +# if defined(__cplusplus) +# define PyMODINIT_FUNC extern "C" Py_EXPORTED_SYMBOL PyObject* +# else /* __cplusplus */ +# define PyMODINIT_FUNC Py_EXPORTED_SYMBOL PyObject* +# endif /* __cplusplus */ +# endif /* Py_BUILD_CORE */ +# endif /* HAVE_DECLSPEC_DLL */ +#endif /* Py_ENABLE_SHARED */ + +/* If no external linkage macros defined by now, create defaults */ +#ifndef PyAPI_FUNC +# define PyAPI_FUNC(RTYPE) Py_EXPORTED_SYMBOL RTYPE +#endif +#ifndef PyAPI_DATA +# define PyAPI_DATA(RTYPE) extern Py_EXPORTED_SYMBOL RTYPE +#endif +#ifndef PyMODINIT_FUNC +# if defined(__cplusplus) +# define PyMODINIT_FUNC extern "C" Py_EXPORTED_SYMBOL PyObject* +# else /* __cplusplus */ +# define PyMODINIT_FUNC Py_EXPORTED_SYMBOL PyObject* +# endif /* __cplusplus */ +#endif +""" + +PYPORT_EXPORT_SEARCH_38 = b""" +#if defined(__CYGWIN__) +# define HAVE_DECLSPEC_DLL +#endif + +/* only get special linkage if built as shared or platform is Cygwin */ +#if defined(Py_ENABLE_SHARED) || defined(__CYGWIN__) +# if defined(HAVE_DECLSPEC_DLL) +# if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) +# define PyAPI_FUNC(RTYPE) __declspec(dllexport) RTYPE +# define PyAPI_DATA(RTYPE) extern __declspec(dllexport) RTYPE + /* module init functions inside the core need no external linkage */ + /* except for Cygwin to handle embedding */ +# if defined(__CYGWIN__) +# define PyMODINIT_FUNC __declspec(dllexport) PyObject* +# else /* __CYGWIN__ */ +# define PyMODINIT_FUNC PyObject* +# endif /* __CYGWIN__ */ +# else /* Py_BUILD_CORE */ + /* Building an extension module, or an embedded situation */ + /* public Python functions and data are imported */ + /* Under Cygwin, auto-import functions to prevent compilation */ + /* failures similar to those described at the bottom of 4.1: */ + /* http://docs.python.org/extending/windows.html#a-cookbook-approach */ +# if !defined(__CYGWIN__) +# define PyAPI_FUNC(RTYPE) __declspec(dllimport) RTYPE +# endif /* !__CYGWIN__ */ +# define PyAPI_DATA(RTYPE) extern __declspec(dllimport) RTYPE + /* module init functions outside the core must be exported */ +# if defined(__cplusplus) +# define PyMODINIT_FUNC extern "C" __declspec(dllexport) PyObject* +# else /* __cplusplus */ +# define PyMODINIT_FUNC __declspec(dllexport) PyObject* +# endif /* __cplusplus */ +# endif /* Py_BUILD_CORE */ +# endif /* HAVE_DECLSPEC_DLL */ +#endif /* Py_ENABLE_SHARED */ + +/* If no external linkage macros defined by now, create defaults */ +#ifndef PyAPI_FUNC +# define PyAPI_FUNC(RTYPE) RTYPE +#endif +#ifndef PyAPI_DATA +# define PyAPI_DATA(RTYPE) extern RTYPE +#endif +#ifndef PyMODINIT_FUNC +# if defined(__cplusplus) +# define PyMODINIT_FUNC extern "C" PyObject* +# else /* __cplusplus */ +# define PyMODINIT_FUNC PyObject* +# endif /* __cplusplus */ +#endif +""" + +PYPORT_EXPORT_SEARCH_37 = b""" +#if defined(__CYGWIN__) +# define HAVE_DECLSPEC_DLL +#endif + +/* only get special linkage if built as shared or platform is Cygwin */ +#if defined(Py_ENABLE_SHARED) || defined(__CYGWIN__) +# if defined(HAVE_DECLSPEC_DLL) +# if defined(Py_BUILD_CORE) || defined(Py_BUILD_CORE_BUILTIN) +# define PyAPI_FUNC(RTYPE) __declspec(dllexport) RTYPE +# define PyAPI_DATA(RTYPE) extern __declspec(dllexport) RTYPE + /* module init functions inside the core need no external linkage */ + /* except for Cygwin to handle embedding */ +# if defined(__CYGWIN__) +# define PyMODINIT_FUNC __declspec(dllexport) PyObject* +# else /* __CYGWIN__ */ +# define PyMODINIT_FUNC PyObject* +# endif /* __CYGWIN__ */ +# else /* Py_BUILD_CORE */ + /* Building an extension module, or an embedded situation */ + /* public Python functions and data are imported */ + /* Under Cygwin, auto-import functions to prevent compilation */ + /* failures similar to those described at the bottom of 4.1: */ + /* http://docs.python.org/extending/windows.html#a-cookbook-approach */ +# if !defined(__CYGWIN__) +# define PyAPI_FUNC(RTYPE) __declspec(dllimport) RTYPE +# endif /* !__CYGWIN__ */ +# define PyAPI_DATA(RTYPE) extern __declspec(dllimport) RTYPE + /* module init functions outside the core must be exported */ +# if defined(__cplusplus) +# define PyMODINIT_FUNC extern "C" __declspec(dllexport) PyObject* +# else /* __cplusplus */ +# define PyMODINIT_FUNC __declspec(dllexport) PyObject* +# endif /* __cplusplus */ +# endif /* Py_BUILD_CORE */ +# endif /* HAVE_DECLSPEC_DLL */ +#endif /* Py_ENABLE_SHARED */ + +/* If no external linkage macros defined by now, create defaults */ +#ifndef PyAPI_FUNC +# define PyAPI_FUNC(RTYPE) RTYPE +#endif +#ifndef PyAPI_DATA +# define PyAPI_DATA(RTYPE) extern RTYPE +#endif +#ifndef PyMODINIT_FUNC +# if defined(__cplusplus) +# define PyMODINIT_FUNC extern "C" PyObject* +# else /* __cplusplus */ +# define PyMODINIT_FUNC PyObject* +# endif /* __cplusplus */ +#endif +""" + +PYPORT_EXPORT_REPLACE_NEW = b""" +#include "exports.h" +#define PyAPI_FUNC(RTYPE) __declspec(dllexport) RTYPE +#define PyAPI_DATA(RTYPE) extern __declspec(dllexport) RTYPE +#define PyMODINIT_FUNC __declspec(dllexport) PyObject* +""" + +PYPORT_EXPORT_REPLACE_OLD = b""" +#define PyAPI_FUNC(RTYPE) __declspec(dllexport) RTYPE +#define PyAPI_DATA(RTYPE) extern __declspec(dllexport) RTYPE +#define PyMODINIT_FUNC __declspec(dllexport) PyObject* +""" + +SYSMODULE_WINVER_SEARCH = b""" +#ifdef MS_COREDLL + SET_SYS("dllhandle", PyLong_FromVoidPtr(PyWin_DLLhModule)); + SET_SYS_FROM_STRING("winver", PyWin_DLLVersionString); +#endif +""" + +SYSMODULE_WINVER_REPLACE = b""" +#ifdef MS_COREDLL + SET_SYS("dllhandle", PyLong_FromVoidPtr(PyWin_DLLhModule)); + SET_SYS_FROM_STRING("winver", PyWin_DLLVersionString); +#else + SET_SYS_FROM_STRING("winver", "%s"); +#endif +""" + +SYSMODULE_WINVER_SEARCH_38 = b""" +#ifdef MS_COREDLL + SET_SYS_FROM_STRING("dllhandle", + PyLong_FromVoidPtr(PyWin_DLLhModule)); + SET_SYS_FROM_STRING("winver", + PyUnicode_FromString(PyWin_DLLVersionString)); +#endif +""" + +SYSMODULE_WINVER_REPLACE_38 = b""" +#ifdef MS_COREDLL + SET_SYS_FROM_STRING("dllhandle", + PyLong_FromVoidPtr(PyWin_DLLhModule)); + SET_SYS_FROM_STRING("winver", + PyUnicode_FromString(PyWin_DLLVersionString)); +#else + SET_SYS_FROM_STRING("winver", PyUnicode_FromString("%s")); +#endif +""" + +# In CPython 3.13+, the export macros were moved from pyport.h to exports.h. +# We add Py_NO_ENABLE_SHARED to the conditions so that our static builds +# (which define Py_NO_ENABLE_SHARED) get proper dllexport on BOTH +# Py_EXPORTED_SYMBOL and Py_IMPORTED_SYMBOL. Using dllexport for BOTH +# (instead of dllimport for Py_IMPORTED_SYMBOL) is critical because static +# libraries don't have __imp_ prefixed symbols that dllimport requires. +# This matches the 3.11/3.12 approach where PyAPI_FUNC is unconditionally +# __declspec(dllexport) for all code. +# We also add Py_NO_ENABLE_SHARED to the PyAPI_FUNC/PyMODINIT_FUNC block +# which activates the Py_BUILD_CORE branch for core code. That branch +# defines PyMODINIT_FUNC as plain PyObject* (no dllexport), matching the +# plain "extern" declarations in internal headers like pycore_warnings.h +# (which changed from PyAPI_FUNC() to plain extern in 3.13). +# Without this, _freeze_module gets C2375 "different linkage" errors. +EXPORTS_H_SEARCH_313 = b"""#if defined(_WIN32) || defined(__CYGWIN__) + #if defined(Py_ENABLE_SHARED) + #define Py_IMPORTED_SYMBOL __declspec(dllimport) + #define Py_EXPORTED_SYMBOL __declspec(dllexport) + #define Py_LOCAL_SYMBOL + #else + #define Py_IMPORTED_SYMBOL + #define Py_EXPORTED_SYMBOL + #define Py_LOCAL_SYMBOL + #endif""" + +EXPORTS_H_REPLACE_313 = b"""#if defined(_WIN32) || defined(__CYGWIN__) + #if defined(Py_ENABLE_SHARED) || defined(Py_NO_ENABLE_SHARED) + #define Py_IMPORTED_SYMBOL __declspec(dllexport) + #define Py_EXPORTED_SYMBOL __declspec(dllexport) + #define Py_LOCAL_SYMBOL + #else + #define Py_IMPORTED_SYMBOL + #define Py_EXPORTED_SYMBOL + #define Py_LOCAL_SYMBOL + #endif""" + +# The second block in exports.h: the PyAPI_FUNC/PyMODINIT_FUNC conditional. +EXPORTS_H_LINKAGE_SEARCH_313 = b"/* only get special linkage if built as shared or platform is Cygwin */\n#if defined(Py_ENABLE_SHARED) || defined(__CYGWIN__)" + +EXPORTS_H_LINKAGE_REPLACE_313 = b"/* only get special linkage if built as shared or platform is Cygwin */\n#if defined(Py_ENABLE_SHARED) || defined(Py_NO_ENABLE_SHARED) || defined(__CYGWIN__)" + + +def hack_source_files(source_path: pathlib.Path, python_version: str): + """Apply source modifications to make things work for static builds.""" + + # The PyAPI_FUNC, PyAPI_DATA, and PyMODINIT_FUNC macros define symbol + # visibility. By default, pyport.h looks at Py_ENABLE_SHARED, __CYGWIN__, + # Py_BUILD_CORE, Py_BUILD_CORE_BUILTIN, etc to determine what the macros + # should be. The logic assumes that Python is being built in a certain + # manner - notably that extensions are standalone dynamic libraries. + # + # We force the use of __declspec(dllexport) in all cases to ensure that + # API symbols are exported. This annotation becomes embedded within the + # object file. When that object file is linked, the symbol is exported + # from the final binary. For statically linked binaries, this behavior + # may not be needed. However, by exporting the symbols we allow downstream + # consumers of the object files to produce a binary that can be + # dynamically linked. This is a useful property to have. + + # In CPython 3.13+, exports moved from pyport.h to exports.h. + exports_h = source_path / "Include" / "exports.h" + pyport_h = source_path / "Include" / "pyport.h" + + if meets_python_minimum_version(python_version, "3.13") and exports_h.exists(): + static_replace_in_file(exports_h, EXPORTS_H_SEARCH_313, EXPORTS_H_REPLACE_313) + # Also patch the PyAPI_FUNC/PyMODINIT_FUNC conditional block. + static_replace_in_file( + exports_h, EXPORTS_H_LINKAGE_SEARCH_313, EXPORTS_H_LINKAGE_REPLACE_313 + ) + else: + try: + static_replace_in_file( + pyport_h, PYPORT_EXPORT_SEARCH_39, PYPORT_EXPORT_REPLACE_NEW + ) + except NoSearchStringError: + try: + static_replace_in_file( + pyport_h, PYPORT_EXPORT_SEARCH_38, PYPORT_EXPORT_REPLACE_OLD + ) + except NoSearchStringError: + static_replace_in_file( + pyport_h, PYPORT_EXPORT_SEARCH_37, PYPORT_EXPORT_REPLACE_OLD + ) + + # Modules/getpath.c unconditionally refers to PyWin_DLLhModule, which is + # conditionally defined behind Py_ENABLE_SHARED. Change its usage + # accordingly. This regressed as part of upstream commit + # 99fcf1505218464c489d419d4500f126b6d6dc28. But it was fixed + # in 3.12 by c6858d1e7f4cd3184d5ddea4025ad5dfc7596546. + if meets_python_minimum_version( + python_version, "3.11" + ) and not meets_python_minimum_version(python_version, "3.12"): + try: + static_replace_in_file( + source_path / "Modules" / "getpath.c", + b"#ifdef MS_WINDOWS\n extern HMODULE PyWin_DLLhModule;", + b"#if defined MS_WINDOWS && defined Py_ENABLE_SHARED\n extern HMODULE PyWin_DLLhModule;", + ) + except NoSearchStringError: + pass + + # Similar deal as above. Regression also introduced in upstream commit + # 99fcf1505218464c489d419d4500f126b6d6dc28. + if meets_python_minimum_version(python_version, "3.11"): + try: + static_replace_in_file( + source_path / "Python" / "dynload_win.c", + b"extern HMODULE PyWin_DLLhModule;\n", + b"#ifdef Py_ENABLE_SHARED\nextern HMODULE PyWin_DLLhModule;\n#else\n#define PyWin_DLLhModule NULL\n#endif\n", + ) + except NoSearchStringError: + pass + + # Modules/_winapi.c and Modules/overlapped.c both define an + # ``OverlappedType`` symbol. We rename one to make the symbol conflict + # go away. + try: + overlapped_c = source_path / "Modules" / "overlapped.c" + static_replace_in_file(overlapped_c, b"OverlappedType", b"OOverlappedType") + except NoSearchStringError: + pass + + # Modules/ctypes/callbacks.c has lines like the following: + # #ifndef Py_NO_ENABLE_SHARED + # BOOL WINAPI DllMain(HINSTANCE hinstDLL, DWORD fdwReason, LPVOID lpvRes) + # We currently define Py_ENABLE_SHARED. And I /think/ this check should + # also check against Py_BUILD_CORE_BUILTIN because Py_BUILD_CORE_BUILTIN + # with Py_ENABLE_SHARED is theoretically a valid configuration. + try: + callbacks_c = source_path / "Modules" / "_ctypes" / "callbacks.c" + static_replace_in_file( + callbacks_c, + b"#ifndef Py_NO_ENABLE_SHARED\nBOOL WINAPI DllMain(", + b"#if !defined(Py_NO_ENABLE_SHARED) && !defined(Py_BUILD_CORE_BUILTIN)\nBOOL WINAPI DllMain(", + ) + except NoSearchStringError: + pass + + # Lib/ctypes/__init__.py needs to populate the Python API version. On + # Windows, it assumes a ``pythonXY`` is available. On Cygwin, a + # ``libpythonXY`` DLL. The former assumes that ``sys.dllhandle`` is + # available. And ``sys.dllhandle`` is only populated if ``MS_COREDLL`` + # (a deprecated symbol) is defined. And ``MS_COREDLL`` is not defined + # if ``Py_NO_ENABLE_SHARED`` is defined. The gist of it is that ctypes + # assumes that Python on Windows will use a Python DLL. + # + # The ``pythonapi`` handle obtained in ``ctypes/__init__.py`` needs to + # expose a handle on the Python API. If we have a static library, that + # handle should be the current binary. So all the fancy logic to find + # the DLL can be simplified. + # + # But, ``PyDLL(None)`` doesn't work out of the box because this is + # translated into a call to ``LoadLibrary(NULL)``. Unlike ``dlopen()``, + # ``LoadLibrary()`` won't accept a NULL value. So, we need a way to + # get an ``HMODULE`` for the current executable. Arguably the best way + # to do this is with ``GetModuleHandleEx()`` using the following C code: + # + # HMODULE hModule = NULL; + # GetModuleHandleEx( + # GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS, + # (LPCSTR)SYMBOL_IN_CURRENT_MODULE, + # &hModule); + # + # The ``ctypes`` module has handles on function pointers in the current + # binary. One would think we'd be able to use ``ctypes.cast()`` + + # ``ctypes.addressof()`` to get a pointer to a symbol in the current + # executable. But the addresses appear to be to heap allocated PyObject + # instances, which won't work. + # + # An ideal solution would be to expose the ``HMODULE`` of the current + # module. We /should/ be able to change the behavior of ``sys.dllhandle`` + # to facilitate this. But this is a bit more work. Our hack is to instead + # use ``sys.executable`` with ``LoadLibrary()``. This should hopefully be + # "good enough." + try: + ctypes_init = source_path / "Lib" / "ctypes" / "__init__.py" + static_replace_in_file( + ctypes_init, + b'pythonapi = PyDLL("python dll", None, _sys.dllhandle)', + b"pythonapi = PyDLL(_sys.executable)", + ) + except NoSearchStringError: + pass + + # Python 3.11 made _Py_IDENTIFIER hidden by default. Source files need to + # opt in to unmasking it. Our static build tickles this into not working. + try: + static_replace_in_file( + source_path / "PC" / "_msi.c", + b"#include \n", + b"#define NEEDS_PY_IDENTIFIER\n#include \n", + ) + except (NoSearchStringError, FileNotFoundError): + pass + + # The `sys` module only populates `sys.winver` if MS_COREDLL is defined, + # which it isn't in static builds. We know what the version should be, so + # we go ahead and set it. + majmin = ".".join(python_version.split(".")[0:2]) + # Source changed in 3.10. + try: + static_replace_in_file( + source_path / "Python" / "sysmodule.c", + SYSMODULE_WINVER_SEARCH, + SYSMODULE_WINVER_REPLACE % majmin.encode("ascii"), + ) + except NoSearchStringError: + try: + static_replace_in_file( + source_path / "Python" / "sysmodule.c", + SYSMODULE_WINVER_SEARCH_38, + SYSMODULE_WINVER_REPLACE_38 % majmin.encode("ascii"), + ) + except NoSearchStringError: + pass + + # Producing statically linked binaries invalidates assumptions in the + # layout tool. Update the tool accordingly. + try: + layout_main = source_path / "PC" / "layout" / "main.py" + + # We no longer have a pythonXX.dll file. + try: + # 3.13+ has an if/else block for freethreaded DLL name. + static_replace_in_file( + layout_main, + b" if ns.include_freethreaded:\n yield from in_build(FREETHREADED_PYTHON_DLL_NAME)\n else:\n yield from in_build(PYTHON_DLL_NAME)\n", + b"", + ) + except NoSearchStringError: + static_replace_in_file( + layout_main, b" yield from in_build(PYTHON_DLL_NAME)\n", b"" + ) + except NoSearchStringError: + pass diff --git a/pythonbuild/utils.py b/pythonbuild/utils.py index b22387cb1..08781f582 100644 --- a/pythonbuild/utils.py +++ b/pythonbuild/utils.py @@ -423,40 +423,6 @@ def extract_zip_to_directory(source: pathlib.Path, dest: pathlib.Path): zf.extractall(dest) -def pystandalone_archive_filter(name: str) -> bool: - if name.startswith( - ( - "python/build", - "python/install/include", - "python/install/share", - "python/install/libs", - ) - ) and not name.endswith(".empty"): - return True - - if name.startswith("python/install/lib") and not name.startswith( - "python/install/lib/python" - ): - return True - - if name.startswith(("python/install/lib/python", "python/install/Lib")): - if "_testclinic" in name: - return True - - module_name = name.split("/")[4 if name.startswith("python/install/lib") else 3] - if module_name in ( - "ensurepip", - "test", - "pydoc_data", - "lib2to3", - "tkinter", - "turtle", - ) or module_name.startswith("config-"): - return True - - return False - - def pystandalone_json_filter(info): info["build_info"]["core"]["objs"] = [] for ext in info["build_info"]["extensions"].values(): @@ -483,11 +449,6 @@ def normalize_tar_archive(data: io.BytesIO) -> io.BytesIO: if ti.isdir(): continue - # PYSTANDALONE: we use this place to slim down our archive by removing files we don't care about - # The JSON will be wrong, but the diff with upstream will be cleaner - if pystandalone_archive_filter(ti.name): - continue - filedata = tf.extractfile(ti) if filedata is not None: filedata = io.BytesIO(filedata.read()) diff --git a/src/release.rs b/src/release.rs index de074b32a..af45ad5a6 100644 --- a/src/release.rs +++ b/src/release.rs @@ -461,6 +461,74 @@ const INSTALL_ONLY_DROP_EXTENSIONS: &[&str] = &[ "_testsinglephase", ]; +fn pystandalone_drop(path_bytes: &[u8]) -> bool { + // Drop all `__pycache__` files and directories. + if path_bytes + .windows(b"__pycache__".len()) + .any(|part| part == b"__pycache__") + { + return true; + } + + // Drop pythonw.exe and the include, share and libs directories + if [ + b"python/install/pythonw.exe".as_slice(), + b"python/install/include", + b"python/install/share", + b"python/install/libs", + ] + .iter() + .any(|prefix| path_bytes.starts_with(prefix)) + { + return true; + } + + // Drop anything in `python/install/lib` that isn't `python/install/lib/python` + if path_bytes.starts_with(b"python/install/lib") + && !path_bytes.starts_with(b"python/install/lib/python") + { + return true; + } + + let module_index = if path_bytes.starts_with(b"python/install/lib/python") { + Some(4) + } else if path_bytes.starts_with(b"python/install/Lib") { + Some(3) + } else { + None + }; + + if let Some(module_index) = module_index { + if path_bytes + .windows(b"_testclinic".len()) + .any(|part| part == b"_testclinic") + { + return true; + } + + let drop_modules = [ + b"ensurepip".as_slice(), + b"idlelib", + b"lib2to3", + b"profiling", + b"pydoc_data", + b"test", + b"tkinter", + b"turtle", + b"turtledemo", + b"venv", + ]; + + if let Some(module_name) = path_bytes.split(|byte| *byte == b'/').nth(module_index) { + if drop_modules.contains(&module_name) || module_name.starts_with(b"config-") { + return true; + } + } + } + + false +} + /// Convert a .tar.zst archive to an install-only .tar.gz archive. pub fn convert_to_install_only(reader: impl BufRead, writer: W) -> Result { let dctx = zstd::stream::Decoder::new(reader)?; @@ -504,6 +572,26 @@ pub fn convert_to_install_only(reader: impl BufRead, writer: W) -> Res } } + // PYSTANDALONE: create a PYSTANDALONE.json with a few useful fields from PYTHON.json + let pystandalone_json = serde_json::to_string(&serde_json::json!({ + "target_triple": json_main.target_triple, + "python_version": json_main.python_version, + "python_exe": json_main.python_exe.strip_prefix("install/").unwrap(), + "python_stdlib": stdlib_path.strip_prefix("install/").unwrap(), + "python_bytecode_magic_number": json_main.python_bytecode_magic_number, + }))?; + + builder.append( + &{ + let mut header = tar::Header::new_gnu(); + header.set_path("python/PYSTANDALONE.json")?; + header.set_size(pystandalone_json.len() as u64); + header.set_cksum(); + header + }, + std::io::Cursor::new(pystandalone_json), + )?; + for entry in entries { let mut entry = entry?; @@ -539,6 +627,11 @@ pub fn convert_to_install_only(reader: impl BufRead, writer: W) -> Res continue; } + // PYSTANDALONE: drop files we don't care about + if pystandalone_drop(&path_bytes) { + continue; + } + if drop_paths.contains(&path_bytes.to_vec()) { continue; } diff --git a/src/validation.rs b/src/validation.rs index 7339885ec..c72804cad 100644 --- a/src/validation.rs +++ b/src/validation.rs @@ -2171,66 +2171,7 @@ fn validate_distribution( } // Validate extension module metadata. - for (name, variants) in json.as_ref().unwrap().build_info.extensions.iter() { - for ext in variants { - if let Some(shared) = &ext.shared_lib { - if !seen_paths.contains(&PathBuf::from("python").join(shared)) { - context.errors.push(format!( - "extension module {name} references missing shared library path {shared}" - )); - } - } - - #[allow(clippy::if_same_then_else)] - // Static builds never have shared library extension modules. - let want_shared = if is_static { - false - // Extension modules in libpython core are never shared libraries. - } else if ext.in_core { - false - // All remaining extensions are shared on Windows. - } else if triple.contains("windows") { - true - // On POSIX platforms we maintain a list. - } else { - SHARED_LIBRARY_EXTENSIONS.contains(&name.as_str()) - }; - - if want_shared && ext.shared_lib.is_none() { - context.errors.push(format!( - "extension module {name} does not have a shared library" - )); - } else if !want_shared && ext.shared_lib.is_some() { - context.errors.push(format!( - "extension module {name} contains a shared library unexpectedly" - )); - } - - // Ensure initialization functions are exported. - - // Note that we export PyInit_* functions from libpython on POSIX whereas these - // aren't exported from official Python builds. We may want to consider changing - // this. - if ext.init_fn == "NULL" { - continue; - } - - let exported = context.libpython_exported_symbols.contains(&ext.init_fn); - - #[allow(clippy::needless_bool, clippy::if_same_then_else)] - // PYSTANDALONE: we don't export symbols - let wanted = false; - - if exported != wanted { - context.errors.push(format!( - "libpython {} {} for extension module {}", - if wanted { "doesn't export" } else { "exports" }, - ext.init_fn, - name - )); - } - } - } + // PYSTANDALONE: skip checking extension module metadata // Validate Mach-O symbols and libraries against what the SDKs say. This is only supported // on macOS.