diff --git a/.agents/skills/third-party-package-patches/SKILL.md b/.agents/skills/third-party-package-patches/SKILL.md index e79e4c3ae5..d9b3c51f2d 100644 --- a/.agents/skills/third-party-package-patches/SKILL.md +++ b/.agents/skills/third-party-package-patches/SKILL.md @@ -1,6 +1,6 @@ --- name: third-party-package-patches -description: Create or update GraalPy third-party package compatibility patches under graalpython/lib-graalpython/patches, including PyPI source preparation, rebasing existing patches, metadata.toml updates, license checks, version-range validation, and verify_patches.py validation. +description: Create or update GraalPy third-party package and Rust crate compatibility patches under graalpython/lib-graalpython/patches, including PyPI source preparation, Cargo crate autopatching, rebasing existing patches, metadata.toml updates, license checks, version-range validation, and verify_patches.py validation. --- # Third-Party Package Patches @@ -11,6 +11,8 @@ Use this skill when creating or updating compatibility patches for packages inst - Source preparation: `scripts/get_pypi_source.py` - Patch metadata: `graalpython/lib-graalpython/patches/metadata.toml` - Patch directory: `graalpython/lib-graalpython/patches/` +- Cargo autopatcher: `graalpython/lib-graalpython/modules/autopatch_cargo.py` +- Crate patch metadata: `graalpython/lib-graalpython/patches/crates/metadata.toml` - Metadata verifier: `mx.graalpython/verify_patches.py` ## Workflow @@ -20,7 +22,7 @@ Use this skill when creating or updating compatibility patches for packages inst ```bash python scripts/get_pypi_source.py package==version ``` -The script prints `Prepared source at: ...`. Use that directory as the working tree. It is already a temporary git repository with an initial commit, and it has already been processed by `graalpython/lib-graalpython/modules/autopatch_capi.py`. +The script prints `Prepared source at: ...`. Use that directory as the working tree. It is already a temporary git repository with an initial commit, and it has already been processed by `autopatch_capi.py` and `autopatch_cargo.py`. 3. Inspect `graalpython/lib-graalpython/patches/metadata.toml` for existing `[[package.rules]]` entries. - If a matching patch exists for the requested version, apply it first. @@ -74,6 +76,19 @@ python mx.graalpython/verify_patches.py graalpython/lib-graalpython/patches 12. If you were asked to build or test the patched package, you need to rebuild GraalPy with `mx python-jvm` to pick up the changes. Create a venv with `mx python -m venv venv_name` and use it for building and testing. +## Rust Crate Patches + +Use `autopatch_cargo.py` when the incompatibility is in a transitive crates.io dependency rather than the Python package source. It finds matching crates in `Cargo.lock`, copies only those crates into the source tree, applies registered patches, and adds Cargo path overrides. + +1. Add rules to `graalpython/lib-graalpython/patches/crates/metadata.toml`. Use the crate package name as the table key and specify `version`, `patch`, and `license` for every rule. +2. Generate crate patch files from a clean checkout of the published crate. Patch paths must be relative to the crate root. Do not hand-edit generated patch files. +3. Run the tool directly when validating a prepared source tree: +```bash +python graalpython/lib-graalpython/modules/autopatch_cargo.py /path/to/source +``` +4. Test every crate release covered by the version range, then build representative Python packages that resolve those releases. +5. Use the Python package rule's `autopatch = false` only when both `autopatch_capi` and `autopatch_cargo` must be disabled for that package version. + ## Metadata Reference Rule keys accepted by the verifier are: - `version` @@ -83,6 +98,7 @@ Rule keys accepted by the verifier are: - `dist-type` - `install-priority` - `note` +- `autopatch` Allowed `dist-type` values are `wheel` and `sdist`. diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ad7267ec2a..305ead54ab 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,7 +5,7 @@ repos: name: Copyright check language: system require_serial: true - entry: bash scripts/pre-commit-checkcopyrights.sh + entry: env VERIFY_PATCHES_GIT=1 bash scripts/pre-commit-checkcopyrights.sh types: [text] - id: eclipseformat name: Eclipse formatter diff --git a/AGENTS.md b/AGENTS.md index c99af625c0..aecfd71849 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -67,9 +67,6 @@ It consists of: Java (Truffle) + C (CPython C-API compatibility) + Python stdlib `mx graalpytest TEST-SELECTOR` * Run JUnit tests `mx python-gate --tags python-junit` -* Style / formatting - `mx python-style --fix` - `mx python-gate --tags style` * Building standalones for benchmarking - use `mx --env native-ee sforceimports && mx --env native-ee checkout-downstream compiler graal-enterprise` to get the right revisions - use `mx --env jvm-ee-libgraal` and `mx --env native-ee` to build the JAVA and NATIVE standalone distributions diff --git a/graalpython/com.oracle.graal.python.shell/src/com/oracle/graal/python/shell/JLineConsoleHandler.java b/graalpython/com.oracle.graal.python.shell/src/com/oracle/graal/python/shell/JLineConsoleHandler.java index 16d25a78a4..5cfec4a0cc 100644 --- a/graalpython/com.oracle.graal.python.shell/src/com/oracle/graal/python/shell/JLineConsoleHandler.java +++ b/graalpython/com.oracle.graal.python.shell/src/com/oracle/graal/python/shell/JLineConsoleHandler.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2025, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2026, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * The Universal Permissive License (UPL), Version 1.0 @@ -74,8 +74,11 @@ import org.graalvm.shadowed.org.jline.terminal.Terminal; import org.graalvm.shadowed.org.jline.terminal.TerminalBuilder; import org.graalvm.shadowed.org.jline.utils.InputStreamReader; +import org.graalvm.shadowed.org.jline.utils.Signals; public class JLineConsoleHandler extends ConsoleHandler { + private static final String[] SIGNALS = {"INT", "QUIT", "TSTP", "CONT", "WINCH"}; + private final InputStream inputStream; private final OutputStream outputStream; private LineReader reader; @@ -93,10 +96,13 @@ public class JLineConsoleHandler extends ConsoleHandler { @SuppressWarnings("unused") public void initializeReadline(Value readlineModule) { Terminal terminal; + Object[] signalHandlers = stashSignalHandlers(); try { - terminal = TerminalBuilder.builder().jna(false).streams(inputStream, outputStream).system(true).signalHandler(Terminal.SignalHandler.SIG_IGN).build(); + terminal = TerminalBuilder.builder().jna(false).streams(inputStream, outputStream).system(true).build(); } catch (IOException ex) { throw new RuntimeException("unexpected error opening console reader", ex); + } finally { + restoreSignalHandlers(signalHandlers); } final Value getCompleter = readlineModule.getMember("get_completer"); final Value shouldRecord = readlineModule.getMember("get_auto_history"); @@ -226,6 +232,7 @@ public String readLine(String prompt) { } } if (lineBuffer.isEmpty()) { + Object[] signalHandlers = stashSignalHandlers(); try { String lines = reader.readLine(prompt); Collections.addAll(lineBuffer, lines.split("\n")); @@ -235,11 +242,27 @@ public String readLine(String prompt) { return null; } catch (Exception ex) { throw new RuntimeException(ex); + } finally { + restoreSignalHandlers(signalHandlers); } } return lineBuffer.poll(); } + private static Object[] stashSignalHandlers() { + Object[] handlers = new Object[SIGNALS.length]; + for (int i = 0; i < SIGNALS.length; i++) { + handlers[i] = Signals.registerDefault(SIGNALS[i]); + } + return handlers; + } + + private static void restoreSignalHandlers(Object[] handlers) { + for (int i = 0; i < SIGNALS.length; i++) { + Signals.unregister(SIGNALS[i], handlers[i]); + } + } + // Used via interop @SuppressWarnings("unused") public CompletionState getCompletionState() { diff --git a/graalpython/com.oracle.graal.python.test/src/runner.py b/graalpython/com.oracle.graal.python.test/src/runner.py index 8725692a1c..c030062570 100644 --- a/graalpython/com.oracle.graal.python.test/src/runner.py +++ b/graalpython/com.oracle.graal.python.test/src/runner.py @@ -82,7 +82,7 @@ CURRENT_PLATFORM_KEYS = frozenset({CURRENT_PLATFORM}) RUNNER_ENV = {} -DISABLE_JIT_ENV = {'GRAAL_PYTHON_VM_ARGS': '--experimental-options --engine.Compilation=false'} +DISABLE_JIT_ENV = {'GRAAL_PYTHON_VM_ARGS': '-X jit=0'} # The worker transport sends pickled data, so keep it on loopback only. WORKER_SERVER_HOST = '127.0.0.1' @@ -93,12 +93,8 @@ # We leave the JIT enabled for the tests themselves, but disable it for subprocesses # noinspection PyUnresolvedReferences -if IS_GRAALPY and __graalpython__.is_native and 'GRAAL_PYTHON_VM_ARGS' not in os.environ: - try: - subprocess.check_output([sys.executable, '--version'], env={**os.environ, **DISABLE_JIT_ENV}) - RUNNER_ENV = DISABLE_JIT_ENV - except subprocess.CalledProcessError: - pass +if IS_GRAALPY and 'GRAAL_PYTHON_VM_ARGS' not in os.environ: + RUNNER_ENV = DISABLE_JIT_ENV class Logger: diff --git a/graalpython/com.oracle.graal.python.test/src/tests/test_autopatch_cargo.py b/graalpython/com.oracle.graal.python.test/src/tests/test_autopatch_cargo.py new file mode 100644 index 0000000000..9ca2e21508 --- /dev/null +++ b/graalpython/com.oracle.graal.python.test/src/tests/test_autopatch_cargo.py @@ -0,0 +1,262 @@ +# Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved. +# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. +# +# The Universal Permissive License (UPL), Version 1.0 +# +# Subject to the condition set forth below, permission is hereby granted to any +# person obtaining a copy of this software, associated documentation and/or +# data (collectively the "Software"), free of charge and under any and all +# copyright rights in the Software, and any and all patent rights owned or +# freely licensable by each licensor hereunder covering either (i) the +# unmodified Software as contributed to or provided by such licensor, or (ii) +# the Larger Works (as defined below), to deal in both +# +# (a) the Software, and +# +# (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if +# one is included with the Software each a "Larger Work" to which the Software +# is contributed by such licensors), +# +# without restriction, including without limitation the rights to copy, create +# derivative works of, display, perform, and distribute the Software and make, +# use, sell, offer for sale, import, export, have made, and have sold the +# Software and the Larger Work(s), and to sublicense the foregoing rights on +# either these or other terms. +# +# This license is subject to the following condition: +# +# The above copyright notice and either this complete permission notice or at a +# minimum a reference to the UPL must be included in all copies or substantial +# portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +import hashlib +import json +import os +import shutil +import subprocess +import sys +import tarfile +import tempfile +import textwrap +import tomllib +import unittest +from pathlib import Path +from unittest.mock import patch + + +if sys.implementation.name == "graalpy": + import autopatch_cargo + + + class AutoPatchCargoTest(unittest.TestCase): + def setUp(self): + self.tempdir = Path(tempfile.mkdtemp()) + self.cargo_home = self.tempdir / "cargo-home" + self.workspace = self.tempdir / "project" + self.patch_dir = self.tempdir / "patches" + self.workspace.mkdir() + self.patch_dir.mkdir() + self.env_patch = patch.dict(os.environ, {"CARGO_HOME": str(self.cargo_home)}) + self.env_patch.start() + + def tearDown(self): + self.env_patch.stop() + shutil.rmtree(self.tempdir) + + @staticmethod + def write(path, contents): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(textwrap.dedent(contents).lstrip()) + + def test_version_specifiers(self): + contains = autopatch_cargo._specifier_contains + assert contains(">=0.28,<0.29", "0.28.0") + assert contains(">=0.28,<0.29", "0.28.3") + assert not contains(">=0.28,<0.29", "0.27.2") + assert not contains(">=0.28,<0.29", "0.29.0") + assert contains("==0.29.0", "0.29") + assert contains(">0.28.1,!=0.28.2,<=0.28.3", "0.28.3") + assert not contains(">=0.28", "0.29.0-alpha.1") + with self.assertRaises(ValueError): + contains("~=0.28", "0.28.3") + + def prepare_crate_archive(self, name="made-up-crate", version="1.2.3", message="unpatched", cached=True): + crate = self.tempdir / "archive-source" / f"{name}-{version}" + self.write( + crate / "Cargo.toml", + f""" + [package] + name = "{name}" + version = "{version}" + edition = "2021" + """, + ) + self.write(crate / "src" / "lib.rs", f'pub fn message() -> &\'static str {{ "{message}" }}\n') + if cached: + archive = self.cargo_home / "registry" / "cache" / "made-up-index" / f"{name}-{version}.crate" + else: + archive = self.tempdir / f"{name}-{version}.crate" + archive.parent.mkdir(parents=True, exist_ok=True) + with tarfile.open(archive, "w:gz") as tar: + tar.add(crate, arcname=crate.name) + return archive, hashlib.sha256(archive.read_bytes()).hexdigest() + + def prepare_repository(self, version=">=1,<2"): + self.write( + self.patch_dir / "metadata.toml", + f""" + [[made-up-crate.rules]] + version = "{version}" + patch = "made-up-crate.patch" + license = "MIT" + """, + ) + self.write( + self.patch_dir / "made-up-crate.patch", + """ + --- a/src/lib.rs + +++ b/src/lib.rs + @@ -1 +1 @@ + -pub fn message() -> &'static str { "unpatched" } + +pub fn message() -> &'static str { "patched" } + """, + ) + return autopatch_cargo.DirectoryPatchRepository(self.patch_dir) + + def prepare_workspace(self, checksum=None): + if checksum is None: + checksum = "0" * 64 + existing_patch = self.workspace / "existing-patch" + self.write( + existing_patch / "Cargo.toml", + """ + [package] + name = "existing-patch" + version = "1.0.0" + edition = "2021" + """, + ) + self.write(existing_patch / "src" / "lib.rs", "") + self.write( + self.workspace / "Cargo.toml", + """ + [package] + name = "made-up-project" + version = "0.1.0" + edition = "2021" + + [dependencies] + made-up-crate = "=1.2.3" + existing-patch = "=1.0.0" + + [patch.crates-io] + existing-patch = { path = "existing-patch" } + """, + ) + self.write(self.workspace / "src" / "lib.rs", "pub use made_up_crate::message;\n") + self.write( + self.workspace / "Cargo.lock", + """ + version = 4 + + [[package]] + name = "existing-patch" + version = "1.0.0" + + [[package]] + name = "made-up-crate" + version = "1.2.3" + source = "registry+https://github.com/rust-lang/crates.io-index" + checksum = "{checksum}" + + [[package]] + name = "made-up-project" + version = "0.1.0" + dependencies = [ + "existing-patch", + "made-up-crate", + ] + """.format(checksum=checksum), + ) + + def test_patches_locked_crate_and_adds_cargo_override(self): + archive, checksum = self.prepare_crate_archive() + cached_crate = self.cargo_home / "registry" / "src" / "made-up-index" / "made-up-crate-1.2.3" + self.write(cached_crate / "Cargo.toml", "[package]\nname = \"made-up-crate\"\nversion = \"1.2.3\"\n") + self.write(cached_crate / "src" / "lib.rs", 'pub fn message() -> &\'static str { "global-cache" }\n') + archive_checksum = hashlib.sha256(archive.read_bytes()).hexdigest() + repository = self.prepare_repository() + self.prepare_workspace(checksum) + + with patch.object(autopatch_cargo.urllib.request, "urlopen") as urlopen: + assert autopatch_cargo.auto_patch_tree(self.workspace, repository) == 1 + urlopen.assert_not_called() + patched_crate = self.workspace / ".graalpy" / "crates" / "made-up-crate-1.2.3" + assert '"patched"' in (patched_crate / "src" / "lib.rs").read_text() + assert '"global-cache"' in (cached_crate / "src" / "lib.rs").read_text() + assert hashlib.sha256(archive.read_bytes()).hexdigest() == archive_checksum + + manifest = tomllib.loads((self.workspace / "Cargo.toml").read_text()) + cargo_patches = manifest["patch"]["crates-io"] + assert cargo_patches["existing-patch"]["path"] == "existing-patch" + override = cargo_patches["graalpy-made-up-crate-1_2_3"] + assert override == { + "package": "made-up-crate", + "path": ".graalpy/crates/made-up-crate-1.2.3", + } + lock_packages = tomllib.loads((self.workspace / "Cargo.lock").read_text())["package"] + locked_crate = next(package for package in lock_packages if package["name"] == "made-up-crate") + assert "source" not in locked_crate + assert "checksum" not in locked_crate + + if cargo := shutil.which("cargo"): + metadata = subprocess.run( + [cargo, "metadata", "--format-version", "1", "--locked", "--offline"], + cwd=self.workspace, + check=True, + stdout=subprocess.PIPE, + text=True, + ) + packages = json.loads(metadata.stdout)["packages"] + crate = next(package for package in packages if package["name"] == "made-up-crate") + assert Path(crate["manifest_path"]) == patched_crate / "Cargo.toml" + + assert autopatch_cargo.auto_patch_tree(self.workspace, repository) == 0 + + def test_downloads_and_verifies_uncached_crate(self): + archive, checksum = self.prepare_crate_archive(cached=False) + repository = self.prepare_repository() + self.prepare_workspace(checksum) + + with open(archive, "rb") as response: + with patch.object(autopatch_cargo.urllib.request, "urlopen", return_value=response) as urlopen: + assert autopatch_cargo.auto_patch_tree(self.workspace, repository) == 1 + + urlopen.assert_called_once() + patched_crate = self.workspace / ".graalpy" / "crates" / "made-up-crate-1.2.3" + assert '"patched"' in (patched_crate / "src" / "lib.rs").read_text() + + def test_accepts_patch_already_applied_to_cached_archive(self): + _, checksum = self.prepare_crate_archive(message="patched") + repository = self.prepare_repository() + self.prepare_workspace(checksum) + + assert autopatch_cargo.auto_patch_tree(self.workspace, repository) == 1 + patched_crate = self.workspace / ".graalpy" / "crates" / "made-up-crate-1.2.3" + assert '"patched"' in (patched_crate / "src" / "lib.rs").read_text() + + def test_ignores_crate_outside_registered_version_range(self): + repository = self.prepare_repository(version=">=2") + self.prepare_workspace() + + assert autopatch_cargo.auto_patch_tree(self.workspace, repository) == 0 + assert not (self.workspace / ".graalpy").exists() + assert "graalpy-made-up-crate" not in (self.workspace / "Cargo.toml").read_text() diff --git a/graalpython/com.oracle.graal.python.test/src/tests/test_repl.py b/graalpython/com.oracle.graal.python.test/src/tests/test_repl.py index 882d267dba..aa2ab1d425 100644 --- a/graalpython/com.oracle.graal.python.test/src/tests/test_repl.py +++ b/graalpython/com.oracle.graal.python.test/src/tests/test_repl.py @@ -227,6 +227,18 @@ def test_exceptions(): """)) + def test_syntax_error_does_not_exit_repl(): + validate_repl(dedent("""\ + >>> if + File "", line 1 + if + ^ + SyntaxError: invalid syntax + >>> 6 * 7 + 42 + """)) + + def test_inspect_flag(): with tempfile.NamedTemporaryFile('w') as f: f.write('a = 1\n') diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/PythonLanguage.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/PythonLanguage.java index 947138e0be..400f9a969a 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/PythonLanguage.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/PythonLanguage.java @@ -598,7 +598,7 @@ public RootCallTarget parse(PythonContext context, Source source, InputType type return compileModule(context, mod, source, topLevel, optimize, argumentNames, errorCb, futureFeatures); } catch (PException e) { if (topLevel) { - PythonUtils.getOrCreateCallTarget(new TopLevelExceptionHandler(this, e)).call(); + PythonUtils.getOrCreateCallTarget(new TopLevelExceptionHandler(this, e, source)).call(); } throw e; } @@ -646,7 +646,7 @@ public RootCallTarget compileModule(PythonContext context, ModTy modIn, Source s return PythonUtils.getOrCreateCallTarget(rootNode); } catch (PException e) { if (topLevel) { - PythonUtils.getOrCreateCallTarget(new TopLevelExceptionHandler(this, e)).call(); + PythonUtils.getOrCreateCallTarget(new TopLevelExceptionHandler(this, e, source)).call(); } throw e; } diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/codecs/ErrorHandlers.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/codecs/ErrorHandlers.java index 445cc11c6d..d8471b5e45 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/codecs/ErrorHandlers.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/codecs/ErrorHandlers.java @@ -51,6 +51,7 @@ import static com.oracle.graal.python.nodes.StringLiterals.T_QUESTIONMARK; import static com.oracle.graal.python.nodes.StringLiterals.T_REPLACE; import static com.oracle.graal.python.nodes.StringLiterals.T_STRICT; +import static com.oracle.graal.python.nodes.StringLiterals.T_SURROGATEESCAPE; import static com.oracle.graal.python.nodes.StringLiterals.T_SURROGATEPASS; import static com.oracle.graal.python.nodes.StringLiterals.T_XMLCHARREFREPLACE; import static com.oracle.graal.python.runtime.exception.PythonErrorType.TypeError; @@ -181,7 +182,7 @@ static ErrorHandler doIt(Node inliningTarget, TruffleString errors, if (strictProfile.profile(inliningTarget, equalNode.execute(T_STRICT, errors, TS_ENCODING))) { return ErrorHandler.STRICT; } - if (surrogateEscapeProfile.profile(inliningTarget, equalNode.execute(T_SURROGATEPASS, errors, TS_ENCODING))) { + if (surrogateEscapeProfile.profile(inliningTarget, equalNode.execute(T_SURROGATEESCAPE, errors, TS_ENCODING))) { return ErrorHandler.SURROGATEESCAPE; } if (replaceProfile.profile(inliningTarget, equalNode.execute(T_REPLACE, errors, TS_ENCODING))) { diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/bytes/BytesCommonBuiltins.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/bytes/BytesCommonBuiltins.java index 16bd5e6bd4..48a4df703e 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/bytes/BytesCommonBuiltins.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/bytes/BytesCommonBuiltins.java @@ -620,13 +620,13 @@ PTuple partition(VirtualFrame frame, Object self, Object sep, second = createBytesNode.execute(inliningTarget, self, sepBytes); if (idx == 0) { first = createBytesNode.execute(inliningTarget, self, PythonUtils.EMPTY_BYTE_ARRAY); - third = createBytesNode.execute(inliningTarget, self, Arrays.copyOfRange(bytes, lenSep, len)); + third = createBytesNode.execute(inliningTarget, self, PythonUtils.arrayCopyOfRange(bytes, lenSep, len)); } else if (idx == len - 1) { - first = createBytesNode.execute(inliningTarget, self, Arrays.copyOfRange(bytes, 0, len - lenSep)); + first = createBytesNode.execute(inliningTarget, self, PythonUtils.arrayCopyOfRange(bytes, 0, len - lenSep)); third = createBytesNode.execute(inliningTarget, self, PythonUtils.EMPTY_BYTE_ARRAY); } else { - first = createBytesNode.execute(inliningTarget, self, Arrays.copyOfRange(bytes, 0, idx)); - third = createBytesNode.execute(inliningTarget, self, Arrays.copyOfRange(bytes, idx + lenSep, len)); + first = createBytesNode.execute(inliningTarget, self, PythonUtils.arrayCopyOfRange(bytes, 0, idx)); + third = createBytesNode.execute(inliningTarget, self, PythonUtils.arrayCopyOfRange(bytes, idx + lenSep, len)); } } return PFactory.createTuple(language, new Object[]{first, second, third}); @@ -2278,7 +2278,7 @@ protected ArgumentClinicProvider getArgumentClinic() { @TruffleBoundary static byte[] copyOfRange(byte[] bytes, int from, int to) { - return Arrays.copyOfRange(bytes, from, to); + return PythonUtils.arrayCopyOfRange(bytes, from, to); } @TruffleBoundary(allowInlining = true) @@ -2334,7 +2334,7 @@ static PBytesLike remove(VirtualFrame frame, Object self, Object prefix, for (int i = 0; i < selfBsLen; i++) { if (i < prefixBsLen) { if (selfBs[i] != prefixBs[i]) { - return create.execute(node, self, selfBs); + return create.execute(node, self, PythonUtils.arrayCopyOf(selfBs, selfBsLen)); } } else { result[j++] = selfBs[i]; @@ -2342,7 +2342,7 @@ static PBytesLike remove(VirtualFrame frame, Object self, Object prefix, } return create.execute(node, self, result); } - return create.execute(node, self, selfBs); + return create.execute(node, self, PythonUtils.arrayCopyOf(selfBs, selfBsLen)); } finally { bufferLib.release(selfBuffer, frame, callData); bufferLib.release(prefixBuffer, frame, callData); @@ -2375,7 +2375,7 @@ static PBytesLike remove(VirtualFrame frame, Object self, Object suffix, for (int i = selfBsLen - 1, j = 1; i >= 0; i--, j++) { if (i >= selfBsLen - suffixBsLen) { if (selfBs[i] != suffixBs[suffixBsLen - j]) { - return create.execute(node, self, selfBs); + return create.execute(node, self, PythonUtils.arrayCopyOf(selfBs, selfBsLen)); } } else { result[result.length - k++] = selfBs[i]; @@ -2383,7 +2383,7 @@ static PBytesLike remove(VirtualFrame frame, Object self, Object suffix, } return create.execute(node, self, result); } - return create.execute(node, self, selfBs); + return create.execute(node, self, PythonUtils.arrayCopyOf(selfBs, selfBsLen)); } finally { bufferLib.release(selfBuffer, frame, callData); bufferLib.release(suffixBuffer, frame, callData); diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/traceback/PTraceback.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/traceback/PTraceback.java index 26bfddc890..f117036bc6 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/traceback/PTraceback.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/traceback/PTraceback.java @@ -91,6 +91,9 @@ public void copyFrom(PTraceback other) { frame = other.frame; frameInfo = other.frameInfo; lineno = other.lineno; + bci = other.bci; + bytecodeNode = other.bytecodeNode; + lasti = other.lasti; next = other.next; } diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/exception/TopLevelExceptionHandler.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/exception/TopLevelExceptionHandler.java index 2f395900bd..2987653790 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/exception/TopLevelExceptionHandler.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/exception/TopLevelExceptionHandler.java @@ -109,12 +109,12 @@ public TopLevelExceptionHandler(PythonLanguage language, RootNode child, Source this.newGlobals = source.getOptions(language).get(PythonSourceOptions.NewGlobals); } - public TopLevelExceptionHandler(PythonLanguage language, PException exception) { + public TopLevelExceptionHandler(PythonLanguage language, PException exception, Source source) { super(language); this.sourceSection = exception.getLocation().getEncapsulatingSourceSection(); this.innerCallTarget = null; this.exception = exception; - this.source = null; + this.source = source; this.newGlobals = false; } diff --git a/graalpython/lib-graalpython/modules/autopatch_cargo.py b/graalpython/lib-graalpython/modules/autopatch_cargo.py new file mode 100644 index 0000000000..32be96915a --- /dev/null +++ b/graalpython/lib-graalpython/modules/autopatch_cargo.py @@ -0,0 +1,376 @@ +# Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved. +# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. +# +# The Universal Permissive License (UPL), Version 1.0 +# +# Subject to the condition set forth below, permission is hereby granted to any +# person obtaining a copy of this software, associated documentation and/or +# data (collectively the "Software"), free of charge and under any and all +# copyright rights in the Software, and any and all patent rights owned or +# freely licensable by each licensor hereunder covering either (i) the +# unmodified Software as contributed to or provided by such licensor, or (ii) +# the Larger Works (as defined below), to deal in both +# +# (a) the Software, and +# +# (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if +# one is included with the Software each a "Larger Work" to which the Software +# is contributed by such licensors), +# +# without restriction, including without limitation the rights to copy, create +# derivative works of, display, perform, and distribute the Software and make, +# use, sell, offer for sale, import, export, have made, and have sold the +# Software and the Larger Work(s), and to sublicense the foregoing rights on +# either these or other terms. +# +# This license is subject to the following condition: +# +# The above copyright notice and either this complete permission notice or at a +# minimum a reference to the UPL must be included in all copies or substantial +# portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +import argparse +import hashlib +import json +import logging +import os +import re +import shutil +import subprocess +import tarfile +import tempfile +import tomllib +import urllib.parse +import urllib.request +from contextlib import contextmanager +from pathlib import Path + + +logger = logging.getLogger(__name__) + +CRATES_IO_SOURCE = "registry+https://github.com/rust-lang/crates.io-index" +AUTOPATCH_DIR = ".graalpy" +IGNORED_DIRECTORIES = {AUTOPATCH_DIR, ".git", ".hg", ".svn", "target"} +PATCH_TABLE_RE = re.compile( + r'^\s*\[(?:patch|"patch")\.(?:crates-io|"crates-io")\]\s*(?:#.*)?$', + re.MULTILINE, +) +VERSION_RE = re.compile(r'^\d+(?:\.\d+)*$') +VERSION_SPECIFIER_RE = re.compile(r'^(==|!=|<=|>=|<|>)\s*(\d+(?:\.\d+)*)$') + + +def _parse_version(version): + if not VERSION_RE.fullmatch(version): + return None + return tuple(int(part) for part in version.split('.')) + + +def _compare_versions(left, right): + length = max(len(left), len(right)) + left += (0,) * (length - len(left)) + right += (0,) * (length - len(right)) + return (left > right) - (left < right) + + +def _specifier_contains(specifier, version): + version = _parse_version(version) + if version is None: + return False + comparisons = { + '==': lambda result: result == 0, + '!=': lambda result: result != 0, + '<': lambda result: result < 0, + '<=': lambda result: result <= 0, + '>': lambda result: result > 0, + '>=': lambda result: result >= 0, + } + for constraint in specifier.split(','): + if not (match := VERSION_SPECIFIER_RE.fullmatch(constraint.strip())): + raise ValueError(f"Unsupported crate version constraint: {constraint!r}") + operator, expected = match.groups() + if not comparisons[operator](_compare_versions(version, _parse_version(expected))): + return False + return True + + +class DirectoryPatchRepository: + def __init__(self, path): + self.path = Path(path) + with open(self.path / "metadata.toml", "rb") as metadata_file: + self.metadata = tomllib.load(metadata_file) + + def get_rules(self, name): + if metadata := self.metadata.get(name): + return metadata.get("rules") + + @contextmanager + def resolve_patch(self, patch_name): + yield self.path / patch_name + + +def _matching_rule(repository, name, version): + for rule in repository.get_rules(name) or (): + if not rule.get("version") or _specifier_contains(rule["version"], version): + return rule + + +def _cargo_home(): + if cargo_home := os.environ.get("CARGO_HOME"): + return Path(cargo_home) + return Path.home() / ".cargo" + + +def _find_cached_crate_archive(name, version): + registry_cache = _cargo_home() / "registry" / "cache" + if not registry_cache.is_dir(): + return None + archive_name = f"{name}-{version}.crate" + for registry in registry_cache.iterdir(): + candidate = registry / archive_name + if candidate.is_file(): + return candidate + return None + + +def _safe_extract(archive, target): + with tarfile.open(archive, "r:gz") as tar: + tar.extractall(target, filter="data") + + +def _extract_crate_archive(archive, name, version, checksum, destination): + actual_checksum = hashlib.sha256(archive.read_bytes()).hexdigest() + if actual_checksum != checksum: + raise RuntimeError( + f"Checksum mismatch for {name} {version}: Cargo.lock has {checksum}, archive has {actual_checksum}" + ) + with tempfile.TemporaryDirectory(prefix="graalpy-crate-") as tempdir: + extracted = Path(tempdir) / "extracted" + extracted.mkdir() + _safe_extract(archive, extracted) + crate_root = extracted / f"{name}-{version}" + if not (crate_root / "Cargo.toml").is_file(): + raise RuntimeError(f"Archive for {name} {version} has an unexpected layout") + shutil.copytree(crate_root, destination) + + +def _download_crate(name, version, checksum, destination): + quoted_name = urllib.parse.quote(name, safe="") + quoted_version = urllib.parse.quote(version, safe="") + url = f"https://crates.io/api/v1/crates/{quoted_name}/{quoted_version}/download" + request = urllib.request.Request(url, headers={"User-Agent": "GraalPy autopatch_cargo"}) + with tempfile.TemporaryDirectory(prefix="graalpy-crate-") as tempdir: + archive = Path(tempdir) / f"{name}-{version}.crate" + with urllib.request.urlopen(request) as response, open(archive, "wb") as output: + shutil.copyfileobj(response, output) + _extract_crate_archive(archive, name, version, checksum, destination) + + +def _copy_crate(name, version, checksum, destination): + if not checksum: + raise RuntimeError(f"Cargo.lock does not contain a checksum for {name} {version}") + if cached_archive := _find_cached_crate_archive(name, version): + _extract_crate_archive(cached_archive, name, version, checksum, destination) + return + _download_crate(name, version, checksum, destination) + + +def _apply_patch(crate_dir, patch_path): + executable = "patch.exe" if os.name == "nt" else "patch" + command = [executable, "-f", "-d", str(crate_dir), "-p1", "-i", str(patch_path)] + try: + subprocess.run( + command, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + ) + except subprocess.CalledProcessError: + already_applied = subprocess.run( + [executable, "--dry-run", "-f", "-R", "-d", str(crate_dir), "-p1", "-i", str(patch_path)], + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + ) + if already_applied.returncode: + raise + + +def _find_lockfiles(location): + location = Path(location) + if location.is_file(): + return [location] if location.name == "Cargo.lock" else [] + lockfiles = [] + for root, directories, files in os.walk(location): + directories[:] = [directory for directory in directories if directory not in IGNORED_DIRECTORIES] + if "Cargo.lock" in files: + lockfiles.append(Path(root) / "Cargo.lock") + return lockfiles + + +def _load_locked_crates(lockfile, repository): + with open(lockfile, "rb") as cargo_lock: + lock_data = tomllib.load(cargo_lock) + result = [] + for package in lock_data.get("package", ()): + name = package.get("name") + version = package.get("version") + if not name or not version or package.get("source") != CRATES_IO_SOURCE: + continue + if rule := _matching_rule(repository, name, version): + if rule.get("patch"): + result.append((name, version, package.get("checksum"), rule)) + return result + + +def _existing_crates_io_patches(manifest_data): + result = set() + for key, value in manifest_data.get("patch", {}).get("crates-io", {}).items(): + if isinstance(value, dict): + result.add(value.get("package", key)) + else: + result.add(key) + return result + + +def _patch_entry(name, version, relative_path): + alias = re.sub(r"[^A-Za-z0-9_-]", "_", f"graalpy-{name}-{version}") + return ( + f'{json.dumps(alias)} = {{ package = {json.dumps(name)}, ' + f'path = {json.dumps(relative_path)} }}\n' + ) + + +def _add_manifest_overrides(manifest, overrides): + contents = manifest.read_text() + manifest_data = tomllib.loads(contents) + existing_patches = _existing_crates_io_patches(manifest_data) + overrides = [override for override in overrides if override[0] not in existing_patches] + if not overrides: + return 0 + + entries = "".join(_patch_entry(*override) for override in overrides) + if match := PATCH_TABLE_RE.search(contents): + insertion_point = match.end() + contents = contents[:insertion_point] + "\n" + entries + contents[insertion_point:] + elif manifest_data.get("patch", {}).get("crates-io"): + raise RuntimeError("Cargo.toml defines patch.crates-io without a table header that autopatch_cargo can extend") + else: + if contents and not contents.endswith("\n"): + contents += "\n" + contents += "\n# Added by GraalPy autopatch_cargo\n[patch.crates-io]\n" + entries + manifest.write_text(contents) + return len(overrides) + + +def _use_path_packages_in_lockfile(lockfile, overrides): + contents = lockfile.read_text() + targets = {(name, version) for name, version, _ in overrides} + replaced = set() + + def replace_package(match): + block = match.group() + package = tomllib.loads(block)["package"][0] + target = (package.get("name"), package.get("version")) + if target not in targets or package.get("source") != CRATES_IO_SOURCE: + return block + block = re.sub(r'^source\s*=.*\n', '', block, flags=re.MULTILINE) + block = re.sub(r'^checksum\s*=.*\n', '', block, flags=re.MULTILINE) + replaced.add(target) + return block + + contents = re.sub( + r'^\[\[package\]\]\n.*?(?=^\[\[package\]\]\n|\Z)', + replace_package, + contents, + flags=re.MULTILINE | re.DOTALL, + ) + if missing := targets - replaced: + raise RuntimeError(f"Cannot find patched crates in Cargo.lock: {sorted(missing)}") + lockfile.write_text(contents) + + +def _autopatch_workspace(lockfile, repository): + manifest = lockfile.with_name("Cargo.toml") + if not manifest.is_file(): + logger.warning("Skipping Cargo.lock without a Cargo.toml in the same directory: %s", lockfile) + return 0 + + try: + locked_crates = _load_locked_crates(lockfile, repository) + manifest_data = tomllib.loads(manifest.read_text()) + except (OSError, tomllib.TOMLDecodeError) as error: + logger.warning("Cannot inspect Cargo workspace at %s: %s", lockfile.parent, error) + return 0 + + existing_patches = _existing_crates_io_patches(manifest_data) + locked_crates = [crate for crate in locked_crates if crate[0] not in existing_patches] + overrides = [] + for name, version, checksum, rule in locked_crates: + crate_dir = lockfile.parent / AUTOPATCH_DIR / "crates" / f"{name}-{version}" + if crate_dir.exists(): + logger.warning("Refusing to overwrite existing autopatch directory %s", crate_dir) + continue + try: + crate_dir.parent.mkdir(parents=True, exist_ok=True) + _copy_crate(name, version, checksum, crate_dir) + with repository.resolve_patch(rule["patch"]) as patch_path: + if not patch_path: + raise RuntimeError(f"Cannot resolve crate patch {rule['patch']}") + _apply_patch(crate_dir, patch_path) + except (OSError, RuntimeError, subprocess.SubprocessError) as error: + shutil.rmtree(crate_dir, ignore_errors=True) + logger.warning("Could not autopatch crate %s %s: %s", name, version, error) + continue + relative_path = Path(os.path.relpath(crate_dir, manifest.parent)).as_posix() + overrides.append((name, version, relative_path)) + logger.info("Autopatched Rust crate %s %s in %s", name, version, crate_dir) + + original_manifest = manifest.read_text() + original_lockfile = lockfile.read_text() + try: + added = _add_manifest_overrides(manifest, overrides) + _use_path_packages_in_lockfile(lockfile, overrides) + except (OSError, RuntimeError, tomllib.TOMLDecodeError) as error: + manifest.write_text(original_manifest) + lockfile.write_text(original_lockfile) + for name, version, _ in overrides: + shutil.rmtree(lockfile.parent / AUTOPATCH_DIR / "crates" / f"{name}-{version}", ignore_errors=True) + logger.warning("Could not add Cargo path overrides to %s: %s", manifest, error) + return 0 + return added + + +def auto_patch_tree(location, repository): + """Patch registered crates used by Cargo workspaces under location. Returns the number of overrides added.""" + patched = 0 + for lockfile in _find_lockfiles(location): + patched += _autopatch_workspace(lockfile, repository) + return patched + + +def main(argv=None): + parser = argparse.ArgumentParser(description="Auto-patch registered Rust crates used under the given directory.") + parser.add_argument("path", help="source tree containing one or more Cargo.lock files") + parser.add_argument( + "--patches-dir", + type=Path, + default=Path(__file__).resolve().parent.parent / "patches" / "crates", + help="directory containing crate metadata.toml and patches", + ) + args = parser.parse_args(argv) + logging.basicConfig(level=logging.INFO) + auto_patch_tree(args.path, DirectoryPatchRepository(args.patches_dir)) + + +if __name__ == "__main__": + main() diff --git a/graalpython/lib-graalpython/modules/graalpy_pip_extensions.py b/graalpython/lib-graalpython/modules/graalpy_pip_extensions.py index 86fcd4ddbe..a03bd459ca 100644 --- a/graalpython/lib-graalpython/modules/graalpy_pip_extensions.py +++ b/graalpython/lib-graalpython/modules/graalpy_pip_extensions.py @@ -60,6 +60,7 @@ MARKER_FILE_NAME = 'GRAALPY_MARKER' METADATA_FILENAME = 'metadata.toml' DEFAULT_PATCHES_PATH = Path(__graalpython__.core_home) / 'patches' +CARGO_PATCHES_SUBDIR = 'crates' VERSION_PARAMETER = '' DEFAULT_PATCHES_URL = f'https://raw.githubusercontent.com/oracle/graalpython/refs/heads/github/patches/{VERSION_PARAMETER}/graalpython/lib-graalpython/patches/' @@ -240,6 +241,7 @@ def resolve_patch(self, patch_name: str): __PATCH_REPOSITORY = None +__CARGO_PATCH_REPOSITORY = None def repository_from_url_or_path(url_or_path): @@ -255,7 +257,7 @@ def repository_from_url_or_path(url_or_path): return RemotePatchRepository.from_url(patches_url) -def create_patch_repository(patches_url): +def create_patch_repository(patches_url, default_patches_path=DEFAULT_PATCHES_PATH): if patches_url and VERSION_PARAMETER in patches_url: if not GRAALPY_VERSION.endswith('-dev'): patches_url = patches_url.replace(VERSION_PARAMETER, GRAALPY_VERSION) @@ -276,7 +278,7 @@ def create_patch_repository(patches_url): logger.warning("Failed to load GraalPy patch repository: %s", e) logger.warning("Falling back to bundled GraalPy patch repository") try: - return LocalPatchRepository.from_path(DEFAULT_PATCHES_PATH) + return LocalPatchRepository.from_path(default_patches_path) except RepositoryException as e: logger.warning("Failed to load internal GraalPy patch repository: %s", e) return EmptyRepository() @@ -289,6 +291,22 @@ def get_patch_repository(): return __PATCH_REPOSITORY +def get_cargo_patch_repository(): + global __CARGO_PATCH_REPOSITORY + if not __CARGO_PATCH_REPOSITORY: + if PATCHES_URL == DISABLED_PATCHES_URL: + cargo_patches_url = PATCHES_URL + elif '://' in PATCHES_URL: + cargo_patches_url = f'{PATCHES_URL.rstrip("/")}/{CARGO_PATCHES_SUBDIR}/' + else: + cargo_patches_url = str(Path(PATCHES_URL) / CARGO_PATCHES_SUBDIR) + __CARGO_PATCH_REPOSITORY = create_patch_repository( + cargo_patches_url, + DEFAULT_PATCHES_PATH / CARGO_PATCHES_SUBDIR, + ) + return __CARGO_PATCH_REPOSITORY + + def apply_graalpy_patches(filename, location, warn_suggested_versions=False): """ Applies any GraalPy patches to package extracted from 'filename' into 'location'. @@ -321,6 +339,7 @@ def apply_graalpy_patches(filename, location, warn_suggested_versions=False): version = name_ver_match.group('version') suffix = name_ver_match.group('suffix') is_wheel = suffix == "whl" + cargo_location = location if is_wheel and is_wheel_marked(filename): # We already processed it when building from source @@ -342,10 +361,13 @@ def apply_graalpy_patches(filename, location, warn_suggested_versions=False): # with a patch intended for a binary distribution, because in the source # distribution the actual deployed sources may be in a subdirectory (typically "src") location = os.path.join(location, subdir) - if not rule or rule.get('autopatch', True): - import autopatch_capi + if not rule or rule.get('autopatch', True): + import autopatch_capi + import autopatch_cargo + + autopatch_capi.auto_patch_tree(location) + autopatch_cargo.auto_patch_tree(cargo_location, get_cargo_patch_repository()) - autopatch_capi.auto_patch_tree(location) if rule: if patch := rule.get('patch'): with repository.resolve_patch(patch) as patch_path: diff --git a/graalpython/lib-graalpython/patches/README.md b/graalpython/lib-graalpython/patches/README.md index 4aa59f8beb..ca091098ba 100644 --- a/graalpython/lib-graalpython/patches/README.md +++ b/graalpython/lib-graalpython/patches/README.md @@ -29,7 +29,8 @@ subdir = 'src' # a number greater than 1 if you want given version to be preferred to other entries. Additionally, if you set the # priority to 0, the version will not be shown in the suggestion list we display when we didn't find an applicable patch install-priority = 1 -# Optional. Whether to run the `autopatch_capi` before applying the explicit patch. Defaults to true. +# Optional. Whether to run `autopatch_capi` and `autopatch_cargo` on a source distribution before applying the +# explicit patch. Autopatching is never applied to wheels. Defaults to true. autopatch = true # The next entry will apply to all other versions of foo that didn't get matched by the previous rule @@ -37,3 +38,7 @@ autopatch = true patch = 'foo.patch' license = 'MIT' ``` + +Rust crate patches used by `autopatch_cargo` live in `crates/`. Its `metadata.toml` uses the same `rules`, `version`, +`patch`, and `license` fields. The tool finds matching crates.io dependencies in `Cargo.lock`, patches local copies of +the crates, and adds `[patch.crates-io]` path overrides to the workspace `Cargo.toml`. diff --git a/graalpython/lib-graalpython/patches/coverage-7.10.0.patch b/graalpython/lib-graalpython/patches/coverage-7.10.0.patch new file mode 100644 index 0000000000..a951849e90 --- /dev/null +++ b/graalpython/lib-graalpython/patches/coverage-7.10.0.patch @@ -0,0 +1,93 @@ +diff --git a/coverage/bytecode.py b/coverage/bytecode.py +index bea039c..772689b 100644 +--- a/coverage/bytecode.py ++++ b/coverage/bytecode.py +@@ -5,8 +5,6 @@ + + from __future__ import annotations + +-import dis +- + from types import CodeType + from typing import Iterable, Optional + from collections.abc import Iterator +@@ -32,7 +30,7 @@ def op_set(*op_names: str) -> set[int]: + + The names might not exist in this version of Python, skip those if not. + """ +- return {op for name in op_names if (op := dis.opmap.get(name))} ++ return set() + + + # Opcodes that are unconditional jumps elsewhere. +@@ -108,6 +106,7 @@ def branch_trails(code: CodeType) -> TBranchTrails: + arc from the original instruction's line to the new source line. + + """ ++ return {} + the_trails: TBranchTrails = {} + iwalker = InstructionWalker(code) + for inst in iwalker.walk(follow_jumps=False): +diff --git a/coverage/pytracer.py b/coverage/pytracer.py +index 31b4866..644cd4b 100644 +--- a/coverage/pytracer.py ++++ b/coverage/pytracer.py +@@ -6,7 +6,6 @@ + from __future__ import annotations + + import atexit +-import dis + import itertools + import sys + import threading +@@ -39,11 +38,11 @@ set_TArc = set[TArc] + + # We need the YIELD_VALUE opcode below, in a comparison-friendly form. + # PYVERSIONS: RESUME is new in Python3.11 +-RESUME = dis.opmap.get("RESUME") +-RETURN_VALUE = dis.opmap["RETURN_VALUE"] ++RESUME = None ++RETURN_VALUE = 1 + if RESUME is None: +- YIELD_VALUE = dis.opmap["YIELD_VALUE"] +- YIELD_FROM = dis.opmap["YIELD_FROM"] ++ YIELD_VALUE = 0 ++ YIELD_FROM = 0 + YIELD_FROM_OFFSET = 0 if env.PYPY else 2 + else: + YIELD_VALUE = YIELD_FROM = YIELD_FROM_OFFSET = -1 +@@ -107,6 +106,7 @@ class PyTracer(Tracer): + # [2] The last line number executed in this frame. + # [3] Boolean: did this frame start a new context? + self.data_stack: list[tuple[TTraceFileData | None, str | None, TLineNo, bool]] = [] ++ self.resumable_frames: set[int] = set() + self.thread: threading.Thread | None = None + self.stopped = False + self._activity = False +@@ -247,10 +247,12 @@ class PyTracer(Tracer): + if RESUME is not None: + # The current opcode is guaranteed to be RESUME. The argument + # determines what kind of resume it is. +- oparg = frame.f_code.co_code[frame.f_lasti + 1] ++ oparg = 0 + real_call = (oparg == 0) + else: +- real_call = (getattr(frame, "f_lasti", -1) < 0) ++ frame_id = id(frame) ++ real_call = frame_id not in self.resumable_frames ++ self.resumable_frames.add(frame_id) + if real_call: + self.last_line = -frame.f_code.co_firstlineno + else: +@@ -271,8 +273,9 @@ class PyTracer(Tracer): + if self.trace_arcs and self.cur_file_data: + # Record an arc leaving the function, but beware that a + # "return" event might just mean yielding from a generator. +- code = frame.f_code.co_code +- lasti = frame.f_lasti ++ resumable = frame.f_code.co_flags & (0x20 | 0x80 | 0x100 | 0x200) ++ code = (YIELD_VALUE,) if resumable else (RETURN_VALUE,) ++ lasti = 0 + if RESUME is not None: + if len(code) == lasti + 2: + # A return from the end of a code object is a real return. diff --git a/graalpython/lib-graalpython/patches/crates/metadata.toml b/graalpython/lib-graalpython/patches/crates/metadata.toml new file mode 100644 index 0000000000..89490e0ada --- /dev/null +++ b/graalpython/lib-graalpython/patches/crates/metadata.toml @@ -0,0 +1,9 @@ +[[pyo3-ffi.rules]] +version = '>=0.28,<0.29' +patch = 'pyo3-ffi-0.28.patch' +license = 'Apache-2.0 OR MIT' + +[[pyo3-ffi.rules]] +version = '==0.29.0' +patch = 'pyo3-ffi-0.29.0.patch' +license = 'Apache-2.0 OR MIT' diff --git a/graalpython/lib-graalpython/patches/crates/pyo3-ffi-0.28.patch b/graalpython/lib-graalpython/patches/crates/pyo3-ffi-0.28.patch new file mode 100644 index 0000000000..477becdbb4 --- /dev/null +++ b/graalpython/lib-graalpython/patches/crates/pyo3-ffi-0.28.patch @@ -0,0 +1,13 @@ +diff --git a/src/cpython/dictobject.rs b/src/cpython/dictobject.rs +index 37991ee..5f76604 100644 +--- a/src/cpython/dictobject.rs ++++ b/src/cpython/dictobject.rs +@@ -2,6 +2,8 @@ + use crate::object::*; + #[cfg(not(any(PyPy, GraalPy)))] + use crate::pyport::Py_ssize_t; ++#[cfg(all(GraalPy, Py_3_13))] ++use crate::PyObject; + + #[cfg(not(PyPy))] + opaque_struct!(pub PyDictKeysObject); diff --git a/graalpython/lib-graalpython/patches/crates/pyo3-ffi-0.29.0.patch b/graalpython/lib-graalpython/patches/crates/pyo3-ffi-0.29.0.patch new file mode 100644 index 0000000000..9ec56796db --- /dev/null +++ b/graalpython/lib-graalpython/patches/crates/pyo3-ffi-0.29.0.patch @@ -0,0 +1,13 @@ +diff --git a/src/cpython/dictobject.rs b/src/cpython/dictobject.rs +index 1bd75d8..df98a94 100644 +--- a/src/cpython/dictobject.rs ++++ b/src/cpython/dictobject.rs +@@ -2,6 +2,8 @@ + use crate::object::*; + #[cfg(not(any(PyPy, GraalPy)))] + use crate::pyport::Py_ssize_t; ++#[cfg(all(GraalPy, Py_3_13))] ++use crate::PyObject; + + #[cfg(all(not(PyPy), Py_3_13))] + use core::ffi::c_char; diff --git a/graalpython/lib-graalpython/patches/metadata.toml b/graalpython/lib-graalpython/patches/metadata.toml index 0dc34eb600..8b65732ce7 100644 --- a/graalpython/lib-graalpython/patches/metadata.toml +++ b/graalpython/lib-graalpython/patches/metadata.toml @@ -50,6 +50,12 @@ version = '== 2.2.1' patch = 'cloudpickle-2.2.1.patch' license = 'BSD-3-Clause' +[[coverage.rules]] +version = '== 7.10.0' +patch = 'coverage-7.10.0.patch' +license = 'Apache-2.0' +dist-type = 'wheel' + [[cramjam.rules]] version = '== 2.7.0' patch = 'cramjam-2.7.0.patch' diff --git a/graalpython/lib-graalpython/patches/numpy-2.4.4.patch b/graalpython/lib-graalpython/patches/numpy-2.4.4.patch index 8a18791753..e1331821a1 100644 --- a/graalpython/lib-graalpython/patches/numpy-2.4.4.patch +++ b/graalpython/lib-graalpython/patches/numpy-2.4.4.patch @@ -34,6 +34,19 @@ index 3a0b52c..1819ec6 100644 include_directories: [ 'include', 'src/common', +diff --git a/numpy/_core/src/common/pythoncapi-compat/pythoncapi_compat.h b/numpy/_core/src/common/pythoncapi-compat/pythoncapi_compat.h +index 10b7bf7..fc61d66 100644 +--- a/numpy/_core/src/common/pythoncapi-compat/pythoncapi_compat.h ++++ b/numpy/_core/src/common/pythoncapi-compat/pythoncapi_compat.h +@@ -2684,7 +2684,7 @@ PyUnstable_SetImmortal(PyObject *op) + op->ob_ref_local = _Py_IMMORTAL_REFCNT_LOCAL; + op->ob_ref_shared = 0; + #else +- Py_REFCNT(op) = _Py_IMMORTAL_REFCNT; ++ Py_SET_REFCNT(op, _Py_IMMORTAL_REFCNT); + #endif + #endif + return 1; diff --git a/numpy/_core/src/multiarray/compiled_base.c b/numpy/_core/src/multiarray/compiled_base.c index 6e37968..010bd03 100644 --- a/numpy/_core/src/multiarray/compiled_base.c diff --git a/mx.graalpython/downstream_tests.py b/mx.graalpython/downstream_tests.py index a9415d007e..004e05307a 100644 --- a/mx.graalpython/downstream_tests.py +++ b/mx.graalpython/downstream_tests.py @@ -161,6 +161,9 @@ def downstream_test_virtualenv(graalpy, testdir): def downstream_test_pyo3(graalpy, testdir): run(['git', 'clone', 'https://github.com/PyO3/pyo3.git', '-b', 'main', '--depth', '1'], cwd=testdir) src = testdir / 'pyo3' + # The runtime test session does not run mypy. Avoid installing it because its + # librt dependency relies on CPython-internal APIs that are incompatible with GraalPy. + replace_in_file(src / 'pytests/pyproject.toml', r'^\s*"mypy[^\n]*\n', '', flags=re.MULTILINE) venv = src / 'venv' run([graalpy, '-m', 'venv', str(venv)]) run_in_venv(venv, ['python', '-m', 'pip', 'install', '--upgrade', 'pip', 'nox[uv]']) @@ -230,10 +233,8 @@ def downstream_test_cython(graalpy, testdir): run([graalpy, '-m', 'venv', str(venv)]) if not CI: replace_in_file(src / 'Tools/ci-run.sh', r'^\s*sudo', '# sudo', flags=re.MULTILINE) - try: - run([graalpy, '--version', '--experimental-options', '--engine.Compilation=false']) - except subprocess.CalledProcessError: - replace_in_file(src / 'Tools/ci-run.sh', r'--engine\.Compilation=false', '') + replace_in_file(src / 'Tools/ci-run.sh', r'--engine.Compilation=false', '-X jit=0') + replace_in_file(src / 'Tools/ci-run.sh', r'--no-cache-dir', '') run_in_venv(venv, ["bash", "./Tools/ci-run.sh"], cwd=src, env=env) diff --git a/mx.graalpython/mx_graalpython.py b/mx.graalpython/mx_graalpython.py index 8762b61ae1..3386775244 100644 --- a/mx.graalpython/mx_graalpython.py +++ b/mx.graalpython/mx_graalpython.py @@ -2447,6 +2447,7 @@ def dev_tag(_=None): rev_list = [ os.path.join('graalpython', 'lib-graalpython', 'patches'), os.path.join('graalpython', 'lib-graalpython', 'modules', 'autopatch_capi.py'), + os.path.join('graalpython', 'lib-graalpython', 'modules', 'autopatch_cargo.py'), os.path.join('graalpython', 'com.oracle.graal.python.cext', 'include'), os.path.join('graalpython', 'com.oracle.graal.python.cext', 'src', 'capi.h'), ] diff --git a/mx.graalpython/verify_patches.py b/mx.graalpython/verify_patches.py index db3506fa07..e0d83a553e 100644 --- a/mx.graalpython/verify_patches.py +++ b/mx.graalpython/verify_patches.py @@ -37,7 +37,9 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. import argparse +import os import re +import subprocess from pathlib import Path from pip._vendor import tomli # pylint: disable=no-name-in-module @@ -61,9 +63,24 @@ SECTIONS = frozenset({'rules', 'add-sources'}) RULE_KEYS = frozenset({'version', 'patch', 'license', 'subdir', 'dist-type', 'install-priority', 'note', 'autopatch'}) +CRATE_VERSION_SPECIFIER_RE = re.compile(r'(==|!=|<=|>=|<|>)\s*\d+(?:\.\d+)*') -def validate_metadata(patches_dir): +def validate_metadata(patches_dir, crates=False): + verify_git = os.environ.get('VERIFY_PATCHES_GIT') + if verify_git: + patch_files = { + patches_dir / file + for file in subprocess.run( + ['git', '-C', patches_dir, 'ls-files'], + check=True, + stdout=subprocess.PIPE, + text=True, + ).stdout.splitlines() + if Path(file).parent == Path('.') + } + else: + patch_files = set(patches_dir.iterdir()) with open(patches_dir / 'metadata.toml', 'rb') as f: all_metadata = tomli.load(f) patches = set() @@ -78,6 +95,8 @@ def validate_metadata(patches_dir): if patch := rule.get('patch'): patch_path = patches_dir / patch assert patch_path.is_file(), f"Patch file does not exists: {patch_path}" + if verify_git: + assert patch_path in patch_files, f"Patch file is not tracked by git: {patch_path}" patches.add(patch_path) license_id = rule.get('license') assert license_id, f"'license' not specified for patch {patch}" @@ -97,8 +116,13 @@ def validate_metadata(patches_dir): if 'autopatch' in rule: assert isinstance(rule['autopatch'], bool), "'rules.autopatch' must be a bool" if version := rule.get('version'): - # Just try that it doesn't raise - SpecifierSet(version) + if crates: + for constraint in version.split(','): + assert CRATE_VERSION_SPECIFIER_RE.fullmatch(constraint.strip()), \ + f"Unsupported crate version constraint: {constraint!r}" + else: + # Just try that it doesn't raise + SpecifierSet(version) if add_sources := metadata.get('add-sources'): for add_source in add_sources: if unexpected_keys := set(add_source) - {'version', 'url'}: @@ -107,8 +131,12 @@ def validate_metadata(patches_dir): assert add_source.get('url'), "Missing 'add_sources.url' key" except Exception as e: raise AssertionError(f"{package}: {e}") - for file in patches_dir.iterdir(): - assert not file.name.endswith('patch') or file in patches, f"Dangling patch file: {file}" + for patch_file in patch_files: + if patch_file.is_file() and patch_file.name.endswith('patch'): + assert patch_file in patches, f"Dangling patch file: {patch_file}" + crates_dir = patches_dir / 'crates' + if crates_dir.is_dir(): + validate_metadata(crates_dir, crates=True) def main(): diff --git a/scripts/get_pypi_source.py b/scripts/get_pypi_source.py index 03e58eeeb7..06a9d07d73 100755 --- a/scripts/get_pypi_source.py +++ b/scripts/get_pypi_source.py @@ -136,8 +136,10 @@ def get_matching_rule(name, version, dist_type): def should_autopatch(name, version, artifact_type): + if artifact_type != "sdist": + return False rule = get_matching_rule(name, version, artifact_type) - if artifact_type == "sdist" and not rule: + if not rule: rule = get_matching_rule(name, version, "wheel") return not rule or rule.get("autopatch", True) @@ -237,6 +239,16 @@ def autopatch_capi(target_dir): run(["git", "commit", "--quiet", "-m", "Autopatched"], cwd=target_dir) +def autopatch_cargo(target_dir): + from autopatch_cargo import auto_patch_tree + + crate_patches = PATCHES_DIR / "crates" + auto_patch_tree(target_dir, LocalPatchRepository.from_path(crate_patches)) + if has_git_changes(target_dir): + run(["git", "add", "-A", "-f"], cwd=target_dir) + run(["git", "commit", "--quiet", "-m", "Autopatched Cargo crates"], cwd=target_dir) + + def main(argv=None): parser = argparse.ArgumentParser( description="Download and extract a PyPI package artifact into a temporary directory." @@ -271,6 +283,7 @@ def main(argv=None): raise if should_autopatch(name, version, artifact_type): autopatch_capi(target_dir) + autopatch_cargo(target_dir) else: eprint(f"Skipping autopatch for {name}=={version}") diff --git a/scripts/wheelbuilder/repair_wheels.py b/scripts/wheelbuilder/repair_wheels.py index c92c58aad1..c85d1f658a 100644 --- a/scripts/wheelbuilder/repair_wheels.py +++ b/scripts/wheelbuilder/repair_wheels.py @@ -1,4 +1,4 @@ -# Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2025, 2026, Oracle and/or its affiliates. All rights reserved. # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. # # The Universal Permissive License (UPL), Version 1.0 @@ -81,7 +81,9 @@ def repair_wheels(wheelhouse): env=env, ) elif sys.platform == "linux": - ensure_installed("auditwheel", "patchelf") + ensure_installed("auditwheel") + if shutil.which("patchelf") is None: + subprocess.check_call([sys.executable, "-m", "pip", "install", "patchelf"]) p = subprocess.run( [ join(dirname(sys.executable), "auditwheel"),