diff --git a/CHANGELOG.md b/CHANGELOG.md index b440331622..cc0bee2296 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,9 @@ This changelog summarizes major changes between GraalVM versions of the Python language runtime. The main focus is on user-observable behavior of the engine. +## Version 25.2.4 +* Windows standalone builds now use a native OS backend by default, improving compatibility for files, sockets, subprocesses, `mmap`, and Windows-specific modules compared to the Java-based backend. + ## Version 25.1.3 * The standalone artifacts now include the Python version name before the Graal version. The new artifacts now start with `graalpy---`. * Standalone JVM artifacts are no longer released as separate distributions. For standalone deployments, use the GraalPy native artifacts. If you require Java interoperability, use our jbang launcher (`jbang graalpy@oracle/graalpython -c "print('hello from GraalPy')"`) or a custom embedding. diff --git a/docs/contributor/IMPLEMENTATION_DETAILS.md b/docs/contributor/IMPLEMENTATION_DETAILS.md index 7b9cef7609..07f995c539 100644 --- a/docs/contributor/IMPLEMENTATION_DETAILS.md +++ b/docs/contributor/IMPLEMENTATION_DETAILS.md @@ -91,6 +91,44 @@ The patches are regular POSIX `patch` command compatible diffs located in `lib-graalpython/patches`. The directory has a `README.md` file describing how the patches are applied. +## Operating System Backends + +GraalPy exposes operating system functionality through a common POSIX-support abstraction with native and Java implementations. +The native implementation is provided by the `python-libposix` shim. +On Windows, the shim translates between CRT file descriptors, Win32 handles, and Winsock sockets, and implements or emulates Windows equivalents for operations such as `stat`, `rename`, `replace`, pipes, `select`, `mmap`, process handling, and semaphores. +Windows modules and platform-specific builtins include `nt`, `msvcrt`, `_winapi`, and `_overlapped`. + +### Native Strings and Filesystem Paths + +Filesystem paths are kept separate from ordinary native C strings because they have different encoding contracts on Windows. +Paths are passed to Windows as UTF-16 wide strings, while interfaces such as networking, environment handling on non-Windows platforms, and command or argument APIs may require narrow, NUL-terminated byte strings. + +### Windows Error Handling + +Windows has three relevant thread-local error channels: + +- `errno` for CRT and POSIX-style APIs +- `GetLastError()` for Win32 APIs +- `WSAGetLastError()` for Winsock APIs + +Native wrappers capture all three immediately after a failing call and record which channel is authoritative. +When GraalPy turns a native failure into a Python `OSError`, the Java POSIX error bridge checks the captured source. +CRT failures retain `errno`; Win32 and Winsock failures are mapped to their closest POSIX error numbers, while the original Windows value is retained as `OSError.winerror`. +This follows CPython's general `winerror`-to-`errno` model and includes compatibility-specific mappings, such as treating `WSAEWOULDBLOCK` from a non-blocking Winsock connect as `EINPROGRESS`, and the "too many posts" error from `ReleaseSemaphore` as `EOVERFLOW`. + +Unlike CPython, which performs much of this flow directly within its C runtime and exception helpers, GraalPy must explicitly preserve error state across a C-to-Java boundary before constructing the Python exception. +The mapping is intentionally explicit and currently covers the Windows errors used by the backend; unknown Win32 errors retain their numeric value rather than receiving a broader fallback mapping. + +### Windows Descriptor Model + +Windows sockets are not CRT file descriptors. +The backend maps GraalPy-visible integer file descriptors to native `SOCKET` values backed by placeholder CRT descriptors. +Python code therefore retains a unified file-descriptor model while native operations dispatch to either CRT file APIs or Winsock APIs. +The mapping also centralizes socket closing, blocking mode, inheritance, and handle lookup. + +The backend provides a Windows-compatible `mmap` implementation and keeps `os.replace` separate from `os.rename`, since Windows distinguishes their behavior more strictly. +CRT descriptors are opened or normalized in binary mode where Python expects byte-exact I/O, preventing CRT newline translation from changing data written to pipes, files, or standard streams. + ## The GIL We always run with a GIL, because C extensions in CPython expect to do so and diff --git a/docs/user/Embedding-Permissions.md b/docs/user/Embedding-Permissions.md index 316714b44c..161b7404d4 100644 --- a/docs/user/Embedding-Permissions.md +++ b/docs/user/Embedding-Permissions.md @@ -22,7 +22,8 @@ The `PosixModuleBackend` option determines which backend is used: `native` or `j ### Native Backend -The native backend directly calls the POSIX API in mostly the same way as CPython (the reference Python implementation). +The native backend directly calls the operating system API in mostly the same way as CPython (the reference Python implementation). +On POSIX platforms this means POSIX APIs; on Windows this means a Windows-native backend that maps Python's OS interface to the CRT, Win32, and Winsock APIs. This approach is the most compatible with CPython and provides bare access to the underlying OS interface without an intermediate emulation layer. diff --git a/docs/user/Platform-Support.md b/docs/user/Platform-Support.md index bf8f1ef1a5..3be1b485bd 100644 --- a/docs/user/Platform-Support.md +++ b/docs/user/Platform-Support.md @@ -4,4 +4,13 @@ GraalPy is mostly written in Java and Python, but the Python package ecosystem i The main operating system is Oracle Linux, the CPU architectures are AMD64 and ARM, and the primary JDK is Oracle GraalVM. **Linux is recommended for getting started with GraalPy.** Windows and macOS with GraalVM JDK are less well tested, and outside of those combinations only basic test coverage is provided. As macOS and other platforms are not prioritized, some GraalPy features may not work on these platforms. -See [Test Tiers](../user/Test-Tiers.md) for a detailed breakdown. \ No newline at end of file +See [Test Tiers](../user/Test-Tiers.md) for a detailed breakdown. + +## Windows + +GraalPy standalone builds on Windows use a native OS backend by default. +The backend maps Python's OS interface to the Windows CRT, Win32, and Winsock APIs instead of relying on the Java-based backend. +It provides native or emulated Windows implementations for files and paths, sockets, pipes, `select`, `mmap`, subprocesses, multiprocessing semaphores, and Windows-specific modules such as `nt`, `msvcrt`, `_winapi`, and `_overlapped`. + +Filesystem paths are passed to wide-character Windows APIs as UTF-16, while other native interfaces continue to use narrow strings where required. +Errors from the CRT, Win32, and Winsock APIs are translated to Python `OSError` instances using CPython-compatible error mappings, with the original Windows error retained as `OSError.winerror`. diff --git a/graalpython/com.oracle.graal.python.shell/src/com/oracle/graal/python/shell/DefaultConsoleHandler.java b/graalpython/com.oracle.graal.python.shell/src/com/oracle/graal/python/shell/DefaultConsoleHandler.java index 60d80c85b5..6421c85099 100644 --- a/graalpython/com.oracle.graal.python.shell/src/com/oracle/graal/python/shell/DefaultConsoleHandler.java +++ b/graalpython/com.oracle.graal.python.shell/src/com/oracle/graal/python/shell/DefaultConsoleHandler.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2025, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2026, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * The Universal Permissive License (UPL), Version 1.0 @@ -48,9 +48,11 @@ class DefaultConsoleHandler extends ConsoleHandler { private final BufferedReader in; + private final boolean eofOnIOException; - DefaultConsoleHandler(InputStream in) { + DefaultConsoleHandler(InputStream in, boolean eofOnIOException) { this.in = new BufferedReader(new InputStreamReader(in)); + this.eofOnIOException = eofOnIOException; } @Override @@ -58,6 +60,9 @@ public String readLine(String prompt) { try { return in.readLine(); } catch (IOException e) { + if (eofOnIOException) { + return null; + } throw new RuntimeException(e); } } diff --git a/graalpython/com.oracle.graal.python.shell/src/com/oracle/graal/python/shell/GraalPythonMain.java b/graalpython/com.oracle.graal.python.shell/src/com/oracle/graal/python/shell/GraalPythonMain.java index 76798264ac..f77512b4c6 100644 --- a/graalpython/com.oracle.graal.python.shell/src/com/oracle/graal/python/shell/GraalPythonMain.java +++ b/graalpython/com.oracle.graal.python.shell/src/com/oracle/graal/python/shell/GraalPythonMain.java @@ -620,7 +620,12 @@ private String getLauncherExecName() { * @return The absolute path to the program or {@code null}. */ private String calculateProgramFullPath(String program, Predicate isExecutable, String envPath) { - Path programPath = Paths.get(program); + Path programPath; + try { + programPath = Paths.get(program); + } catch (InvalidPathException e) { + return null; + } // If this is an absolute path, we are already fine. if (programPath.isAbsolute()) { @@ -877,10 +882,6 @@ protected void launch(Builder contextBuilder) { setOptionIfNotSetViaCommandLine(contextBuilder, "AllowSignalHandlers", "true"); - if (IS_WINDOWS) { - contextBuilder.option("python.PosixModuleBackend", "java"); - } - setOptionIfNotSetViaCommandLine(contextBuilder, "WarnExperimentalFeatures", "false"); setOptionIfNotSetViaCommandLine(contextBuilder, "UnicodeCharacterDatabaseNativeFallback", "true"); @@ -1023,6 +1024,9 @@ private static void applyRaisedJitThresholds(Map polyglotOptions } private static String toAbsolutePath(String executable) { + if (executable.isEmpty()) { + return executable; + } if (executable.contains(":")) { // this is either already an absolute windows path, or not a single executable return executable; @@ -1285,7 +1289,7 @@ protected void collectArguments(Set options) { private static ConsoleHandler createConsoleHandler(InputStream inStream, OutputStream outStream) { if (!isTTY()) { - return new DefaultConsoleHandler(inStream); + return new DefaultConsoleHandler(inStream, IS_WINDOWS); } else { return new JLineConsoleHandler(inStream, outStream); } @@ -1470,7 +1474,7 @@ protected String getResolvedExecutableName() { if (ImageInfo.inImageRuntimeCode()) { // This is the fastest, most straightforward way to get the name of the actual // executable, i.e., with symlinks resolved all the way down to graalpy - String executableName = ProcessProperties.getExecutableName(); + String executableName = getNativeExecutableName(); if (executableName != null) { return executableName; } @@ -1499,7 +1503,7 @@ protected String getResolvedExecutableName() { private List getCmdline(List args, List subProcessDefs) { List cmd = new ArrayList<>(); if (isAOT()) { - cmd.add(ProcessProperties.getExecutableName()); + cmd.add(getNativeExecutableName()); for (String subProcArg : subProcessDefs) { assert subProcArg.startsWith("D"); cmd.add("--native." + subProcArg); @@ -1530,6 +1534,20 @@ private List getCmdline(List args, List subProcessDefs) return cmd; } + private static String getNativeExecutableName() { + String executableName = ProcessProperties.getExecutableName(); + if (executableName != null) { + try { + if (Files.isExecutable(Path.of(executableName))) { + return executableName; + } + } catch (InvalidPathException e) { + // Try the process handle below. + } + } + return ProcessHandle.current().info().command().orElse(executableName); + } + private static final class ExitException extends RuntimeException { private static final long serialVersionUID = 1L; private final int code; diff --git a/graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/test/SocketTests.java b/graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/test/SocketTests.java index df3344a028..286ee3f46a 100644 --- a/graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/test/SocketTests.java +++ b/graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/test/SocketTests.java @@ -1060,11 +1060,11 @@ private byte[] getNulTerminatedTempFileName() throws IOException { } private Object s2p(String s) { - return lib.createPathFromString(posixSupport, toTruffleStringUncached(s)); + return lib.createCStringFromString(posixSupport, toTruffleStringUncached(s)); } private String p2s(Object p) { - return lib.getPathAsString(posixSupport, p).toJavaStringUncached(); + return lib.getCStringAsString(posixSupport, p).toJavaStringUncached(); } private static void expectErrno(ThrowingRunnable runnable, OSErrorEnum... expectedErrorCodes) { diff --git a/graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/test/builtin/modules/PathConversionNodeTests.java b/graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/test/builtin/modules/PathConversionNodeTests.java index f6d9b140f8..a3308be538 100644 --- a/graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/test/builtin/modules/PathConversionNodeTests.java +++ b/graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/test/builtin/modules/PathConversionNodeTests.java @@ -67,7 +67,7 @@ import com.oracle.graal.python.builtins.objects.PNone; import com.oracle.graal.python.builtins.objects.common.SequenceStorageNodes; import com.oracle.graal.python.builtins.objects.tuple.PTuple; -import com.oracle.graal.python.runtime.PosixSupportLibrary.Buffer; +import com.oracle.graal.python.runtime.PosixSupportLibrary; import com.oracle.graal.python.runtime.PythonContext; import com.oracle.graal.python.runtime.object.PFactory; import com.oracle.graal.python.test.PythonTests; @@ -91,10 +91,9 @@ public static String[] params() { public void setUp() { org.junit.Assume.assumeTrue(backendName.equals("java") || !IS_WINDOWS); PythonTests.enterContext(Collections.singletonMap("python.PosixModuleBackend", backendName), new String[0]); - pathToString = backendName.equals("java") ? p -> (String) p.value : p -> { - Buffer b = (Buffer) p.value; - return new String(b.data, 0, (int) b.length); - }; + Object posixSupport = PythonContext.get(null).getPosixSupport(); + PosixSupportLibrary posixLib = PosixSupportLibrary.getUncached(); + pathToString = p -> posixLib.getPathAsString(posixSupport, p.value).toJavaStringUncached(); } @After diff --git a/graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/test/shell/TestPathResolution.java b/graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/test/shell/TestPathResolution.java index 54c2b236c8..ded3fd6284 100644 --- a/graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/test/shell/TestPathResolution.java +++ b/graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/test/shell/TestPathResolution.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2024, 2025, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2024, 2026, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * The Universal Permissive License (UPL), Version 1.0 @@ -53,12 +53,15 @@ public class TestPathResolution { private Method calculateProgramFullPathMethod; + private Method toAbsolutePathMethod; @Before public void setup() throws NoSuchMethodException { // Use reflection to avoid exposing the method in public API calculateProgramFullPathMethod = GraalPythonMain.class.getDeclaredMethod("calculateProgramFullPath", String.class, Predicate.class, String.class); calculateProgramFullPathMethod.setAccessible(true); + toAbsolutePathMethod = GraalPythonMain.class.getDeclaredMethod("toAbsolutePath", String.class); + toAbsolutePathMethod.setAccessible(true); } public String calculateProgramFullPath(String executable, Predicate isExecutable, String path) { @@ -69,6 +72,14 @@ public String calculateProgramFullPath(String executable, Predicate isExec } } + public String toAbsolutePath(String executable) { + try { + return (String) toAbsolutePathMethod.invoke(null, executable); + } catch (Throwable e) { + throw new RuntimeException(e); + } + } + @Test public void testProgramNameResolutionFullPath() { Assert.assertEquals("/absolute/path/graalpy", @@ -78,6 +89,11 @@ public void testProgramNameResolutionFullPath() { calculateProgramFullPath("./blah/graalpy", path -> true, null)); } + @Test + public void testEmptyExecutableStaysEmpty() { + Assert.assertEquals("", toAbsolutePath("")); + } + @Test public void testProgramNameResolutionEndOfPath() { Assert.assertEquals("/last/in/path/graalpy", diff --git a/graalpython/com.oracle.graal.python.test/src/tests/__init__.py b/graalpython/com.oracle.graal.python.test/src/tests/__init__.py index 454d5678dc..063cb75f85 100644 --- a/graalpython/com.oracle.graal.python.test/src/tests/__init__.py +++ b/graalpython/com.oracle.graal.python.test/src/tests/__init__.py @@ -144,6 +144,31 @@ def get_setuptools(setuptools='67.6.1'): ensure_packages(setuptools=setuptools) +def _system_python_for_venv(): + python = os.environ.get("MX_PYTHON") or "python" + resolved = shutil.which(python) + if resolved and (not sys.platform.startswith('win32') or Path(resolved).suffix.lower() == ".exe"): + return resolved + if sys.platform.startswith('win32'): + version = os.environ.get("MX_PYTHON_VERSION") + if not version and python.startswith("python") and len(python) > len("python"): + version = python[len("python"):] + if version: + version_dir = version.replace(".", "") + for candidate in ( + f"C:/Python{version_dir}/python.exe", + f"C:/Python{version_dir}/python3.exe", + f"C:/Python{version_dir}/{python}.exe", + ): + if Path(candidate).is_file(): + return candidate + for executable in ("python.exe", "python3.exe", "python"): + fallback = shutil.which(executable) + if fallback and Path(fallback).suffix.lower() == ".exe": + return fallback + return python + + def install_venv(venv_path: Path) -> bool: """Installs a virtual environment at the given path.""" if not sys.executable or (sys.platform.startswith('win32') and sys.implementation.name == "graalpy"): @@ -151,7 +176,7 @@ def install_venv(venv_path: Path) -> bool: # And thus we must defer to the system's python # Deferring to the system's python is fine as it will only be used to install setuptools import subprocess - subprocess.run(["python", "-m", "venv", str(venv_path)], check=True) + subprocess.run([_system_python_for_venv(), "-m", "venv", str(venv_path)], check=True) return True else: import venv diff --git a/graalpython/com.oracle.graal.python.test/src/tests/cpyext/test_tuple.py b/graalpython/com.oracle.graal.python.test/src/tests/cpyext/test_tuple.py index 45f894ffda..d7b45d22d3 100644 --- a/graalpython/com.oracle.graal.python.test/src/tests/cpyext/test_tuple.py +++ b/graalpython/com.oracle.graal.python.test/src/tests/cpyext/test_tuple.py @@ -391,16 +391,19 @@ def test_memoryview_cast_shape_native_tuple(self): def test_utime_ns_divmod_native_tuple(self): class NativeDivmod: + def __init__(self, value): + self.value = value + def __divmod__(self, other): assert other == 1000000000 - result = TupleSubclass(1, 234567890) + result = TupleSubclass(*divmod(self.value, other)) assert is_native_object(result) return result with tempfile.NamedTemporaryFile() as f: - os.utime(f.name, ns=(NativeDivmod(), NativeDivmod())) + expected = 1700000001234567890 + os.utime(f.name, ns=(NativeDivmod(expected), NativeDivmod(expected))) stat = os.stat(f.name) - expected = 1234567890 # We don't care about the exact value. It's about if native tuples are accepted. tolerance = 1_000_000_000 assert abs(stat.st_atime_ns - expected) <= tolerance diff --git a/graalpython/com.oracle.graal.python.test/src/tests/test_autopatch_capi.py b/graalpython/com.oracle.graal.python.test/src/tests/test_autopatch_capi.py index b44a86890d..cd54956ec2 100644 --- a/graalpython/com.oracle.graal.python.test/src/tests/test_autopatch_capi.py +++ b/graalpython/com.oracle.graal.python.test/src/tests/test_autopatch_capi.py @@ -1,4 +1,4 @@ -# Copyright (c) 2024, 2025, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2024, 2026, Oracle and/or its affiliates. All rights reserved. # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. # # The Universal Permissive License (UPL), Version 1.0 @@ -38,6 +38,7 @@ # SOFTWARE. import sys +import os import tempfile import textwrap @@ -48,13 +49,17 @@ def check_autopatched(source, expected): source = textwrap.dedent(source).rstrip() expected = textwrap.dedent(expected).rstrip() - with tempfile.NamedTemporaryFile('r+') as f: + with tempfile.NamedTemporaryFile('w+', delete=False) as f: f.write(source) f.flush() - autopatch_capi.auto_patch(f.name, False) - f.seek(0) - actual = f.read() + name = f.name + try: + autopatch_capi.auto_patch(name, False) + with open(name) as f: + actual = f.read() assert actual == expected, f"Autopatch didn't make expected changes. Expected:\n{expected}\nActual:\n{actual}" + finally: + os.unlink(name) def test_replace_field_access(): diff --git a/graalpython/com.oracle.graal.python.test/src/tests/test_fcntl.py b/graalpython/com.oracle.graal.python.test/src/tests/test_fcntl.py index 2f6ea64bdf..fad169e069 100644 --- a/graalpython/com.oracle.graal.python.test/src/tests/test_fcntl.py +++ b/graalpython/com.oracle.graal.python.test/src/tests/test_fcntl.py @@ -89,21 +89,25 @@ def python_flock_interacts_with_sh_flock(python_flock_type, sh_flock_type, shoul class FcntlTests(unittest.TestCase): @unittest.skipUnless(__graalpython__.posix_module_backend() != 'java', 'No support in Truffle API (GR-28740)') @unittest.skipUnless(sys.platform != 'darwin', 'MacOSX does not have flock utility') + @unittest.skipUnless(sys.platform != 'win32', 'Windows does not have flock utility') def test_flock_x_and_x(self): python_flock_interacts_with_sh_flock(fcntl.LOCK_EX, 'x', should_block=True) @unittest.skipUnless(__graalpython__.posix_module_backend() != 'java', 'No support in Truffle API (GR-28740)') @unittest.skipUnless(sys.platform != 'darwin', 'MacOSX does not have flock utility') + @unittest.skipUnless(sys.platform != 'win32', 'Windows does not have flock utility') def test_flock_x_and_s(self): python_flock_interacts_with_sh_flock(fcntl.LOCK_EX, 's', should_block=True) @unittest.skipUnless(__graalpython__.posix_module_backend() != 'java', 'No support in Truffle API (GR-28740)') @unittest.skipUnless(sys.platform != 'darwin', 'MacOSX does not have flock utility') + @unittest.skipUnless(sys.platform != 'win32', 'Windows does not have flock utility') def test_flock_s_and_x(self): python_flock_interacts_with_sh_flock(fcntl.LOCK_SH, 'x', should_block=True) @unittest.skipUnless(__graalpython__.posix_module_backend() != 'java', 'No support in Truffle API (GR-28740)') @unittest.skipUnless(sys.platform != 'darwin', 'MacOSX does not have flock utility') + @unittest.skipUnless(sys.platform != 'win32', 'Windows does not have flock utility') @unittest.skipUnless("graalpython" in os.environ.get("BITBUCKET_REPO_URL", "graalpython"), "Do not run this in auxillary CI jobs, it can be flaky") def test_flock_s_and_s(self): python_flock_interacts_with_sh_flock(fcntl.LOCK_SH, 's', should_block=False) diff --git a/graalpython/com.oracle.graal.python.test/src/tests/test_json.py b/graalpython/com.oracle.graal.python.test/src/tests/test_json.py index cf85d43747..56ef337f6a 100644 --- a/graalpython/com.oracle.graal.python.test/src/tests/test_json.py +++ b/graalpython/com.oracle.graal.python.test/src/tests/test_json.py @@ -83,8 +83,10 @@ def test_invalid_object_pairs_hook(self): def test_dump(self): cwd = os.getcwd() new_file_path = os.path.join(cwd, 'myFile.json') - json.dump(['a', 'b', 'c'], open(new_file_path, 'w')) - assert json.load(open(new_file_path)) == ['a', 'b', 'c'] + with open(new_file_path, 'w') as f: + json.dump(['a', 'b', 'c'], f) + with open(new_file_path) as f: + assert json.load(f) == ['a', 'b', 'c'] os.remove(new_file_path) def test_load_bigint(self): diff --git a/graalpython/com.oracle.graal.python.test/src/tests/test_mmap.py b/graalpython/com.oracle.graal.python.test/src/tests/test_mmap.py index 150333a6ae..eb3b129e5e 100644 --- a/graalpython/com.oracle.graal.python.test/src/tests/test_mmap.py +++ b/graalpython/com.oracle.graal.python.test/src/tests/test_mmap.py @@ -38,12 +38,50 @@ # SOFTWARE. import mmap +import os import sys +import time +import unittest PAGESIZE = mmap.PAGESIZE FIND_BUFFER_SIZE = 1024 # keep in sync with FindNode#BUFFER_SIZE +@unittest.skipUnless(sys.platform == "win32", "requires Windows named mmap support") +def test_named_mmap_clears_windows_last_error(): + import ctypes + import _winapi + + # A use_last_error ctypes call must not make _winapi.GetLastError return the stale private + # ctypes copy after mmap has cleared the actual OS thread state. + ctypes.set_last_error(_winapi.ERROR_ALREADY_EXISTS) + try: + tagname = f"graalpy-mmap-test-{os.getpid()}-{time.time_ns()}" + m = mmap.mmap(-1, 1, tagname=tagname) + try: + assert _winapi.GetLastError() == 0 + finally: + m.close() + finally: + ctypes.set_last_error(0) + + +@unittest.skipUnless(sys.platform == "win32", "requires Windows named mmap support") +def test_windows_tagname_as_third_positional_argument(): + data = b"named mmap" + tagname = f"graalpy-mmap-test-{os.getpid()}-{time.time_ns()}" + m1 = mmap.mmap(-1, len(data), tagname) + try: + m1[:] = data + m2 = mmap.mmap(-1, len(data), tagname=tagname) + try: + assert m2[:] == data + finally: + m2.close() + finally: + m1.close() + + def test_map_private_constant_matches_platform(): assert hasattr(mmap, "MAP_PRIVATE") == (sys.platform != "win32") diff --git a/graalpython/com.oracle.graal.python.test/src/tests/test_posix.py b/graalpython/com.oracle.graal.python.test/src/tests/test_posix.py index 4b3070171d..2f30af866e 100644 --- a/graalpython/com.oracle.graal.python.test/src/tests/test_posix.py +++ b/graalpython/com.oracle.graal.python.test/src/tests/test_posix.py @@ -55,6 +55,9 @@ def posix_module_backend(self): import stat import tempfile import io +import glob +import shutil +import socket from contextlib import contextmanager @@ -270,6 +273,59 @@ def test_open_bytes_path(self): except Exception: pass + @unittest.skipUnless(sys.platform == 'win32' and __graalpython__.posix_module_backend() == 'native', + 'requires the Windows native POSIX backend') + def test_windows_native_lone_surrogate_paths(self): + root = tempfile.mkdtemp() + old_cwd = os.getcwd() + filenames = ['suffix-\udc80', 'middle-\udc80-name', 'non-bmp-\U0001f600-\udc80'] + dirname = 'directory-\udc80' + try: + for filename in filenames: + path = os.path.join(root, filename) + fd = os.open(path, os.O_CREAT | os.O_WRONLY) + os.write(fd, b'content') + os.close(fd) + self.assertTrue(os.path.isfile(path)) + os.stat(path) + os.chmod(path, stat.S_IWRITE | stat.S_IREAD) + os.utime(path, (1, 2)) + self.assertIn(filename, os.listdir(root)) + self.assertIn(filename.encode('utf-8', 'surrogatepass'), os.listdir(os.fsencode(root))) + self.assertEqual([path], glob.glob(path)) + with os.scandir(root) as entries: + self.assertIn(filename, [entry.name for entry in entries]) + + copied = path + '.copy' + shutil.copyfile(path, copied) + with open(copied, 'rb') as copied_file: + self.assertEqual(b'content', copied_file.read()) + os.unlink(copied) + + renamed = path + '.new' + os.rename(path, renamed) + self.assertTrue(os.path.isfile(renamed)) + os.unlink(renamed) + + encoded_path = path.encode('utf-8', 'surrogatepass') + fd = os.open(encoded_path, os.O_CREAT | os.O_WRONLY) + os.close(fd) + self.assertTrue(os.path.isfile(path)) + os.unlink(encoded_path) + + directory_path = os.path.join(root, dirname) + os.mkdir(directory_path) + os.chdir(directory_path) + self.assertEqual(dirname, os.path.basename(os.getcwd())) + os.chdir(old_cwd) + os.rmdir(directory_path) + + with self.assertRaises(UnicodeDecodeError): + os.stat(os.fsencode(root) + b'\\malformed-\xff') + finally: + os.chdir(old_cwd) + os.rmdir(root) + def test_failed_read_write_errno(self): read_fd, write_fd = os.pipe() try: @@ -284,6 +340,143 @@ def test_failed_read_write_errno(self): os.close(read_fd) os.close(write_fd) + @unittest.skipUnless(sys.platform == 'win32' and __graalpython__.posix_module_backend() != 'java', + 'requires the Windows native POSIX backend') + def test_windows_native_error_codes(self): + missing_path = os.path.join(TEMP_DIR, 'graalpython_missing_path_for_winerror') + try: + os.unlink(missing_path) + except FileNotFoundError: + pass + + with self.assertRaises(OSError) as missing_error: + os.stat(missing_path) + self.assertEqual(errno.ENOENT, missing_error.exception.errno) + self.assertEqual(2, missing_error.exception.winerror) # ERROR_FILE_NOT_FOUND + + # A CRT failure following a WinAPI failure must not reuse the earlier winerror. + read_fd, write_fd = os.pipe() + os.close(read_fd) + os.close(write_fd) + with self.assertRaises(OSError) as crt_error: + os.read(read_fd, 1) + self.assertEqual(errno.EBADF, crt_error.exception.errno) + self.assertIsNone(crt_error.exception.winerror) + + def assert_crt_error(operation, expected_errno): + # Seed the native last-error state before every CRT failure. + with self.assertRaises(OSError) as winapi_error: + os.stat(missing_path) + self.assertEqual(2, winapi_error.exception.winerror) + with self.assertRaises(OSError) as error: + operation() + self.assertEqual(expected_errno, error.exception.errno) + self.assertIsNone(error.exception.winerror) + + assert_crt_error(lambda: os.rename(missing_path, missing_path + '.renamed'), errno.ENOENT) + assert_crt_error(lambda: os.chmod(missing_path, 0o600), errno.ENOENT) + assert_crt_error(lambda: os.listdir(''), errno.ENOENT) + + import socket + first = socket.socket() + second = socket.socket() + try: + first.bind(('127.0.0.1', 0)) + with self.assertRaises(OSError) as socket_error: + second.bind(first.getsockname()) + self.assertEqual(errno.EADDRINUSE, socket_error.exception.errno) + self.assertEqual(10048, socket_error.exception.winerror) # WSAEADDRINUSE + finally: + first.close() + second.close() + + # Winsock reports WSAEWOULDBLOCK while a timed connect is in progress. The socket layer + # must treat that as EINPROGRESS and wait for completion instead of raising immediately. + server = socket.socket() + try: + server.bind(('127.0.0.1', 0)) + server.listen() + with socket.create_connection(server.getsockname(), timeout=10) as client: + accepted, _ = server.accept() + accepted.close() + finally: + server.close() + + @unittest.skipUnless(sys.platform == 'win32' and __graalpython__.posix_module_backend() == 'native', + 'requires the Windows native POSIX backend') + def test_windows_native_dup2_socket(self): + source = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + target = os.open(os.devnull, os.O_RDONLY) + try: + source.bind(('127.0.0.1', 0)) + self.assertEqual(target, os.dup2(source.fileno(), target)) + duplicate = socket.socket(fileno=target) + target = -1 + with duplicate: + self.assertEqual(source.getsockname(), duplicate.getsockname()) + finally: + source.close() + if target >= 0: + os.close(target) + + @unittest.skipUnless(sys.platform == 'win32' and __graalpython__.posix_module_backend() == 'native', + 'requires the Windows native POSIX backend') + def test_windows_native_dup2_file_over_socket(self): + read_fd, write_fd = os.pipe() + target = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + target_fd = target.detach() + try: + self.assertEqual(target_fd, os.dup2(read_fd, target_fd)) + os.close(read_fd) + read_fd = -1 + os.write(write_fd, b'x') + self.assertEqual(b'x', os.read(target_fd, 1)) + finally: + if read_fd >= 0: + os.close(read_fd) + os.close(write_fd) + os.close(target_fd) + + @unittest.skipUnless(sys.platform == 'win32' and __graalpython__.posix_module_backend() == 'native', + 'requires the Windows native POSIX backend') + def test_windows_native_dup2_socket_over_socket(self): + source = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + target = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + target.bind(('127.0.0.1', 0)) + target_fd = target.detach() + try: + source.bind(('127.0.0.1', 0)) + self.assertEqual(target_fd, os.dup2(source.fileno(), target_fd)) + duplicate = socket.socket(fileno=target_fd) + target_fd = -1 + with duplicate: + self.assertEqual(source.getsockname(), duplicate.getsockname()) + finally: + source.close() + if target_fd >= 0: + os.close(target_fd) + + @unittest.skipUnless(sys.platform == 'win32' and __graalpython__.posix_module_backend() == 'native', + 'requires the Windows native POSIX backend') + def test_windows_native_dup2_socket_to_self_non_inheritable(self): + with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as sock: + fd = sock.fileno() + os.set_inheritable(fd, True) + self.assertEqual(fd, os.dup2(fd, fd, inheritable=False)) + self.assertFalse(os.get_inheritable(fd)) + sock.getsockname() + + @unittest.skipUnless(sys.platform == 'win32' and __graalpython__.posix_module_backend() == 'native', + 'requires the Windows native POSIX backend') + def test_windows_native_dup_nonblocking_socket(self): + source = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + source.setblocking(False) + try: + with socket.socket(fileno=os.dup(source.fileno())) as duplicate: + self.assertFalse(duplicate.getblocking()) + finally: + source.close() + def test_fd_converter(self): class MyInt(int): def fileno(self): return 0 @@ -440,6 +633,15 @@ def test_fstat(self): with open(TEST_FULL_PATH2, 0) as fd: # follows symlink self.assertEqual(inode, os.fstat(fd).st_ino) + @unittest.skipUnless(sys.platform == 'win32' and __graalpython__.posix_module_backend() == 'native', + 'test requires the Windows native POSIX backend') + def test_windows_native_stat_uses_utc_timestamps(self): + timestamp = 1700000000 + os.utime(TEST_FULL_PATH1, (timestamp, timestamp)) + self.assertEqual(timestamp, int(os.stat(TEST_FULL_PATH1).st_mtime)) + with open(TEST_FULL_PATH1, 0) as fd: + self.assertEqual(timestamp, int(os.fstat(fd).st_mtime)) + def test_statvfs(self): res = os.statvfs(TEST_FULL_PATH1) with open(TEST_FULL_PATH1, 0) as fd: diff --git a/graalpython/com.oracle.graal.python.test/src/tests/test_reparse.py b/graalpython/com.oracle.graal.python.test/src/tests/test_reparse.py index 575e7f9a30..ef1d84adb9 100644 --- a/graalpython/com.oracle.graal.python.test/src/tests/test_reparse.py +++ b/graalpython/com.oracle.graal.python.test/src/tests/test_reparse.py @@ -125,7 +125,7 @@ def pyc_reparse(test_content, expect_success=True, python_options=()): if expect_success: assert proc.wait() == 0, out else: - assert proc.wait() == 1 and re.search(r"SystemError:.*--python\.KeepBytecodeInMemory", out), out + assert proc.wait() == 1 and re.search(r"SystemError:[\s\S]*--python\.KeepBytecodeInMemory", out), repr(out) TRACING_TEST = ''' @@ -182,7 +182,7 @@ def test_reparse_modified(): def test_reparse_disabled(): - with pyc_reparse(TRACING_TEST, python_options=["--python.KeepBytecodeInMemory"], expect_success=True) \ + with pyc_reparse(TRACING_TEST, python_options=["--experimental-options=true", "--python.KeepBytecodeInMemory"], expect_success=True) \ as (example_file, pyc_file): pyc_file.unlink() @@ -192,9 +192,15 @@ def foo(): a = 42 return a -assert foo() == 42 -foo.__code__ = foo.__code__.replace(co_code=foo.__code__.co_code) -assert foo() == 42 +try: + assert foo() == 42 + foo.__code__ = foo.__code__.replace(co_code=foo.__code__.co_code) + assert foo() == 42 +except SystemError as e: + # Traceback rendering may itself need the deleted bytecode file and stop before printing the + # exception. Emit it first so that the parent can verify the expected reparse failure. + print(f"SystemError: {e}", flush=True) + raise ''' diff --git a/graalpython/com.oracle.graal.python.test/src/tests/test_resource.py b/graalpython/com.oracle.graal.python.test/src/tests/test_resource.py index 4ae2b0179b..c761954560 100644 --- a/graalpython/com.oracle.graal.python.test/src/tests/test_resource.py +++ b/graalpython/com.oracle.graal.python.test/src/tests/test_resource.py @@ -60,6 +60,8 @@ def test_import(): def test_getrusage(): + if sys.platform not in ['darwin', 'linux']: + raise unittest.SkipTest("resource.getrusage is not available on this platform") from resource import getrusage, RUSAGE_SELF try: from resource import RUSAGE_THREAD diff --git a/graalpython/com.oracle.graal.python.test/src/tests/test_select.py b/graalpython/com.oracle.graal.python.test/src/tests/test_select.py index ef3bc9b662..2bf2ebd4cb 100644 --- a/graalpython/com.oracle.graal.python.test/src/tests/test_select.py +++ b/graalpython/com.oracle.graal.python.test/src/tests/test_select.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020, 2024, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2020, 2026, Oracle and/or its affiliates. All rights reserved. # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. # # The Universal Permissive License (UPL), Version 1.0 @@ -39,6 +39,7 @@ import os import select +import sys import tempfile import unittest @@ -68,7 +69,11 @@ def __index__(self): return 1 v = MyVal() - select.select([], [], [], v) + if sys.platform == 'win32': + with self.assertRaises(OSError): + select.select([], [], [], v) + else: + select.select([], [], [], v) assert v.called_index == 1 def test_select_timeout_arg_validation(self): @@ -79,6 +84,7 @@ def test_select_timeout_arg_validation(self): @unittest.skipUnless(__graalpython__.posix_module_backend() != 'java', 'The java backend does not support select for ordinary files, only sockets.') + @unittest.skipUnless(sys.platform != 'win32', 'Windows select only supports sockets.') def test_select_result_duplicate_fds_preserve_objs_order(self): class F: def __init__(self, fd): diff --git a/graalpython/com.oracle.graal.python.test/src/tests/test_startup.py b/graalpython/com.oracle.graal.python.test/src/tests/test_startup.py index d3dbbff6d3..fead003f14 100644 --- a/graalpython/com.oracle.graal.python.test/src/tests/test_startup.py +++ b/graalpython/com.oracle.graal.python.test/src/tests/test_startup.py @@ -48,16 +48,7 @@ if IS_WINDOWS and sys.implementation.name == 'graalpy' and __graalpython__.native_access_is_available(): WINDOWS_CORE_MODULES = ['_nt', '_winapi', '_overlapped', 'winreg', '_winreg'] - WINDOWS_FULL_STARTUP_MODULES = [ - '_datetime', - 'datetime', - '_ctypes', - '_struct', - 'struct', - 'ctypes._endian', - 'ctypes', - 'ctypes.wintypes', - ] + WINDOWS_FULL_STARTUP_MODULES = [] else: WINDOWS_CORE_MODULES = [] WINDOWS_FULL_STARTUP_MODULES = ['_winapi'] if IS_WINDOWS else [] diff --git a/graalpython/com.oracle.graal.python.test/src/tests/test_subprocess.py b/graalpython/com.oracle.graal.python.test/src/tests/test_subprocess.py index 80d7f523a6..ec5e722c4e 100644 --- a/graalpython/com.oracle.graal.python.test/src/tests/test_subprocess.py +++ b/graalpython/com.oracle.graal.python.test/src/tests/test_subprocess.py @@ -104,6 +104,22 @@ def test_check_output_stdout_arg(self): self.fail("Expected ValueError when stdout arg supplied.") self.assertIn('stdout', c.exception.args[0]) + @unittest.skipUnless(sys.platform == 'win32', "Windows handle-list specific") + def test_close_fds_with_handle_list(self): + with tempfile.TemporaryDirectory() as tmp_dir: + leak_path = os.path.join(tmp_dir, "leak") + out_path = os.path.join(tmp_dir, "out") + leak = open(leak_path, "w+") + os.set_inheritable(leak.fileno(), True) + with open(out_path, "w+") as out: + p = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(5)"], stdout=out, stderr=out) + leak.close() + try: + os.unlink(leak_path) + finally: + p.terminate() + p.wait() + def test_kill(self): p = subprocess.Popen([sys.executable, "-c", "print('oh no')"]) p.kill() diff --git a/graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_fcntl.txt b/graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_fcntl.txt index 393bd810bb..557d5a85f2 100644 --- a/graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_fcntl.txt +++ b/graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_fcntl.txt @@ -1,5 +1,5 @@ # Disabled due to relatively frequent transient failures !test.test_fcntl.FcntlTests.test_flock_x_and_s -test.test_fcntl.TestFcntl.test_flock @ darwin-arm64,linux-aarch64,linux-aarch64-github,linux-x86_64,linux-x86_64-github,win32-AMD64,win32-AMD64-github +test.test_fcntl.TestFcntl.test_flock @ darwin-arm64,linux-aarch64,linux-aarch64-github,linux-x86_64,linux-x86_64-github test.test_fcntl.TestFcntl.test_lockf_exclusive @ darwin-arm64,linux-aarch64,linux-aarch64-github,linux-x86_64,linux-x86_64-github test.test_fcntl.TestFcntl.test_lockf_share @ darwin-arm64,linux-aarch64,linux-aarch64-github,linux-x86_64,linux-x86_64-github diff --git a/graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_lzma.txt b/graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_lzma.txt index 2f46223ff9..32158564ae 100644 --- a/graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_lzma.txt +++ b/graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_lzma.txt @@ -111,9 +111,9 @@ test.test_lzma.OpenTestCase.test_bad_params @ darwin-arm64,linux-aarch64,linux-a test.test_lzma.OpenTestCase.test_binary_modes @ darwin-arm64,linux-aarch64,linux-aarch64-github,linux-x86_64,linux-x86_64-github,win32-AMD64,win32-AMD64-github test.test_lzma.OpenTestCase.test_encoding @ darwin-arm64,linux-aarch64,linux-aarch64-github,linux-x86_64,linux-x86_64-github,win32-AMD64,win32-AMD64-github test.test_lzma.OpenTestCase.test_encoding_error_handler @ darwin-arm64,linux-aarch64,linux-aarch64-github,linux-x86_64,linux-x86_64-github,win32-AMD64,win32-AMD64-github -test.test_lzma.OpenTestCase.test_filename @ darwin-arm64,linux-aarch64,linux-aarch64-github,linux-x86_64,linux-x86_64-github,win32-AMD64,win32-AMD64-github +test.test_lzma.OpenTestCase.test_filename @ darwin-arm64,linux-aarch64,linux-aarch64-github,linux-x86_64,linux-x86_64-github test.test_lzma.OpenTestCase.test_format_and_filters @ darwin-arm64,linux-aarch64,linux-aarch64-github,linux-x86_64,linux-x86_64-github test.test_lzma.OpenTestCase.test_newline @ darwin-arm64,linux-aarch64,linux-aarch64-github,linux-x86_64,linux-x86_64-github,win32-AMD64,win32-AMD64-github test.test_lzma.OpenTestCase.test_text_modes @ darwin-arm64,linux-aarch64,linux-aarch64-github,linux-x86_64,linux-x86_64-github,win32-AMD64,win32-AMD64-github -test.test_lzma.OpenTestCase.test_with_pathlike_filename @ darwin-arm64,linux-aarch64,linux-aarch64-github,linux-x86_64,linux-x86_64-github,win32-AMD64,win32-AMD64-github +test.test_lzma.OpenTestCase.test_with_pathlike_filename @ darwin-arm64,linux-aarch64,linux-aarch64-github,linux-x86_64,linux-x86_64-github test.test_lzma.OpenTestCase.test_x_mode @ darwin-arm64,linux-aarch64,linux-aarch64-github,linux-x86_64,linux-x86_64-github,win32-AMD64,win32-AMD64-github diff --git a/graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_zipfile.txt b/graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_zipfile.txt index eae188b815..a30d0151bd 100644 --- a/graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_zipfile.txt +++ b/graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_zipfile.txt @@ -3,7 +3,7 @@ test.test_zipfile._path.test_path.TestPath.test_backslash_not_separator @ darwin-arm64,linux-aarch64,linux-aarch64-github,linux-x86_64,linux-x86_64-github,win32-AMD64,win32-AMD64-github test.test_zipfile._path.test_path.TestPath.test_dir_parent @ darwin-arm64,linux-aarch64,linux-aarch64-github,linux-x86_64,linux-x86_64-github,win32-AMD64,win32-AMD64-github test.test_zipfile._path.test_path.TestPath.test_eq_hash @ darwin-arm64,linux-aarch64,linux-aarch64-github,linux-x86_64,linux-x86_64-github,win32-AMD64,win32-AMD64-github -test.test_zipfile._path.test_path.TestPath.test_extract_orig_with_implied_dirs @ darwin-arm64,linux-aarch64,linux-aarch64-github,linux-x86_64,linux-x86_64-github,win32-AMD64,win32-AMD64-github +test.test_zipfile._path.test_path.TestPath.test_extract_orig_with_implied_dirs @ darwin-arm64,linux-aarch64,linux-aarch64-github,linux-x86_64,linux-x86_64-github test.test_zipfile._path.test_path.TestPath.test_filename @ darwin-arm64,linux-aarch64,linux-aarch64-github,linux-x86_64,linux-x86_64-github,win32-AMD64,win32-AMD64-github test.test_zipfile._path.test_path.TestPath.test_getinfo_missing @ darwin-arm64,linux-aarch64,linux-aarch64-github,linux-x86_64,linux-x86_64-github,win32-AMD64,win32-AMD64-github test.test_zipfile._path.test_path.TestPath.test_glob_chars @ darwin-arm64,linux-aarch64,linux-aarch64-github,linux-x86_64,linux-x86_64-github,win32-AMD64,win32-AMD64-github @@ -35,7 +35,7 @@ test.test_zipfile._path.test_path.TestPath.test_open_extant_directory @ darwin-a test.test_zipfile._path.test_path.TestPath.test_open_missing_directory @ darwin-arm64,linux-aarch64,linux-aarch64-github,linux-x86_64,linux-x86_64-github,win32-AMD64,win32-AMD64-github test.test_zipfile._path.test_path.TestPath.test_open_write @ darwin-arm64,linux-aarch64,linux-aarch64-github,linux-x86_64,linux-x86_64-github,win32-AMD64,win32-AMD64-github test.test_zipfile._path.test_path.TestPath.test_parent @ darwin-arm64,linux-aarch64,linux-aarch64-github,linux-x86_64,linux-x86_64-github,win32-AMD64,win32-AMD64-github -test.test_zipfile._path.test_path.TestPath.test_pathlike_construction @ darwin-arm64,linux-aarch64,linux-aarch64-github,linux-x86_64,linux-x86_64-github,win32-AMD64,win32-AMD64-github +test.test_zipfile._path.test_path.TestPath.test_pathlike_construction @ darwin-arm64,linux-aarch64,linux-aarch64-github,linux-x86_64,linux-x86_64-github test.test_zipfile._path.test_path.TestPath.test_pickle @ darwin-arm64,linux-aarch64,linux-aarch64-github,linux-x86_64,linux-x86_64-github # Times out on Windows in CI !test.test_zipfile._path.test_path.TestPath.test_pickle @ win32-AMD64,win32-AMD64-github diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/GraalPythonModuleBuiltins.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/GraalPythonModuleBuiltins.java index 08b0421d06..1fe6e2be2c 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/GraalPythonModuleBuiltins.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/GraalPythonModuleBuiltins.java @@ -511,7 +511,7 @@ private static Object doLoadBytecodeFile(Object bytecodePath, Object sourcePath, try { // get_data TruffleString strBytecodePath = PyObjectStrAsTruffleStringNode.executeUncached(bytecodePath); - TruffleFile bytecodeFile = context.getPublicTruffleFileRelaxed(strBytecodePath); + TruffleFile bytecodeFile = context.getPublicTruffleFileRelaxed(strBytecodePath).getAbsoluteFile().normalize(); byte[] bytes = bytecodeFile.readAllBytes(); // _classify_pyc if (bytes.length < 16 || !Arrays.equals(bytes, 0, 4, MAGIC_NUMBER_BYTES, 0, 4)) { @@ -536,7 +536,7 @@ private static Object doLoadBytecodeFile(Object bytecodePath, Object sourcePath, if (!checkHashBasedPycs.equals("never") && (checkSource || checkHashBasedPycs.equals("always"))) { // get_data TruffleString strSourcePath = PyObjectStrAsTruffleStringNode.executeUncached(sourcePath); - TruffleFile sourceFile = context.getEnv().getPublicTruffleFile(strSourcePath.toJavaStringUncached()); + TruffleFile sourceFile = context.getEnv().getPublicTruffleFile(strSourcePath.toJavaStringUncached()).getAbsoluteFile().normalize(); byte[] sourceBytes = sourceFile.readAllBytes(); long sourceHash = ARRAY_ACCESSOR_LE.getLong(ImpModuleBuiltins.SourceHashNode.hashSource(MAGIC_NUMBER, sourceBytes, sourceBytes.length), 0); // _validate_hash_pyc @@ -571,7 +571,7 @@ private static Object doLoadBytecodeFile(Object bytecodePath, Object sourcePath, if (sourcePath != PNone.NONE) { try { TruffleString strSourcePath = PyObjectStrAsTruffleStringNode.executeUncached(sourcePath); - sourceFile = context.getPublicTruffleFileRelaxed(strSourcePath); + sourceFile = context.getPublicTruffleFileRelaxed(strSourcePath).getAbsoluteFile().normalize(); } catch (SecurityException | UnsupportedOperationException | IllegalArgumentException ignored) { // Fall back to Marshal's empty source. } diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/MMapModuleBuiltins.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/MMapModuleBuiltins.java index 87de8bda5d..ae318297ba 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/MMapModuleBuiltins.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/MMapModuleBuiltins.java @@ -43,6 +43,8 @@ import java.util.Collections; import java.util.List; +import com.oracle.graal.python.PythonLanguage; +import com.oracle.graal.python.annotations.PythonOS; import com.oracle.graal.python.builtins.CoreFunctions; import com.oracle.graal.python.builtins.Python3Core; import com.oracle.graal.python.builtins.PythonBuiltinClassType; @@ -86,7 +88,8 @@ public MMapModuleBuiltins() { // 'MADV_DONTNEED': 4, // 'MADV_FREE': 5 - addBuiltinConstant("ALLOCATIONGRANULARITY", 4096); + addBuiltinConstant("ALLOCATIONGRANULARITY", + PythonLanguage.getPythonOS() == PythonOS.PLATFORM_WIN32 ? 65536 : 4096); addBuiltinConstant("PAGESIZE", 4096); for (PosixConstants.IntConstant c : PosixConstants.mmapFlags) { diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/MsvcrtModuleBuiltins.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/MsvcrtModuleBuiltins.java index 4ef96d0031..9fd598de48 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/MsvcrtModuleBuiltins.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/MsvcrtModuleBuiltins.java @@ -53,7 +53,9 @@ import com.oracle.graal.python.builtins.objects.PNone; import com.oracle.graal.python.nodes.PConstructAndRaiseNode; import com.oracle.graal.python.nodes.function.PythonBuiltinBaseNode; +import com.oracle.graal.python.nodes.function.builtins.PythonBinaryClinicBuiltinNode; import com.oracle.graal.python.nodes.function.builtins.PythonTernaryClinicBuiltinNode; +import com.oracle.graal.python.nodes.function.builtins.PythonUnaryClinicBuiltinNode; import com.oracle.graal.python.nodes.function.builtins.clinic.ArgumentClinicProvider; import com.oracle.graal.python.runtime.PosixSupportLibrary; import com.oracle.truffle.api.dsl.Bind; @@ -73,6 +75,8 @@ public final class MsvcrtModuleBuiltins extends PythonBuiltins { public static final int LK_NBLCK = 2; public static final int LK_UNLOCK = 3; private static final TruffleString T_MSVCRT_LOCKING = tsLiteral("msvcrt.locking"); + private static final TruffleString T_MSVCRT_GET_OSFHANDLE = tsLiteral("msvcrt.get_osfhandle"); + private static final TruffleString T_MSVCRT_OPEN_OSFHANDLE = tsLiteral("msvcrt.open_osfhandle"); @Override protected List> getNodeFactories() { @@ -101,7 +105,7 @@ Object locking(VirtualFrame frame, int fd, int mode, long nbytes, @Cached PConstructAndRaiseNode.Lazy constructAndRaiseNode) { auditNode.audit(frame, inliningTarget, T_MSVCRT_LOCKING, fd, mode, nbytes); try { - posixLib.fcntlLock(getPosixSupport(), fd, mode == LK_NBLCK, mode == LK_LOCK ? 1 : 0, 0, 0, nbytes); + posixLib.fcntlLock(getPosixSupport(), fd, mode != LK_NBLCK, mode == LK_UNLOCK ? 0 : 1, 0, 0, nbytes); } catch (PosixSupportLibrary.PosixException e) { throw constructAndRaiseNode.get(inliningTarget).raiseOSErrorFromPosixException(frame, e); } @@ -113,4 +117,53 @@ protected ArgumentClinicProvider getArgumentClinic() { return MsvcrtModuleBuiltinsClinicProviders.LockingNodeClinicProviderGen.INSTANCE; } } + + @Builtin(name = "get_osfhandle", minNumOfPositionalArgs = 1, parameterNames = {"fd"}) + @ArgumentClinic(name = "fd", conversion = ArgumentClinic.ClinicConversion.Int) + @GenerateNodeFactory + public abstract static class GetOsfHandleNode extends PythonUnaryClinicBuiltinNode { + @Specialization + long getOsfHandle(VirtualFrame frame, int fd, + @Bind Node inliningTarget, + @Cached SysModuleBuiltins.AuditNode auditNode, + @CachedLibrary("getPosixSupport()") PosixSupportLibrary posixLib, + @Cached PConstructAndRaiseNode.Lazy constructAndRaiseNode) { + auditNode.audit(frame, inliningTarget, T_MSVCRT_GET_OSFHANDLE, fd); + try { + return posixLib.getOsfHandle(getPosixSupport(), fd); + } catch (PosixSupportLibrary.PosixException e) { + throw constructAndRaiseNode.get(inliningTarget).raiseOSErrorFromPosixException(frame, e); + } + } + + @Override + protected ArgumentClinicProvider getArgumentClinic() { + return MsvcrtModuleBuiltinsClinicProviders.GetOsfHandleNodeClinicProviderGen.INSTANCE; + } + } + + @Builtin(name = "open_osfhandle", minNumOfPositionalArgs = 2, parameterNames = {"handle", "flags"}) + @ArgumentClinic(name = "handle", conversion = ArgumentClinic.ClinicConversion.Long) + @ArgumentClinic(name = "flags", conversion = ArgumentClinic.ClinicConversion.Int) + @GenerateNodeFactory + public abstract static class OpenOsfHandleNode extends PythonBinaryClinicBuiltinNode { + @Specialization + int openOsfHandle(VirtualFrame frame, long handle, int flags, + @Bind Node inliningTarget, + @Cached SysModuleBuiltins.AuditNode auditNode, + @CachedLibrary("getPosixSupport()") PosixSupportLibrary posixLib, + @Cached PConstructAndRaiseNode.Lazy constructAndRaiseNode) { + auditNode.audit(frame, inliningTarget, T_MSVCRT_OPEN_OSFHANDLE, handle, flags); + try { + return posixLib.openOsfHandle(getPosixSupport(), handle, flags); + } catch (PosixSupportLibrary.PosixException e) { + throw constructAndRaiseNode.get(inliningTarget).raiseOSErrorFromPosixException(frame, e); + } + } + + @Override + protected ArgumentClinicProvider getArgumentClinic() { + return MsvcrtModuleBuiltinsClinicProviders.OpenOsfHandleNodeClinicProviderGen.INSTANCE; + } + } } diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/NtModuleBuiltins.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/NtModuleBuiltins.java index 6f6f924f61..237185e05a 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/NtModuleBuiltins.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/NtModuleBuiltins.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, 2025, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2022, 2026, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * The Universal Permissive License (UPL), Version 1.0 @@ -40,7 +40,6 @@ */ package com.oracle.graal.python.builtins.modules; -import static com.oracle.graal.python.nodes.BuiltinNames.T_NT; import static com.oracle.graal.python.nodes.BuiltinNames.T_POSIX; import static com.oracle.graal.python.nodes.StringLiterals.T_EMPTY_STRING; import static com.oracle.graal.python.util.PythonUtils.TS_ENCODING; @@ -78,7 +77,7 @@ import com.oracle.truffle.api.nodes.Node; import com.oracle.truffle.api.strings.TruffleString; -@CoreFunctions(defineModule = "nt", isEager = true) +@CoreFunctions(defineModule = "nt", isEager = true, os = PythonOS.PLATFORM_WIN32) public final class NtModuleBuiltins extends PythonBuiltins { @Override @@ -89,11 +88,7 @@ protected List> getNodeFa @Override public void initialize(Python3Core core) { super.initialize(core); - if (PythonLanguage.getPythonOS() == PythonOS.PLATFORM_WIN32) { - core.removeBuiltinModule(T_POSIX); - } else { - core.removeBuiltinModule(T_NT); - } + core.removeBuiltinModule(T_POSIX); } @Builtin(name = "_getfullpathname", minNumOfPositionalArgs = 1, parameterNames = {"path"}) diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/PosixModuleBuiltins.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/PosixModuleBuiltins.java index a91e20e39b..94b90a0edd 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/PosixModuleBuiltins.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/PosixModuleBuiltins.java @@ -33,11 +33,13 @@ import static com.oracle.graal.python.runtime.PosixConstants.AT_FDCWD; import static com.oracle.graal.python.runtime.PosixConstants.AT_SYMLINK_FOLLOW; import static com.oracle.graal.python.runtime.PosixConstants.O_CLOEXEC; +import static com.oracle.graal.python.lib.PyUnicodeFSDecoderNode.SURROGATE_ESCAPE_TO_UTF8_TRANSCODING_ERROR_HANDLER; import static com.oracle.graal.python.runtime.exception.PythonErrorType.NotImplementedError; import static com.oracle.graal.python.runtime.exception.PythonErrorType.OSError; import static com.oracle.graal.python.runtime.exception.PythonErrorType.OverflowError; import static com.oracle.graal.python.runtime.exception.PythonErrorType.TypeError; import static com.oracle.graal.python.runtime.exception.PythonErrorType.ValueError; +import static com.oracle.graal.python.util.PythonUtils.TS_ENCODING; import static com.oracle.graal.python.util.PythonUtils.toTruffleStringUncached; import static com.oracle.graal.python.util.PythonUtils.tsInternedLiteral; import static com.oracle.graal.python.util.PythonUtils.tsLiteral; @@ -145,6 +147,7 @@ import com.oracle.truffle.api.dsl.GenerateInline; import com.oracle.truffle.api.dsl.GenerateNodeFactory; import com.oracle.truffle.api.dsl.ImportStatic; +import com.oracle.truffle.api.dsl.Idempotent; import com.oracle.truffle.api.dsl.NeverDefault; import com.oracle.truffle.api.dsl.NodeFactory; import com.oracle.truffle.api.dsl.Specialization; @@ -350,8 +353,15 @@ public void postInitialize(Python3Core core) { // overwrite it with our executable to ensure that subprocesses will use us. TruffleString value = core.getContext().getOption(PythonOptions.Executable); try { - Object k = posixLib.createPathFromString(posixSupport, toTruffleStringUncached(pyenvLauncherKey)); - Object v = posixLib.createPathFromString(posixSupport, value); + Object k; + Object v; + if (PythonLanguage.getPythonOS() == PythonOS.PLATFORM_WIN32) { + k = posixLib.createWideStringFromString(posixSupport, toTruffleStringUncached(pyenvLauncherKey)); + v = posixLib.createWideStringFromString(posixSupport, value); + } else { + k = posixLib.createPathFromString(posixSupport, toTruffleStringUncached(pyenvLauncherKey)); + v = posixLib.createPathFromString(posixSupport, value); + } posixLib.setenv(posixSupport, k, v, true); } catch (PosixException ignored) { } @@ -414,9 +424,11 @@ protected ArgumentClinicProvider getArgumentClinic() { } @Specialization - static PNone putenv(VirtualFrame frame, PBytes nameBytes, PBytes valueBytes, + static PNone putenv(VirtualFrame frame, Object nameObject, Object valueObject, @Bind Node inliningTarget, @Cached BytesNodes.ToBytesNode toBytesNode, + @Cached TruffleString.CodePointLengthNode codePointLengthNode, + @Cached TruffleString.IndexOfCodePointNode indexOfCodePointNode, @Cached SysModuleBuiltins.AuditNode auditNode, @Bind PythonContext context, @CachedLibrary("context.getPosixSupport()") PosixSupportLibrary posixLib, @@ -425,13 +437,23 @@ static PNone putenv(VirtualFrame frame, PBytes nameBytes, PBytes valueBytes, // Unlike in other posix builtins, we go through str -> bytes -> byte[] -> String // conversions for emulated backend because the bytes version after fsencode conversion // is subject to sys.audit. - byte[] name = toBytesNode.execute(nameBytes); - byte[] value = toBytesNode.execute(valueBytes); PosixSupport posixSupport = context.getPosixSupport(); - Object nameOpaque = checkNull(inliningTarget, posixLib.createPathFromBytes(posixSupport, name), raiseNode); - Object valueOpaque = checkNull(inliningTarget, posixLib.createPathFromBytes(posixSupport, value), raiseNode); - checkEqualSign(inliningTarget, name, raiseNode); - auditNode.audit(frame, inliningTarget, T_OS_PUTENV, nameBytes, valueBytes); + Object nameOpaque; + Object valueOpaque; + if (PythonLanguage.getPythonOS() == PythonOS.PLATFORM_WIN32) { + TruffleString name = (TruffleString) nameObject; + TruffleString value = (TruffleString) valueObject; + nameOpaque = checkNull(inliningTarget, posixLib.createWideStringFromString(posixSupport, name), raiseNode); + valueOpaque = checkNull(inliningTarget, posixLib.createWideStringFromString(posixSupport, value), raiseNode); + checkWindowsEnvName(inliningTarget, name, codePointLengthNode, indexOfCodePointNode, raiseNode); + } else { + byte[] name = toBytesNode.execute((PBytes) nameObject); + byte[] value = toBytesNode.execute((PBytes) valueObject); + nameOpaque = checkNull(inliningTarget, posixLib.createCStringFromBytes(posixSupport, name), raiseNode); + valueOpaque = checkNull(inliningTarget, posixLib.createCStringFromBytes(posixSupport, value), raiseNode); + checkEqualSign(inliningTarget, name, raiseNode); + } + auditNode.audit(frame, inliningTarget, T_OS_PUTENV, nameObject, valueObject); try { posixLib.setenv(posixSupport, nameOpaque, valueOpaque, true); } catch (PosixException e) { @@ -454,6 +476,17 @@ private static void checkEqualSign(Node inliningTarget, byte[] bytes, PRaiseNode } } } + + private static void checkWindowsEnvName(Node inliningTarget, TruffleString name, TruffleString.CodePointLengthNode codePointLengthNode, + TruffleString.IndexOfCodePointNode indexOfCodePointNode, PRaiseNode raiseNode) { + int length = codePointLengthNode.execute(name, TS_ENCODING); + // Match CPython's Windows-specific name validation in posixmodule.c:win32_putenv: + // the name must not be empty, but an initial '=' is allowed for hidden drive-current + // directory environment variables such as "=C:". + if (length == 0 || (length > 1 && indexOfCodePointNode.execute(name, '=', 1, length, TS_ENCODING) >= 0)) { + throw raiseNode.raise(inliningTarget, ValueError, ErrorMessages.ILLEGAL_ENVIRONMENT_VARIABLE_NAME); + } + } } @Builtin(name = "unsetenv", minNumOfPositionalArgs = 1, parameterNames = {"name"}) @@ -467,17 +500,26 @@ protected ArgumentClinicProvider getArgumentClinic() { } @Specialization - static PNone putenv(VirtualFrame frame, PBytes nameBytes, + static PNone putenv(VirtualFrame frame, Object nameObject, @Bind Node inliningTarget, @Bind PythonContext context, @Cached BytesNodes.ToBytesNode toBytesNode, + @Cached TruffleString.CodePointLengthNode codePointLengthNode, + @Cached TruffleString.IndexOfCodePointNode indexOfCodePointNode, @Cached SysModuleBuiltins.AuditNode auditNode, @CachedLibrary("context.getPosixSupport()") PosixSupportLibrary posixLib, @Cached PConstructAndRaiseNode.Lazy constructAndRaiseNode, @Cached PRaiseNode raiseNode) { - byte[] name = toBytesNode.execute(nameBytes); - Object nameOpaque = checkNull(inliningTarget, posixLib.createPathFromBytes(context.getPosixSupport(), name), raiseNode); - auditNode.audit(frame, inliningTarget, T_OS_UNSETENV, nameBytes); + Object nameOpaque; + if (PythonLanguage.getPythonOS() == PythonOS.PLATFORM_WIN32) { + TruffleString name = (TruffleString) nameObject; + nameOpaque = checkNull(inliningTarget, posixLib.createWideStringFromString(context.getPosixSupport(), name), raiseNode); + PutenvNode.checkWindowsEnvName(inliningTarget, name, codePointLengthNode, indexOfCodePointNode, raiseNode); + } else { + byte[] name = toBytesNode.execute((PBytes) nameObject); + nameOpaque = checkNull(inliningTarget, posixLib.createCStringFromBytes(context.getPosixSupport(), name), raiseNode); + } + auditNode.audit(frame, inliningTarget, T_OS_UNSETENV, nameObject); try { posixLib.unsetenv(context.getPosixSupport(), nameOpaque); } catch (PosixException e) { @@ -534,7 +576,9 @@ static Object execvArgs(VirtualFrame frame, PosixPath path, Object argv, } Object[] opaqueArgs = new Object[args.length]; for (int i = 0; i < args.length; ++i) { - opaqueArgs[i] = toOpaquePathNode.execute(frame, inliningTarget, args[i], i == 0); + Object opaquePath = toOpaquePathNode.execute(frame, inliningTarget, args[i], i == 0); + Buffer bytes = posixLib.getPathAsBytes(context.getPosixSupport(), opaquePath); + opaqueArgs[i] = posixLib.createCStringFromBytes(context.getPosixSupport(), bytes.data); } // TODO ValueError "execv() arg 2 first element cannot be empty" @@ -2280,15 +2324,28 @@ static PNone rename(VirtualFrame frame, PosixPath src, PosixPath dst, int srcDir @ArgumentClinic(name = "src_dir_fd", conversionClass = DirFdConversionNode.class) @ArgumentClinic(name = "dst_dir_fd", conversionClass = DirFdConversionNode.class) @GenerateNodeFactory - abstract static class ReplaceNode extends RenameNode { - - // although this built-in is the same as rename(), we need to provide our own - // ArgumentClinicProvider because the error messages contain the name of the built-in + abstract static class ReplaceNode extends PythonClinicBuiltinNode { @Override protected ArgumentClinicProvider getArgumentClinic() { return PosixModuleBuiltinsClinicProviders.ReplaceNodeClinicProviderGen.INSTANCE; } + + @Specialization + static PNone replace(VirtualFrame frame, PosixPath src, PosixPath dst, int srcDirFd, int dstDirFd, + @Bind PythonContext context, + @CachedLibrary("context.getPosixSupport()") PosixSupportLibrary posixLib, + @Bind Node inliningTarget, + @Cached SysModuleBuiltins.AuditNode auditNode, + @Cached PConstructAndRaiseNode.Lazy constructAndRaiseNode) { + auditNode.audit(frame, inliningTarget, T_OS_RENAME, src.originalObject, dst.originalObject, dirFdForAudit(srcDirFd), dirFdForAudit(dstDirFd)); + try { + posixLib.replaceat(context.getPosixSupport(), srcDirFd, src.value, dstDirFd, dst.value); + } catch (PosixException e) { + throw constructAndRaiseNode.get(inliningTarget).raiseOSErrorFromPosixException(frame, e, src.originalObject, dst.originalObject); + } + return PNone.NONE; + } } @Builtin(name = "access", minNumOfPositionalArgs = 2, parameterNames = {"path", "mode"}, keywordOnlyNames = {"dir_fd", "effective_ids", "follow_symlinks"}) @@ -2852,7 +2909,7 @@ protected ArgumentClinicProvider getArgumentClinic() { } @Specialization - static int system(VirtualFrame frame, PBytes command, + static int system(VirtualFrame frame, Object command, @Bind Node inliningTarget, @Cached BytesNodes.ToBytesNode toBytesNode, @Cached SysModuleBuiltins.AuditNode auditNode, @@ -2863,10 +2920,14 @@ static int system(VirtualFrame frame, PBytes command, // conversions for emulated backend because the bytes version after fsencode conversion // is subject to sys.audit. auditNode.audit(frame, inliningTarget, T_OS_SYSTEM, command); - byte[] bytes = toBytesNode.execute(command); gil.release(true); try { - Object cmdOpaque = posixLib.createPathFromBytes(context.getPosixSupport(), bytes); + Object cmdOpaque; + if (PythonLanguage.getPythonOS() == PythonOS.PLATFORM_WIN32) { + cmdOpaque = posixLib.createWideStringFromString(context.getPosixSupport(), (TruffleString) command); + } else { + cmdOpaque = posixLib.createCStringFromBytes(context.getPosixSupport(), toBytesNode.execute((PBytes) command)); + } return posixLib.system(context.getPosixSupport(), cmdOpaque); } finally { gil.acquire(); @@ -3091,7 +3152,7 @@ static PBytes doString(Node inliningTarget, Object strObj, @Cached TruffleString.SwitchEncodingNode switchEncodingNode, @Cached TruffleString.CopyToByteArrayNode copyToByteArrayNode) { TruffleString str = castToStringNode.execute(inliningTarget, strObj); - TruffleString utf8 = switchEncodingNode.execute(str, Encoding.UTF_8); + TruffleString utf8 = switchEncodingNode.execute(str, Encoding.UTF_8, SURROGATE_ESCAPE_TO_UTF8_TRANSCODING_ERROR_HANDLER); byte[] bytes = new byte[utf8.byteLength(Encoding.UTF_8)]; copyToByteArrayNode.execute(utf8, 0, bytes, 0, bytes.length, Encoding.UTF_8); return PFactory.createBytes(language, bytes); @@ -3326,14 +3387,33 @@ public static PBytes opaquePathToBytes(Object opaquePath, PosixSupportLibrary po // Converters public abstract static class FsConverterNode extends ArgumentCastNode { - @Specialization - static PBytes convert(VirtualFrame frame, Object value, + @Specialization(guards = "!isWindows()") + static PBytes convertPosix(VirtualFrame frame, Object value, @Bind Node inliningTarget, @Cached PyOSFSPathNode fspathNode, @Cached StringOrBytesToBytesNode stringOrBytesToBytesNode) { return stringOrBytesToBytesNode.execute(inliningTarget, fspathNode.execute(frame, inliningTarget, value)); } + @Specialization(guards = {"isWindows()", "isString(value)"}) + static TruffleString convertWindows(Object value, + @Bind Node inliningTarget, + @Cached CastToTruffleStringNode castToStringNode) { + return castToStringNode.execute(inliningTarget, value); + } + + @Specialization(guards = {"isWindows()", "!isString(value)"}) + static Object rejectWindows(Object value, + @Bind Node inliningTarget, + @Cached PRaiseNode raiseNode) { + throw raiseNode.raise(inliningTarget, TypeError, ErrorMessages.MUST_BE_STR_NOT_P, value); + } + + @Idempotent + protected static boolean isWindows() { + return PythonLanguage.getPythonOS() == PythonOS.PLATFORM_WIN32; + } + @ClinicConverterFactory @NeverDefault public static FsConverterNode create() { diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/PosixShMemModuleBuiltins.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/PosixShMemModuleBuiltins.java index fcb3529cc1..e5e4fae2e9 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/PosixShMemModuleBuiltins.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/PosixShMemModuleBuiltins.java @@ -47,8 +47,6 @@ import com.oracle.graal.python.annotations.Builtin; import com.oracle.graal.python.builtins.CoreFunctions; import com.oracle.graal.python.builtins.PythonBuiltins; -import com.oracle.graal.python.builtins.modules.PosixModuleBuiltins.PathConversionNode; -import com.oracle.graal.python.builtins.modules.PosixModuleBuiltins.PosixPath; import com.oracle.graal.python.builtins.objects.PNone; import com.oracle.graal.python.nodes.PConstructAndRaiseNode; import com.oracle.graal.python.nodes.function.PythonBuiltinBaseNode; @@ -66,6 +64,7 @@ import com.oracle.truffle.api.frame.VirtualFrame; import com.oracle.truffle.api.library.CachedLibrary; import com.oracle.truffle.api.nodes.Node; +import com.oracle.truffle.api.strings.TruffleString; @CoreFunctions(defineModule = "_posixshmem") public final class PosixShMemModuleBuiltins extends PythonBuiltins { @@ -76,7 +75,7 @@ protected List> getNodeFa } @Builtin(name = "shm_open", minNumOfPositionalArgs = 2, parameterNames = {"path", "flags", "mode"}) - @ArgumentClinic(name = "path", conversionClass = PathConversionNode.class, args = {"false", "false"}) + @ArgumentClinic(name = "path", conversion = ClinicConversion.TString) @ArgumentClinic(name = "flags", conversion = ClinicConversion.Int) @ArgumentClinic(name = "mode", conversion = ClinicConversion.Int, defaultValue = "0777") @GenerateNodeFactory @@ -88,21 +87,22 @@ protected ArgumentClinicProvider getArgumentClinic() { } @Specialization - static int shmOpen(VirtualFrame frame, PosixPath path, int flags, int mode, + static int shmOpen(VirtualFrame frame, TruffleString path, int flags, int mode, @Bind Node inliningTarget, @Bind PythonContext context, @CachedLibrary("context.getPosixSupport()") PosixSupportLibrary posixLib, @Cached PConstructAndRaiseNode.Lazy constructAndRaiseNode) { try { - return posixLib.shmOpen(context.getPosixSupport(), path.value, flags, mode); + Object name = posixLib.createCStringFromString(context.getPosixSupport(), path); + return posixLib.shmOpen(context.getPosixSupport(), name, flags, mode); } catch (PosixException e) { - throw constructAndRaiseNode.get(inliningTarget).raiseOSErrorFromPosixException(frame, e, path.originalObject); + throw constructAndRaiseNode.get(inliningTarget).raiseOSErrorFromPosixException(frame, e, path); } } } @Builtin(name = "shm_unlink", minNumOfPositionalArgs = 1, parameterNames = {"path"}) - @ArgumentClinic(name = "path", conversionClass = PathConversionNode.class, args = {"false", "false"}) + @ArgumentClinic(name = "path", conversion = ClinicConversion.TString) @GenerateNodeFactory public abstract static class ShmUnlinkNode extends PythonUnaryClinicBuiltinNode { @@ -112,15 +112,16 @@ protected ArgumentClinicProvider getArgumentClinic() { } @Specialization - static PNone shmUnlink(VirtualFrame frame, PosixPath path, + static PNone shmUnlink(VirtualFrame frame, TruffleString path, @Bind Node inliningTarget, @Bind PythonContext context, @CachedLibrary("context.getPosixSupport()") PosixSupportLibrary posixLib, @Cached PConstructAndRaiseNode.Lazy constructAndRaiseNode) { try { - posixLib.shmUnlink(context.getPosixSupport(), path.value); + Object name = posixLib.createCStringFromString(context.getPosixSupport(), path); + posixLib.shmUnlink(context.getPosixSupport(), name); } catch (PosixException e) { - throw constructAndRaiseNode.get(inliningTarget).raiseOSErrorFromPosixException(frame, e, path.originalObject); + throw constructAndRaiseNode.get(inliningTarget).raiseOSErrorFromPosixException(frame, e, path); } return PNone.NONE; } diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/PosixSubprocessModuleBuiltins.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/PosixSubprocessModuleBuiltins.java index 519654fa9d..22b17b9833 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/PosixSubprocessModuleBuiltins.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/PosixSubprocessModuleBuiltins.java @@ -82,6 +82,7 @@ import com.oracle.graal.python.runtime.GilNode; import com.oracle.graal.python.runtime.PosixSupport; import com.oracle.graal.python.runtime.PosixSupportLibrary; +import com.oracle.graal.python.runtime.PosixSupportLibrary.Buffer; import com.oracle.graal.python.runtime.PosixSupportLibrary.PosixException; import com.oracle.graal.python.runtime.PythonContext; import com.oracle.graal.python.runtime.PythonOptions; @@ -109,7 +110,7 @@ protected List> getNodeFa /** * Helper converter which iterates the argv argument and converts each element to the opaque - * path representation used by {@link PosixSupportLibrary}. + * narrow string representation used by {@link PosixSupportLibrary}. */ abstract static class ProcessArgsConversionNode extends ArgumentCastNode { @Specialization @@ -122,10 +123,12 @@ static Object[] doNone(@SuppressWarnings("unused") PNone processArgs) { @Specialization static Object[] doSequence(VirtualFrame frame, Object processArgs, @Bind Node inliningTarget, + @Bind PythonContext context, @Cached FastConstructListNode fastConstructListNode, @Cached GetSequenceStorageNode getSequenceStorageNode, @Cached IsBuiltinObjectProfile isBuiltinClassProfile, @Cached ObjectToOpaquePathNode objectToOpaquePathNode, + @CachedLibrary("context.getPosixSupport()") PosixSupportLibrary posixLib, @Cached("createNotNormalized()") GetItemNode getItemNode, @Cached PRaiseNode raiseNode) { PList argsList; @@ -146,7 +149,9 @@ static Object[] doSequence(VirtualFrame frame, Object processArgs, throw raiseNode.raise(inliningTarget, RuntimeError, ErrorMessages.ARGS_CHANGED_DURING_ITERATION); } Object o = getItemNode.execute(argsStorage, i); - argsArray[i] = objectToOpaquePathNode.execute(frame, inliningTarget, o, false); + Object path = objectToOpaquePathNode.execute(frame, inliningTarget, o, false); + Buffer bytes = posixLib.getPathAsBytes(context.getPosixSupport(), path); + argsArray[i] = posixLib.createCStringFromBytes(context.getPosixSupport(), bytes.data); } return argsArray; } @@ -179,7 +184,7 @@ static Object doSequence(VirtualFrame frame, Object env, for (int i = 0; i < length; ++i) { Object o = getItem.execute(frame, inliningTarget, env, i); byte[] bytes = toBytesNode.execute(frame, o); - Object o1 = posixLib.createPathFromBytes(context.getPosixSupport(), bytes); + Object o1 = posixLib.createCStringFromBytes(context.getPosixSupport(), bytes); if (o1 == null) { throw raiseNode.raise(inliningTarget, ValueError, ErrorMessages.EMBEDDED_NULL_BYTE); } @@ -231,10 +236,10 @@ private static byte[] fsEncode(String s) { return s.getBytes(); } - private static Object createPathFromBytes(Node inliningTarget, byte[] bytes, PosixSupportLibrary posixLib, PRaiseNode raiseNode) { - Object o = posixLib.createPathFromBytes(PosixSupport.get(inliningTarget), bytes); + private static Object createCStringFromBytes(Node inliningTarget, byte[] bytes, PosixSupportLibrary posixLib, PRaiseNode raiseNode) { + Object o = posixLib.createCStringFromBytes(PosixSupport.get(inliningTarget), bytes); if (o == null) { - // TODO reconsider the contract of PosixSupportLibrary#createPathFromBytes w.r.t. + // TODO reconsider the contract of PosixSupportLibrary#createCStringFromBytes w.r.t. // embedded null checks (we need to review that anyway since PosixSupportLibrary // cannot do Python-specific fsencode) throw raiseNode.raise(inliningTarget, ValueError, ErrorMessages.EMBEDDED_NULL_BYTE); @@ -291,7 +296,7 @@ static int forkExec(VirtualFrame frame, Object[] args, Object executableList, bo } Object[] extendedArgs = new Object[additionalArgs.length + (processArgs.length == 0 ? 0 : processArgs.length - 1)]; for (int j = 0; j < additionalArgs.length; ++j) { - extendedArgs[j] = createPathFromBytes(inliningTarget, fsEncode(additionalArgs[j].toJavaStringUncached()), posixLib, raiseNode); + extendedArgs[j] = createCStringFromBytes(inliningTarget, fsEncode(additionalArgs[j].toJavaStringUncached()), posixLib, raiseNode); } if (processArgs.length > 1) { PythonUtils.arraycopy(processArgs, 1, extendedArgs, additionalArgs.length, processArgs.length - 1); @@ -299,7 +304,7 @@ static int forkExec(VirtualFrame frame, Object[] args, Object executableList, bo processArgs = extendedArgs; executables[i] = extendedArgs[0]; } else { - executables[i] = createPathFromBytes(inliningTarget, bytes, posixLib, raiseNode); + executables[i] = createCStringFromBytes(inliningTarget, bytes, posixLib, raiseNode); } } diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/PwdModuleBuiltins.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/PwdModuleBuiltins.java index b1f94c8e3e..b05f6a58d5 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/PwdModuleBuiltins.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/PwdModuleBuiltins.java @@ -72,6 +72,7 @@ import com.oracle.graal.python.nodes.object.BuiltinClassProfiles.IsBuiltinObjectProfile; import com.oracle.graal.python.runtime.GilNode; import com.oracle.graal.python.runtime.PosixSupportLibrary; +import com.oracle.graal.python.runtime.PosixSupportLibrary.Buffer; import com.oracle.graal.python.runtime.PosixSupportLibrary.PosixException; import com.oracle.graal.python.runtime.PosixSupportLibrary.PwdResult; import com.oracle.graal.python.runtime.PythonContext; @@ -202,7 +203,9 @@ static Object doGetpwname(VirtualFrame frame, TruffleString name, @Cached PRaiseNode raiseNode) { // Note: CPython also takes only Strings, not bytes, and then encodes the String // StringOrBytesToOpaquePathNode already checks for embedded '\0' - Object nameEncoded = encodeFSDefault.execute(inliningTarget, name); + Object pathEncoded = encodeFSDefault.execute(inliningTarget, name); + Buffer nameBytes = posixLib.getPathAsBytes(context.getPosixSupport(), pathEncoded); + Object nameEncoded = posixLib.createCStringFromBytes(context.getPosixSupport(), nameBytes.data); PwdResult pwd; try { gil.release(true); diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/SignalModuleBuiltins.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/SignalModuleBuiltins.java index 8b5690aa9b..270af53e82 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/SignalModuleBuiltins.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/SignalModuleBuiltins.java @@ -56,6 +56,7 @@ import com.oracle.graal.python.PythonLanguage; import com.oracle.graal.python.annotations.ArgumentClinic; import com.oracle.graal.python.annotations.Builtin; +import com.oracle.graal.python.annotations.PythonOS; import com.oracle.graal.python.builtins.CoreFunctions; import com.oracle.graal.python.builtins.Python3Core; import com.oracle.graal.python.builtins.PythonBuiltinClassType; @@ -129,6 +130,10 @@ public void initialize(Python3Core core) { addBuiltinConstant("ITIMER_VIRTUAL", ITIMER_VIRTUAL); addBuiltinConstant("ITIMER_PROF", ITIMER_PROF); addBuiltinConstant("NSIG", Signals.SIGMAX + 1); + if (PythonLanguage.getPythonOS() == PythonOS.PLATFORM_WIN32) { + addBuiltinConstant("CTRL_C_EVENT", 0); + addBuiltinConstant("CTRL_BREAK_EVENT", 1); + } } /* diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/SocketModuleBuiltins.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/SocketModuleBuiltins.java index 765a7ab0e2..e838baaaf2 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/SocketModuleBuiltins.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/SocketModuleBuiltins.java @@ -70,6 +70,7 @@ import com.oracle.graal.python.PythonLanguage; import com.oracle.graal.python.annotations.ArgumentClinic; import com.oracle.graal.python.annotations.Builtin; +import com.oracle.graal.python.annotations.PythonOS; import com.oracle.graal.python.builtins.CoreFunctions; import com.oracle.graal.python.builtins.Python3Core; import com.oracle.graal.python.builtins.PythonBuiltinClassType; @@ -198,7 +199,8 @@ public void initialize(Python3Core core) { public void postInitialize(Python3Core core) { PythonModule module = core.lookupBuiltinModule(T__SOCKET); module.setModuleState(-1L); - if (PosixSupportLibrary.getUncached().getBackend(core.getContext().getPosixSupport()).toJavaStringUncached().equals("java")) { + if (PythonLanguage.getPythonOS() == PythonOS.PLATFORM_WIN32 || + PosixSupportLibrary.getUncached().getBackend(core.getContext().getPosixSupport()).toJavaStringUncached().equals("java")) { module.setAttribute(toTruffleStringUncached(PosixConstants.AF_UNIX.name), PNone.NO_VALUE); } } @@ -253,7 +255,7 @@ static TruffleString doGeneric(VirtualFrame frame, try { gil.release(true); try { - return posixLib.getPathAsString(context.getPosixSupport(), posixLib.gethostname(context.getPosixSupport())); + return posixLib.getCStringAsString(context.getPosixSupport(), posixLib.gethostname(context.getPosixSupport())); } finally { gil.acquire(); } @@ -289,7 +291,7 @@ static Object doGeneric(VirtualFrame frame, Object ip, int family = sockAddrLibrary.getFamily(addr); try { Object[] getnameinfoResult = posixLib.getnameinfo(context.getPosixSupport(), addr, NI_NAMEREQD.value); - TruffleString hostname = posixLib.getPathAsString(context.getPosixSupport(), getnameinfoResult[0]); + TruffleString hostname = posixLib.getCStringAsString(context.getPosixSupport(), getnameinfoResult[0]); SequenceStorage storage = new ObjectSequenceStorage(5); @@ -297,7 +299,7 @@ static Object doGeneric(VirtualFrame frame, Object ip, AddrInfoCursor cursor; gil.release(true); try { - cursor = posixLib.getaddrinfo(context.getPosixSupport(), getnameinfoResult[0], posixLib.createPathFromString(context.getPosixSupport(), T_ZERO), + cursor = posixLib.getaddrinfo(context.getPosixSupport(), getnameinfoResult[0], posixLib.createCStringFromString(context.getPosixSupport(), T_ZERO), family, 0, 0, 0); } finally { gil.acquire(); @@ -347,7 +349,7 @@ static TruffleString getHostByName(VirtualFrame frame, Object nameObj, UniversalSockAddr addr = setIpAddrNode.execute(frame, name, AF_INET.value); Inet4SockAddr inet4SockAddr = addrLib.asInet4SockAddr(addr); try { - return posixLib.getPathAsString(context.getPosixSupport(), posixLib.inet_ntop(context.getPosixSupport(), AF_INET.value, inet4SockAddr.getAddressAsBytes())); + return posixLib.getCStringAsString(context.getPosixSupport(), posixLib.inet_ntop(context.getPosixSupport(), AF_INET.value, inet4SockAddr.getAddressAsBytes())); } catch (PosixException e) { throw constructAndRaiseNode.get(inliningTarget).raiseOSErrorFromPosixException(frame, e); } @@ -382,12 +384,12 @@ static Object get(VirtualFrame frame, Object nameObj, */ try { PosixSupport posixSupport = context.getPosixSupport(); - AddrInfoCursor cursor = posixLib.getaddrinfo(posixSupport, posixLib.createPathFromBytes(posixSupport, name), + AddrInfoCursor cursor = posixLib.getaddrinfo(posixSupport, posixLib.createCStringFromBytes(posixSupport, name), null, AF_INET.value, 0, 0, AI_CANONNAME.value); try { - TruffleString canonName = posixLib.getPathAsString(posixSupport, addrInfoCursorLib.getCanonName(cursor)); + TruffleString canonName = posixLib.getCStringAsString(posixSupport, addrInfoCursorLib.getCanonName(cursor)); Inet4SockAddr inet4SockAddr = addrLib.asInet4SockAddr(addrInfoCursorLib.getSockAddr(cursor)); - TruffleString addr = posixLib.getPathAsString(posixSupport, posixLib.inet_ntop(posixSupport, AF_INET.value, inet4SockAddr.getAddressAsBytes())); + TruffleString addr = posixLib.getCStringAsString(posixSupport, posixLib.inet_ntop(posixSupport, AF_INET.value, inet4SockAddr.getAddressAsBytes())); // getaddrinfo doesn't support aliases PList aliases = PFactory.createList(context.getLanguage(inliningTarget)); // we support just one address for now @@ -451,7 +453,7 @@ static Object getServByName(VirtualFrame frame, TruffleString serviceName, Objec AddrInfoCursor cursor; try { PosixSupport posixSupport = context.getPosixSupport(); - cursor = posixLib.getaddrinfo(posixSupport, null, posixLib.createPathFromString(posixSupport, serviceName), AF_INET.value, 0, protocol, 0); + cursor = posixLib.getaddrinfo(posixSupport, null, posixLib.createCStringFromString(posixSupport, serviceName), AF_INET.value, 0, protocol, 0); } finally { gil.acquire(); } @@ -518,7 +520,7 @@ Object getServByPort(VirtualFrame frame, int port, Object protocolNameObj, flags |= NI_DGRAM.value; } Object[] result = posixLib.getnameinfo(getPosixSupport(), addr, flags); - TruffleString name = posixLib.getPathAsString(getPosixSupport(), result[1]); + TruffleString name = posixLib.getCStringAsString(getPosixSupport(), result[1]); checkName(name); return name; } finally { @@ -599,8 +601,8 @@ static Object getNameInfo(VirtualFrame frame, Object sockaddr, int flags, gil.release(true); PosixSupport posixSupport = context.getPosixSupport(); try { - AddrInfoCursor cursor = posixLib.getaddrinfo(posixSupport, posixLib.createPathFromString(posixSupport, address), - posixLib.createPathFromString(posixSupport, fromLongNode.execute(port, TS_ENCODING, false)), + AddrInfoCursor cursor = posixLib.getaddrinfo(posixSupport, posixLib.createCStringFromString(posixSupport, address), + posixLib.createCStringFromString(posixSupport, fromLongNode.execute(port, TS_ENCODING, false)), AF_UNSPEC.value, SOCK_DGRAM.value, 0, AI_NUMERICHOST.value); try { family = addrInfoCursorLib.getFamily(cursor); @@ -628,8 +630,8 @@ static Object getNameInfo(VirtualFrame frame, Object sockaddr, int flags, } Object[] getnameinfo = posixLib.getnameinfo(posixSupport, queryAddr, flags); - TruffleString host = posixLib.getPathAsString(posixSupport, getnameinfo[0]); - TruffleString service = posixLib.getPathAsString(posixSupport, getnameinfo[1]); + TruffleString host = posixLib.getCStringAsString(posixSupport, getnameinfo[0]); + TruffleString service = posixLib.getCStringAsString(posixSupport, getnameinfo[1]); return PFactory.createTuple(context.getLanguage(inliningTarget), new Object[]{host, service}); } catch (GetAddrInfoException e) { throw constructAndRaiseNode.get(inliningTarget).executeWithArgsOnly(frame, SocketGAIError, new Object[]{e.getErrorCode(), e.getMessageAsTruffleString()}); @@ -679,17 +681,17 @@ static Object getAddrInfo(VirtualFrame frame, Object hostObject, Object portObje Object host = null; PosixSupport posixSupport = context.getPosixSupport(); if (hostObject != PNone.NONE) { - host = posixLib.createPathFromBytes(posixSupport, idna.execute(frame, hostObject)); + host = posixLib.createCStringFromBytes(posixSupport, idna.execute(frame, hostObject)); } Object port; Object portObjectProfiled = profile.profile(inliningTarget, portObject); if (PGuards.canBeInteger(portObjectProfiled)) { - port = posixLib.createPathFromString(posixSupport, fromLongNode.execute(asLongNode.execute(frame, inliningTarget, portObjectProfiled), TS_ENCODING, false)); + port = posixLib.createCStringFromString(posixSupport, fromLongNode.execute(asLongNode.execute(frame, inliningTarget, portObjectProfiled), TS_ENCODING, false)); } else if (PGuards.isString(portObjectProfiled)) { - port = posixLib.createPathFromString(posixSupport, castToString.execute(inliningTarget, portObjectProfiled)); + port = posixLib.createCStringFromString(posixSupport, castToString.execute(inliningTarget, portObjectProfiled)); } else if (PGuards.isBytes(portObjectProfiled)) { - port = posixLib.createPathFromBytes(posixSupport, toBytes.execute(frame, portObjectProfiled)); + port = posixLib.createCStringFromBytes(posixSupport, toBytes.execute(frame, portObjectProfiled)); } else if (portObject == PNone.NONE) { port = null; } else { @@ -718,7 +720,7 @@ static Object getAddrInfo(VirtualFrame frame, Object hostObject, Object portObje Object addr = makeSockAddrNode.execute(frame, inliningTarget, cursorLib.getSockAddr(cursor)); TruffleString canonName = T_EMPTY_STRING; if (cursorLib.getCanonName(cursor) != null) { - canonName = posixLib.getPathAsString(posixSupport, cursorLib.getCanonName(cursor)); + canonName = posixLib.getCStringAsString(posixSupport, cursorLib.getCanonName(cursor)); } PTuple tuple = PFactory.createTuple(context.getLanguage(inliningTarget), new Object[]{cursorLib.getFamily(cursor), cursorLib.getSockType(cursor), cursorLib.getProtocol(cursor), canonName, addr}); @@ -817,7 +819,7 @@ static PBytes doConvert(TruffleString addr, @Cached PRaiseNode raiseNode) { try { PosixSupport posixSupport = context.getPosixSupport(); - int converted = posixLib.inet_aton(posixSupport, posixLib.createPathFromString(posixSupport, addr)); + int converted = posixLib.inet_aton(posixSupport, posixLib.createCStringFromString(posixSupport, addr)); byte[] bytes = new byte[4]; ByteArraySupport.bigEndian().putInt(bytes, 0, converted); return PFactory.createBytes(context.getLanguage(inliningTarget), bytes); @@ -853,7 +855,7 @@ static TruffleString doGeneric(VirtualFrame frame, Object addr, } PosixSupport posixSupport = context.getPosixSupport(); Object result = posixLib.inet_ntoa(posixSupport, ByteArraySupport.bigEndian().getInt(bytes, 0)); - return posixLib.getPathAsString(posixSupport, result); + return posixLib.getCStringAsString(posixSupport, result); } finally { bufferLib.release(buffer, frame, callData); } @@ -874,7 +876,7 @@ static PBytes doConvert(VirtualFrame frame, int family, TruffleString addr, @Cached PRaiseNode raiseNode) { try { PosixSupport posixSupport = context.getPosixSupport(); - byte[] bytes = posixLib.inet_pton(posixSupport, family, posixLib.createPathFromString(posixSupport, addr)); + byte[] bytes = posixLib.inet_pton(posixSupport, family, posixLib.createCStringFromString(posixSupport, addr)); return PFactory.createBytes(context.getLanguage(inliningTarget), bytes); } catch (PosixException e) { throw constructAndRaiseNode.get(inliningTarget).raiseOSErrorFromPosixException(frame, e); @@ -921,7 +923,7 @@ static TruffleString doGeneric(VirtualFrame frame, int family, Object obj, try { PosixSupport posixSupport = context.getPosixSupport(); Object result = posixLib.inet_ntop(posixSupport, family, bytes); - return posixLib.getPathAsString(posixSupport, result); + return posixLib.getCStringAsString(posixSupport, result); } catch (PosixException e) { throw constructAndRaiseNode.get(inliningTarget).raiseOSErrorFromPosixException(frame, e); } diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/SysModuleBuiltins.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/SysModuleBuiltins.java index 64ddbec4c2..4eb2280f10 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/SysModuleBuiltins.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/SysModuleBuiltins.java @@ -778,17 +778,17 @@ static void initStd(Python3Core core) { FileIOBuiltins.FileIOInit.internalInit(stdinFileIO, toTruffleStringUncached(""), 0, IOMode.RB); PBuffered stdinBuffer = PFactory.createBufferedReader(language); BufferedReaderBuiltins.BufferedReaderInit.internalInit(stdinBuffer, stdinFileIO, BufferedReaderBuiltins.DEFAULT_BUFFER_SIZE, language, posixSupport, posixLib); - setWrapper(T_STDIN, T___STDIN__, T_R, stdioEncoding, stdioError, stdinBuffer, sysModule, language, true); + setWrapper(T_STDIN, T___STDIN__, T_R, stdioEncoding, stdioError, PNone.NONE, stdinBuffer, sysModule, language, true); PFileIO stdoutFileIO = PFactory.createFileIO(language); FileIOBuiltins.FileIOInit.internalInit(stdoutFileIO, toTruffleStringUncached(""), 1, IOMode.WB); Object stdoutBuffer = createBufferedIO(buffering, language, stdoutFileIO, posixSupport, posixLib); - setWrapper(T_STDOUT, T___STDOUT__, T_W, stdioEncoding, stdioError, stdoutBuffer, sysModule, language, buffering); + setWrapper(T_STDOUT, T___STDOUT__, T_W, stdioEncoding, stdioError, PNone.NONE, stdoutBuffer, sysModule, language, buffering); PFileIO stderr = PFactory.createFileIO(language); FileIOBuiltins.FileIOInit.internalInit(stderr, toTruffleStringUncached(""), 2, IOMode.WB); Object stderrBuffer = createBufferedIO(buffering, language, stderr, posixSupport, posixLib); - setWrapper(T_STDERR, T___STDERR__, T_W, stdioEncoding, T_BACKSLASHREPLACE, stderrBuffer, sysModule, language, buffering); + setWrapper(T_STDERR, T___STDERR__, T_W, stdioEncoding, T_BACKSLASHREPLACE, PNone.NONE, stderrBuffer, sysModule, language, buffering); } private static Object createBufferedIO(boolean buffering, PythonLanguage language, PFileIO fileIo, Object posixSupport, PosixSupportLibrary posixLib) { @@ -800,10 +800,10 @@ private static Object createBufferedIO(boolean buffering, PythonLanguage languag return writer; } - private static PTextIO setWrapper(TruffleString name, TruffleString specialName, TruffleString mode, TruffleString encoding, TruffleString error, Object buffer, PythonModule sysModule, - PythonLanguage language, boolean buffering) { + private static PTextIO setWrapper(TruffleString name, TruffleString specialName, TruffleString mode, TruffleString encoding, TruffleString error, Object newline, Object buffer, + PythonModule sysModule, PythonLanguage language, boolean buffering) { PTextIO textIOWrapper = PFactory.createTextIO(language); - TextIOWrapperInitNodeGen.getUncached().execute(null, null, textIOWrapper, buffer, encoding, error, PNone.NONE, + TextIOWrapperInitNodeGen.getUncached().execute(null, null, textIOWrapper, buffer, encoding, error, newline, /* line_buffering */ buffering, /* write_through */ !buffering); setAttribute(textIOWrapper, T_MODE, mode); diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/WinapiModuleBuiltins.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/WinapiModuleBuiltins.java index 21b03b1f6b..f0f7188a5d 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/WinapiModuleBuiltins.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/WinapiModuleBuiltins.java @@ -92,12 +92,17 @@ int getacp(@Bind PythonContext context) { } } + /* + * Managed fallback for contexts without native access. With native access, Python3Core loads + * modules/_winapi.py into this module and its CloseHandle definition replaces this builtin. + */ @Builtin(name = "CloseHandle", minNumOfPositionalArgs = 1) @GenerateNodeFactory abstract static class CloseHandleNode extends PythonBuiltinNode { @Specialization - static Object closeHandle(@SuppressWarnings("unused") Object handle) { + static PNone closeHandle(@SuppressWarnings("unused") Object handle) { return PNone.NONE; } } + } diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/multiprocessing/MultiprocessingModuleBuiltins.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/multiprocessing/MultiprocessingModuleBuiltins.java index a3eced324a..193609ad2b 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/multiprocessing/MultiprocessingModuleBuiltins.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/multiprocessing/MultiprocessingModuleBuiltins.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, 2025, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2023, 2026, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * The Universal Permissive License (UPL), Version 1.0 @@ -42,16 +42,27 @@ import java.util.List; +import com.oracle.graal.python.PythonLanguage; +import com.oracle.graal.python.annotations.ArgumentClinic; import com.oracle.graal.python.annotations.Builtin; +import com.oracle.graal.python.annotations.PythonOS; import com.oracle.graal.python.builtins.CoreFunctions; import com.oracle.graal.python.builtins.PythonBuiltins; import com.oracle.graal.python.builtins.objects.PNone; +import com.oracle.graal.python.builtins.objects.buffer.PythonBufferAccessLibrary; +import com.oracle.graal.python.builtins.objects.bytes.PBytes; import com.oracle.graal.python.nodes.PConstructAndRaiseNode; import com.oracle.graal.python.nodes.function.PythonBuiltinBaseNode; +import com.oracle.graal.python.nodes.function.builtins.PythonBinaryClinicBuiltinNode; import com.oracle.graal.python.nodes.function.builtins.PythonUnaryBuiltinNode; +import com.oracle.graal.python.nodes.function.builtins.PythonUnaryClinicBuiltinNode; +import com.oracle.graal.python.nodes.function.builtins.clinic.ArgumentClinicProvider; +import com.oracle.graal.python.runtime.IndirectCallData.InteropCallData; +import com.oracle.graal.python.runtime.PythonContext; import com.oracle.graal.python.runtime.PosixSupport; import com.oracle.graal.python.runtime.PosixSupportLibrary; import com.oracle.graal.python.runtime.PosixSupportLibrary.PosixException; +import com.oracle.graal.python.runtime.object.PFactory; import com.oracle.truffle.api.dsl.Bind; import com.oracle.truffle.api.dsl.Cached; import com.oracle.truffle.api.dsl.GenerateNodeFactory; @@ -69,6 +80,91 @@ protected List> getNodeFa return MultiprocessingModuleBuiltinsFactory.getFactories(); } + @GenerateNodeFactory + @Builtin(name = "recv", minNumOfPositionalArgs = 2, parameterNames = {"handle", "size"}, os = PythonOS.PLATFORM_WIN32) + @ArgumentClinic(name = "handle", conversion = ArgumentClinic.ClinicConversion.Int) + @ArgumentClinic(name = "size", conversion = ArgumentClinic.ClinicConversion.Int) + abstract static class Recv extends PythonBinaryClinicBuiltinNode { + @Specialization + PBytes doit(VirtualFrame frame, int handle, int size, + @Bind PythonContext context, + @Bind PythonLanguage language, + @CachedLibrary("context.getPosixSupport()") PosixSupportLibrary posixLib, + @Bind Node inliningTarget, + @Cached PConstructAndRaiseNode.Lazy constructAndRaiseNode) { + byte[] buffer = new byte[size]; + try { + int received = posixLib.recv(context.getPosixSupport(), handle, buffer, 0, size, 0); + if (received == size) { + return PFactory.createBytes(language, buffer); + } + byte[] result = new byte[received]; + System.arraycopy(buffer, 0, result, 0, received); + return PFactory.createBytes(language, result); + } catch (PosixException e) { + throw constructAndRaiseNode.get(inliningTarget).raiseOSErrorFromPosixException(frame, e); + } + } + + @Override + protected ArgumentClinicProvider getArgumentClinic() { + return MultiprocessingModuleBuiltinsClinicProviders.RecvClinicProviderGen.INSTANCE; + } + } + + @GenerateNodeFactory + @Builtin(name = "send", minNumOfPositionalArgs = 2, parameterNames = {"handle", "data"}, os = PythonOS.PLATFORM_WIN32) + @ArgumentClinic(name = "handle", conversion = ArgumentClinic.ClinicConversion.Int) + @ArgumentClinic(name = "data", conversion = ArgumentClinic.ClinicConversion.ReadableBuffer) + abstract static class Send extends PythonBinaryClinicBuiltinNode { + @Specialization(limit = "3") + static int doit(VirtualFrame frame, int handle, Object buffer, + @Bind PythonContext context, + @CachedLibrary("context.getPosixSupport()") PosixSupportLibrary posixLib, + @Bind Node inliningTarget, + @Cached("createFor($node)") InteropCallData callData, + @CachedLibrary("buffer") PythonBufferAccessLibrary bufferLib, + @Cached PConstructAndRaiseNode.Lazy constructAndRaiseNode) { + try { + byte[] bytes = bufferLib.getInternalOrCopiedByteArray(buffer); + return posixLib.send(context.getPosixSupport(), handle, bytes, 0, bufferLib.getBufferLength(buffer), 0); + } catch (PosixException e) { + throw constructAndRaiseNode.get(inliningTarget).raiseOSErrorFromPosixException(frame, e); + } finally { + bufferLib.release(buffer, frame, callData); + } + } + + @Override + protected ArgumentClinicProvider getArgumentClinic() { + return MultiprocessingModuleBuiltinsClinicProviders.SendClinicProviderGen.INSTANCE; + } + } + + @GenerateNodeFactory + @Builtin(name = "closesocket", minNumOfPositionalArgs = 1, parameterNames = {"handle"}, os = PythonOS.PLATFORM_WIN32) + @ArgumentClinic(name = "handle", conversion = ArgumentClinic.ClinicConversion.Int) + abstract static class CloseSocket extends PythonUnaryClinicBuiltinNode { + @Specialization + PNone doit(VirtualFrame frame, int handle, + @Bind PythonContext context, + @CachedLibrary("context.getPosixSupport()") PosixSupportLibrary posixLib, + @Bind Node inliningTarget, + @Cached PConstructAndRaiseNode.Lazy constructAndRaiseNode) { + try { + posixLib.close(context.getPosixSupport(), handle); + } catch (PosixException e) { + throw constructAndRaiseNode.get(inliningTarget).raiseOSErrorFromPosixException(frame, e); + } + return PNone.NONE; + } + + @Override + protected ArgumentClinicProvider getArgumentClinic() { + return MultiprocessingModuleBuiltinsClinicProviders.CloseSocketClinicProviderGen.INSTANCE; + } + } + @GenerateNodeFactory @Builtin(name = "sem_unlink", parameterNames = {"name"}) abstract static class SemUnlink extends PythonUnaryBuiltinNode { @@ -79,7 +175,7 @@ PNone doit(VirtualFrame frame, TruffleString name, @Bind Node inliningTarget, @Cached PConstructAndRaiseNode.Lazy constructAndRaiseNode) { try { - posixLib.semUnlink(posixSupport, posixLib.createPathFromString(posixSupport, name)); + posixLib.semUnlink(posixSupport, posixLib.createCStringFromString(posixSupport, name)); } catch (PosixException e) { throw constructAndRaiseNode.get(inliningTarget).raiseOSErrorFromPosixException(frame, e); } diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/multiprocessing/SemLockBuiltins.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/multiprocessing/SemLockBuiltins.java index 34a1bb2347..c538b9b6dc 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/multiprocessing/SemLockBuiltins.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/multiprocessing/SemLockBuiltins.java @@ -49,8 +49,10 @@ import java.util.List; +import com.oracle.graal.python.PythonLanguage; import com.oracle.graal.python.annotations.ArgumentClinic; import com.oracle.graal.python.annotations.Builtin; +import com.oracle.graal.python.annotations.PythonOS; import com.oracle.graal.python.annotations.Slot; import com.oracle.graal.python.annotations.Slot.SlotKind; import com.oracle.graal.python.annotations.Slot.SlotSignature; @@ -126,7 +128,7 @@ static PSemLock construct(VirtualFrame frame, Object cls, int kind, int value, i if (kind != PSemLock.RECURSIVE_MUTEX && kind != PSemLock.SEMAPHORE) { throw raiseNode.raise(inliningTarget, ValueError, ErrorMessages.UNRECOGNIZED_KIND); } - Object posixName = posixLib.createPathFromString(posixSupport, name); + Object posixName = posixLib.createCStringFromString(posixSupport, name); long handle; try { handle = posixLib.semOpen(posixSupport, posixName, O_CREAT.value | O_EXCL.value, 0600, value); @@ -145,7 +147,8 @@ static PSemLock construct(VirtualFrame frame, Object cls, int kind, int value, i throw constructAndRaiseNode.get(inliningTarget).raiseOSErrorFromPosixException(frame, e); } } - return PFactory.createSemLock(cls, getInstanceShape.execute(cls), handle, kind, maxValue, name); + TruffleString storedName = PythonLanguage.getPythonOS() == PythonOS.PLATFORM_WIN32 ? null : name; + return PFactory.createSemLock(cls, getInstanceShape.execute(cls), handle, kind, maxValue, storedName); } @Override @@ -177,7 +180,8 @@ static Object get(PSemLock self) { abstract static class NameNode extends PythonUnaryBuiltinNode { @Specialization static Object get(PSemLock self) { - return self.getName(); + TruffleString name = self.getName(); + return name == null ? PNone.NONE : name; } } @@ -413,24 +417,27 @@ static Object afterFork(PSemLock self) { @ArgumentClinic(name = "handle", conversion = ArgumentClinic.ClinicConversion.Long) @ArgumentClinic(name = "kind", conversion = ArgumentClinic.ClinicConversion.Int) @ArgumentClinic(name = "maxvalue", conversion = ArgumentClinic.ClinicConversion.Int) - @ArgumentClinic(name = "name", conversion = ArgumentClinic.ClinicConversion.TString) @GenerateNodeFactory abstract static class RebuildNode extends PythonClinicBuiltinNode { @Specialization - static Object rebuild(VirtualFrame frame, Object cls, @SuppressWarnings("unused") long origHandle, int kind, int maxValue, TruffleString name, + static Object rebuild(VirtualFrame frame, Object cls, long origHandle, int kind, int maxValue, Object name, @Bind Node inliningTarget, @Bind("getPosixSupport()") PosixSupport posixSupport, @CachedLibrary("posixSupport") PosixSupportLibrary posixLib, @Cached TypeNodes.GetInstanceShape getInstanceShape, @Cached PConstructAndRaiseNode.Lazy constructAndRaiseNode) { - Object posixName = posixLib.createPathFromString(posixSupport, name); + if (PythonLanguage.getPythonOS() == PythonOS.PLATFORM_WIN32) { + return PFactory.createSemLock(cls, getInstanceShape.execute(cls), origHandle, kind, maxValue, null); + } + TruffleString posixNameString = (TruffleString) name; + Object posixName = posixLib.createCStringFromString(posixSupport, posixNameString); long handle; try { handle = posixLib.semOpen(posixSupport, posixName); } catch (PosixException e) { throw constructAndRaiseNode.get(inliningTarget).raiseOSErrorFromPosixException(frame, e); } - return PFactory.createSemLock(cls, getInstanceShape.execute(cls), handle, kind, maxValue, name); + return PFactory.createSemLock(cls, getInstanceShape.execute(cls), handle, kind, maxValue, posixNameString); } @Override diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/common/CExtCommonNodes.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/common/CExtCommonNodes.java index 731dc4219b..d3689529fd 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/common/CExtCommonNodes.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/common/CExtCommonNodes.java @@ -57,6 +57,7 @@ import static com.oracle.graal.python.runtime.nativeaccess.NativeMemory.readIntArrayElement; import static com.oracle.graal.python.runtime.nativeaccess.NativeMemory.readShortArrayElement; import static com.oracle.graal.python.util.PythonUtils.TS_ENCODING; +import static com.oracle.graal.python.util.PythonUtils.SURROGATE_CODE_POINT_SET; import static com.oracle.graal.python.util.PythonUtils.tsLiteral; import java.nio.charset.Charset; @@ -147,10 +148,6 @@ static Object doObj(Object o) { @GenerateInline(false) // footprint reduction 40 -> 22 @GenerateUncached public abstract static class EncodeNativeStringNode extends PNodeWithContext { - private static final TruffleString.CodePointSet SURROGATES = TruffleString.CodePointSet.fromRanges(new int[]{ - Character.MIN_SURROGATE, Character.MAX_SURROGATE, - }, TS_ENCODING); - public abstract TruffleString execute(TruffleString.Encoding encoding, Object unicodeObject, TruffleString errors); @Specialization @@ -220,7 +217,7 @@ private static PException raiseSurrogatesEncodeError(PConstructAndRaiseNode cons } private static int findFirstSurrogateIndex(TruffleString str, TruffleString.ByteIndexOfCodePointSetNode byteIndexOfCodePointSetNode) { - int byteIndex = byteIndexOfCodePointSetNode.execute(str, 0, str.byteLength(TS_ENCODING), SURROGATES); + int byteIndex = byteIndexOfCodePointSetNode.execute(str, 0, str.byteLength(TS_ENCODING), SURROGATE_CODE_POINT_SET); return byteIndex < 0 ? 0 : byteIndexToCodepointIndex(byteIndex); } diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/exception/OsErrorBuiltins.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/exception/OsErrorBuiltins.java index 1ae58e4674..488c276052 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/exception/OsErrorBuiltins.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/exception/OsErrorBuiltins.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, 2025, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2021, 2026, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * The Universal Permissive License (UPL), Version 1.0 @@ -389,12 +389,7 @@ public abstract static class OSErrorWinerrorNode extends PythonBuiltinNode { Object generic(PBaseException self, Object value, @Cached BaseExceptionAttrNode attrNode) { Object result = attrNode.execute(self, value, IDX_WINERROR, OS_ERROR_ATTR_FACTORY); - if (result instanceof PNone) { - // TODO: fallback to errno for now - return attrNode.execute(self, value, IDX_ERRNO, OS_ERROR_ATTR_FACTORY); - } else { - return result; - } + return result; } } diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/mmap/MMapBuiltins.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/mmap/MMapBuiltins.java index 81067e74c6..399da9ec9c 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/mmap/MMapBuiltins.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/mmap/MMapBuiltins.java @@ -75,6 +75,7 @@ import com.oracle.graal.python.PythonLanguage; import com.oracle.graal.python.annotations.ArgumentClinic; import com.oracle.graal.python.annotations.ArgumentClinic.ClinicConversion; +import com.oracle.graal.python.annotations.PythonOS; import com.oracle.graal.python.annotations.Builtin; import com.oracle.graal.python.annotations.Slot; import com.oracle.graal.python.annotations.Slot.SlotKind; @@ -110,6 +111,7 @@ import com.oracle.graal.python.builtins.objects.type.slots.TpSlotSqAssItem.SqAssItemBuiltinNode; import com.oracle.graal.python.lib.PyBytesCheckNode; import com.oracle.graal.python.lib.PyIndexCheckNode; +import com.oracle.graal.python.lib.PyLongAsIntNode; import com.oracle.graal.python.lib.PyLongAsLongNode; import com.oracle.graal.python.lib.PyNumberAsSizeNode; import com.oracle.graal.python.nodes.ErrorMessages; @@ -126,12 +128,15 @@ import com.oracle.graal.python.nodes.function.builtins.PythonUnaryBuiltinNode; import com.oracle.graal.python.nodes.function.builtins.clinic.ArgumentClinicProvider; import com.oracle.graal.python.nodes.function.builtins.clinic.LongIndexConverterNode; +import com.oracle.graal.python.nodes.util.CannotCastException; +import com.oracle.graal.python.nodes.util.CastToTruffleStringNode; import com.oracle.graal.python.runtime.AsyncHandler; import com.oracle.graal.python.runtime.IndirectCallData.InteropCallData; import com.oracle.graal.python.runtime.PosixSupport; import com.oracle.graal.python.runtime.PosixSupportLibrary; import com.oracle.graal.python.runtime.PosixSupportLibrary.PosixException; import com.oracle.graal.python.runtime.PythonContext; +import com.oracle.graal.python.runtime.nativeaccess.NativeContext; import com.oracle.graal.python.runtime.object.PFactory; import com.oracle.graal.python.runtime.sequence.storage.ByteSequenceStorage; import com.oracle.graal.python.util.OverflowException; @@ -178,12 +183,11 @@ private static byte[] readBytes(VirtualFrame frame, Node inliningTarget, PMMap s @Slot(value = SlotKind.tp_new, isComplex = true) @SlotSignature(name = "mmap", minNumOfPositionalArgs = 3, parameterNames = {"cls", "fd", "length", "flags", "prot", "access", - "offset"}) + "offset"}, keywordOnlyNames = {"tagname"}) @GenerateNodeFactory // Note: it really should not call fileno on fd as per Python spec @ArgumentClinic(name = "fd", conversion = ClinicConversion.Int) @ArgumentClinic(name = "length", conversion = ClinicConversion.LongIndex) - @ArgumentClinic(name = "flags", conversion = ClinicConversion.Int, defaultValue = "FLAGS_DEFAULT") @ArgumentClinic(name = "prot", conversion = ClinicConversion.Int, defaultValue = "PROT_DEFAULT") @ArgumentClinic(name = "access", conversion = ClinicConversion.Int, defaultValue = "ACCESS_ARG_DEFAULT") @ArgumentClinic(name = "offset", conversion = ClinicConversion.Long, defaultValue = "0") @@ -194,6 +198,9 @@ public abstract static class MMapNode extends PythonClinicBuiltinNode { private static final int ANONYMOUS_FD = -1; + private record MMapArgs(int flags, Object tagname) { + } + @Override protected ArgumentClinicProvider getArgumentClinic() { return MMapNodeClinicProviderGen.INSTANCE; @@ -201,20 +208,32 @@ protected ArgumentClinicProvider getArgumentClinic() { // mmap(fileno, length, tagname=None, access=ACCESS_DEFAULT[, offset=0]) @Specialization(guards = "!isIllegal(fd)") - static PMMap doFile(VirtualFrame frame, Object clazz, int fd, long lengthIn, int flagsIn, int protIn, @SuppressWarnings("unused") int accessIn, long offset, + static PMMap doFile(VirtualFrame frame, Object clazz, int fd, long lengthIn, Object flagsIn, int protIn, @SuppressWarnings("unused") int accessIn, long offset, + Object tagname, @Bind Node inliningTarget, @Cached SysModuleBuiltins.AuditNode auditNode, @CachedLibrary("getPosixSupport()") PosixSupportLibrary posixSupport, @Cached PConstructAndRaiseNode.Lazy constructAndRaiseNode, @Cached TypeNodes.GetInstanceShape getInstanceShape, - @Cached PRaiseNode raiseNode) { + @Exclusive @Cached CastToTruffleStringNode castTagnameNode, + @Exclusive @Cached PyLongAsIntNode flagsAsIntNode, + @Exclusive @Cached PRaiseNode raiseNode) { + MMapArgs mmapArgs = parseFlagsAndTagname(frame, inliningTarget, flagsIn, tagname, castTagnameNode, flagsAsIntNode, raiseNode); + Object mmapTagname = PNone.NONE; + if (mmapArgs.tagname() != PNone.NO_VALUE && !isPNone(mmapArgs.tagname())) { + try { + mmapTagname = castTagnameNode.execute(inliningTarget, mmapArgs.tagname()); + } catch (CannotCastException e) { + throw raiseNode.raise(inliningTarget, TypeError, ErrorMessages.MUST_BE_STR_NOT_P, mmapArgs.tagname()); + } + } if (lengthIn < 0) { throw raiseNode.raise(inliningTarget, OverflowError, ErrorMessages.MEM_MAPPED_LENGTH_MUST_BE_POSITIVE); } if (offset < 0) { throw raiseNode.raise(inliningTarget, OverflowError, ErrorMessages.MEM_MAPPED_OFFSET_MUST_BE_POSITIVE); } - int flags = flagsIn; + int flags = mmapArgs.flags(); int prot = protIn; int access = accessIn; switch (access) { @@ -287,21 +306,50 @@ static PMMap doFile(VirtualFrame frame, Object clazz, int fd, long lengthIn, int Object mmapHandle; try { - mmapHandle = posixSupport.mmap(posixSupport1, length, prot, flags, dupFd, offset); + mmapHandle = posixSupport.mmap(posixSupport1, length, prot, flags, dupFd, offset, mmapTagname); } catch (PosixException e) { throw constructAndRaiseNode.get(inliningTarget).raiseOSErrorFromPosixException(frame, e); } PythonContext context = PythonContext.get(inliningTarget); - return PFactory.createMMap(context, clazz, getInstanceShape.execute(clazz), mmapHandle, dupFd, length, access); + PMMap mmap = PFactory.createMMap(context, clazz, getInstanceShape.execute(clazz), mmapHandle, dupFd, length, access); + if (PythonLanguage.getPythonOS() == PythonOS.PLATFORM_WIN32) { + NativeContext.setLastError(0); + } + return mmap; } @Specialization(guards = "isIllegal(fd)") @SuppressWarnings("unused") - static PMMap doIllegal(Object clazz, int fd, long lengthIn, int flagsIn, int protIn, int accessIn, long offset, - @Bind Node inliningTarget) { + static PMMap doIllegal(VirtualFrame frame, Object clazz, int fd, long lengthIn, Object flagsIn, int protIn, int accessIn, long offset, + Object tagname, + @Bind Node inliningTarget, + @Exclusive @Cached CastToTruffleStringNode castTagnameNode, + @Exclusive @Cached PyLongAsIntNode flagsAsIntNode, + @Exclusive @Cached PRaiseNode raiseNode) { + parseFlagsAndTagname(frame, inliningTarget, flagsIn, tagname, castTagnameNode, flagsAsIntNode, raiseNode); throw PRaiseNode.raiseStatic(inliningTarget, PythonBuiltinClassType.OSError); } + private static MMapArgs parseFlagsAndTagname(VirtualFrame frame, Node inliningTarget, Object flagsArg, Object tagnameArg, + CastToTruffleStringNode castTagnameNode, PyLongAsIntNode flagsAsIntNode, PRaiseNode raiseNode) { + Object flags = flagsArg; + Object tagname = tagnameArg; + if (PythonLanguage.getPythonOS() == PythonOS.PLATFORM_WIN32 && tagname == PNone.NO_VALUE && flags != PNone.NO_VALUE) { + try { + if (!isPNone(flags)) { + castTagnameNode.execute(inliningTarget, flags); + } + tagname = flags; + flags = PNone.NO_VALUE; + } catch (CannotCastException ignored) { + } + } + if (tagname != PNone.NO_VALUE && PythonLanguage.getPythonOS() != PythonOS.PLATFORM_WIN32) { + throw raiseNode.raise(inliningTarget, TypeError, ErrorMessages.GOT_UNEXPECTED_KEYWORD_ARG, "mmap", "tagname"); + } + return new MMapArgs(flags == PNone.NO_VALUE ? FLAGS_DEFAULT : flagsAsIntNode.execute(frame, inliningTarget, flags), tagname); + } + protected static boolean isIllegal(int fd) { return fd < -1; } diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/socket/SocketNodes.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/socket/SocketNodes.java index 3db5196ae5..cf57c4364b 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/socket/SocketNodes.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/socket/SocketNodes.java @@ -292,7 +292,7 @@ static UniversalSockAddr setipaddr(VirtualFrame frame, byte[] name, int family, gil.release(true); try { // TODO getaddrinfo lock? - AddrInfoCursor cursor = posixLib.getaddrinfo(posixSupport, null, posixLib.createPathFromString(posixSupport, T_ZERO), + AddrInfoCursor cursor = posixLib.getaddrinfo(posixSupport, null, posixLib.createCStringFromString(posixSupport, T_ZERO), family, SOCK_DGRAM.value, 0, AI_PASSIVE.value); try { if (addrInfoLib.next(cursor)) { @@ -336,7 +336,7 @@ static UniversalSockAddr setipaddr(VirtualFrame frame, byte[] name, int family, gil.release(true); try { // TODO getaddrinfo lock? - AddrInfoCursor cursor = posixLib.getaddrinfo(posixSupport, posixLib.createPathFromBytes(posixSupport, name), null, + AddrInfoCursor cursor = posixLib.getaddrinfo(posixSupport, posixLib.createCStringFromBytes(posixSupport, name), null, family, 0, 0, 0); try { return addrInfoLib.getSockAddr(cursor); @@ -372,7 +372,7 @@ static byte[] cached(PosixSupportLibrary posixLib, Object posixSupport, int fami static byte[] doParse(PosixSupportLibrary posixLib, Object posixSupport, int family, byte[] string) { assert family == AF_INET.value || family == AF_INET6.value; try { - return posixLib.inet_pton(posixSupport, family, posixLib.createPathFromBytes(posixSupport, string)); + return posixLib.inet_pton(posixSupport, family, posixLib.createCStringFromBytes(posixSupport, string)); } catch (PosixException | InvalidAddressException e) { return null; } @@ -412,12 +412,12 @@ static Object makeSockAddr(VirtualFrame frame, Node inliningTarget, UniversalSoc if (family == AF_INET.value) { Inet4SockAddr inet4SockAddr = addrLib.asInet4SockAddr(addr); Object posixSupport = context.getPosixSupport(); - TruffleString addressString = posixLib.getPathAsString(posixSupport, posixLib.inet_ntop(posixSupport, family, inet4SockAddr.getAddressAsBytes())); + TruffleString addressString = posixLib.getCStringAsString(posixSupport, posixLib.inet_ntop(posixSupport, family, inet4SockAddr.getAddressAsBytes())); return PFactory.createTuple(language, new Object[]{addressString, inet4SockAddr.getPort()}); } else if (family == AF_INET6.value) { Inet6SockAddr inet6SockAddr = addrLib.asInet6SockAddr(addr); Object posixSupport = context.getPosixSupport(); - TruffleString addressString = posixLib.getPathAsString(posixSupport, posixLib.inet_ntop(posixSupport, family, inet6SockAddr.getAddress())); + TruffleString addressString = posixLib.getCStringAsString(posixSupport, posixLib.inet_ntop(posixSupport, family, inet6SockAddr.getAddress())); return PFactory.createTuple(language, new Object[]{addressString, inet6SockAddr.getPort(), inet6SockAddr.getFlowInfo(), inet6SockAddr.getScopeId()}); } else if (family == AF_UNIX.value) { UnixSockAddr unixSockAddr = addrLib.asUnixSockAddr(addr); @@ -469,11 +469,11 @@ static Object makeAddr(VirtualFrame frame, Node inliningTarget, UniversalSockAdd if (family == AF_INET.value) { Inet4SockAddr inet4SockAddr = addrLib.asInet4SockAddr(addr); Object posixSupport = context.getPosixSupport(); - return posixLib.getPathAsString(posixSupport, posixLib.inet_ntop(posixSupport, family, inet4SockAddr.getAddressAsBytes())); + return posixLib.getCStringAsString(posixSupport, posixLib.inet_ntop(posixSupport, family, inet4SockAddr.getAddressAsBytes())); } else if (family == AF_INET6.value) { Inet6SockAddr inet6SockAddr = addrLib.asInet6SockAddr(addr); Object posixSupport = context.getPosixSupport(); - return posixLib.getPathAsString(posixSupport, posixLib.inet_ntop(posixSupport, family, inet6SockAddr.getAddress())); + return posixLib.getCStringAsString(posixSupport, posixLib.inet_ntop(posixSupport, family, inet6SockAddr.getAddress())); } else { throw raiseNode.raise(inliningTarget, NotImplementedError, toTruffleStringUncached("makesockaddr: unknown address family")); } diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/PConstructAndRaiseNode.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/PConstructAndRaiseNode.java index 566c350ed5..1eebdddb6c 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/PConstructAndRaiseNode.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/PConstructAndRaiseNode.java @@ -49,6 +49,7 @@ import com.oracle.graal.python.builtins.Python3Core; import com.oracle.graal.python.builtins.PythonBuiltinClassType; +import com.oracle.graal.python.builtins.objects.PNone; import com.oracle.graal.python.builtins.objects.exception.OSErrorEnum; import com.oracle.graal.python.builtins.objects.exception.PBaseException; import com.oracle.graal.python.builtins.objects.function.PKeyword; @@ -194,6 +195,20 @@ private static Object[] createOsErrorArgs(int errno, TruffleString message, Obje return new Object[]{errno, message}; } + private static Object[] createOsErrorArgs(int errno, TruffleString message, Integer winerror, Object filename1, Object filename2) { + if (winerror == null) { + return createOsErrorArgs(errno, message, filename1, filename2); + } + if (filename1 == null) { + assert filename2 == null; + return new Object[]{errno, message, PNone.NONE, winerror}; + } + if (filename2 == null) { + return new Object[]{errno, message, filename1, winerror}; + } + return new Object[]{errno, message, filename1, winerror, filename2}; + } + private static Object[] createOsErrorArgs(OSErrorEnum osErrorEnum) { return new Object[]{osErrorEnum.getNumber(), osErrorEnum.getMessage()}; } @@ -233,7 +248,7 @@ public final PException raiseOSError(VirtualFrame frame, int errno, TruffleStrin public final PException raiseOSErrorFromPosixException(VirtualFrame frame, PosixException e) { if (e instanceof PosixErrnoException errnoException) { - return raiseOSErrorInternal(frame, createOsErrorArgs(errnoException.getErrorCode(), errnoException.getMessageAsTruffleString())); + return raiseOSErrorInternal(frame, createOsErrorArgs(errnoException.getErrorCode(), errnoException.getMessageAsTruffleString(), errnoException.getWinerror(), null, null)); } assert e instanceof UnsupportedPosixFeatureException; return raiseOSErrorUnsupported(frame, (UnsupportedPosixFeatureException) e); @@ -241,7 +256,7 @@ public final PException raiseOSErrorFromPosixException(VirtualFrame frame, Posix public final PException raiseOSErrorFromPosixException(VirtualFrame frame, PosixException e, Object filename1) { if (e instanceof PosixErrnoException errnoException) { - return raiseOSErrorInternal(frame, createOsErrorArgs(errnoException.getErrorCode(), errnoException.getMessageAsTruffleString(), filename1)); + return raiseOSErrorInternal(frame, createOsErrorArgs(errnoException.getErrorCode(), errnoException.getMessageAsTruffleString(), errnoException.getWinerror(), filename1, null)); } assert e instanceof UnsupportedPosixFeatureException; return raiseOSErrorUnsupported(frame, (UnsupportedPosixFeatureException) e); @@ -249,7 +264,7 @@ public final PException raiseOSErrorFromPosixException(VirtualFrame frame, Posix public final PException raiseOSErrorFromPosixException(VirtualFrame frame, PosixException e, Object filename1, Object filename2) { if (e instanceof PosixErrnoException errnoException) { - return raiseOSErrorInternal(frame, createOsErrorArgs(errnoException.getErrorCode(), errnoException.getMessageAsTruffleString(), filename1, filename2)); + return raiseOSErrorInternal(frame, createOsErrorArgs(errnoException.getErrorCode(), errnoException.getMessageAsTruffleString(), errnoException.getWinerror(), filename1, filename2)); } assert e instanceof UnsupportedPosixFeatureException; return raiseOSErrorUnsupported(frame, (UnsupportedPosixFeatureException) e); diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/EmulatedPosixSupport.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/EmulatedPosixSupport.java index db77b96186..cd9bb602eb 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/EmulatedPosixSupport.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/EmulatedPosixSupport.java @@ -587,6 +587,18 @@ public void setInheritable(int fd, boolean inheritable) { compatibilityIgnored("setting inheritable '%b' for file descriptor %d in POSIX emulation layer (not supported)", inheritable, fd); } + @ExportMessage + @SuppressWarnings("static-method") + public long getOsfHandle(int fd) throws UnsupportedPosixFeatureException { + throw createUnsupportedFeature("get_osfhandle"); + } + + @ExportMessage + @SuppressWarnings("static-method") + public int openOsfHandle(long handle, int flags) throws UnsupportedPosixFeatureException { + throw createUnsupportedFeature("open_osfhandle"); + } + @ExportMessage(name = "pipe") public int[] pipeMessage(@Shared("eq") @Cached TruffleString.EqualNode eqNode) throws PosixException { // TODO: will merge with super.pipe once the super class is merged with this class @@ -1887,6 +1899,25 @@ public void renameat(int oldDirFd, Object oldPath, int newDirFd, Object newPath, } } + @ExportMessage + public void replaceat(int oldDirFd, Object oldPath, int newDirFd, Object newPath, + @Bind Node inliningTarget, + @Shared("defaultDirProfile") @Cached InlinedConditionProfile defaultDirFdPofile, + @Shared("eq") @Cached TruffleString.EqualNode eqNode, + @Shared("js2ts") @Cached TruffleString.FromJavaStringNode fromJavaStringNode, + @Shared("ts2js") @Cached TruffleString.ToJavaStringNode toJavaStringNode) throws PosixException { + try { + TruffleFile newFile = resolvePath(inliningTarget, newDirFd, pathToTruffleString(newPath, fromJavaStringNode), defaultDirFdPofile, eqNode, toJavaStringNode); + if (newFile.isDirectory()) { + throw posixException(OSErrorEnum.EISDIR); + } + TruffleFile oldFile = resolvePath(inliningTarget, oldDirFd, pathToTruffleString(oldPath, fromJavaStringNode), defaultDirFdPofile, eqNode, toJavaStringNode); + oldFile.move(newFile, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE); + } catch (Exception e) { + throw posixException(OSErrorEnum.fromException(e, eqNode)); + } + } + @ExportMessage public boolean faccessat(int dirFd, Object path, int mode, boolean effectiveIds, boolean followSymlinks, @Bind Node inliningTarget, @@ -2828,7 +2859,7 @@ public SeekableByteChannel truncate(long size) throws IOException { @ExportMessage @SuppressWarnings("static-method") - final MMapHandle mmap(long length, int prot, int flags, int fd, long offset, + final MMapHandle mmap(long length, int prot, int flags, int fd, long offset, @SuppressWarnings("unused") Object tagname, @Bind Node inliningTarget, @Shared("defaultDirProfile") @Cached InlinedConditionProfile isAnonymousProfile, @Shared("eq") @Cached TruffleString.EqualNode eqNode, @@ -4521,6 +4552,34 @@ public Buffer getPathAsBytes(Object path) { return Buffer.wrap(utf8StringToBytes((String) path)); } + @ExportMessage + public Object createCStringFromString(TruffleString string, + @Shared("ts2js") @Cached TruffleString.ToJavaStringNode toJavaStringNode) { + return checkEmbeddedNulls(toJavaStringNode.execute(string)); + } + + @ExportMessage + public Object createCStringFromBytes(byte[] bytes) { + return checkEmbeddedNulls(createUTF8String(bytes)); + } + + @ExportMessage + public Object createWideStringFromString(TruffleString string, + @Shared("ts2js") @Cached TruffleString.ToJavaStringNode toJavaStringNode) { + return checkEmbeddedNulls(toJavaStringNode.execute(string)); + } + + @ExportMessage + public TruffleString getCStringAsString(Object string, + @Shared("js2ts") @Cached TruffleString.FromJavaStringNode fromJavaStringNode) { + return fromJavaStringNode.execute((String) string, TS_ENCODING); + } + + @ExportMessage + public Buffer getCStringAsBytes(Object string) { + return Buffer.wrap(utf8StringToBytes((String) string)); + } + @TruffleBoundary private static String createUTF8String(byte[] retbuf) { return new String(retbuf, StandardCharsets.UTF_8); diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/LoggingPosixSupport.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/LoggingPosixSupport.java index ee8f3f62ed..e43257605d 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/LoggingPosixSupport.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/LoggingPosixSupport.java @@ -241,6 +241,28 @@ final void setInheritable(int fd, boolean inheritable, } } + @ExportMessage + final long getOsfHandle(int fd, + @CachedLibrary("this.delegate") PosixSupportLibrary lib) throws PosixException { + logEnter("getOsfHandle", "%d", fd); + try { + return logExit("getOsfHandle", "%d", lib.getOsfHandle(delegate, fd)); + } catch (PosixException e) { + throw logException("getOsfHandle", e); + } + } + + @ExportMessage + final int openOsfHandle(long handle, int flags, + @CachedLibrary("this.delegate") PosixSupportLibrary lib) throws PosixException { + logEnter("openOsfHandle", "%d, %d", handle, flags); + try { + return logExit("openOsfHandle", "%d", lib.openOsfHandle(delegate, handle, flags)); + } catch (PosixException e) { + throw logException("openOsfHandle", e); + } + } + @ExportMessage final int[] pipe( @CachedLibrary("this.delegate") PosixSupportLibrary lib) throws PosixException { @@ -681,6 +703,17 @@ final void renameat(int oldDirFd, Object oldPath, int newDirFd, Object newPath, } } + @ExportMessage + final void replaceat(int oldDirFd, Object oldPath, int newDirFd, Object newPath, + @CachedLibrary("this.delegate") PosixSupportLibrary lib) throws PosixException { + logEnter("replaceAt", "%d, %s, %d, %s", oldDirFd, oldPath, newDirFd, newPath); + try { + lib.replaceat(delegate, oldDirFd, oldPath, newDirFd, newPath); + } catch (PosixException e) { + throw logException("replaceAt", e); + } + } + @ExportMessage final boolean faccessat(int dirFd, Object path, int mode, boolean effectiveIds, boolean followSymlinks, @CachedLibrary("this.delegate") PosixSupportLibrary lib) throws UnsupportedPosixFeatureException { @@ -825,11 +858,11 @@ final void killpg(long pgid, int signal, } @ExportMessage - public Object mmap(long length, int prot, int flags, int fd, long offset, + public Object mmap(long length, int prot, int flags, int fd, long offset, Object tagname, @CachedLibrary("this.delegate") PosixSupportLibrary lib) throws PosixException { - logEnter("mmap", "%d, %d, %d, %d, %d", length, prot, flags, fd, offset); + logEnter("mmap", "%d, %d, %d, %d, %d, %s", length, prot, flags, fd, offset, tagname); try { - return logExit("mmap", "%s", lib.mmap(delegate, length, prot, flags, fd, offset)); + return logExit("mmap", "%s", lib.mmap(delegate, length, prot, flags, fd, offset, tagname)); } catch (PosixException e) { throw logException("mmap", e); } @@ -1654,6 +1687,36 @@ final Buffer getPathAsBytes(Object path, return logExit(Level.FINEST, "getPathAsBytes", "%s", lib.getPathAsBytes(delegate, path)); } + @ExportMessage + final Object createCStringFromString(TruffleString string, + @CachedLibrary("this.delegate") PosixSupportLibrary lib) { + return lib.createCStringFromString(delegate, string); + } + + @ExportMessage + final Object createCStringFromBytes(byte[] bytes, + @CachedLibrary("this.delegate") PosixSupportLibrary lib) { + return lib.createCStringFromBytes(delegate, bytes); + } + + @ExportMessage + final Object createWideStringFromString(TruffleString string, + @CachedLibrary("this.delegate") PosixSupportLibrary lib) { + return lib.createWideStringFromString(delegate, string); + } + + @ExportMessage + final TruffleString getCStringAsString(Object string, + @CachedLibrary("this.delegate") PosixSupportLibrary lib) { + return lib.getCStringAsString(delegate, string); + } + + @ExportMessage + final Buffer getCStringAsBytes(Object string, + @CachedLibrary("this.delegate") PosixSupportLibrary lib) { + return lib.getCStringAsBytes(delegate, string); + } + @TruffleBoundary private static void logEnter(Level level, String msg, String argFmt, Object... args) { if (LOGGER.isLoggable(level)) { diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/NativeLZMASupport.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/NativeLZMASupport.java index 116d90c42f..edcb51767b 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/NativeLZMASupport.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/NativeLZMASupport.java @@ -438,7 +438,8 @@ public int lzmaAloneDecoder(long lzmast, long memlimit) { } public int decompress(long lzmast, byte[] inputBuffer, long offset, long maxLength, long bufsize, long lzsAvailIn) { - long nativeInputBuffer = copyToNativeByteArray(inputBuffer); + int inputLength = Math.toIntExact(Math.addExact(offset, lzsAvailIn)); + long nativeInputBuffer = copyToNativeByteArray(inputBuffer, inputLength); try { return nativeFunctions.lzma_decompress(lzmast, nativeInputBuffer, offset, maxLength, bufsize, lzsAvailIn); } finally { diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/NativePosixSupport.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/NativePosixSupport.java index 2364319073..81867d6e74 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/NativePosixSupport.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/NativePosixSupport.java @@ -75,14 +75,15 @@ import static com.oracle.graal.python.runtime.PosixConstants.PATH_MAX; import static com.oracle.graal.python.runtime.PosixConstants.WNOHANG; import static com.oracle.graal.python.runtime.PosixConstants._POSIX_HOST_NAME_MAX; -import static com.oracle.graal.python.runtime.PosixSupportLibrary.POSIX_FILENAME_SEPARATOR; import static com.oracle.graal.python.runtime.PosixSupportLibrary.UnsupportedPosixFeatureException; import static com.oracle.graal.python.runtime.nativeaccess.NativeMemory.NULLPTR; import static com.oracle.graal.python.util.PythonUtils.ARRAY_ACCESSOR; import static com.oracle.graal.python.util.PythonUtils.ARRAY_ACCESSOR_BE; import static com.oracle.graal.python.util.PythonUtils.EMPTY_LONG_ARRAY; +import static com.oracle.graal.python.util.PythonUtils.SURROGATE_CODE_POINT_SET; import static com.oracle.graal.python.util.PythonUtils.TS_ENCODING; import static com.oracle.truffle.api.CompilerDirectives.shouldNotReachHere; +import static com.oracle.truffle.api.strings.TruffleString.Encoding.UTF_16LE; import static com.oracle.truffle.api.strings.TruffleString.Encoding.UTF_8; import java.util.ArrayList; @@ -98,6 +99,7 @@ import com.oracle.graal.python.lib.PyUnicodeFSDecoderNode; import com.oracle.graal.python.nodes.ErrorMessages; import com.oracle.graal.python.nodes.PRaiseNode; +import com.oracle.graal.python.nodes.call.CallNode; import com.oracle.graal.python.runtime.PosixSupportLibrary.AcceptResult; import com.oracle.graal.python.runtime.PosixSupportLibrary.AddrInfoCursor; import com.oracle.graal.python.runtime.PosixSupportLibrary.AddrInfoCursorLibrary; @@ -118,9 +120,11 @@ import com.oracle.graal.python.runtime.PosixSupportLibrary.UniversalSockAddr; import com.oracle.graal.python.runtime.PosixSupportLibrary.UniversalSockAddrLibrary; import com.oracle.graal.python.runtime.PosixSupportLibrary.UnixSockAddr; +import com.oracle.graal.python.runtime.exception.PException; import com.oracle.graal.python.runtime.nativeaccess.NativeLibrary; import com.oracle.graal.python.runtime.nativeaccess.NativeLibraryLoadException; import com.oracle.graal.python.runtime.nativeaccess.NativeMemory; +import com.oracle.graal.python.runtime.object.PFactory; import com.oracle.graal.python.util.OverflowException; import com.oracle.graal.python.util.PythonUtils; import com.oracle.truffle.api.ArrayUtils; @@ -140,6 +144,9 @@ import com.oracle.truffle.api.library.ExportLibrary; import com.oracle.truffle.api.library.ExportMessage; import com.oracle.truffle.api.nodes.Node; +import com.oracle.truffle.api.strings.AbstractTruffleString; +import com.oracle.truffle.api.strings.InternalByteArray; +import com.oracle.truffle.api.strings.TranscodingErrorHandler; import com.oracle.truffle.api.strings.TruffleString; import sun.misc.Unsafe; @@ -160,6 +167,10 @@ public final class NativePosixSupport extends PosixSupport { private static final int PWD_OUTPUT_LEN = 5; private static final int PWD_BUFFER_MAX_SIZE = Integer.MAX_VALUE >> 2; private static final int STRERROR_BUF_LENGTH = 1024; + private static final int ERROR_SOURCE_ERRNO = 0; + private static final int ERROR_SOURCE_WINAPI = 1; + private static final int ERROR_SOURCE_WINSOCK = 2; + private static final boolean WINDOWS = PythonLanguage.getPythonOS() == PythonOS.PLATFORM_WIN32; private static final int MAX_READ = Integer.MAX_VALUE / 2; @@ -170,14 +181,26 @@ public final class NativePosixSupport extends PosixSupport { private static final Object CRYPT_LOCK = new Object(); abstract static class PosixNativeFunctionInvoker { + @DowncallSignature(returnType = VOID) + abstract void call_initialize(); + @DowncallSignature(returnType = SINT32, argumentTypes = {POINTER, SINT32}) abstract int init_constants(long out, int len); @DowncallSignature(critical = true, returnType = SINT32) abstract int get_errno(); - @DowncallSignature(returnType = SINT64, argumentTypes = {SINT64, SINT32, SINT32, SINT32, SINT64}) - abstract long call_mmap(long length, int prot, int flags, int fd, long offset); + @DowncallSignature(critical = true, returnType = SINT32) + abstract int get_winerror(); + + @DowncallSignature(critical = true, returnType = SINT32) + abstract int get_wsaerror(); + + @DowncallSignature(critical = true, returnType = SINT32) + abstract int get_error_source(); + + @DowncallSignature(returnType = SINT64, argumentTypes = {SINT64, SINT32, SINT32, SINT32, SINT64, POINTER}) + abstract long call_mmap(long length, int prot, int flags, int fd, long offset, long tagname); @DowncallSignature(returnType = SINT32, argumentTypes = {SINT64, SINT64}) abstract int call_munmap(long address, long length); @@ -212,6 +235,12 @@ abstract static class PosixNativeFunctionInvoker { @DowncallSignature(returnType = SINT32, argumentTypes = {SINT32, SINT32, SINT32}) abstract int call_dup2(int oldfd, int newfd, int inheritable); + @DowncallSignature(returnType = SINT32, argumentTypes = {SINT32, POINTER}) + abstract int call_get_osfhandle(int fd, long out); + + @DowncallSignature(returnType = SINT32, argumentTypes = {SINT64, SINT32}) + abstract int call_open_osfhandle(long handle, int flags); + @DowncallSignature(returnType = SINT32, argumentTypes = {POINTER}) abstract int call_pipe2(long pipefd); @@ -311,6 +340,9 @@ abstract static class PosixNativeFunctionInvoker { @DowncallSignature(returnType = SINT32, argumentTypes = {SINT32, POINTER, SINT32, POINTER}) abstract int call_renameat(int oldDirFd, long oldPath, int newDirFd, long newPath); + @DowncallSignature(returnType = SINT32, argumentTypes = {SINT32, POINTER, SINT32, POINTER}) + abstract int call_replaceat(int oldDirFd, long oldPath, int newDirFd, long newPath); + @DowncallSignature(returnType = SINT32, argumentTypes = {SINT32, POINTER, SINT32, SINT32, SINT32}) abstract int call_faccessat(int dirFd, long path, int mode, int effectiveIds, int followSymlinks); @@ -670,6 +702,9 @@ public void setEnv(Env env) { if (env.isPreInitialization()) { return; } + if (context.isNativeAccessAllowed()) { + posixNativeFunctionInvoker.call_initialize(); + } // Java NIO (and TruffleFile) do not expect/support changing native working directory since // it is inherently thread-unsafe operation. It is not defined how NIO behaves when native // cwd changes, thus we need to prevent TruffleFile from resolving relative paths using @@ -807,6 +842,28 @@ public void setInheritable(int fd, boolean inheritable) throws PosixException { } } + @ExportMessage + public long getOsfHandle(int fd) throws PosixException { + long nativeOut = NativeMemory.mallocLongArray(1); + try { + if (posixNativeFunctionInvoker.call_get_osfhandle(fd, nativeOut) != 0) { + throw getErrnoAndThrowPosixException(); + } + return NativeMemory.readLong(nativeOut); + } finally { + NativeMemory.free(nativeOut); + } + } + + @ExportMessage + public int openOsfHandle(long handle, int flags) throws PosixException { + int fd = posixNativeFunctionInvoker.call_open_osfhandle(handle, flags); + if (fd < 0) { + throw getErrnoAndThrowPosixException(); + } + return fd; + } + @ExportMessage public int[] pipe() throws PosixException { int[] fds = new int[2]; @@ -1170,14 +1227,17 @@ public void mkdirat(int dirFd, Object pathname, int mode) throws PosixException @ExportMessage public Object getcwd() throws PosixException { for (int bufLen = 1024;; bufLen += 1024) { - Buffer buffer = Buffer.allocate(bufLen); - long nativeBuffer = NativeMemory.mallocByteArray(bufLen); + int byteLen = WINDOWS ? bufLen * 2 : bufLen; + byte[] buffer = new byte[byteLen]; + long nativeBuffer = NativeMemory.mallocByteArray(byteLen); try { int n = posixNativeFunctionInvoker.call_getcwd(nativeBuffer, bufLen); if (n == 0) { - NativeMemory.readByteArrayElements(nativeBuffer, 0, buffer.data, 0, bufLen); - buffer = buffer.withLength(findZero(buffer.data)); - return buffer; + NativeMemory.readByteArrayElements(nativeBuffer, 0, buffer, 0, byteLen); + int length = WINDOWS ? findWideZero(buffer) : findZero(buffer); + byte[] result = new byte[length]; + PythonUtils.arraycopy(buffer, 0, result, 0, length); + return WINDOWS ? new WidePath(result) : new NarrowPath(result); } int errno = getErrno(); if (errno != OSErrorEnum.ERANGE.getNumber()) { @@ -1248,24 +1308,28 @@ public void closedir(Object dirStreamObj) throws PosixException { @ExportMessage public Object readdir(Object dirStreamObj) throws PosixException { - Buffer name = Buffer.allocate(DIRENT_NAME_BUF_LENGTH); + int nameBufBytes = WINDOWS ? DIRENT_NAME_BUF_LENGTH * 2 : DIRENT_NAME_BUF_LENGTH; + byte[] name = new byte[nameBufBytes]; long[] out = new long[2]; long dirStream = ((Long) dirStreamObj).longValue(); long nativeName = NULLPTR; long nativeOut = NULLPTR; try { - nativeName = NativeMemory.mallocByteArray(DIRENT_NAME_BUF_LENGTH); + nativeName = NativeMemory.mallocByteArray(nameBufBytes); nativeOut = NativeMemory.mallocLongArray(out.length); int result; do { result = posixNativeFunctionInvoker.call_readdir(dirStream, nativeName, DIRENT_NAME_BUF_LENGTH, nativeOut); if (result > 0) { - NativeMemory.readByteArrayElements(nativeName, 0, name.data, 0, name.data.length); + NativeMemory.readByteArrayElements(nativeName, 0, name, 0, name.length); } - } while (result > 0 && name.data[0] == '.' && (name.data[1] == 0 || (name.data[1] == '.' && name.data[2] == 0))); + } while (result > 0 && isDotOrDotDot(name)); if (result > 0) { NativeMemory.readLongArrayElements(nativeOut, 0, out, 0, out.length); - return new DirEntry(name.withLength(findZero(name.data)), out[0], (int) out[1]); + int length = WINDOWS ? findWideZero(name) : findZero(name); + byte[] resultName = new byte[length]; + PythonUtils.arraycopy(name, 0, resultName, 0, length); + return new DirEntry(WINDOWS ? new WidePath(resultName) : new NarrowPath(resultName), out[0], (int) out[1]); } if (result < 0) { throw getErrnoAndThrowPosixException(); @@ -1292,31 +1356,37 @@ public Object dirEntryGetName(Object dirEntryObj) { @ExportMessage public static class DirEntryGetPath { @Specialization(guards = "endsWithSlash(scandirPath)") - static Buffer withSlash(@SuppressWarnings("unused") NativePosixSupport receiver, DirEntry dirEntry, Object scandirPath) { - Buffer scandirPathBuffer = (Buffer) scandirPath; + static NativePath withSlash(@SuppressWarnings("unused") NativePosixSupport receiver, DirEntry dirEntry, Object scandirPath) { + NativePath scandirPathBuffer = (NativePath) scandirPath; int pathLen = scandirPathBuffer.data.length; - int nameLen = (int) dirEntry.name.length; + int nameLen = dirEntry.name.data.length; byte[] buf = new byte[pathLen + nameLen]; PythonUtils.arraycopy(scandirPathBuffer.data, 0, buf, 0, pathLen); PythonUtils.arraycopy(dirEntry.name.data, 0, buf, pathLen, nameLen); - return Buffer.wrap(buf); + return createLike(scandirPathBuffer, buf); } @Specialization(guards = "!endsWithSlash(scandirPath)") - static Buffer withoutSlash(@SuppressWarnings("unused") NativePosixSupport receiver, DirEntry dirEntry, Object scandirPath) { - Buffer scandirPathBuffer = (Buffer) scandirPath; + static NativePath withoutSlash(@SuppressWarnings("unused") NativePosixSupport receiver, DirEntry dirEntry, Object scandirPath) { + NativePath scandirPathBuffer = (NativePath) scandirPath; int pathLen = scandirPathBuffer.data.length; - int nameLen = (int) dirEntry.name.length; - byte[] buf = new byte[pathLen + 1 + nameLen]; + int nameLen = dirEntry.name.data.length; + int separatorBytes = scandirPathBuffer instanceof WidePath ? 2 : 1; + byte[] buf = new byte[pathLen + separatorBytes + nameLen]; PythonUtils.arraycopy(scandirPathBuffer.data, 0, buf, 0, pathLen); - buf[pathLen] = POSIX_FILENAME_SEPARATOR; - PythonUtils.arraycopy(dirEntry.name.data, 0, buf, pathLen + 1, nameLen); - return Buffer.wrap(buf); + buf[pathLen] = (byte) (scandirPathBuffer instanceof WidePath ? '\\' : '/'); + PythonUtils.arraycopy(dirEntry.name.data, 0, buf, pathLen + separatorBytes, nameLen); + return createLike(scandirPathBuffer, buf); } protected static boolean endsWithSlash(Object path) { - Buffer b = (Buffer) path; - return b.data[b.data.length - 1] == POSIX_FILENAME_SEPARATOR; + NativePath b = (NativePath) path; + byte last = b.data[b.data.length - (b instanceof WidePath ? 2 : 1)]; + return last == '/' || last == '\\'; + } + + private static NativePath createLike(NativePath path, byte[] data) { + return path instanceof WidePath ? new WidePath(data) : new NarrowPath(data); } } @@ -1435,6 +1505,23 @@ public void renameat(int oldDirFd, Object oldPath, int newDirFd, Object newPath) } } + @ExportMessage + public void replaceat(int oldDirFd, Object oldPath, int newDirFd, Object newPath) throws PosixException { + long oldPathPtr = NULLPTR; + long newPathPtr = NULLPTR; + try { + oldPathPtr = pathToNativeCString(oldPath); + newPathPtr = pathToNativeCString(newPath); + int ret = posixNativeFunctionInvoker.call_replaceat(oldDirFd, oldPathPtr, newDirFd, newPathPtr); + if (ret != 0) { + throw getErrnoAndThrowPosixException(); + } + } finally { + NativeMemory.free(newPathPtr); + NativeMemory.free(oldPathPtr); + } + } + @ExportMessage public boolean faccessat(int dirFd, Object path, int mode, boolean effectiveIds, boolean followSymlinks) { long pathPtr = pathToNativeCString(path); @@ -1494,17 +1581,20 @@ public void fchown(int fd, long owner, long group) throws PosixException { @ExportMessage public Object readlinkat(int dirFd, Object path) throws PosixException { - Buffer buffer = Buffer.allocate(PATH_MAX.value); + int bufferBytes = WINDOWS ? PATH_MAX.value * 2 : PATH_MAX.value; + byte[] buffer = new byte[bufferBytes]; long pathPtr = pathToNativeCString(path); try { - long nativeBuffer = NativeMemory.mallocByteArrayOrNull(PATH_MAX.value); + long nativeBuffer = NativeMemory.mallocByteArrayOrNull(bufferBytes); try { long n = posixNativeFunctionInvoker.call_readlinkat(dirFd, pathPtr, nativeBuffer, PATH_MAX.value); if (n < 0) { throw getErrnoAndThrowPosixException(); } - NativeMemory.readByteArrayElements(nativeBuffer, 0, buffer.data, 0, (int) n); - return buffer.withLength(n); + int byteLength = WINDOWS ? Math.toIntExact(n * 2) : Math.toIntExact(n); + NativeMemory.readByteArrayElements(nativeBuffer, 0, buffer, 0, byteLength); + byte[] result = PythonUtils.arrayCopyOfRange(buffer, 0, byteLength); + return WINDOWS ? new WidePath(result) : new NarrowPath(result); } finally { NativeMemory.free(nativeBuffer); } @@ -1802,8 +1892,8 @@ public void setenv(Object name, Object value, boolean overwrite) throws PosixExc long namePtr = NULLPTR; long valuePtr = NULLPTR; try { - namePtr = pathToNativeCString(name); - valuePtr = pathToNativeCString(value); + namePtr = opaqueStringToNative(name); + valuePtr = opaqueStringToNative(value); int res = posixNativeFunctionInvoker.call_setenv(namePtr, valuePtr, overwrite ? 1 : 0); if (res == -1) { throw getErrnoAndThrowPosixException(); @@ -1816,7 +1906,7 @@ public void setenv(Object name, Object value, boolean overwrite) throws PosixExc @ExportMessage public void unsetenv(Object name) throws PosixException { - long namePtr = pathToNativeCString(name); + long namePtr = opaqueStringToNative(name); try { int res = posixNativeFunctionInvoker.call_unsetenv(namePtr); if (res == -1) { @@ -1879,7 +1969,7 @@ public int forkExec(Object[] executables, Object[] args, Object cwd, Object[] en offsetsLen += 1; // The +1 in the second argument can overflow only if the buffer contains 2^63-1 // bytes, which is impossible since we are using Java arrays limited to 2^31-1. - dataLen = PythonUtils.addExact(dataLen, ((Buffer) cwd).length + 1L); + dataLen = PythonUtils.addExact(dataLen, ((NativePath) cwd).data.length + 1L); } else { cwdPos = -1; } @@ -1904,8 +1994,8 @@ public int forkExec(Object[] executables, Object[] args, Object cwd, Object[] en offset = encodeCStringArray(data, offset, offsets, envPos, env); } if (cwd != null) { - Buffer buf = (Buffer) cwd; - int strLen = (int) buf.length; + NativePath buf = (NativePath) cwd; + int strLen = buf.data.length; PythonUtils.arraycopy(buf.data, 0, data, (int) offset, strLen); offsets[cwdPos] = offset; offset += strLen + 1L; @@ -1944,6 +2034,10 @@ public int forkExec(Object[] executables, Object[] args, Object cwd, Object[] en @ExportMessage public void execv(Object pathname, Object[] args) throws PosixException { + if (WINDOWS) { + throw newPosixException(OSErrorEnum.ENOSYS.getNumber()); + } + // The following strings and string arrays need to be present in the native function: // - char* - the pathname ('\0'-terminated string) // - char** of arguments ('\0'-terminated strings with an extra NULL at the end) @@ -1958,7 +2052,7 @@ public void execv(Object pathname, Object[] args) throws PosixException { // First we calculate the lengths of the offsets array and the string buffer (dataLen). int offsetsLen = 1 + args.length + 1; - long pathnameLen = ((Buffer) pathname).length; + long pathnameLen = ((NarrowPath) pathname).data.length; long dataLen; try { @@ -1981,7 +2075,7 @@ public void execv(Object pathname, Object[] args) throws PosixException { byte[] data = new byte[(int) dataLen]; long[] offsets = new long[offsetsLen]; - PythonUtils.arraycopy(((Buffer) pathname).data, 0, data, 0, (int) pathnameLen); + PythonUtils.arraycopy(((NarrowPath) pathname).data, 0, data, 0, (int) pathnameLen); long offset = encodeCStringArray(data, pathnameLen + 1L, offsets, 1, args); assert offset == dataLen; @@ -2000,7 +2094,7 @@ public void execv(Object pathname, Object[] args) throws PosixException { @ExportMessage public int system(Object command) { - long commandPtr = pathToNativeCString(command); + long commandPtr = opaqueStringToNative(command); try { return posixNativeFunctionInvoker.call_system(commandPtr); } finally { @@ -2048,12 +2142,26 @@ public MMapHandle(long pointer, long length) { } @ExportMessage - public Object mmap(long length, int prot, int flags, int fd, long offset) throws PosixException { - long address = posixNativeFunctionInvoker.call_mmap(length, prot, flags, fd, offset); - if (address == 0) { - throw getErrnoAndThrowPosixException(); + public Object mmap(long length, int prot, int flags, int fd, long offset, Object tagname, + @Bind Node inliningTarget, + @Exclusive @Cached TruffleString.SwitchEncodingNode switchEncodingNode, + @Exclusive @Cached TruffleString.IsValidNode isValidNode, + @Exclusive @Cached TruffleString.CopyToByteArrayNode copyToByteArrayNode) throws PosixException { + long tagnamePtr = NULLPTR; + try { + if (tagname instanceof TruffleString tagnameString) { + tagnamePtr = WINDOWS + ? stringToNativeUTF16CString(tagnameString, switchEncodingNode, copyToByteArrayNode) + : stringToNativeUTF8CString(inliningTarget, tagnameString, switchEncodingNode, isValidNode, copyToByteArrayNode); + } + long address = posixNativeFunctionInvoker.call_mmap(length, prot, flags, fd, offset, tagnamePtr); + if (address == 0) { + throw getErrnoAndThrowPosixException(); + } + return new MMapHandle(address, length); + } finally { + NativeMemory.free(tagnamePtr); } - return new MMapHandle(address, length); } @ExportMessage @@ -2184,7 +2292,7 @@ public void connect(int sockfd, UniversalSockAddr usa) throws PosixException { nativeAddr = NativeMemory.copyToNativeByteArray(addr.data, 0, addrLen); int result = posixNativeFunctionInvoker.call_connect(sockfd, nativeAddr, addrLen); if (result == -1) { - throw getErrnoAndThrowPosixException(); + throw getConnectErrnoAndThrowPosixException(); } } finally { NativeMemory.free(nativeAddr); @@ -2371,7 +2479,7 @@ public void setsockopt(int sockfd, int level, int optname, byte[] optval, int op @ExportMessage public int inet_addr(Object src) { - long srcPtr = pathToNativeCString(src); + long srcPtr = bufferToNativeCString((Buffer) src); try { return posixNativeFunctionInvoker.call_inet_addr(srcPtr); } finally { @@ -2381,7 +2489,7 @@ public int inet_addr(Object src) { @ExportMessage public int inet_aton(Object src) throws InvalidAddressException { - long srcPtr = pathToNativeCString(src); + long srcPtr = bufferToNativeCString((Buffer) src); try { long r = posixNativeFunctionInvoker.call_inet_aton(srcPtr); if (r < 0) { @@ -2414,7 +2522,7 @@ public byte[] inet_pton(int family, Object src) throws PosixException, InvalidAd long srcPtr = NULLPTR; long nativeBuf = NULLPTR; try { - srcPtr = pathToNativeCString(src); + srcPtr = bufferToNativeCString((Buffer) src); nativeBuf = NativeMemory.mallocByteArray(buf.length); int res = posixNativeFunctionInvoker.call_inet_pton(family, srcPtr, nativeBuf); // Rather unusually, the return value of 0 does not indicate success but is used by @@ -2461,15 +2569,21 @@ public Object inet_ntop(int family, byte[] src) throws PosixException { @ExportMessage public Object gethostname() throws PosixException { int maxLen = (HOST_NAME_MAX.defined ? HOST_NAME_MAX.getValueIfDefined() : _POSIX_HOST_NAME_MAX.value) + 1; - Buffer buf = Buffer.allocate(maxLen); - long nativeBuf = NativeMemory.mallocByteArray(maxLen); + if (maxLen <= 1) { + maxLen = NI_MAXHOST.value; + } + int bufferBytes = WINDOWS ? maxLen * 2 : maxLen; + byte[] buf = new byte[bufferBytes]; + long nativeBuf = NativeMemory.mallocByteArray(bufferBytes); try { int res = posixNativeFunctionInvoker.call_gethostname(nativeBuf, maxLen); if (res != 0) { throw getErrnoAndThrowPosixException(); } - NativeMemory.readByteArrayElements(nativeBuf, 0, buf.data, 0, buf.data.length); - return buf.withLength(findZero(buf.data)); + NativeMemory.readByteArrayElements(nativeBuf, 0, buf, 0, buf.length); + int length = WINDOWS ? findWideZero(buf) : findZero(buf); + byte[] result = PythonUtils.arrayCopyOfRange(buf, 0, length); + return WINDOWS ? new WideString(result) : Buffer.wrap(result); } finally { NativeMemory.free(nativeBuf); } @@ -2514,8 +2628,8 @@ public AddrInfoCursor getaddrinfo(Object node, Object service, int family, int s long servicePtr = NULLPTR; long nativePtr = NULLPTR; try { - nodePtr = pathToNativeCStringOrNull(node); - servicePtr = pathToNativeCStringOrNull(service); + nodePtr = bufferToNativeCStringOrNull(node); + servicePtr = bufferToNativeCStringOrNull(service); nativePtr = NativeMemory.mallocLongArray(1); int res = posixNativeFunctionInvoker.call_getaddrinfo(nodePtr, servicePtr, family, sockType, protocol, flags, nativePtr); if (res != 0) { @@ -2535,6 +2649,7 @@ public AddrInfoCursor getaddrinfo(Object node, Object service, int family, int s public TruffleString crypt(TruffleString word, TruffleString salt, @Bind Node raisingNode, @Exclusive @Cached TruffleString.SwitchEncodingNode switchEncodingToUtf8Node, + @Exclusive @Cached TruffleString.IsValidNode isValidNode, @Exclusive @Cached TruffleString.CopyToByteArrayNode copyToByteArrayNode, @Exclusive @Cached NativeMemory.ZeroTerminatedUtf8ToTruffleStringNode zeroTerminatedUtf8ToTruffleStringNode) throws PosixException { /* @@ -2548,8 +2663,8 @@ public TruffleString crypt(TruffleString word, TruffleString salt, long wordPtr = NULLPTR; long saltPtr = NULLPTR; try { - wordPtr = stringToNativeUTF8CString(word, switchEncodingToUtf8Node, copyToByteArrayNode); - saltPtr = stringToNativeUTF8CString(salt, switchEncodingToUtf8Node, copyToByteArrayNode); + wordPtr = stringToNativeUTF8CString(raisingNode, word, switchEncodingToUtf8Node, isValidNode, copyToByteArrayNode); + saltPtr = stringToNativeUTF8CString(raisingNode, salt, switchEncodingToUtf8Node, isValidNode, copyToByteArrayNode); // Note GIL is not enough as crypt is using global memory, we need a really global lock synchronized (CRYPT_LOCK) { long resultPtr; @@ -2922,7 +3037,7 @@ void setLen(int len) { @ExportMessage long semOpen(Object name, int openFlags, int mode, int value) throws PosixException { - long namePtr = pathToNativeCString(name); + long namePtr = bufferToNativeCString((Buffer) name); try { long ptr = posixNativeFunctionInvoker.call_sem_open(namePtr, openFlags, mode, value); if (ptr == NULLPTR) { @@ -2944,7 +3059,7 @@ void semClose(long handle) throws PosixException { @ExportMessage void semUnlink(Object name) throws PosixException { - long namePtr = pathToNativeCString(name); + long namePtr = bufferToNativeCString((Buffer) name); try { int res = posixNativeFunctionInvoker.call_sem_unlink(namePtr); if (res < 0) { @@ -2957,7 +3072,7 @@ void semUnlink(Object name) throws PosixException { @ExportMessage int shmOpen(Object name, int openFlags, int mode) throws PosixException { - long namePtr = pathToNativeCString(name); + long namePtr = bufferToNativeCString((Buffer) name); try { int fd = posixNativeFunctionInvoker.call_shm_open(namePtr, openFlags, mode); if (fd < 0) { @@ -2971,7 +3086,7 @@ int shmOpen(Object name, int openFlags, int mode) throws PosixException { @ExportMessage void shmUnlink(Object name) throws PosixException { - long namePtr = pathToNativeCString(name); + long namePtr = bufferToNativeCString((Buffer) name); try { int res = posixNativeFunctionInvoker.call_shm_unlink(namePtr); if (res < 0) { @@ -2987,10 +3102,11 @@ void shmUnlink(Object name) throws PosixException { @ExportMessage int semGetValue(long handle) throws PosixException { /* - * msimacek: It works on Linux, and it doesn't work on Darwin. It might work on some other - * Unix-likes, but it's hard to check, so let's assume it only works on Linux for now + * This works on Linux and is emulated with Windows semaphore APIs as on CPython. It + * doesn't work on Darwin. It might work on some other Unix-likes, but it's hard to check, + * so keep the allow-list narrow. */ - if (PythonLanguage.getPythonOS() != PythonOS.PLATFORM_LINUX) { + if (PythonLanguage.getPythonOS() != PythonOS.PLATFORM_LINUX && PythonLanguage.getPythonOS() != PythonOS.PLATFORM_WIN32) { throw NO_SEM_GETVALUE_EXCEPTION; } long nativeValue = NativeMemory.mallocIntArray(1); @@ -3009,7 +3125,7 @@ int semGetValue(long handle) throws PosixException { void semPost(long handle) throws PosixException { int res = posixNativeFunctionInvoker.call_sem_post(handle); if (res < 0) { - throw getErrnoAndThrowPosixException(); + throw getSemPostErrnoAndThrowPosixException(); } } @@ -3077,7 +3193,7 @@ public PwdResult getpwuid(long uid, public PwdResult getpwnam(Object name, @Shared("tsFromBytes") @Cached TruffleString.FromByteArrayNode fromByteArrayNode, @Shared("fromUtf8") @Cached TruffleString.SwitchEncodingNode switchEncodingFromUtf8Node) throws PosixException { - long namePtr = pathToNativeCString(name); + long namePtr = bufferToNativeCString((Buffer) name); try { return getpw(-1, namePtr, fromByteArrayNode, switchEncodingFromUtf8Node); } finally { @@ -3265,14 +3381,27 @@ private int getSysConfPwdSizeMax() throws PosixException { public Object createPathFromString(TruffleString path, @Exclusive @Cached TruffleString.SwitchEncodingNode switchEncodingNode, @Exclusive @Cached TruffleString.CopyToByteArrayNode copyToByteArrayNode) { + if (WINDOWS) { + TruffleString utf16 = switchEncodingNode.execute(path, UTF_16LE, TranscodingErrorHandler.DEFAULT_KEEP_SURROGATES_IN_UTF8); + return checkWidePath(copyToByteArrayNode.execute(utf16, UTF_16LE)); + } TruffleString utf8 = switchEncodingNode.execute(path, UTF_8, SURROGATE_ESCAPE_TO_UTF8_TRANSCODING_ERROR_HANDLER); - return checkPath(copyToByteArrayNode.execute(utf8, UTF_8)); + return checkNarrowPath(copyToByteArrayNode.execute(utf8, UTF_8)); } @ExportMessage @SuppressWarnings("static-method") - public Object createPathFromBytes(byte[] path) { - return checkPath(path); + public Object createPathFromBytes(byte[] path, + @Bind Node inliningTarget, + @Exclusive @Cached TruffleString.FromByteArrayNode fromByteArrayNode, + @Exclusive @Cached TruffleString.SwitchEncodingNode switchEncodingNode, + @Exclusive @Cached TruffleString.CopyToByteArrayNode copyToByteArrayNode) { + if (WINDOWS) { + TruffleString utf8 = fromByteArrayNode.execute(path, UTF_8, true); + TruffleString utf16 = switchEncodingNode.execute(utf8, UTF_16LE, windowsPathDecodeErrorHandler(inliningTarget, path)); + return checkWidePath(copyToByteArrayNode.execute(utf16, UTF_16LE)); + } + return checkNarrowPath(path); } @ExportMessage @@ -3281,24 +3410,94 @@ public TruffleString getPathAsString(Object path, @Exclusive @Cached TruffleString.FromByteArrayNode fromByteArrayNode, @Exclusive @Cached TruffleString.IsValidNode isValidNode, @Exclusive @Cached TruffleString.SwitchEncodingNode switchEncodingNode) { - Buffer result = (Buffer) path; - if (result.length > Integer.MAX_VALUE) { - // sanity check that it is safe to cast result.length to int, to be removed once - // we support large arrays - throw CompilerDirectives.shouldNotReachHere("Posix path cannot fit into a Java array"); - } - int length = (int) result.length; - TruffleString utf8 = fromByteArrayNode.execute(result.data, 0, length, UTF_8, true); + NativePath result = (NativePath) path; + TruffleString encoded = fromByteArrayNode.execute(result.data, 0, result.data.length, result.encoding(), true); + if (result instanceof WidePath) { + return switchEncodingNode.execute(encoded, TS_ENCODING, TranscodingErrorHandler.DEFAULT_KEEP_SURROGATES_IN_UTF8); + } + TruffleString utf8 = encoded; if (isValidNode.execute(utf8, UTF_8)) { return switchEncodingNode.execute(utf8, TS_ENCODING); } - return switchEncodingNode.execute(utf8, TS_ENCODING, PyUnicodeFSDecoderNode.SURROGATE_ESCAPE_FROM_UTF8_TRANSCODING_ERROR_HANDLER); + TranscodingErrorHandler errorHandler = PythonLanguage.getPythonOS() == PythonOS.PLATFORM_WIN32 + ? TranscodingErrorHandler.DEFAULT_KEEP_SURROGATES_IN_UTF8 + : PyUnicodeFSDecoderNode.SURROGATE_ESCAPE_FROM_UTF8_TRANSCODING_ERROR_HANDLER; + return switchEncodingNode.execute(utf8, TS_ENCODING, errorHandler); } @ExportMessage @SuppressWarnings("static-method") public Buffer getPathAsBytes(Object path) { - return (Buffer) path; + NativePath nativePath = (NativePath) path; + if (nativePath instanceof NarrowPath) { + return Buffer.wrap(nativePath.data); + } + TruffleString utf16 = TruffleString.fromByteArrayUncached(nativePath.data, UTF_16LE); + TruffleString utf8 = utf16.switchEncodingUncached(UTF_8, TranscodingErrorHandler.DEFAULT_KEEP_SURROGATES_IN_UTF8); + InternalByteArray bytes = utf8.getInternalByteArrayUncached(UTF_8); + return Buffer.wrap(PythonUtils.arrayCopyOfRange(bytes.getArray(), bytes.getOffset(), bytes.getEnd())); + } + + private static TranscodingErrorHandler windowsPathDecodeErrorHandler(Node inliningTarget, byte[] input) { + return (AbstractTruffleString sourceString, int byteIndex, int estimatedByteLength, TruffleString.Encoding sourceEncoding, + TruffleString.Encoding targetEncoding) -> { + if (byteIndex + 2 < input.length && (input[byteIndex] & 0xff) == 0xed && + (input[byteIndex + 1] & 0xe0) == 0xa0 && (input[byteIndex + 2] & 0xc0) == 0x80) { + int codePoint = ((input[byteIndex] & 0x0f) << 12) | ((input[byteIndex + 1] & 0x3f) << 6) | (input[byteIndex + 2] & 0x3f); + return new TranscodingErrorHandler.ReplacementString(TruffleString.fromCodePointUncached(codePoint, UTF_16LE, true), 3); + } + Object exception = CallNode.executeUncached(PythonBuiltinClassType.UnicodeDecodeError, + PythonUtils.toTruffleStringUncached("utf-8"), PFactory.createBytes(PythonLanguage.get(inliningTarget), input), + byteIndex, Math.min(input.length, byteIndex + Math.max(1, estimatedByteLength)), + PythonUtils.toTruffleStringUncached("invalid UTF-8 path")); + throw PRaiseNode.raiseExceptionObjectStatic(inliningTarget, exception); + }; + } + + @ExportMessage + public Object createCStringFromString(TruffleString string, + @Bind Node inliningTarget, + @Exclusive @Cached TruffleString.SwitchEncodingNode switchEncodingNode, + @Exclusive @Cached TruffleString.IsValidNode isValidNode, + @Exclusive @Cached TruffleString.CopyToByteArrayNode copyToByteArrayNode) { + return checkCString(getUTF8StringBytes(inliningTarget, string, switchEncodingNode, isValidNode, copyToByteArrayNode)); + } + + @ExportMessage + public Object createCStringFromBytes(byte[] bytes) { + return checkCString(bytes); + } + + @ExportMessage + public Object createWideStringFromString(TruffleString string, + @Exclusive @Cached TruffleString.SwitchEncodingNode switchEncodingNode, + @Exclusive @Cached TruffleString.CopyToByteArrayNode copyToByteArrayNode) { + TruffleString utf16 = switchEncodingNode.execute(string, UTF_16LE, TranscodingErrorHandler.DEFAULT_KEEP_SURROGATES_IN_UTF8); + byte[] bytes = copyToByteArrayNode.execute(utf16, UTF_16LE); + return checkWideString(bytes); + } + + @ExportMessage + public TruffleString getCStringAsString(Object string, + @Exclusive @Cached TruffleString.FromByteArrayNode fromByteArrayNode, + @Exclusive @Cached TruffleString.SwitchEncodingNode switchEncodingNode) { + if (string instanceof WideString wideString) { + TruffleString utf16 = fromByteArrayNode.execute(wideString.data, UTF_16LE, true); + return switchEncodingNode.execute(utf16, TS_ENCODING, TranscodingErrorHandler.DEFAULT_KEEP_SURROGATES_IN_UTF8); + } + Buffer buffer = (Buffer) string; + return createString(buffer.data, 0, (int) buffer.length, true, fromByteArrayNode, switchEncodingNode); + } + + @ExportMessage + public Buffer getCStringAsBytes(Object string) { + if (string instanceof WideString wideString) { + TruffleString utf16 = TruffleString.fromByteArrayUncached(wideString.data, UTF_16LE); + TruffleString utf8 = utf16.switchEncodingUncached(UTF_8, TranscodingErrorHandler.DEFAULT_KEEP_SURROGATES_IN_UTF8); + InternalByteArray bytes = utf8.getInternalByteArrayUncached(UTF_8); + return Buffer.wrap(PythonUtils.arrayCopyOfRange(bytes.getArray(), bytes.getOffset(), bytes.getEnd())); + } + return (Buffer) string; } private static TruffleString createString(byte[] src, int offset, int length, boolean copy, TruffleString.FromByteArrayNode fromByteArrayNode, @@ -3307,14 +3506,37 @@ private static TruffleString createString(byte[] src, int offset, int length, bo return switchEncodingNode.execute(utf8, TS_ENCODING); } - private static byte[] getStringBytes(TruffleString str, TruffleString.SwitchEncodingNode switchEncodingNode, TruffleString.CopyToByteArrayNode copyToByteArrayNode) { + private static byte[] getUTF8StringBytes(Node node, TruffleString str, TruffleString.SwitchEncodingNode switchEncodingNode, TruffleString.IsValidNode isValidNode, + TruffleString.CopyToByteArrayNode copyToByteArrayNode) { + if (!isValidNode.execute(str, TS_ENCODING)) { + throw raiseSurrogatesEncodeError(node, str); + } TruffleString utf8 = switchEncodingNode.execute(str, UTF_8); byte[] bytes = new byte[utf8.byteLength(UTF_8)]; copyToByteArrayNode.execute(utf8, 0, bytes, 0, bytes.length, UTF_8); return bytes; } - private static Buffer checkPath(byte[] path) { + @TruffleBoundary + private static PException raiseSurrogatesEncodeError(Node node, TruffleString str) { + int byteIndex = TruffleString.ByteIndexOfCodePointSetNode.getUncached().execute(str, 0, str.byteLength(TS_ENCODING), SURROGATE_CODE_POINT_SET); + int start = byteIndex < 0 ? 0 : byteIndex / 4; + int length = str.codePointLengthUncached(TS_ENCODING); + int end = Math.min(start + 1, length); + while (end < length) { + int codePoint = str.codePointAtIndexUncached(end, TS_ENCODING); + if (codePoint < Character.MIN_SURROGATE || codePoint > Character.MAX_SURROGATE) { + break; + } + end++; + } + Object exception = CallNode.executeUncached(PythonBuiltinClassType.UnicodeEncodeError, + PythonUtils.toTruffleStringUncached("utf-8"), str, start, end, + PythonUtils.toTruffleStringUncached("surrogates not allowed")); + return PRaiseNode.raiseExceptionObjectStatic(node, exception); + } + + private static Buffer checkCString(byte[] path) { for (byte b : path) { if (b == 0) { return null; @@ -3326,15 +3548,79 @@ private static Buffer checkPath(byte[] path) { return Buffer.wrap(path); } + private static NarrowPath checkNarrowPath(byte[] path) { + return checkCString(path) == null ? null : new NarrowPath(path); + } + + private static WidePath checkWidePath(byte[] path) { + assert (path.length & 1) == 0; + for (int i = 0; i < path.length; i += 2) { + if (path[i] == 0 && path[i + 1] == 0) { + return null; + } + } + return new WidePath(path); + } + + private static WideString checkWideString(byte[] string) { + assert (string.length & 1) == 0; + for (int i = 0; i < string.length; i += 2) { + if (string[i] == 0 && string[i + 1] == 0) { + return null; + } + } + return new WideString(string); + } + // ------------------ // Objects/handles/pointers + protected abstract static class NativePath { + final byte[] data; + + NativePath(byte[] data) { + this.data = data; + } + + abstract TruffleString.Encoding encoding(); + } + + private static final class NarrowPath extends NativePath { + NarrowPath(byte[] data) { + super(data); + } + + @Override + TruffleString.Encoding encoding() { + return UTF_8; + } + } + + private static final class WidePath extends NativePath { + WidePath(byte[] data) { + super(data); + } + + @Override + TruffleString.Encoding encoding() { + return UTF_16LE; + } + } + + private static final class WideString { + final byte[] data; + + WideString(byte[] data) { + this.data = data; + } + } + protected static class DirEntry { - final Buffer name; + final NativePath name; final long ino; final int type; - DirEntry(Buffer name, long ino, int type) { + DirEntry(NativePath name, long ino, int type) { this.name = name; this.ino = ino; this.type = type; @@ -3343,7 +3629,7 @@ protected static class DirEntry { @Override public String toString() { return "DirEntry{" + - "name='" + new String(name.data, 0, (int) name.length) + "'" + + "name='" + name + "'" + ", ino=" + ino + ", type=" + type + '}'; @@ -3353,13 +3639,170 @@ public String toString() { // ------------------ // Helpers + @TruffleBoundary private PosixException getErrnoAndThrowPosixException() throws PosixException { throw newPosixException(getErrno()); } + @TruffleBoundary + private PosixException getConnectErrnoAndThrowPosixException() throws PosixException { + if (PythonLanguage.getPythonOS() == PythonOS.PLATFORM_WIN32 && posixNativeFunctionInvoker.get_error_source() == ERROR_SOURCE_WINSOCK) { + int winerror = posixNativeFunctionInvoker.get_wsaerror(); + // A nonblocking Winsock connect reports WSAEWOULDBLOCK, but the socket layer uses + // EINPROGRESS to decide whether to wait for the connection to complete. + if (winerror == WSAError.WSAEWOULDBLOCK.getNumber()) { + throw newPosixException(OSErrorEnum.EINPROGRESS.getNumber(), winerror); + } + } + throw newPosixException(getErrno()); + } + + @TruffleBoundary + private PosixException getSemPostErrnoAndThrowPosixException() throws PosixException { + if (PythonLanguage.getPythonOS() == PythonOS.PLATFORM_WIN32 && posixNativeFunctionInvoker.get_error_source() == ERROR_SOURCE_WINAPI) { + int winerror = posixNativeFunctionInvoker.get_winerror(); + // ReleaseSemaphore reports ERROR_TOO_MANY_POSTS when sem_post would exceed the + // semaphore's maximum count. POSIX exposes that condition as EOVERFLOW. + if (winerror == WinAPIError.ERROR_TOO_MANY_POSTS.getNumber()) { + throw newPosixException(OSErrorEnum.EOVERFLOW.getNumber(), winerror); + } + } + throw newPosixException(getErrno()); + } + @TruffleBoundary private PosixException newPosixException(int errno) throws PosixException { - throw new PosixErrnoException(errno, strerror(errno, null, NativeMemory.ZeroTerminatedUtf8ToTruffleStringNode.getUncached())); + Integer winerror = null; + if (PythonLanguage.getPythonOS() == PythonOS.PLATFORM_WIN32) { + switch (posixNativeFunctionInvoker.get_error_source()) { + case ERROR_SOURCE_WINAPI -> { + winerror = posixNativeFunctionInvoker.get_winerror(); + errno = winapiErrorToErrno(winerror); + } + case ERROR_SOURCE_WINSOCK -> { + winerror = posixNativeFunctionInvoker.get_wsaerror(); + errno = winsockErrorToErrno(winerror); + } + default -> { + // everything already in-place + } + } + } + throw newPosixException(errno, winerror); + } + + @TruffleBoundary + private PosixException newPosixException(int errno, Integer winerror) throws PosixException { + throw new PosixErrnoException(errno, strerror(errno, null, NativeMemory.ZeroTerminatedUtf8ToTruffleStringNode.getUncached()), winerror); + } + + // CPython performs this mapping in PC/errmap.h:winerror_to_errno(). + private static int winapiErrorToErrno(int error) { + WinAPIError winAPIError = WinAPIError.fromNumber(error); + return winAPIError == null ? error : winAPIError.getDefaultErrnoNumber(); + } + + // CPython handles Winsock errors in PC/errmap.h:winerror_to_errno(). + private static int winsockErrorToErrno(int error) { + WSAError wsaError = WSAError.fromNumber(error); + return wsaError == null ? error : wsaError.getDefaultErrnoNumber(); + } + + private enum WinAPIError { + ERROR_FILE_NOT_FOUND(2, OSErrorEnum.ENOENT), + ERROR_PATH_NOT_FOUND(3, OSErrorEnum.ENOENT), + ERROR_ACCESS_DENIED(5, OSErrorEnum.EACCES), + ERROR_INVALID_PARAMETER(87, OSErrorEnum.EINVAL), + ERROR_INVALID_NAME(123, OSErrorEnum.EINVAL), + ERROR_DIRECTORY(267, OSErrorEnum.ENOTDIR), + ERROR_TOO_MANY_POSTS(298), + ERROR_NO_UNICODE_TRANSLATION(1113, OSErrorEnum.ENOENT); + + private final int number; + private final OSErrorEnum defaultErrno; + + WinAPIError(int number) { + this(number, null); + } + + WinAPIError(int number, OSErrorEnum defaultErrno) { + this.number = number; + this.defaultErrno = defaultErrno; + } + + int getNumber() { + return number; + } + + int getDefaultErrnoNumber() { + return defaultErrno == null ? number : defaultErrno.getNumber(); + } + + static WinAPIError fromNumber(int number) { + for (WinAPIError error : values()) { + if (error.number == number) { + return error; + } + } + return null; + } + } + + private enum WSAError { + WSAEINTR(10004, OSErrorEnum.EINTR), + WSAEBADF(10009, OSErrorEnum.EBADF), + WSAEACCES(10013, OSErrorEnum.EACCES), + WSAEFAULT(10014, OSErrorEnum.EFAULT), + WSAEINVAL(10022, OSErrorEnum.EINVAL), + WSAEMFILE(10024, OSErrorEnum.EMFILE), + WSAEWOULDBLOCK(10035, OSErrorEnum.EWOULDBLOCK), + WSAEINPROGRESS(10036, OSErrorEnum.EINPROGRESS), + WSAEALREADY(10037, OSErrorEnum.EALREADY), + WSAENOTSOCK(10038, OSErrorEnum.ENOTSOCK), + WSAEDESTADDRREQ(10039, OSErrorEnum.EDESTADDRREQ), + WSAEMSGSIZE(10040, OSErrorEnum.EMSGSIZE), + WSAEPROTOTYPE(10041, OSErrorEnum.EPROTOTYPE), + WSAENOPROTOOPT(10042, OSErrorEnum.ENOPROTOOPT), + WSAEPROTONOSUPPORT(10043, OSErrorEnum.EPROTONOSUPPORT), + WSAEAFNOSUPPORT(10047, OSErrorEnum.EAFNOSUPPORT), + WSAEADDRINUSE(10048, OSErrorEnum.EADDRINUSE), + WSAEADDRNOTAVAIL(10049, OSErrorEnum.EADDRNOTAVAIL), + WSAENETDOWN(10050, OSErrorEnum.ENETDOWN), + WSAENETUNREACH(10051, OSErrorEnum.ENETUNREACH), + WSAENETRESET(10052, OSErrorEnum.ENETRESET), + WSAECONNABORTED(10053, OSErrorEnum.ECONNABORTED), + WSAECONNRESET(10054, OSErrorEnum.ECONNRESET), + WSAENOBUFS(10055, OSErrorEnum.ENOBUFS), + WSAEISCONN(10056, OSErrorEnum.EISCONN), + WSAENOTCONN(10057, OSErrorEnum.ENOTCONN), + WSAETIMEDOUT(10060, OSErrorEnum.ETIMEDOUT), + WSAECONNREFUSED(10061, OSErrorEnum.ECONNREFUSED), + WSAEHOSTUNREACH(10065, OSErrorEnum.EHOSTUNREACH); + + private final int number; + private final OSErrorEnum defaultErrno; + + WSAError(int number, OSErrorEnum defaultErrno) { + this.number = number; + this.defaultErrno = defaultErrno; + } + + int getNumber() { + return number; + } + + int getDefaultErrnoNumber() { + return defaultErrno.getNumber(); + } + + static WSAError fromNumber(int number) { + for (WSAError error : values()) { + if (error.number == number) { + return error; + } + } + return null; + } } private static long copyTimevalArrayToNativeOrNull(Timeval[] timeval) { @@ -3402,25 +3845,80 @@ private static int findZero(byte[] buf) { return buf.length; } + private static int findWideZero(byte[] buf) { + for (int i = 0; i + 1 < buf.length; i += 2) { + if (buf[i] == 0 && buf[i + 1] == 0) { + return i; + } + } + return buf.length; + } + + private static boolean isDotOrDotDot(byte[] name) { + if (WINDOWS) { + return name[0] == '.' && name[1] == 0 && ((name[2] == 0 && name[3] == 0) || + (name[2] == '.' && name[3] == 0 && name[4] == 0 && name[5] == 0)); + } + return name[0] == '.' && (name[1] == 0 || (name[1] == '.' && name[2] == 0)); + } + private long pathToNativeCStringOrNull(Object path) { - return path == null ? NULLPTR : bufferToNativeCString((Buffer) path); + return path == null ? NULLPTR : pathToNativeCString(path); } private long pathToNativeCString(Object path) { - return bufferToNativeCString((Buffer) path); + NativePath nativePath = (NativePath) path; + if (nativePath instanceof NarrowPath) { + return NativeMemory.copyToNativeZeroTerminatedByteArray(nativePath.data, 0, nativePath.data.length); + } + long result = NativeMemory.mallocByteArray(nativePath.data.length + 2L); + NativeMemory.writeByteArrayElements(result, 0, nativePath.data, 0, nativePath.data.length); + NativeMemory.writeByteArrayElement(result, nativePath.data.length, (byte) 0); + NativeMemory.writeByteArrayElement(result, nativePath.data.length + 1L, (byte) 0); + return result; } private static long bufferToNativeCString(Buffer path) { return NativeMemory.copyToNativeZeroTerminatedByteArray(path.data, 0, (int) path.length); } - private long stringToNativeUTF8CString(TruffleString input, + private static long bufferToNativeCStringOrNull(Object string) { + return string == null ? NULLPTR : bufferToNativeCString((Buffer) string); + } + + private static long opaqueStringToNative(Object string) { + if (string instanceof WideString wideString) { + return bytesToNativeUTF16CString(wideString.data); + } + if (string instanceof NarrowPath narrowPath) { + return NativeMemory.copyToNativeZeroTerminatedByteArray(narrowPath.data, 0, narrowPath.data.length); + } + return bufferToNativeCString((Buffer) string); + } + + private static long stringToNativeUTF8CString(Node node, TruffleString input, @Cached TruffleString.SwitchEncodingNode switchEncodingToUtf8Node, + @Cached TruffleString.IsValidNode isValidNode, @Cached TruffleString.CopyToByteArrayNode copyToByteArrayNode) { - byte[] utf8 = getStringBytes(input, switchEncodingToUtf8Node, copyToByteArrayNode); + byte[] utf8 = getUTF8StringBytes(node, input, switchEncodingToUtf8Node, isValidNode, copyToByteArrayNode); return NativeMemory.copyToNativeZeroTerminatedByteArray(utf8, 0, utf8.length); } + private static long stringToNativeUTF16CString(TruffleString input, + TruffleString.SwitchEncodingNode switchEncodingNode, + TruffleString.CopyToByteArrayNode copyToByteArrayNode) { + TruffleString utf16 = switchEncodingNode.execute(input, UTF_16LE, TranscodingErrorHandler.DEFAULT_KEEP_SURROGATES_IN_UTF8); + return bytesToNativeUTF16CString(copyToByteArrayNode.execute(utf16, UTF_16LE)); + } + + private static long bytesToNativeUTF16CString(byte[] utf16) { + long result = NativeMemory.mallocByteArray(utf16.length + 2L); + NativeMemory.writeByteArrayElements(result, 0, utf16, 0, utf16.length); + NativeMemory.writeByteArrayElement(result, utf16.length, (byte) 0); + NativeMemory.writeByteArrayElement(result, utf16.length + 1L, (byte) 0); + return result; + } + private static void checkBounds(byte[] buf, int offset, int length) { if (length < 0) { CompilerDirectives.transferToInterpreterAndInvalidate(); diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PosixSupportLibrary.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PosixSupportLibrary.java index 3b987d2294..db73974bb8 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PosixSupportLibrary.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PosixSupportLibrary.java @@ -104,6 +104,10 @@ public abstract class PosixSupportLibrary extends Library { public abstract void setInheritable(Object receiver, int fd, boolean inheritable) throws PosixException; + public abstract long getOsfHandle(Object receiver, int fd) throws PosixException; + + public abstract int openOsfHandle(Object receiver, long handle, int flags) throws PosixException; + public abstract int[] pipe(Object receiver) throws PosixException; public abstract SelectResult select(Object receiver, int[] readfds, int[] writefds, int[] errorfds, Timeval timeout) throws PosixException; @@ -247,6 +251,8 @@ public abstract class PosixSupportLibrary extends Library { public abstract void renameat(Object receiver, int oldDirFd, Object oldPath, int newDirFd, Object newPath) throws PosixException; + public abstract void replaceat(Object receiver, int oldDirFd, Object oldPath, int newDirFd, Object newPath) throws PosixException; + public abstract boolean faccessat(Object receiver, int dirFd, Object path, int mode, boolean effectiveIds, boolean followSymlinks) throws UnsupportedPosixFeatureException; public abstract void fchmodat(Object receiver, int dirFd, Object path, int mode, boolean followSymlinks) throws PosixException; @@ -364,7 +370,7 @@ public abstract int forkExec(Object receiver, Object[] executables, Object[] arg // does not throw, because posix does not exactly define the return value public abstract int system(Object receiver, Object command); - public abstract Object mmap(Object receiver, long length, int prot, int flags, int fd, long offset) throws PosixException; + public abstract Object mmap(Object receiver, long length, int prot, int flags, int fd, long offset, Object tagname) throws PosixException; public abstract byte mmapReadByte(Object receiver, Object mmap, long index) throws PosixException; @@ -490,6 +496,19 @@ public String toString() { public abstract Buffer getPathAsBytes(Object receiver, Object path); + /** Converts a string to strict UTF-8 for non-filesystem APIs that use CPython's {@code s} conversion. */ + public abstract Object createCStringFromString(Object receiver, TruffleString string); + + /** Wraps already-encoded bytes for narrow native APIs without applying a filesystem conversion. */ + public abstract Object createCStringFromBytes(Object receiver, byte[] bytes); + + /** Converts a string to the UTF-16 representation used by Windows wide-character APIs. */ + public abstract Object createWideStringFromString(Object receiver, TruffleString string); + + public abstract TruffleString getCStringAsString(Object receiver, Object string); + + public abstract Buffer getCStringAsBytes(Object receiver, Object string); + // region Socket addresses /** @@ -739,7 +758,8 @@ public String toString() { * information. * * @param src the IPv4 address in numbers-and-dots notation (converted to opaque object using - * createPathFromBytes or createPathFromString) + * {@link #createCStringFromBytes(Object, byte[])} or + * {@link #createCStringFromString(Object, TruffleString)}) * @return address in host byte order or {@link PosixConstants#INADDR_NONE} if the input is * invalid * @see "inet(3) man pages" @@ -752,7 +772,8 @@ public String toString() { * further conversions. * * @param src the IPv4 address in numbers-and-dots notation (converted to opaque object using - * createPathFromBytes or createPathFromString) + * {@link #createCStringFromBytes(Object, byte[])} or + * {@link #createCStringFromString(Object, TruffleString)}) * @return address in host byte order * @throws InvalidAddressException if {@code cp} is not a valid representation of an IPv4 * address @@ -766,8 +787,8 @@ public String toString() { * * @param address tha IPv4 address in host byte order * @return opaque string in IPv4 dotted-decimal notation to be converted using - * {@link PosixSupportLibrary#getPathAsString(Object, Object)} or - * {@link PosixSupportLibrary#getPathAsBytes(Object, Object)} + * {@link #getCStringAsString(Object, Object)} or + * {@link #getCStringAsBytes(Object, Object)} * @see "inet(3) man pages" */ public abstract Object inet_ntoa(Object receiver, int address); @@ -776,7 +797,8 @@ public String toString() { * Corresponds to POSIX {@code inet_pton} function. * * @param family {@code AF_INET} or {@code AF_INET6} - * @param src opaque string (converted using createPathFromBytes or createPathFromString) + * @param src opaque string (converted using {@link #createCStringFromBytes(Object, byte[])} or + * {@link #createCStringFromString(Object, TruffleString)}) * @return the binary address in network order (4 bytes for {@code AF_INET}, 16 bytes for * {@code AF_INET6}) * @throws PosixException with {@code EAFNOSUPPORT} if the {@code family} is not supported @@ -791,7 +813,8 @@ public String toString() { * @param family {@code AF_INET} or {@code AF_INET6} * @param src the address in network order, must be at least 4 (for {@code AF_INET}) or 16 (for * {@code AF_INET6}) bytes long, extra bytes are ignored - * @return an opaque string to be converted using getPathAsString or getPathAsBytes + * @return an opaque string to be converted using {@link #getCStringAsString(Object, Object)} or + * {@link #getCStringAsBytes(Object, Object)} * @throws PosixException with {@code EAFNOSUPPORT} if the {@code family} is not supported * @throws IllegalArgumentException if {@code src} does not satisfy the requirements stated * above @@ -799,7 +822,8 @@ public String toString() { public abstract Object inet_ntop(Object receiver, int family, byte[] src) throws PosixException; /** - * @return an opaque string to be converted using getPathAsString or getPathAsBytes + * @return an opaque string to be converted using {@link #getCStringAsString(Object, Object)} or + * {@link #getCStringAsBytes(Object, Object)} */ public abstract Object gethostname(Object receiver) throws PosixException; @@ -809,8 +833,9 @@ public String toString() { * * @param addr socket address to convert * @param flags a combination of {@code NI_xxx} flags - * @return an array of two (host, service) opaque strings to be converted using getPathAsString - * or getPathAsBytes + * @return an array of two (host, service) opaque strings to be converted using + * {@link #getCStringAsString(Object, Object)} or + * {@link #getCStringAsBytes(Object, Object)} * @throws GetAddrInfoException when an error occurs (PosixException is not thrown because * getnameinfo uses its own error codes and gai_strerror instead of the usual errno * and strerror) @@ -822,11 +847,11 @@ public String toString() { * {@code hints} parameter. * * @param node {@code null} or the host name converted using - * {@link PosixSupportLibrary#createPathFromBytes(Object, byte[])} or - * {@link PosixSupportLibrary#createPathFromString(Object, TruffleString)} + * {@link #createCStringFromBytes(Object, byte[])} or + * {@link #createCStringFromString(Object, TruffleString)} * @param service {@code null} or the service name converted using - * {@link PosixSupportLibrary#createPathFromBytes(Object, byte[])} or - * {@link PosixSupportLibrary#createPathFromString(Object, TruffleString)} + * {@link #createCStringFromBytes(Object, byte[])} or + * {@link #createCStringFromString(Object, TruffleString)} * @param family one of the {@code AF_xxx} constants, or {@link PosixConstants#AF_UNSPEC} to get * addresses of any family * @param sockType one of the {@code SOCK_xxx} constants, or 0 to get addresses of any type @@ -1055,10 +1080,21 @@ public static final class PosixErrnoException extends PosixException { private final int errorCode; private final transient TruffleString msg; + /* + * Windows APIs expose both POSIX errno and a native Win32/Winsock error code. Store the + * native code as java.lang.Integer rather than int because most PosixErrnoException + * instances are not backed by a Windows error, so this field can be null. + */ + private final Integer winerror; public PosixErrnoException(int errorCode, TruffleString message) { + this(errorCode, message, null); + } + + public PosixErrnoException(int errorCode, TruffleString message, Integer winerror) { this.errorCode = errorCode; msg = message; + this.winerror = winerror; } public TruffleString getMessageAsTruffleString() { @@ -1073,6 +1109,10 @@ public String getMessage() { public int getErrorCode() { return errorCode; } + + public Integer getWinerror() { + return winerror; + } } /** diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PreInitPosixSupport.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PreInitPosixSupport.java index 6f01ee26fe..74bd0e165f 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PreInitPosixSupport.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PreInitPosixSupport.java @@ -242,6 +242,20 @@ final void setInheritable(int fd, boolean inheritable, nativeLib.setInheritable(nativePosixSupport, fd, inheritable); } + @ExportMessage + final long getOsfHandle(int fd, + @CachedLibrary("this.nativePosixSupport") PosixSupportLibrary nativeLib) throws PosixException { + checkNotInPreInitialization(); + return nativeLib.getOsfHandle(nativePosixSupport, fd); + } + + @ExportMessage + final int openOsfHandle(long handle, int flags, + @CachedLibrary("this.nativePosixSupport") PosixSupportLibrary nativeLib) throws PosixException { + checkNotInPreInitialization(); + return nativeLib.openOsfHandle(nativePosixSupport, handle, flags); + } + @ExportMessage final int[] pipe(@CachedLibrary("this.nativePosixSupport") PosixSupportLibrary nativeLib) throws PosixException { checkNotInPreInitialization(); @@ -547,6 +561,13 @@ final void renameat(int oldDirFd, Object oldPath, int newDirFd, Object newPath, nativeLib.renameat(nativePosixSupport, oldDirFd, oldPath, newDirFd, newPath); } + @ExportMessage + final void replaceat(int oldDirFd, Object oldPath, int newDirFd, Object newPath, + @CachedLibrary("this.nativePosixSupport") PosixSupportLibrary nativeLib) throws PosixException { + checkNotInPreInitialization(); + nativeLib.replaceat(nativePosixSupport, oldDirFd, oldPath, newDirFd, newPath); + } + @ExportMessage final boolean faccessat(int dirFd, Object path, int mode, boolean effectiveIds, boolean followSymlinks, @CachedLibrary("this.nativePosixSupport") PosixSupportLibrary nativeLib) throws UnsupportedPosixFeatureException { @@ -829,10 +850,10 @@ final int system(Object command, } @ExportMessage - final Object mmap(long length, int prot, int flags, int fd, long offset, + final Object mmap(long length, int prot, int flags, int fd, long offset, Object tagname, @CachedLibrary("this.nativePosixSupport") PosixSupportLibrary nativeLib) throws PosixException { checkNotInPreInitialization(); - return nativeLib.mmap(nativePosixSupport, length, prot, flags, fd, offset); + return nativeLib.mmap(nativePosixSupport, length, prot, flags, fd, offset, tagname); } @ExportMessage @@ -1210,4 +1231,49 @@ final Buffer getPathAsBytes(Object path, } return nativeLib.getPathAsBytes(nativePosixSupport, path); } + + @ExportMessage + final Object createCStringFromString(TruffleString string, + @CachedLibrary("this.nativePosixSupport") PosixSupportLibrary nativeLib) { + if (inPreInitialization) { + return PosixSupportLibrary.getUncached().createCStringFromString(emulatedPosixSupport, string); + } + return nativeLib.createCStringFromString(nativePosixSupport, string); + } + + @ExportMessage + final Object createCStringFromBytes(byte[] bytes, + @CachedLibrary("this.nativePosixSupport") PosixSupportLibrary nativeLib) { + if (inPreInitialization) { + return PosixSupportLibrary.getUncached().createCStringFromBytes(emulatedPosixSupport, bytes); + } + return nativeLib.createCStringFromBytes(nativePosixSupport, bytes); + } + + @ExportMessage + final Object createWideStringFromString(TruffleString string, + @CachedLibrary("this.nativePosixSupport") PosixSupportLibrary nativeLib) { + if (inPreInitialization) { + return PosixSupportLibrary.getUncached().createWideStringFromString(emulatedPosixSupport, string); + } + return nativeLib.createWideStringFromString(nativePosixSupport, string); + } + + @ExportMessage + final TruffleString getCStringAsString(Object string, + @CachedLibrary("this.nativePosixSupport") PosixSupportLibrary nativeLib) { + if (inPreInitialization) { + return PosixSupportLibrary.getUncached().getCStringAsString(emulatedPosixSupport, string); + } + return nativeLib.getCStringAsString(nativePosixSupport, string); + } + + @ExportMessage + final Buffer getCStringAsBytes(Object string, + @CachedLibrary("this.nativePosixSupport") PosixSupportLibrary nativeLib) { + if (inPreInitialization) { + return PosixSupportLibrary.getUncached().getCStringAsBytes(emulatedPosixSupport, string); + } + return nativeLib.getCStringAsBytes(nativePosixSupport, string); + } } diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PythonContext.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PythonContext.java index 805db631f1..7f09720d57 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PythonContext.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PythonContext.java @@ -1018,9 +1018,9 @@ public void setEnv(TruffleLanguage.Env newEnv) { in = env.in(); out = env.out(); err = env.err(); + nativeAccessAllowed = newEnv.isNativeAccessAllowed() && !PythonOS.isUnsupported() && NativeAccessSupport.isAvailable(); posixSupport.setEnv(env); optionValues = PythonOptions.createOptionValuesStorage(newEnv); - nativeAccessAllowed = newEnv.isNativeAccessAllowed() && !PythonOS.isUnsupported() && NativeAccessSupport.isAvailable(); } /** diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/nativeaccess/NativeContext.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/nativeaccess/NativeContext.java index acc3a6a888..63206f5df1 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/nativeaccess/NativeContext.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/nativeaccess/NativeContext.java @@ -43,6 +43,7 @@ import static com.oracle.graal.python.annotations.NativeSimpleType.POINTER; import static com.oracle.graal.python.annotations.NativeSimpleType.SINT32; import static com.oracle.graal.python.annotations.NativeSimpleType.SINT64; +import static com.oracle.graal.python.annotations.NativeSimpleType.VOID; import java.lang.invoke.MethodHandle; import java.util.concurrent.ConcurrentLinkedQueue; @@ -174,6 +175,7 @@ private static boolean isWindows() { private static final MethodHandle DLSYM = NativeAccessSupport.createDowncallHandle(false, false, SINT64, SINT64, POINTER); private static final MethodHandle FREE_LIBRARY = NativeAccessSupport.createDowncallHandle(false, false, SINT32, SINT64); private static final MethodHandle GET_PROC_ADDRESS = NativeAccessSupport.createDowncallHandle(false, false, SINT64, SINT64, POINTER); + private static final MethodHandle SET_LAST_ERROR = NativeAccessSupport.createDowncallHandle(false, false, VOID, SINT32); private static final MethodHandle FORMAT_MESSAGE = NativeAccessSupport.createDowncallHandle(false, false, SINT32, SINT32, POINTER, SINT32, SINT32, POINTER, SINT32, POINTER); private static final MethodHandle DLERROR = NativeAccessSupport.createDowncallHandle(false, false, SINT64); private static final MethodHandle LOAD_LIBRARY_EX_CAPTURED = isWindows() ? NativeAccessSupport.createDowncallHandle(false, true, SINT64, POINTER, POINTER, SINT32) : null; @@ -185,6 +187,7 @@ private static boolean isWindows() { private static long loadLibraryExPtr; private static long freeLibraryPtr; private static long getProcAddressPtr; + private static long setLastErrorPtr; private static long formatMessagePtr; private static Object windowsLookupArena; private static NativeLibraryLookup windowsLookup; @@ -194,6 +197,7 @@ private static void ensureLoader() throws UnsupportedOperationException { if (loadLibraryExPtr != 0) { assert freeLibraryPtr != 0; assert getProcAddressPtr != 0; + assert setLastErrorPtr != 0; assert formatMessagePtr != 0; assert LOAD_LIBRARY_EX_CAPTURED != null; return; @@ -205,6 +209,7 @@ private static void ensureLoader() throws UnsupportedOperationException { loadLibraryExPtr = NativeAccessSupport.lookupSymbol(windowsLookup, "LoadLibraryExW"); freeLibraryPtr = NativeAccessSupport.lookupSymbol(windowsLookup, "FreeLibrary"); getProcAddressPtr = NativeAccessSupport.lookupSymbol(windowsLookup, "GetProcAddress"); + setLastErrorPtr = NativeAccessSupport.lookupSymbol(windowsLookup, "SetLastError"); formatMessagePtr = NativeAccessSupport.lookupSymbol(windowsLookup, "FormatMessageW"); return; } @@ -234,6 +239,17 @@ private static NativeLibraryLoadException createLoadLibraryException(int windows return new NativeLibraryLoadException(detail); } + @TruffleBoundary + public static void setLastError(int errorCode) { + assert isWindows(); + ensureLoader(); + try { + SET_LAST_ERROR.invokeExact(setLastErrorPtr, errorCode); + } catch (Throwable e) { + throw CompilerDirectives.shouldNotReachHere(e); + } + } + private static String getDlError() { try { long ptr = (long) DLERROR.invokeExact(dlerrorPtr); diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/util/PythonUtils.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/util/PythonUtils.java index 830912d32c..290d46df54 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/util/PythonUtils.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/util/PythonUtils.java @@ -131,6 +131,9 @@ private PythonUtils() { * Encoding of all {@link TruffleString} instances. */ public static final TruffleString.Encoding TS_ENCODING = TruffleString.Encoding.UTF_32; + public static final TruffleString.CodePointSet SURROGATE_CODE_POINT_SET = TruffleString.CodePointSet.fromRanges(new int[]{ + Character.MIN_SURROGATE, Character.MAX_SURROGATE, + }, TS_ENCODING); /** * Encoding for {@link TruffleString}-representations of bytes-like objects. */ diff --git a/graalpython/lib-graalpython/modules/_overlapped.py b/graalpython/lib-graalpython/modules/_overlapped.py index d3b98d4139..fd5a14d876 100644 --- a/graalpython/lib-graalpython/modules/_overlapped.py +++ b/graalpython/lib-graalpython/modules/_overlapped.py @@ -47,12 +47,42 @@ from _winapi import INFINITE, INVALID_HANDLE_VALUE, NULL +ERROR_SUCCESS = 0 +ERROR_FILE_NOT_FOUND = 2 +ERROR_PATH_NOT_FOUND = 3 +ERROR_INVALID_HANDLE = 6 +ERROR_BROKEN_PIPE = 109 ERROR_NETNAME_DELETED = 64 ERROR_SEM_TIMEOUT = 121 +ERROR_MORE_DATA = 234 +ERROR_NOT_FOUND = 1168 +ERROR_CONNECTION_REFUSED = 1225 +ERROR_CONNECTION_ABORTED = 1236 ERROR_OPERATION_ABORTED = 995 +ERROR_IO_INCOMPLETE = 996 ERROR_IO_PENDING = 997 ERROR_PORT_UNREACHABLE = 1234 ERROR_PIPE_BUSY = 231 +ERROR_NO_DATA = 232 + +WSA_ERROR = -1 +EBADF = 9 +ENOTSOCK = 88 + +_TYPE_NONE = 0 +_TYPE_NOT_STARTED = 1 +_TYPE_READ = 2 +_TYPE_READINTO = 3 +_TYPE_WRITE = 4 +_TYPE_CONNECT = 6 +_TYPE_ACCEPT = 7 +_TYPE_TRANSMIT_FILE = 8 +_TYPE_CONNECT_NAMED_PIPE = 9 +_TYPE_READ_FROM = 11 +_TYPE_WRITE_TO = 12 +_TYPE_READ_FROM_INTO = 13 + +SIO_GET_EXTENSION_FUNCTION_POINTER = 0xC8000006 SO_UPDATE_ACCEPT_CONTEXT = 0x700B SO_UPDATE_CONNECT_CONTEXT = 0x7010 @@ -60,21 +90,36 @@ _ctypes = None _wintypes = None _kernel32 = None +_ws2_32 = None +_posix = None _PostQueuedCompletionStatus = None _UnregisterWait = None _UnregisterWaitEx = None +_CancelIoEx = None +_RegisterWaitForSingleObject = None +_OVERLAPPED = None +_WSABUF = None +_SOCKADDR_IN = None +_SOCKADDR_IN6 = None +_GUID = None +_AcceptEx = None +_ConnectEx = None +_TransmitFile = None +_wait_callbacks = {} def _native(): - global _ctypes, _wintypes, _kernel32 + global _ctypes, _wintypes, _kernel32, _ws2_32 global _PostQueuedCompletionStatus, _UnregisterWait, _UnregisterWaitEx + global _CancelIoEx, _RegisterWaitForSingleObject if _kernel32 is not None: - return _ctypes, _wintypes, _kernel32 + return _ctypes, _wintypes, _kernel32, _ws2_32 import ctypes from ctypes import wintypes kernel32 = ctypes.WinDLL("kernel32", use_last_error=True) + ws2_32 = ctypes.WinDLL("ws2_32", use_last_error=True) kernel32.GetLastError.argtypes = [] kernel32.GetLastError.restype = wintypes.DWORD kernel32.CreateIoCompletionPort.argtypes = [wintypes.HANDLE, wintypes.HANDLE, ctypes.c_size_t, wintypes.DWORD] @@ -89,6 +134,50 @@ def _native(): kernel32.GetQueuedCompletionStatus.restype = wintypes.BOOL kernel32.CreateEventW.argtypes = [wintypes.LPVOID, wintypes.BOOL, wintypes.BOOL, wintypes.LPCWSTR] kernel32.CreateEventW.restype = wintypes.HANDLE + kernel32.ReadFile.argtypes = [ + wintypes.HANDLE, + ctypes.c_void_p, + wintypes.DWORD, + ctypes.POINTER(wintypes.DWORD), + ctypes.c_void_p, + ] + kernel32.ReadFile.restype = wintypes.BOOL + kernel32.WriteFile.argtypes = [ + wintypes.HANDLE, + ctypes.c_void_p, + wintypes.DWORD, + ctypes.POINTER(wintypes.DWORD), + ctypes.c_void_p, + ] + kernel32.WriteFile.restype = wintypes.BOOL + kernel32.GetOverlappedResult.argtypes = [ + wintypes.HANDLE, + ctypes.c_void_p, + ctypes.POINTER(wintypes.DWORD), + wintypes.BOOL, + ] + kernel32.GetOverlappedResult.restype = wintypes.BOOL + kernel32.CloseHandle.argtypes = [wintypes.HANDLE] + kernel32.CloseHandle.restype = wintypes.BOOL + kernel32.SetEvent.argtypes = [wintypes.HANDLE] + kernel32.SetEvent.restype = wintypes.BOOL + kernel32.ConnectNamedPipe.argtypes = [wintypes.HANDLE, ctypes.c_void_p] + kernel32.ConnectNamedPipe.restype = wintypes.BOOL + kernel32.CreateFileW.argtypes = [ + wintypes.LPCWSTR, + wintypes.DWORD, + wintypes.DWORD, + wintypes.LPVOID, + wintypes.DWORD, + wintypes.DWORD, + wintypes.HANDLE, + ] + kernel32.CreateFileW.restype = wintypes.HANDLE + + ws2_32.WSAGetLastError.argtypes = [] + ws2_32.WSAGetLastError.restype = ctypes.c_int + ws2_32.WSASetLastError.argtypes = [ctypes.c_int] + ws2_32.WSASetLastError.restype = None try: post_queued_completion_status = kernel32.PostQueuedCompletionStatus @@ -110,13 +199,169 @@ def _native(): unregister_wait_ex.argtypes = [wintypes.HANDLE, wintypes.HANDLE] unregister_wait_ex.restype = wintypes.BOOL + try: + register_wait = kernel32.RegisterWaitForSingleObject + except AttributeError: + register_wait = None + else: + register_wait.argtypes = [ + ctypes.POINTER(wintypes.HANDLE), + wintypes.HANDLE, + ctypes.c_void_p, + ctypes.c_void_p, + wintypes.ULONG, + wintypes.ULONG, + ] + register_wait.restype = wintypes.BOOL + + try: + cancel_io_ex = kernel32.CancelIoEx + except AttributeError: + cancel_io_ex = None + else: + cancel_io_ex.argtypes = [wintypes.HANDLE, ctypes.c_void_p] + cancel_io_ex.restype = wintypes.BOOL + _ctypes = ctypes _wintypes = wintypes _kernel32 = kernel32 + _ws2_32 = ws2_32 _PostQueuedCompletionStatus = post_queued_completion_status _UnregisterWait = unregister_wait _UnregisterWaitEx = unregister_wait_ex - return _ctypes, _wintypes, _kernel32 + _RegisterWaitForSingleObject = register_wait + _CancelIoEx = cancel_io_ex + _configure_ws2_32() + return _ctypes, _wintypes, _kernel32, _ws2_32 + + +def _structures(): + global _OVERLAPPED, _WSABUF, _SOCKADDR_IN, _SOCKADDR_IN6, _GUID + if _OVERLAPPED is not None: + return _OVERLAPPED, _WSABUF, _SOCKADDR_IN, _SOCKADDR_IN6, _GUID + + ctypes, wintypes, _, _ = _native() + + class OVERLAPPED(ctypes.Structure): + _fields_ = [ + ("Internal", ctypes.c_size_t), + ("InternalHigh", ctypes.c_size_t), + ("Offset", wintypes.DWORD), + ("OffsetHigh", wintypes.DWORD), + ("hEvent", wintypes.HANDLE), + ] + + class WSABUF(ctypes.Structure): + _fields_ = [ + ("len", wintypes.ULONG), + ("buf", ctypes.c_void_p), + ] + + class SOCKADDR_IN(ctypes.Structure): + _fields_ = [ + ("sin_family", ctypes.c_ushort), + ("sin_port", ctypes.c_ushort), + ("sin_addr", ctypes.c_ubyte * 4), + ("sin_zero", ctypes.c_char * 8), + ] + + class SOCKADDR_IN6(ctypes.Structure): + _fields_ = [ + ("sin6_family", ctypes.c_ushort), + ("sin6_port", ctypes.c_ushort), + ("sin6_flowinfo", ctypes.c_ulong), + ("sin6_addr", ctypes.c_ubyte * 16), + ("sin6_scope_id", ctypes.c_ulong), + ] + + class GUID(ctypes.Structure): + _fields_ = [ + ("Data1", ctypes.c_ulong), + ("Data2", ctypes.c_ushort), + ("Data3", ctypes.c_ushort), + ("Data4", ctypes.c_ubyte * 8), + ] + + _OVERLAPPED = OVERLAPPED + _WSABUF = WSABUF + _SOCKADDR_IN = SOCKADDR_IN + _SOCKADDR_IN6 = SOCKADDR_IN6 + _GUID = GUID + return _OVERLAPPED, _WSABUF, _SOCKADDR_IN, _SOCKADDR_IN6, _GUID + + +def _configure_ws2_32(): + ctypes, wintypes, _, ws2_32 = _ctypes, _wintypes, _kernel32, _ws2_32 + OVERLAPPED, WSABUF, _, _, _ = _structures() + socket_t = ctypes.c_size_t + ws2_32.bind.argtypes = [socket_t, ctypes.c_void_p, ctypes.c_int] + ws2_32.bind.restype = ctypes.c_int + ws2_32.WSAConnect.argtypes = [ + socket_t, + ctypes.c_void_p, + ctypes.c_int, + ctypes.c_void_p, + ctypes.c_void_p, + ctypes.c_void_p, + ctypes.c_void_p, + ] + ws2_32.WSAConnect.restype = ctypes.c_int + ws2_32.WSAIoctl.argtypes = [ + socket_t, + wintypes.DWORD, + ctypes.c_void_p, + wintypes.DWORD, + ctypes.c_void_p, + wintypes.DWORD, + ctypes.POINTER(wintypes.DWORD), + ctypes.c_void_p, + ctypes.c_void_p, + ] + ws2_32.WSAIoctl.restype = ctypes.c_int + ws2_32.WSARecv.argtypes = [ + socket_t, + ctypes.POINTER(WSABUF), + wintypes.DWORD, + ctypes.POINTER(wintypes.DWORD), + ctypes.POINTER(wintypes.DWORD), + ctypes.POINTER(OVERLAPPED), + ctypes.c_void_p, + ] + ws2_32.WSARecv.restype = ctypes.c_int + ws2_32.WSASend.argtypes = [ + socket_t, + ctypes.POINTER(WSABUF), + wintypes.DWORD, + ctypes.POINTER(wintypes.DWORD), + wintypes.DWORD, + ctypes.POINTER(OVERLAPPED), + ctypes.c_void_p, + ] + ws2_32.WSASend.restype = ctypes.c_int + ws2_32.WSARecvFrom.argtypes = [ + socket_t, + ctypes.POINTER(WSABUF), + wintypes.DWORD, + ctypes.POINTER(wintypes.DWORD), + ctypes.POINTER(wintypes.DWORD), + ctypes.c_void_p, + ctypes.POINTER(ctypes.c_int), + ctypes.POINTER(OVERLAPPED), + ctypes.c_void_p, + ] + ws2_32.WSARecvFrom.restype = ctypes.c_int + ws2_32.WSASendTo.argtypes = [ + socket_t, + ctypes.POINTER(WSABUF), + wintypes.DWORD, + ctypes.POINTER(wintypes.DWORD), + wintypes.DWORD, + ctypes.c_void_p, + ctypes.c_int, + ctypes.POINTER(OVERLAPPED), + ctypes.c_void_p, + ] + ws2_32.WSASendTo.restype = ctypes.c_int def _as_handle(handle): @@ -126,16 +371,234 @@ def _as_handle(handle): def _winerror(code=None): - ctypes, _, kernel32 = _native() + ctypes, _, kernel32, _ = _native() if code is None: code = kernel32.GetLastError() if hasattr(ctypes, "WinError"): - return ctypes.WinError(code) + error = ctypes.WinError(code) + if code in (ERROR_FILE_NOT_FOUND, ERROR_PATH_NOT_FOUND): + return FileNotFoundError(error.errno, error.strerror, error.filename, code) + if code in (ERROR_BROKEN_PIPE, ERROR_NO_DATA): + return BrokenPipeError(error.errno, error.strerror, error.filename, code) + if code == ERROR_CONNECTION_REFUSED: + return ConnectionRefusedError(error.errno, error.strerror, error.filename, code) + if code == ERROR_CONNECTION_ABORTED: + return ConnectionAbortedError(error.errno, error.strerror, error.filename, code) + return error + if code in (ERROR_FILE_NOT_FOUND, ERROR_PATH_NOT_FOUND): + return FileNotFoundError(code, f"Windows error {code}") + if code in (ERROR_BROKEN_PIPE, ERROR_NO_DATA): + return BrokenPipeError(code, f"Windows error {code}") return OSError(code, f"Windows error {code}") +def _posix_native(): + global _posix + if _posix is not None: + return _posix + if __graalpython__.posix_module_backend() != "native": + raise RuntimeError("_overlapped socket operations require --python.PosixModuleBackend=native") + if not __graalpython__.native_access_is_available(): + raise RuntimeError("_overlapped socket operations require native access") + + ctypes, _, _, _ = _native() + import os + + posix_path = os.path.join(__graalpython__.core_home, "posix.dll") + posix = ctypes.WinDLL(posix_path, use_errno=True) + posix.graalpy_get_socket_handle.argtypes = [ctypes.c_int32, ctypes.POINTER(ctypes.c_int64)] + posix.graalpy_get_socket_handle.restype = ctypes.c_int32 + posix.get_errno.argtypes = [] + posix.get_errno.restype = ctypes.c_int32 + _posix = posix + return _posix + + +def _socket_handle(fd, required=False): + ctypes, _, _, _ = _native() + handle = ctypes.c_int64() + posix = _posix_native() + if posix.graalpy_get_socket_handle(int(fd), ctypes.byref(handle)) == 0: + return handle.value + errno = posix.get_errno() + if not required and errno in (EBADF, ENOTSOCK): + return int(fd) + if errno == EBADF: + raise OSError(errno, "Bad file descriptor") + if errno == ENOTSOCK: + raise OSError(errno, "Socket operation on non-socket") + raise OSError(errno, "Could not get native socket handle") + + +def _pack_address(address): + import socket + + ctypes, _, _, _ = _native() + _, _, SOCKADDR_IN, SOCKADDR_IN6, _ = _structures() + if len(address) == 2: + host, port = address + packed_host = socket.inet_pton(socket.AF_INET, host) + sockaddr = SOCKADDR_IN() + sockaddr.sin_family = socket.AF_INET + sockaddr.sin_port = socket.htons(port) + sockaddr.sin_addr[:] = packed_host + return sockaddr, ctypes.sizeof(sockaddr) + if len(address) == 4: + host, port, flowinfo, scope_id = address + packed_host = socket.inet_pton(socket.AF_INET6, host) + sockaddr = SOCKADDR_IN6() + sockaddr.sin6_family = socket.AF_INET6 + sockaddr.sin6_port = socket.htons(port) + sockaddr.sin6_flowinfo = flowinfo + sockaddr.sin6_addr[:] = packed_host + sockaddr.sin6_scope_id = scope_id + return sockaddr, ctypes.sizeof(sockaddr) + raise ValueError("illegal address_as_bytes argument") + + +def _unpack_address(address_buffer): + import socket + + ctypes, _, _, _ = _native() + _, _, SOCKADDR_IN, SOCKADDR_IN6, _ = _structures() + family = ctypes.c_ushort.from_buffer(address_buffer).value + if family == socket.AF_INET: + sockaddr = SOCKADDR_IN.from_buffer(address_buffer) + host = socket.inet_ntop(socket.AF_INET, bytes(sockaddr.sin_addr)) + return host, socket.ntohs(sockaddr.sin_port) + if family == socket.AF_INET6: + sockaddr = SOCKADDR_IN6.from_buffer(address_buffer) + host = socket.inet_ntop(socket.AF_INET6, bytes(sockaddr.sin6_addr)) + return host, socket.ntohs(sockaddr.sin6_port), socket.ntohl(sockaddr.sin6_flowinfo), sockaddr.sin6_scope_id + raise ValueError("recvfrom returned unsupported address family") + + +def _writable_buffer(buffer): + ctypes, _, _, _ = _native() + view = memoryview(buffer) + if view.readonly: + raise TypeError("underlying buffer is not writable") + if not view.contiguous: + raise BufferError("memoryview: underlying buffer is not contiguous") + size = view.nbytes + if size == 0: + return view, None, NULL, 0 + c_buffer = (ctypes.c_char * size).from_buffer(view) + return view, c_buffer, ctypes.addressof(c_buffer), size + + +def _connect_ex(socket_handle): + global _ConnectEx + if _ConnectEx is not None: + return _ConnectEx + _load_mswsock_functions(socket_handle) + return _ConnectEx + + +def _accept_ex(socket_handle): + global _AcceptEx + if _AcceptEx is not None: + return _AcceptEx + _load_mswsock_functions(socket_handle) + return _AcceptEx + + +def _transmit_file(socket_handle): + global _TransmitFile + if _TransmitFile is not None: + return _TransmitFile + _load_mswsock_functions(socket_handle) + return _TransmitFile + + +def _load_mswsock_functions(socket_handle): + global _AcceptEx, _ConnectEx, _TransmitFile + if _AcceptEx is not None and _ConnectEx is not None and _TransmitFile is not None: + return + + ctypes, wintypes, _, ws2_32 = _native() + OVERLAPPED, _, _, _, GUID = _structures() + + def load(guid): + function_pointer = ctypes.c_void_p() + bytes_returned = wintypes.DWORD() + result = ws2_32.WSAIoctl( + ctypes.c_size_t(socket_handle), + wintypes.DWORD(SIO_GET_EXTENSION_FUNCTION_POINTER), + ctypes.byref(guid), + wintypes.DWORD(ctypes.sizeof(guid)), + ctypes.byref(function_pointer), + wintypes.DWORD(ctypes.sizeof(function_pointer)), + ctypes.byref(bytes_returned), + None, + None, + ) + if result == WSA_ERROR: + raise _winerror(ws2_32.WSAGetLastError()) + return function_pointer.value + + guid_accept_ex = GUID( + 0xB5367DF1, + 0xCBAC, + 0x11CF, + (ctypes.c_ubyte * 8)(0x95, 0xCA, 0x00, 0x80, 0x5F, 0x48, 0xA1, 0x92), + ) + guid_connect_ex = GUID( + 0x25A207B9, + 0xDDF3, + 0x4660, + (ctypes.c_ubyte * 8)(0x8E, 0xE9, 0x76, 0xE5, 0x8C, 0x74, 0x06, 0x3E), + ) + guid_transmit_file = GUID( + 0xB5367DF0, + 0xCBAC, + 0x11CF, + (ctypes.c_ubyte * 8)(0x95, 0xCA, 0x00, 0x80, 0x5F, 0x48, 0xA1, 0x92), + ) + + accept_ex_prototype = ctypes.WINFUNCTYPE( + wintypes.BOOL, + ctypes.c_size_t, + ctypes.c_size_t, + ctypes.c_void_p, + wintypes.DWORD, + wintypes.DWORD, + wintypes.DWORD, + ctypes.POINTER(wintypes.DWORD), + ctypes.POINTER(OVERLAPPED), + ) + + connect_ex_prototype = ctypes.WINFUNCTYPE( + wintypes.BOOL, + ctypes.c_size_t, + ctypes.c_void_p, + ctypes.c_int, + ctypes.c_void_p, + wintypes.DWORD, + ctypes.POINTER(wintypes.DWORD), + ctypes.POINTER(OVERLAPPED), + ) + + transmit_file_prototype = ctypes.WINFUNCTYPE( + wintypes.BOOL, + ctypes.c_size_t, + wintypes.HANDLE, + wintypes.DWORD, + wintypes.DWORD, + ctypes.POINTER(OVERLAPPED), + ctypes.c_void_p, + wintypes.DWORD, + ) + + _AcceptEx = accept_ex_prototype(load(guid_accept_ex)) + _ConnectEx = connect_ex_prototype(load(guid_connect_ex)) + _TransmitFile = transmit_file_prototype(load(guid_transmit_file)) + + def CreateIoCompletionPort(file_handle, existing_completion_port, completion_key, number_of_concurrent_threads): - ctypes, wintypes, kernel32 = _native() + ctypes, wintypes, kernel32, _ = _native() + if _as_handle(file_handle) != INVALID_HANDLE_VALUE: + file_handle = _socket_handle(file_handle) handle = kernel32.CreateIoCompletionPort( wintypes.HANDLE(_as_handle(file_handle)), wintypes.HANDLE(_as_handle(existing_completion_port)), @@ -148,7 +611,7 @@ def CreateIoCompletionPort(file_handle, existing_completion_port, completion_key def GetQueuedCompletionStatus(completion_port, milliseconds=INFINITE): - ctypes, wintypes, kernel32 = _native() + ctypes, wintypes, kernel32, _ = _native() transferred = wintypes.DWORD() key = ctypes.c_size_t() overlapped = ctypes.c_void_p() @@ -168,7 +631,7 @@ def GetQueuedCompletionStatus(completion_port, milliseconds=INFINITE): def PostQueuedCompletionStatus(completion_port, transferred, completion_key, overlapped): - ctypes, wintypes, _ = _native() + ctypes, wintypes, _, _ = _native() if _PostQueuedCompletionStatus is None: raise NotImplementedError("_overlapped.PostQueuedCompletionStatus is not available") result = _PostQueuedCompletionStatus( @@ -182,7 +645,7 @@ def PostQueuedCompletionStatus(completion_port, transferred, completion_key, ove def CreateEvent(event_attributes, manual_reset, initial_state, name): - _, wintypes, kernel32 = _native() + _, wintypes, kernel32, _ = _native() handle = kernel32.CreateEventW(event_attributes, bool(manual_reset), bool(initial_state), name) if not handle: raise _winerror() @@ -190,19 +653,21 @@ def CreateEvent(event_attributes, manual_reset, initial_state, name): def UnregisterWait(wait_handle): - _, wintypes, _ = _native() + _, wintypes, _, _ = _native() if _UnregisterWait is None: raise NotImplementedError("_overlapped.UnregisterWait is not available") if not _UnregisterWait(wintypes.HANDLE(_as_handle(wait_handle))): raise _winerror() + _wait_callbacks.pop(_as_handle(wait_handle), None) def UnregisterWaitEx(wait_handle, completion_event): - _, wintypes, _ = _native() + _, wintypes, _, _ = _native() if _UnregisterWaitEx is None: raise NotImplementedError("_overlapped.UnregisterWaitEx is not available") if not _UnregisterWaitEx(wintypes.HANDLE(_as_handle(wait_handle)), wintypes.HANDLE(_as_handle(completion_event))): raise _winerror() + _wait_callbacks.pop(_as_handle(wait_handle), None) def _not_implemented(name): @@ -213,36 +678,558 @@ def function(*args, **kwargs): return function -RegisterWaitWithQueue = _not_implemented("RegisterWaitWithQueue") -WSAConnect = _not_implemented("WSAConnect") -BindLocal = _not_implemented("BindLocal") -ConnectPipe = _not_implemented("ConnectPipe") +def RegisterWaitWithQueue(handle, completion_port, overlapped, milliseconds): + ctypes, wintypes, _, _ = _native() + if _RegisterWaitForSingleObject is None: + raise NotImplementedError("_overlapped.RegisterWaitWithQueue is not available") + if _PostQueuedCompletionStatus is None: + raise NotImplementedError("_overlapped.PostQueuedCompletionStatus is not available") + + callback_type = ctypes.WINFUNCTYPE(None, ctypes.c_void_p, wintypes.BOOLEAN) + data = (ctypes.c_size_t(_as_handle(completion_port)), ctypes.c_void_p(_as_handle(overlapped))) + + def post_to_queue(parameter, timer_or_wait_fired): + port, address = data + _PostQueuedCompletionStatus( + wintypes.HANDLE(port.value), + wintypes.DWORD(1 if timer_or_wait_fired else 0), + ctypes.c_size_t(0), + address, + ) + + callback = callback_type(post_to_queue) + wait_handle = wintypes.HANDLE() + WT_EXECUTEINWAITTHREAD = 0x00000004 + WT_EXECUTEONLYONCE = 0x00000008 + result = _RegisterWaitForSingleObject( + ctypes.byref(wait_handle), + wintypes.HANDLE(_as_handle(handle)), + callback, + None, + wintypes.ULONG(milliseconds), + wintypes.ULONG(WT_EXECUTEINWAITTHREAD | WT_EXECUTEONLYONCE), + ) + if not result: + raise _winerror() + wait_handle_value = _as_handle(wait_handle.value) + _wait_callbacks[wait_handle_value] = callback, data + return wait_handle_value + + +def ConnectPipe(address): + _, wintypes, kernel32, _ = _native() + GENERIC_READ = 0x80000000 + GENERIC_WRITE = 0x40000000 + OPEN_EXISTING = 3 + FILE_FLAG_OVERLAPPED = 0x40000000 + handle = _as_handle( + kernel32.CreateFileW( + address, + wintypes.DWORD(GENERIC_READ | GENERIC_WRITE), + wintypes.DWORD(0), + None, + wintypes.DWORD(OPEN_EXISTING), + wintypes.DWORD(FILE_FLAG_OVERLAPPED), + wintypes.HANDLE(NULL), + ) + ) + if handle == INVALID_HANDLE_VALUE: + raise _winerror() + return handle + + +def BindLocal(handle, family): + import socket + + ctypes, _, _, ws2_32 = _native() + if family == socket.AF_INET: + sockaddr, sockaddr_len = _pack_address(("0.0.0.0", 0)) + elif family == socket.AF_INET6: + sockaddr, sockaddr_len = _pack_address(("::", 0, 0, 0)) + else: + raise ValueError("Only AF_INET and AF_INET6 families are supported") + result = ws2_32.bind( + ctypes.c_size_t(_socket_handle(handle, required=True)), + ctypes.byref(sockaddr), + ctypes.c_int(sockaddr_len), + ) + if result == WSA_ERROR: + raise _winerror(ws2_32.WSAGetLastError()) + + +def WSAConnect(handle, address): + ctypes, _, _, ws2_32 = _native() + sockaddr, sockaddr_len = _pack_address(address) + result = ws2_32.WSAConnect( + ctypes.c_size_t(_socket_handle(handle, required=True)), + ctypes.byref(sockaddr), + ctypes.c_int(sockaddr_len), + None, + None, + None, + None, + ) + if result == WSA_ERROR: + raise _winerror(ws2_32.WSAGetLastError()) class Overlapped: - def __init__(self, event): - self.event = event - self.pending = False - self.address = id(self) + def __init__(self, event=INVALID_HANDLE_VALUE): + ctypes, wintypes, kernel32, _ = _native() + OVERLAPPED, _, _, _, _ = _structures() + if event == INVALID_HANDLE_VALUE: + event = kernel32.CreateEventW(None, True, False, None) + if not event: + raise _winerror() + self.event = _as_handle(event) + self._pending = False + self.error = ERROR_SUCCESS + self._type = _TYPE_NONE + self._handle = NULL + self._buffer = None + self._user_buffer = None + self._user_buffer_view = None + self._address_buffer = None + self._address_length = None + self._last_transferred = 0 + self._overlapped = OVERLAPPED() + if self.event: + self._overlapped.hEvent = wintypes.HANDLE(self.event) + self.address = ctypes.addressof(self._overlapped) def cancel(self): - self.pending = False + ctypes, wintypes, _, _ = _native() + if self._type in (_TYPE_NONE, _TYPE_NOT_STARTED): + return + if self.pending and _CancelIoEx is not None: + result = _CancelIoEx(wintypes.HANDLE(_as_handle(self._handle)), ctypes.byref(self._overlapped)) + if not result: + code = _kernel32.GetLastError() + if code != ERROR_NOT_FOUND: + raise _winerror(code) def getresult(self, wait=False): - return 0 + ctypes, wintypes, kernel32, _ = _native() + if self._type == _TYPE_NONE: + raise ValueError("operation not yet attempted") + if self._type == _TYPE_NOT_STARTED: + raise ValueError("operation failed to start") + transferred = wintypes.DWORD() + result = kernel32.GetOverlappedResult( + wintypes.HANDLE(_as_handle(self._handle)), + ctypes.byref(self._overlapped), + ctypes.byref(transferred), + bool(wait), + ) + self._last_transferred = transferred.value + self.error = ERROR_SUCCESS if result else kernel32.GetLastError() + if self.error in (ERROR_SUCCESS, ERROR_MORE_DATA): + self.pending = False + elif self.error == ERROR_BROKEN_PIPE and self._type in (_TYPE_READ, _TYPE_READINTO): + self.pending = False + else: + if self.error != ERROR_IO_INCOMPLETE: + self.pending = False + raise _winerror(self.error) + + if self._type == _TYPE_READ: + return self._buffer.raw[: transferred.value] + if self._type == _TYPE_READ_FROM: + return self._buffer.raw[: transferred.value], _unpack_address(self._address_buffer) + if self._type == _TYPE_READ_FROM_INTO: + return transferred.value, _unpack_address(self._address_buffer) + return transferred.value GetOverlappedResult = getresult - WSARecv = _not_implemented("Overlapped.WSARecv") - WSARecvInto = _not_implemented("Overlapped.WSARecvInto") - WSARecvFrom = _not_implemented("Overlapped.WSARecvFrom") - WSARecvFromInto = _not_implemented("Overlapped.WSARecvFromInto") - WSASend = _not_implemented("Overlapped.WSASend") - WSASendTo = _not_implemented("Overlapped.WSASendTo") - ReadFile = _not_implemented("Overlapped.ReadFile") - ReadFileInto = _not_implemented("Overlapped.ReadFileInto") - WriteFile = _not_implemented("Overlapped.WriteFile") - AcceptEx = _not_implemented("Overlapped.AcceptEx") - ConnectEx = _not_implemented("Overlapped.ConnectEx") - TransmitFile = _not_implemented("Overlapped.TransmitFile") - ConnectNamedPipe = _not_implemented("Overlapped.ConnectNamedPipe") + @property + def pending(self): + if not self._pending: + return False + if self._type in (_TYPE_NONE, _TYPE_NOT_STARTED): + self._pending = False + return False + ctypes, wintypes, kernel32, _ = _native() + transferred = wintypes.DWORD() + result = kernel32.GetOverlappedResult( + wintypes.HANDLE(_as_handle(self._handle)), + ctypes.byref(self._overlapped), + ctypes.byref(transferred), + False, + ) + if result: + self._last_transferred = transferred.value + self._pending = False + return False + if kernel32.GetLastError() != ERROR_IO_INCOMPLETE: + self._pending = False + return False + return True + + @pending.setter + def pending(self, value): + self._pending = bool(value) + + def _start_socket_operation(self, handle, operation_type): + if self._type != _TYPE_NONE: + raise ValueError("operation already attempted") + self._type = operation_type + self._handle = _socket_handle(handle, required=True) + self.pending = False + + def _start_handle_operation(self, handle, operation_type): + if self._type != _TYPE_NONE: + raise ValueError("operation already attempted") + self._type = operation_type + self._handle = _as_handle(handle) + self.pending = False + + def ReadFile(self, handle, size): + ctypes, wintypes, kernel32, _ = _native() + size = int(size) + self._start_handle_operation(handle, _TYPE_READ) + self._buffer = ctypes.create_string_buffer(max(size, 1)) + transferred = wintypes.DWORD() + result = kernel32.ReadFile( + wintypes.HANDLE(_as_handle(self._handle)), + ctypes.cast(self._buffer, ctypes.c_void_p), + wintypes.DWORD(size), + ctypes.byref(transferred), + ctypes.byref(self._overlapped), + ) + self.error = ERROR_SUCCESS if result else kernel32.GetLastError() + if self.error in (ERROR_SUCCESS, ERROR_MORE_DATA, ERROR_IO_PENDING): + self.pending = True + self._last_transferred = transferred.value + return None + self._type = _TYPE_NOT_STARTED + self.pending = False + raise _winerror(self.error) + + def ReadFileInto(self, handle, buffer): + ctypes, wintypes, kernel32, _ = _native() + self._user_buffer_view, self._user_buffer, address, size = _writable_buffer(buffer) + self._start_handle_operation(handle, _TYPE_READINTO) + transferred = wintypes.DWORD() + result = kernel32.ReadFile( + wintypes.HANDLE(_as_handle(self._handle)), + ctypes.c_void_p(address), + wintypes.DWORD(size), + ctypes.byref(transferred), + ctypes.byref(self._overlapped), + ) + self.error = ERROR_SUCCESS if result else kernel32.GetLastError() + if self.error in (ERROR_SUCCESS, ERROR_MORE_DATA, ERROR_IO_PENDING): + self.pending = True + self._last_transferred = transferred.value + return None + self._type = _TYPE_NOT_STARTED + self.pending = False + raise _winerror(self.error) + + def WriteFile(self, handle, buffer): + ctypes, wintypes, kernel32, _ = _native() + data = bytes(buffer) + self._start_handle_operation(handle, _TYPE_WRITE) + self._buffer = ctypes.create_string_buffer(data, len(data)) + transferred = wintypes.DWORD() + result = kernel32.WriteFile( + wintypes.HANDLE(_as_handle(self._handle)), + ctypes.cast(self._buffer, ctypes.c_void_p), + wintypes.DWORD(len(data)), + ctypes.byref(transferred), + ctypes.byref(self._overlapped), + ) + self.error = ERROR_SUCCESS if result else kernel32.GetLastError() + if self.error in (ERROR_SUCCESS, ERROR_IO_PENDING): + self.pending = True + self._last_transferred = transferred.value + return None + self._type = _TYPE_NOT_STARTED + self.pending = False + raise _winerror(self.error) + + def WSARecv(self, handle, size, flags=0): + ctypes, wintypes, _, ws2_32 = _native() + _, WSABUF, _, _, _ = _structures() + size = int(size) + self._start_socket_operation(handle, _TYPE_READ) + self._buffer = ctypes.create_string_buffer(max(size, 1)) + transferred = wintypes.DWORD() + flags_value = wintypes.DWORD(flags) + wsabuf = WSABUF(wintypes.ULONG(size), ctypes.cast(self._buffer, ctypes.c_void_p)) + result = ws2_32.WSARecv( + ctypes.c_size_t(self._handle), + ctypes.byref(wsabuf), + wintypes.DWORD(1), + ctypes.byref(transferred), + ctypes.byref(flags_value), + ctypes.byref(self._overlapped), + None, + ) + self.error = ws2_32.WSAGetLastError() if result == WSA_ERROR else ERROR_SUCCESS + if self.error in (ERROR_SUCCESS, ERROR_MORE_DATA, ERROR_IO_PENDING): + self.pending = True + self._last_transferred = transferred.value + return None + self._type = _TYPE_NOT_STARTED + self.pending = False + raise _winerror(self.error) + + def WSARecvInto(self, handle, buffer, flags=0): + ctypes, wintypes, _, ws2_32 = _native() + _, WSABUF, _, _, _ = _structures() + self._start_socket_operation(handle, _TYPE_READINTO) + self._user_buffer_view, self._user_buffer, address, size = _writable_buffer(buffer) + transferred = wintypes.DWORD() + flags_value = wintypes.DWORD(flags) + wsabuf = WSABUF(wintypes.ULONG(size), ctypes.c_void_p(address)) + result = ws2_32.WSARecv( + ctypes.c_size_t(self._handle), + ctypes.byref(wsabuf), + wintypes.DWORD(1), + ctypes.byref(transferred), + ctypes.byref(flags_value), + ctypes.byref(self._overlapped), + None, + ) + self.error = ws2_32.WSAGetLastError() if result == WSA_ERROR else ERROR_SUCCESS + if self.error in (ERROR_SUCCESS, ERROR_MORE_DATA, ERROR_IO_PENDING): + self.pending = True + self._last_transferred = transferred.value + return None + self._type = _TYPE_NOT_STARTED + self.pending = False + raise _winerror(self.error) + + def WSARecvFrom(self, handle, size, flags=0): + ctypes, wintypes, _, ws2_32 = _native() + _, WSABUF, _, SOCKADDR_IN6, _ = _structures() + size = int(size) + self._start_socket_operation(handle, _TYPE_READ_FROM) + self._buffer = ctypes.create_string_buffer(max(size, 1)) + self._address_buffer = ctypes.create_string_buffer(ctypes.sizeof(SOCKADDR_IN6)) + self._address_length = ctypes.c_int(ctypes.sizeof(self._address_buffer)) + transferred = wintypes.DWORD() + flags_value = wintypes.DWORD(flags) + wsabuf = WSABUF(wintypes.ULONG(size), ctypes.cast(self._buffer, ctypes.c_void_p)) + result = ws2_32.WSARecvFrom( + ctypes.c_size_t(self._handle), + ctypes.byref(wsabuf), + wintypes.DWORD(1), + ctypes.byref(transferred), + ctypes.byref(flags_value), + ctypes.cast(self._address_buffer, ctypes.c_void_p), + ctypes.byref(self._address_length), + ctypes.byref(self._overlapped), + None, + ) + self.error = ws2_32.WSAGetLastError() if result == WSA_ERROR else ERROR_SUCCESS + if self.error in (ERROR_SUCCESS, ERROR_MORE_DATA, ERROR_IO_PENDING): + self.pending = True + self._last_transferred = transferred.value + return None + self._type = _TYPE_NOT_STARTED + self.pending = False + raise _winerror(self.error) + + def WSARecvFromInto(self, handle, buffer, size=0, flags=0): + ctypes, wintypes, _, ws2_32 = _native() + _, WSABUF, _, SOCKADDR_IN6, _ = _structures() + self._user_buffer_view, self._user_buffer, address, buffer_size = _writable_buffer(buffer) + size = int(size) if size else buffer_size + if size > buffer_size: + raise ValueError("nbytes is greater than the length of the buffer") + self._start_socket_operation(handle, _TYPE_READ_FROM_INTO) + self._address_buffer = ctypes.create_string_buffer(ctypes.sizeof(SOCKADDR_IN6)) + self._address_length = ctypes.c_int(ctypes.sizeof(self._address_buffer)) + transferred = wintypes.DWORD() + flags_value = wintypes.DWORD(flags) + wsabuf = WSABUF(wintypes.ULONG(size), ctypes.c_void_p(address)) + result = ws2_32.WSARecvFrom( + ctypes.c_size_t(self._handle), + ctypes.byref(wsabuf), + wintypes.DWORD(1), + ctypes.byref(transferred), + ctypes.byref(flags_value), + ctypes.cast(self._address_buffer, ctypes.c_void_p), + ctypes.byref(self._address_length), + ctypes.byref(self._overlapped), + None, + ) + self.error = ws2_32.WSAGetLastError() if result == WSA_ERROR else ERROR_SUCCESS + if self.error in (ERROR_SUCCESS, ERROR_MORE_DATA, ERROR_IO_PENDING): + self.pending = True + self._last_transferred = transferred.value + return None + self._type = _TYPE_NOT_STARTED + self.pending = False + raise _winerror(self.error) + + + def WSASend(self, handle, buffer, flags=0): + ctypes, wintypes, _, ws2_32 = _native() + _, WSABUF, _, _, _ = _structures() + data = bytes(buffer) + self._start_socket_operation(handle, _TYPE_WRITE) + self._buffer = ctypes.create_string_buffer(data, len(data)) + transferred = wintypes.DWORD() + wsabuf = WSABUF(wintypes.ULONG(len(data)), ctypes.cast(self._buffer, ctypes.c_void_p)) + result = ws2_32.WSASend( + ctypes.c_size_t(self._handle), + ctypes.byref(wsabuf), + wintypes.DWORD(1), + ctypes.byref(transferred), + wintypes.DWORD(flags), + ctypes.byref(self._overlapped), + None, + ) + self.error = ws2_32.WSAGetLastError() if result == WSA_ERROR else ERROR_SUCCESS + if self.error in (ERROR_SUCCESS, ERROR_IO_PENDING): + self.pending = True + self._last_transferred = transferred.value + return None + self._type = _TYPE_NOT_STARTED + self.pending = False + raise _winerror(self.error) + + def WSASendTo(self, handle, buffer, flags, address): + ctypes, wintypes, _, ws2_32 = _native() + _, WSABUF, _, _, _ = _structures() + data = bytes(buffer) + sockaddr, sockaddr_len = _pack_address(address) + self._start_socket_operation(handle, _TYPE_WRITE_TO) + self._buffer = ctypes.create_string_buffer(data, len(data)) + self._address_buffer = sockaddr + transferred = wintypes.DWORD() + wsabuf = WSABUF(wintypes.ULONG(len(data)), ctypes.cast(self._buffer, ctypes.c_void_p)) + result = ws2_32.WSASendTo( + ctypes.c_size_t(self._handle), + ctypes.byref(wsabuf), + wintypes.DWORD(1), + ctypes.byref(transferred), + wintypes.DWORD(flags), + ctypes.byref(sockaddr), + ctypes.c_int(sockaddr_len), + ctypes.byref(self._overlapped), + None, + ) + self.error = ws2_32.WSAGetLastError() if result == WSA_ERROR else ERROR_SUCCESS + if self.error in (ERROR_SUCCESS, ERROR_IO_PENDING): + self.pending = True + self._last_transferred = transferred.value + return None + self._type = _TYPE_NOT_STARTED + self.pending = False + raise _winerror(self.error) + + def AcceptEx(self, listen_handle, accept_handle): + ctypes, wintypes, _, ws2_32 = _native() + _, _, _, SOCKADDR_IN6, _ = _structures() + if self._type != _TYPE_NONE: + raise ValueError("operation already attempted") + listen_socket = _socket_handle(listen_handle, required=True) + accept_socket = _socket_handle(accept_handle, required=True) + address_size = ctypes.sizeof(SOCKADDR_IN6) + 16 + self._start_socket_operation(listen_handle, _TYPE_ACCEPT) + self._buffer = ctypes.create_string_buffer(address_size * 2) + accept_ex = _accept_ex(listen_socket) + transferred = wintypes.DWORD() + ws2_32.WSASetLastError(ERROR_SUCCESS) + result = accept_ex( + ctypes.c_size_t(listen_socket), + ctypes.c_size_t(accept_socket), + ctypes.cast(self._buffer, ctypes.c_void_p), + wintypes.DWORD(0), + wintypes.DWORD(address_size), + wintypes.DWORD(address_size), + ctypes.byref(transferred), + ctypes.byref(self._overlapped), + ) + self.error = ERROR_SUCCESS if result else ws2_32.WSAGetLastError() + if self.error == ERROR_SUCCESS and not result: + self.error = ERROR_IO_PENDING + if self.error in (ERROR_SUCCESS, ERROR_IO_PENDING): + self.pending = True + self._last_transferred = transferred.value + return None + self._type = _TYPE_NOT_STARTED + self.pending = False + raise _winerror(self.error) + + def ConnectEx(self, handle, address): + ctypes, wintypes, _, ws2_32 = _native() + if self._type != _TYPE_NONE: + raise ValueError("operation already attempted") + sockaddr, sockaddr_len = _pack_address(address) + self._type = _TYPE_CONNECT + self._handle = _socket_handle(handle, required=True) + self._address_buffer = sockaddr + connect_ex = _connect_ex(self._handle) + bytes_sent = wintypes.DWORD() + result = connect_ex( + ctypes.c_size_t(self._handle), + ctypes.byref(sockaddr), + ctypes.c_int(sockaddr_len), + None, + wintypes.DWORD(0), + ctypes.byref(bytes_sent), + ctypes.byref(self._overlapped), + ) + self.error = ERROR_SUCCESS if result else ws2_32.WSAGetLastError() + if self.error in (ERROR_SUCCESS, ERROR_IO_PENDING): + self.pending = True + return None + self._type = _TYPE_NOT_STARTED + self.pending = False + raise _winerror(self.error) + + def TransmitFile(self, socket, file, offset, offset_high, count_to_write, count_per_send, flags): + ctypes, wintypes, _, ws2_32 = _native() + self._start_socket_operation(socket, _TYPE_TRANSMIT_FILE) + self._overlapped.Offset = wintypes.DWORD(offset) + self._overlapped.OffsetHigh = wintypes.DWORD(offset_high) + transmit_file = _transmit_file(self._handle) + ws2_32.WSASetLastError(ERROR_SUCCESS) + result = transmit_file( + ctypes.c_size_t(self._handle), + wintypes.HANDLE(_as_handle(file)), + wintypes.DWORD(count_to_write), + wintypes.DWORD(count_per_send), + ctypes.byref(self._overlapped), + None, + wintypes.DWORD(flags), + ) + self.error = ERROR_SUCCESS if result else ws2_32.WSAGetLastError() + if self.error == ERROR_SUCCESS and not result: + self.error = ERROR_IO_PENDING + if self.error in (ERROR_SUCCESS, ERROR_IO_PENDING): + self.pending = True + return None + self._type = _TYPE_NOT_STARTED + self.pending = False + raise _winerror(self.error) + + def ConnectNamedPipe(self, handle): + ctypes, wintypes, kernel32, _ = _native() + self._start_handle_operation(handle, _TYPE_CONNECT_NAMED_PIPE) + result = kernel32.ConnectNamedPipe(wintypes.HANDLE(_as_handle(self._handle)), ctypes.byref(self._overlapped)) + self.error = ERROR_SUCCESS if result else kernel32.GetLastError() + if self.error == 535: # ERROR_PIPE_CONNECTED + self.pending = False + return True + if self.error in (ERROR_SUCCESS, ERROR_IO_PENDING): + self.pending = True + return False + self._type = _TYPE_NOT_STARTED + self.pending = False + raise _winerror(self.error) + + def __del__(self): + try: + if self.event: + _, wintypes, kernel32, _ = _native() + kernel32.CloseHandle(wintypes.HANDLE(self.event)) + self.event = NULL + except Exception: + pass diff --git a/graalpython/lib-graalpython/modules/_winapi.py b/graalpython/lib-graalpython/modules/_winapi.py index b1a3bb8997..0fdd49c317 100644 --- a/graalpython/lib-graalpython/modules/_winapi.py +++ b/graalpython/lib-graalpython/modules/_winapi.py @@ -79,6 +79,8 @@ def _unsigned_pointer(value): CREATE_DEFAULT_ERROR_MODE = 0x04000000 CREATE_BREAKAWAY_FROM_JOB = 0x01000000 CREATE_UNICODE_ENVIRONMENT = 0x00000400 +EXTENDED_STARTUPINFO_PRESENT = 0x00080000 +PROC_THREAD_ATTRIBUTE_HANDLE_LIST = 0x00020002 SYNCHRONIZE = 0x00100000 PROCESS_DUP_HANDLE = 0x00000040 @@ -123,6 +125,7 @@ def _unsigned_pointer(value): ERROR_PATH_NOT_FOUND = 3 ERROR_ACCESS_DENIED = 5 ERROR_INVALID_HANDLE = 6 +ERROR_NETNAME_DELETED = 64 ERROR_BROKEN_PIPE = 109 ERROR_SEM_TIMEOUT = 121 ERROR_ALREADY_EXISTS = 183 @@ -142,6 +145,7 @@ def _unsigned_pointer(value): _ctypes = None _wintypes = None _kernel32 = None +_GetLastError = None _NeedCurrentDirectoryForExePathW = None _CancelIoEx = None _SECURITY_ATTRIBUTES = None @@ -151,7 +155,7 @@ def _unsigned_pointer(value): def _native(): - global _ctypes, _wintypes, _kernel32, _NeedCurrentDirectoryForExePathW, _CancelIoEx + global _ctypes, _wintypes, _kernel32, _GetLastError, _NeedCurrentDirectoryForExePathW, _CancelIoEx if _kernel32 is not None: return _ctypes, _wintypes, _kernel32 @@ -221,6 +225,25 @@ def _native(): wintypes.LPVOID, ] kernel32.CreateProcessW.restype = wintypes.BOOL + kernel32.InitializeProcThreadAttributeList.argtypes = [ + wintypes.LPVOID, + wintypes.DWORD, + wintypes.DWORD, + ctypes.POINTER(ctypes.c_size_t), + ] + kernel32.InitializeProcThreadAttributeList.restype = wintypes.BOOL + kernel32.UpdateProcThreadAttribute.argtypes = [ + wintypes.LPVOID, + wintypes.DWORD, + ctypes.c_size_t, + wintypes.LPVOID, + ctypes.c_size_t, + wintypes.LPVOID, + wintypes.LPVOID, + ] + kernel32.UpdateProcThreadAttribute.restype = wintypes.BOOL + kernel32.DeleteProcThreadAttributeList.argtypes = [wintypes.LPVOID] + kernel32.DeleteProcThreadAttributeList.restype = None kernel32.CreateFileW.argtypes = [ wintypes.LPCWSTR, wintypes.DWORD, @@ -306,6 +329,12 @@ def _native(): ] kernel32.LCMapStringEx.restype = ctypes.c_int + # _winapi.GetLastError must read the actual OS thread state. Calling it through the + # use_last_error=True DLL would swap in ctypes' private last-error copy first. + get_last_error = ctypes.WinDLL("kernel32").GetLastError + get_last_error.argtypes = [] + get_last_error.restype = wintypes.DWORD + try: need_current_directory = kernel32.NeedCurrentDirectoryForExePathW except AttributeError: @@ -325,6 +354,7 @@ def _native(): _ctypes = ctypes _wintypes = wintypes _kernel32 = kernel32 + _GetLastError = get_last_error _NeedCurrentDirectoryForExePathW = need_current_directory _CancelIoEx = cancel_io_ex return _ctypes, _wintypes, _kernel32 @@ -550,8 +580,8 @@ def CloseHandle(handle): def GetLastError(): - _, _, kernel32 = _native() - return kernel32.GetLastError() + _native() + return _GetLastError() def GetCurrentProcess(): @@ -693,8 +723,22 @@ def CreateProcess( command_line = _optional_path(command_line, "command_line") current_directory = _optional_path(current_directory, "current_directory") - startup = STARTUPINFOW() - startup.cb = ctypes.sizeof(startup) + class STARTUPINFOEXW(ctypes.Structure): + _fields_ = [ + ("StartupInfo", STARTUPINFOW), + ("lpAttributeList", wintypes.LPVOID), + ] + + attribute_list = None + handle_list = [] + if startup_info is not None: + attribute_list = getattr(startup_info, "lpAttributeList", None) + if attribute_list is not None: + handle_list = list(attribute_list.get("handle_list", [])) + + startup_ex = STARTUPINFOEXW() if handle_list else None + startup = startup_ex.StartupInfo if startup_ex is not None else STARTUPINFOW() + startup.cb = ctypes.sizeof(startup_ex) if startup_ex is not None else ctypes.sizeof(startup) if startup_info is not None: startup.dwFlags = getattr(startup_info, "dwFlags", 0) startup.wShowWindow = getattr(startup_info, "wShowWindow", 0) @@ -709,20 +753,52 @@ def CreateProcess( creation_flags |= CREATE_UNICODE_ENVIRONMENT process_information = PROCESS_INFORMATION() - result = kernel32.CreateProcessW( - application_name, - command_line_buffer, - None, - None, - wintypes.BOOL(inherit_handles), - wintypes.DWORD(creation_flags), - environment_buffer, - current_directory, - ctypes.byref(startup), - ctypes.byref(process_information), - ) - if not result: - _raise_last_error() + handle_array = None + attribute_buffer = None + if handle_list: + handle_array = (wintypes.HANDLE * len(handle_list))(*(_as_handle(handle) for handle in handle_list)) + attribute_list_size = ctypes.c_size_t() + kernel32.InitializeProcThreadAttributeList(None, 1, 0, ctypes.byref(attribute_list_size)) + attribute_buffer = ctypes.create_string_buffer(attribute_list_size.value) + _raise_if_zero( + kernel32.InitializeProcThreadAttributeList(attribute_buffer, 1, 0, ctypes.byref(attribute_list_size)) + ) + try: + _raise_if_zero( + kernel32.UpdateProcThreadAttribute( + attribute_buffer, + 0, + PROC_THREAD_ATTRIBUTE_HANDLE_LIST, + handle_array, + ctypes.sizeof(handle_array), + None, + None, + ) + ) + except BaseException: + kernel32.DeleteProcThreadAttributeList(attribute_buffer) + raise + startup_ex.lpAttributeList = ctypes.cast(attribute_buffer, wintypes.LPVOID) + creation_flags |= EXTENDED_STARTUPINFO_PRESENT + + try: + result = kernel32.CreateProcessW( + application_name, + command_line_buffer, + None, + None, + wintypes.BOOL(inherit_handles), + wintypes.DWORD(creation_flags), + environment_buffer, + current_directory, + ctypes.byref(startup_ex) if startup_ex is not None else ctypes.byref(startup), + ctypes.byref(process_information), + ) + if not result: + _raise_last_error() + finally: + if attribute_buffer is not None: + kernel32.DeleteProcThreadAttributeList(attribute_buffer) return ( _as_handle(process_information.hProcess), _as_handle(process_information.hThread), @@ -975,11 +1051,13 @@ def GetModuleFileName(module_handle): def LCMapStringEx(locale_name, flags, src): - ctypes, _, kernel32 = _native() if not isinstance(src, str): raise TypeError("src must be str") if locale_name is None: locale_name = LOCALE_NAME_INVARIANT + if locale_name == LOCALE_NAME_INVARIANT and flags == LCMAP_LOWERCASE: + return src.lower() + ctypes, _, kernel32 = _native() buffer = ctypes.create_unicode_buffer(len(src) + 1) result = kernel32.LCMapStringEx(locale_name, flags, src, len(src), buffer, len(buffer), None, None, 0) if result == 0: diff --git a/graalpython/lib-python/3/multiprocessing/connection.py b/graalpython/lib-python/3/multiprocessing/connection.py index e3b7a95d34..7438affad4 100644 --- a/graalpython/lib-python/3/multiprocessing/connection.py +++ b/graalpython/lib-python/3/multiprocessing/connection.py @@ -35,12 +35,6 @@ raise _winapi = None -# GraalPy change: temporary until we implement proper multiprocessing for Windows -_winapi = None -# -# -# - BUFSIZE = 8192 # A very generous timeout when it comes to local connections... CONNECTION_TIMEOUT = 20. diff --git a/graalpython/lib-python/3/subprocess.py b/graalpython/lib-python/3/subprocess.py index f6c6313a27..3afb8cb0ed 100644 --- a/graalpython/lib-python/3/subprocess.py +++ b/graalpython/lib-python/3/subprocess.py @@ -72,7 +72,8 @@ except ModuleNotFoundError: _mswindows = False else: - # GraalPy change: under emulated posix, use the posix APIs even under windows + # GraalPy change: the Java posix backend is pure Java and does not expose + # CPython-style Windows handles. Keep it on the POSIX subprocess path. _mswindows = not (sys.implementation.name == 'graalpy' and __graalpython__.posix_module_backend() == 'java') # wasm32-emscripten and wasm32-wasi do not support processes @@ -1407,7 +1408,8 @@ def _get_handles(self, stdin, stdout, stderr): else: # Assuming file-like object errwrite = msvcrt.get_osfhandle(stderr.fileno()) - errwrite = self._make_inheritable(errwrite) + if stderr != STDOUT: + errwrite = self._make_inheritable(errwrite) return (p2cread, p2cwrite, c2pread, c2pwrite, @@ -1420,6 +1422,11 @@ def _make_inheritable(self, handle): _winapi.GetCurrentProcess(), handle, _winapi.GetCurrentProcess(), 0, 1, _winapi.DUPLICATE_SAME_ACCESS) + if isinstance(handle, Handle): + # CPython closes this original handle immediately via refcounted + # finalization when the local variable is overwritten. GraalPy + # needs the close to be explicit so pipe EOF is observable. + handle.Close() return Handle(h) @@ -1831,8 +1838,11 @@ def _execute_child(self, args, executable, preexec_fn, close_fds, comspec = os.environ.get("COMSPEC", "cmd.exe") executable = comspec # This very specific replace is for the pattern in distutils. - # We really ought to finally implement enough of the winapi - # to use the Windows codepaths... + # This is all working around the fact that emulating CreateProcess + # through an EmulatedPosixBackend that goes through Truffle API + # to a Java API introduces layers of escaping that are deeply troubling + # and cannot possibly be correctly handled in all cases. But we do our, + # best, oh don't you know, it we do. args = [arg.replace(" && ", " ^&^& ") for arg in args] if len(args) == 1: args = [comspec, "/u", "/c", *args] diff --git a/graalpython/lib-python/3/test/test_repl.py b/graalpython/lib-python/3/test/test_repl.py index f2f18a2581..553867aac4 100644 --- a/graalpython/lib-python/3/test/test_repl.py +++ b/graalpython/lib-python/3/test/test_repl.py @@ -124,7 +124,7 @@ def test_close_stdin(self): ''') process = spawn_repl('-c', prepare_repl) output = process.communicate(user_input)[0] - self.assertEqual(process.returncode, 0) + self.assertEqual(process.returncode, 0, output) self.assertIn('before close', output) @@ -160,7 +160,7 @@ def test_toplevel_contextvars_sync(self): """) p.stdin.write(user_input2) output = kill_python(p) - self.assertEqual(p.returncode, 0) + self.assertEqual(p.returncode, 0, output) expected = "toplevel contextvar test: ok" self.assertIn(expected, output, expected) diff --git a/graalpython/python-libposix/src/errno_capture.h b/graalpython/python-libposix/src/errno_capture.h index c1696a1957..2a1f927b73 100644 --- a/graalpython/python-libposix/src/errno_capture.h +++ b/graalpython/python-libposix/src/errno_capture.h @@ -53,11 +53,30 @@ #endif extern THREAD_LOCAL int errno_capture; +#ifdef _WIN32 +extern THREAD_LOCAL int winerror_capture; +extern THREAD_LOCAL int wsaerror_capture; +extern THREAD_LOCAL int error_source_capture; +#endif + +enum error_capture_source { + ERROR_CAPTURE_ERRNO = 0, + ERROR_CAPTURE_WINAPI = 1, + ERROR_CAPTURE_WINSOCK = 2, +}; -static inline void capture_errno(void) { +static inline void capture_errors(void) { errno_capture = errno; +#ifdef _WIN32 + winerror_capture = (int) GetLastError(); + wsaerror_capture = WSAGetLastError(); + error_source_capture = ERROR_CAPTURE_ERRNO; +#endif } +/* Temporary compatibility alias while existing wrappers migrate to the plural API. */ +#define capture_errno capture_errors + #define CAPTURE_ERRNO_AND_RETURN(indicator, expr) do { \ __typeof__(expr) return_value = (expr); \ if (return_value == (indicator)) { \ diff --git a/graalpython/python-libposix/src/fork_exec.c b/graalpython/python-libposix/src/fork_exec.c index 46005b0b44..dfb660f9a8 100644 --- a/graalpython/python-libposix/src/fork_exec.c +++ b/graalpython/python-libposix/src/fork_exec.c @@ -3,6 +3,30 @@ * * Licensed under the PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2 */ +#ifdef _WIN32 + +#include +#include + +__declspec(dllexport) int32_t fork_exec( + char *data, int64_t *offsets, int32_t offsetsLen, int32_t argsPos, int32_t envPos, int32_t cwdPos, + int32_t stdinRdFd, int32_t stdinWrFd, + int32_t stdoutRdFd, int32_t stdoutWrFd, + int32_t stderrRdFd, int32_t stderrWrFd, + int32_t errPipeRdFd, int32_t errPipeWrFd, + int32_t closeFds, + int32_t restoreSignals, + int32_t callSetsid, + int32_t pgidToSet, + int32_t allowVFork, + int32_t *fdsToKeep, int64_t fdsToKeepLen + ) { + errno = ENOSYS; + return -1; +} + +#else + #if defined(__gnu_linux__) && !defined(_GNU_SOURCE) #define _GNU_SOURCE 1 #endif @@ -637,3 +661,5 @@ int32_t fork_exec( capture_errno(); return pid; } + +#endif diff --git a/graalpython/python-libposix/src/posix.c b/graalpython/python-libposix/src/posix.c index 44fedc800b..59190c2f55 100644 --- a/graalpython/python-libposix/src/posix.c +++ b/graalpython/python-libposix/src/posix.c @@ -42,6 +42,1992 @@ // Helper functions that mostly delegate to POSIX functions // These functions are called from NativePosixSupport Java class using native-access downcalls +#ifdef _WIN32 + +#define WIN32_LEAN_AND_MEAN +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "errno_capture.h" + +#ifndef SO_UPDATE_ACCEPT_CONTEXT +#define SO_UPDATE_ACCEPT_CONTEXT 0x700B +#endif +#ifndef ENOTSUP +#define ENOTSUP ENOSYS +#endif +#ifndef EWOULDBLOCK +#define EWOULDBLOCK 11 +#endif +#ifndef EOVERFLOW +#define EOVERFLOW 75 +#endif +#ifndef EALREADY +#define EALREADY 114 +#endif +#ifndef EINPROGRESS +#define EINPROGRESS 115 +#endif +#ifndef ENOTSOCK +#define ENOTSOCK 88 +#endif +#ifndef EDESTADDRREQ +#define EDESTADDRREQ 89 +#endif +#ifndef EMSGSIZE +#define EMSGSIZE 90 +#endif +#ifndef EPROTOTYPE +#define EPROTOTYPE 91 +#endif +#ifndef ENOPROTOOPT +#define ENOPROTOOPT 92 +#endif +#ifndef EPROTONOSUPPORT +#define EPROTONOSUPPORT 93 +#endif +#ifndef EAFNOSUPPORT +#define EAFNOSUPPORT 97 +#endif +#ifndef EADDRINUSE +#define EADDRINUSE 98 +#endif +#ifndef EADDRNOTAVAIL +#define EADDRNOTAVAIL 99 +#endif +#ifndef ENETDOWN +#define ENETDOWN 100 +#endif +#ifndef ENETUNREACH +#define ENETUNREACH 101 +#endif +#ifndef ENETRESET +#define ENETRESET 102 +#endif +#ifndef ECONNABORTED +#define ECONNABORTED 103 +#endif +#ifndef ECONNRESET +#define ECONNRESET 104 +#endif +#ifndef ENOBUFS +#define ENOBUFS 105 +#endif +#ifndef EISCONN +#define EISCONN 106 +#endif +#ifndef ENOTCONN +#define ENOTCONN 107 +#endif +#ifndef ETIMEDOUT +#define ETIMEDOUT 110 +#endif +#ifndef ECONNREFUSED +#define ECONNREFUSED 111 +#endif +#ifndef EHOSTUNREACH +#define EHOSTUNREACH 113 +#endif + +#define GP_EWOULDBLOCK 11 +#define GP_EINPROGRESS 115 +#define GP_EALREADY 114 +#define GP_ENOTSOCK 88 +#define GP_EDESTADDRREQ 89 +#define GP_EMSGSIZE 90 +#define GP_EPROTOTYPE 91 +#define GP_ENOPROTOOPT 92 +#define GP_EPROTONOSUPPORT 93 +#define GP_EAFNOSUPPORT 97 +#define GP_EADDRINUSE 98 +#define GP_EADDRNOTAVAIL 99 +#define GP_ENETDOWN 100 +#define GP_ENETUNREACH 101 +#define GP_ENETRESET 102 +#define GP_ECONNABORTED 103 +#define GP_ECONNRESET 104 +#define GP_ENOBUFS 105 +#define GP_EISCONN 106 +#define GP_ENOTCONN 107 +#define GP_ETIMEDOUT 110 +#define GP_ECONNREFUSED 111 +#define GP_EHOSTUNREACH 113 + +#define GP_EXPORT __declspec(dllexport) + +THREAD_LOCAL int errno_capture = 0; +THREAD_LOCAL int winerror_capture = 0; +THREAD_LOCAL int wsaerror_capture = 0; +THREAD_LOCAL int error_source_capture = ERROR_CAPTURE_ERRNO; + +#define WIN_AT_FDCWD (-100) +#define WIN_GRAALPY_DEFAULT_DIR_FD (-1) +#define WIN_DT_UNKNOWN 0 + +typedef struct { + HANDLE handle; + WIN32_FIND_DATAW data; + int first; + int exhausted; +} win_dir_t; + +static void silent_invalid_parameter_handler(const wchar_t *expression, const wchar_t *function, const wchar_t *file, unsigned int line, uintptr_t reserved) { + (void) expression; + (void) function; + (void) file; + (void) line; + (void) reserved; +} + +#if defined(_MSC_VER) && _MSC_VER >= 1900 +#define BEGIN_SUPPRESS_IPH \ + { _invalid_parameter_handler old_handler = _set_thread_local_invalid_parameter_handler(silent_invalid_parameter_handler); +#define END_SUPPRESS_IPH \ + _set_thread_local_invalid_parameter_handler(old_handler); } +#else +#define BEGIN_SUPPRESS_IPH +#define END_SUPPRESS_IPH +#endif + +static int is_default_dir_fd(int32_t dirFd) { + return dirFd == 0 || dirFd < 0 || dirFd == WIN_AT_FDCWD || dirFd == WIN_GRAALPY_DEFAULT_DIR_FD; +} + +static int unsupported(void) { + errno = ENOSYS; + capture_errno(); + return -1; +} + +static void set_posix_errno(int error) { + errno = error; + capture_errors(); +} + +static void set_win_errno(DWORD error) { + capture_errors(); + winerror_capture = (int) error; + error_source_capture = ERROR_CAPTURE_WINAPI; +} + +static int ensure_winsock(void) { + static int initialized = 0; + if (!initialized) { + WSADATA wsa_data; + int err = WSAStartup(MAKEWORD(2, 2), &wsa_data); + if (err != 0) { + capture_errors(); + wsaerror_capture = err; + error_source_capture = ERROR_CAPTURE_WINSOCK; + return -1; + } + initialized = 1; + } + return 0; +} + +static int set_wsa_errno_from_error(int error) { + capture_errors(); + wsaerror_capture = error; + error_source_capture = ERROR_CAPTURE_WINSOCK; + return -1; +} + +static int set_wsa_errno(void) { + return set_wsa_errno_from_error(WSAGetLastError()); +} + +static int close_noraise(int32_t fd) { + int result; + BEGIN_SUPPRESS_IPH + result = _close(fd); + END_SUPPRESS_IPH + if (result < 0) { + capture_errno(); + } + return result; +} + +static int64_t read_noraise(int32_t fd, void *buf, unsigned int count) { + int result; + BEGIN_SUPPRESS_IPH + result = _read(fd, buf, count); + END_SUPPRESS_IPH + if (result < 0) { + capture_errno(); + } + return result; +} + +static int64_t write_noraise(int32_t fd, void *buf, unsigned int count) { + int result; + BEGIN_SUPPRESS_IPH + result = _write(fd, buf, count); + END_SUPPRESS_IPH + if (result < 0) { + capture_errno(); + } + return result; +} + +static int64_t lseek_noraise(int32_t fd, int64_t offset, int32_t whence) { + int64_t result; + BEGIN_SUPPRESS_IPH + result = _lseeki64(fd, offset, whence); + END_SUPPRESS_IPH + if (result < 0) { + capture_errno(); + } + return result; +} + +static int chsize_noraise(int32_t fd, int64_t length) { + int result; + BEGIN_SUPPRESS_IPH + result = _chsize_s(fd, length); + END_SUPPRESS_IPH + if (result != 0) { + errno = result; + capture_errno(); + } + return result; +} + +static int commit_noraise(int32_t fd) { + int result; + BEGIN_SUPPRESS_IPH + result = _commit(fd); + END_SUPPRESS_IPH + if (result < 0) { + capture_errno(); + } + return result; +} + +static int dup_noraise(int32_t fd) { + int result; + BEGIN_SUPPRESS_IPH + result = _dup(fd); + END_SUPPRESS_IPH + if (result < 0) { + capture_errno(); + } + return result; +} + +static int dup2_noraise(int32_t oldfd, int32_t newfd) { + int result; + BEGIN_SUPPRESS_IPH + result = _dup2(oldfd, newfd); + END_SUPPRESS_IPH + if (result < 0) { + capture_errno(); + } + return result; +} + +typedef struct { + int fd; + SOCKET socket; + int blocking; +} win_socket_entry_t; + +typedef struct { + void *view; + HANDLE mapping; +} win_mmap_entry_t; + +#define WIN_SOCKET_TABLE_SIZE 1024 +#define WIN_MMAP_TABLE_SIZE 1024 + +static win_socket_entry_t win_socket_table[WIN_SOCKET_TABLE_SIZE]; +static INIT_ONCE win_socket_table_init_once = INIT_ONCE_STATIC_INIT; +static CRITICAL_SECTION win_socket_table_lock; + +static win_mmap_entry_t win_mmap_table[WIN_MMAP_TABLE_SIZE]; +static INIT_ONCE win_mmap_table_init_once = INIT_ONCE_STATIC_INIT; +static CRITICAL_SECTION win_mmap_table_lock; + +static BOOL CALLBACK win_socket_table_init(PINIT_ONCE init_once, PVOID parameter, PVOID *context) { + (void) init_once; + (void) parameter; + (void) context; + InitializeCriticalSection(&win_socket_table_lock); + for (int i = 0; i < WIN_SOCKET_TABLE_SIZE; i++) { + win_socket_table[i].fd = -1; + win_socket_table[i].socket = INVALID_SOCKET; + win_socket_table[i].blocking = 1; + } + return TRUE; +} + +static void ensure_socket_table(void) { + InitOnceExecuteOnce(&win_socket_table_init_once, win_socket_table_init, NULL, NULL); +} + +static BOOL CALLBACK win_mmap_table_init(PINIT_ONCE init_once, PVOID parameter, PVOID *context) { + (void) init_once; + (void) parameter; + (void) context; + InitializeCriticalSection(&win_mmap_table_lock); + for (int i = 0; i < WIN_MMAP_TABLE_SIZE; i++) { + win_mmap_table[i].view = NULL; + win_mmap_table[i].mapping = NULL; + } + return TRUE; +} + +static void ensure_mmap_table(void) { + InitOnceExecuteOnce(&win_mmap_table_init_once, win_mmap_table_init, NULL, NULL); +} + +static int win_add_mmap(void *view, HANDLE mapping) { + ensure_mmap_table(); + EnterCriticalSection(&win_mmap_table_lock); + for (int i = 0; i < WIN_MMAP_TABLE_SIZE; i++) { + if (win_mmap_table[i].view == NULL) { + win_mmap_table[i].view = view; + win_mmap_table[i].mapping = mapping; + LeaveCriticalSection(&win_mmap_table_lock); + return 0; + } + } + LeaveCriticalSection(&win_mmap_table_lock); + set_posix_errno(ENOMEM); + return -1; +} + +static HANDLE win_remove_mmap(void *view) { + HANDLE mapping = NULL; + ensure_mmap_table(); + EnterCriticalSection(&win_mmap_table_lock); + for (int i = 0; i < WIN_MMAP_TABLE_SIZE; i++) { + if (win_mmap_table[i].view == view) { + mapping = win_mmap_table[i].mapping; + win_mmap_table[i].view = NULL; + win_mmap_table[i].mapping = NULL; + break; + } + } + LeaveCriticalSection(&win_mmap_table_lock); + return mapping; +} + +static SOCKET win_socket_from_fd(int32_t fd) { + SOCKET result = INVALID_SOCKET; + ensure_socket_table(); + EnterCriticalSection(&win_socket_table_lock); + for (int i = 0; i < WIN_SOCKET_TABLE_SIZE; i++) { + if (win_socket_table[i].fd == fd) { + result = win_socket_table[i].socket; + break; + } + } + LeaveCriticalSection(&win_socket_table_lock); + return result; +} + +static int win_socket_blocking_from_fd(int32_t fd) { + int result = 1; + ensure_socket_table(); + EnterCriticalSection(&win_socket_table_lock); + for (int i = 0; i < WIN_SOCKET_TABLE_SIZE; i++) { + if (win_socket_table[i].fd == fd) { + result = win_socket_table[i].blocking; + break; + } + } + LeaveCriticalSection(&win_socket_table_lock); + return result; +} + +static int win_socket_entry_index(int32_t fd) { + int result = -1; + ensure_socket_table(); + EnterCriticalSection(&win_socket_table_lock); + for (int i = 0; i < WIN_SOCKET_TABLE_SIZE; i++) { + if (win_socket_table[i].fd == fd) { + result = i; + break; + } + } + LeaveCriticalSection(&win_socket_table_lock); + return result; +} + +static int win_alloc_socket_fd(SOCKET s, int blocking) { + int fd = _open("NUL", _O_RDONLY | _O_BINARY | _O_NOINHERIT); + if (fd < 0) { + capture_errno(); + closesocket(s); + return -1; + } + + ensure_socket_table(); + EnterCriticalSection(&win_socket_table_lock); + for (int i = 0; i < WIN_SOCKET_TABLE_SIZE; i++) { + if (win_socket_table[i].fd < 0) { + win_socket_table[i].fd = fd; + win_socket_table[i].socket = s; + win_socket_table[i].blocking = blocking; + LeaveCriticalSection(&win_socket_table_lock); + return fd; + } + } + LeaveCriticalSection(&win_socket_table_lock); + + close_noraise(fd); + closesocket(s); + set_posix_errno(EMFILE); + return -1; +} + +static SOCKET win_remove_socket_fd(int32_t fd) { + SOCKET result = INVALID_SOCKET; + ensure_socket_table(); + EnterCriticalSection(&win_socket_table_lock); + for (int i = 0; i < WIN_SOCKET_TABLE_SIZE; i++) { + if (win_socket_table[i].fd == fd) { + result = win_socket_table[i].socket; + win_socket_table[i].fd = -1; + win_socket_table[i].socket = INVALID_SOCKET; + win_socket_table[i].blocking = 1; + break; + } + } + LeaveCriticalSection(&win_socket_table_lock); + return result; +} + +GP_EXPORT int32_t set_inheritable(int32_t fd, int32_t inheritable); + +/* The caller must hold win_socket_table_lock. */ +static int win_replace_socket_fd_locked(int32_t fd, SOCKET s, int blocking, SOCKET *replaced) { + assert(win_socket_table_lock.OwningThread == (HANDLE) (uintptr_t) GetCurrentThreadId()); + int free_index = -1; + for (int i = 0; i < WIN_SOCKET_TABLE_SIZE; i++) { + if (win_socket_table[i].fd == fd) { + *replaced = win_socket_table[i].socket; + win_socket_table[i].socket = s; + win_socket_table[i].blocking = blocking; + return 0; + } + if (free_index < 0 && win_socket_table[i].fd < 0) { + free_index = i; + } + } + if (free_index < 0) { + set_posix_errno(EMFILE); + return -1; + } + win_socket_table[free_index].fd = fd; + win_socket_table[free_index].socket = s; + win_socket_table[free_index].blocking = blocking; + *replaced = INVALID_SOCKET; + return 0; +} + +static SOCKET win_duplicate_socket(SOCKET s) { + WSAPROTOCOL_INFOA protocol_info; + if (WSADuplicateSocketA(s, GetCurrentProcessId(), &protocol_info) == SOCKET_ERROR) { + set_wsa_errno(); + return INVALID_SOCKET; + } + SOCKET dup = WSASocketA(FROM_PROTOCOL_INFO, FROM_PROTOCOL_INFO, FROM_PROTOCOL_INFO, &protocol_info, 0, WSA_FLAG_NO_HANDLE_INHERIT); + if (dup == INVALID_SOCKET) { + set_wsa_errno(); + return INVALID_SOCKET; + } + return dup; +} + +static int win_socket_dup(int32_t fd) { + SOCKET s = win_socket_from_fd(fd); + int blocking = win_socket_blocking_from_fd(fd); + SOCKET dup = win_duplicate_socket(s); + if (dup == INVALID_SOCKET) { + return -1; + } + return win_alloc_socket_fd(dup, blocking); +} + +static int win_socket_dup2(int32_t oldfd, int32_t newfd, int32_t inheritable) { + SOCKET dup = win_duplicate_socket(win_socket_from_fd(oldfd)); + if (dup == INVALID_SOCKET) { + return -1; + } + int blocking = win_socket_blocking_from_fd(oldfd); + + /* + * Keep the table locked while replacing the dummy CRT fd. This ensures that + * a full table is reported before _dup2 changes newfd and that the table + * entry is updated atomically with respect to other table users. + */ + ensure_socket_table(); + EnterCriticalSection(&win_socket_table_lock); + int table_has_slot = 0; + for (int i = 0; i < WIN_SOCKET_TABLE_SIZE; i++) { + if (win_socket_table[i].fd == newfd || win_socket_table[i].fd < 0) { + table_has_slot = 1; + break; + } + } + if (!table_has_slot) { + LeaveCriticalSection(&win_socket_table_lock); + closesocket(dup); + set_posix_errno(EMFILE); + return -1; + } + + if (dup2_noraise(oldfd, newfd) != 0) { + LeaveCriticalSection(&win_socket_table_lock); + closesocket(dup); + return -1; + } + + SOCKET replaced = INVALID_SOCKET; + int replaced_result = win_replace_socket_fd_locked(newfd, dup, blocking, &replaced); + LeaveCriticalSection(&win_socket_table_lock); + if (replaced_result < 0) { + closesocket(dup); + return -1; + } + if (replaced != INVALID_SOCKET) { + closesocket(replaced); + } + if (!inheritable) { + return set_inheritable(newfd, 0); + } + return 0; +} + +static intptr_t get_osfhandle_noraise(int32_t fd) { + intptr_t handle; + BEGIN_SUPPRESS_IPH + handle = _get_osfhandle(fd); + END_SUPPRESS_IPH + if (handle == -1) { + capture_errno(); + } + return handle; +} + +static int open_osfhandle_noraise(int64_t handle, int32_t flags) { + int fd; + BEGIN_SUPPRESS_IPH + fd = _open_osfhandle((intptr_t) handle, flags); + END_SUPPRESS_IPH + if (fd < 0) { + capture_errno(); + } + return fd; +} + +static void unix_time_to_filetime(int64_t seconds, int64_t nanoseconds, FILETIME *out) { + int64_t ticks = (seconds + 11644473600LL) * 10000000LL + nanoseconds / 100; + out->dwLowDateTime = (DWORD) ticks; + out->dwHighDateTime = (DWORD) (ticks >> 32); +} + +static int64_t filetime_to_unix_time(FILETIME time) { + ULARGE_INTEGER ticks; + ticks.LowPart = time.dwLowDateTime; + ticks.HighPart = time.dwHighDateTime; + return (int64_t) (ticks.QuadPart / 10000000ULL) - 11644473600LL; +} + +static int set_file_times(HANDLE handle, int64_t *times, int32_t nanosecond_resolution) { + FILETIME atime; + FILETIME mtime; + if (times == NULL) { + GetSystemTimeAsFileTime(&mtime); + atime = mtime; + } else { + unix_time_to_filetime(times[0], nanosecond_resolution ? times[1] : times[1] * 1000, &atime); + unix_time_to_filetime(times[2], nanosecond_resolution ? times[3] : times[3] * 1000, &mtime); + } + if (!SetFileTime(handle, NULL, &atime, &mtime)) { + set_win_errno(GetLastError()); + return -1; + } + return 0; +} + +static int set_path_times(const wchar_t *path, int64_t *times, int32_t nanosecond_resolution, int32_t followSymlinks) { + DWORD flags = FILE_FLAG_BACKUP_SEMANTICS; + if (!followSymlinks) { + flags |= FILE_FLAG_OPEN_REPARSE_POINT; + } + HANDLE handle = CreateFileW(path, FILE_WRITE_ATTRIBUTES, FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + NULL, OPEN_EXISTING, flags, NULL); + if (handle == INVALID_HANDLE_VALUE) { + set_win_errno(GetLastError()); + return -1; + } + int result = set_file_times(handle, times, nanosecond_resolution); + CloseHandle(handle); + return result; +} + +static int root_path_from_path(const wchar_t *path, wchar_t *root, DWORD root_size) { + wchar_t full_path[32768]; + DWORD full_len = GetFullPathNameW(path, (DWORD) (sizeof(full_path) / sizeof(wchar_t)), full_path, NULL); + if (full_len == 0 || full_len >= sizeof(full_path) / sizeof(wchar_t)) { + set_win_errno(GetLastError()); + return -1; + } + if (!GetVolumePathNameW(full_path, root, root_size)) { + set_win_errno(GetLastError()); + return -1; + } + return 0; +} + +static int statvfs_from_root(const wchar_t *root, int64_t *out) { + ULARGE_INTEGER free_bytes_available; + ULARGE_INTEGER total_number_of_bytes; + ULARGE_INTEGER total_number_of_free_bytes; + DWORD sectors_per_cluster = 0; + DWORD bytes_per_sector = 0; + DWORD number_of_free_clusters = 0; + DWORD total_number_of_clusters = 0; + if (!GetDiskFreeSpaceExW(root, &free_bytes_available, &total_number_of_bytes, &total_number_of_free_bytes)) { + set_win_errno(GetLastError()); + return -1; + } + if (!GetDiskFreeSpaceW(root, §ors_per_cluster, &bytes_per_sector, &number_of_free_clusters, &total_number_of_clusters)) { + sectors_per_cluster = 1; + bytes_per_sector = 4096; + } + uint64_t block_size = (uint64_t) sectors_per_cluster * bytes_per_sector; + if (block_size == 0) { + block_size = 4096; + } + out[0] = (int64_t) block_size; + out[1] = (int64_t) block_size; + out[2] = (int64_t) (total_number_of_bytes.QuadPart / block_size); + out[3] = (int64_t) (total_number_of_free_bytes.QuadPart / block_size); + out[4] = (int64_t) (free_bytes_available.QuadPart / block_size); + out[5] = 0; + out[6] = 0; + out[7] = 0; + out[8] = 0; + out[9] = 255; + out[10] = 0; + return 0; +} + +static int statvfs_from_path(const wchar_t *path, int64_t *out) { + DWORD attributes = GetFileAttributesW(path); + if (attributes == INVALID_FILE_ATTRIBUTES) { + set_win_errno(GetLastError()); + return -1; + } + + wchar_t root[MAX_PATH]; + if (root_path_from_path(path, root, MAX_PATH) != 0) { + return -1; + } + return statvfs_from_root(root, out); +} + +static int copy_string_to_buffer(char *buffer, int32_t bufferSize, size_t *offset, const char *value, uint64_t *out, int32_t outIndex) { + size_t len = strlen(value) + 1; + if (*offset + len > (size_t) bufferSize) { + return ERANGE; + } + out[outIndex] = *offset; + memcpy(buffer + *offset, value, len); + *offset += len; + return 0; +} + +static int get_current_username(char *buffer, DWORD size) { + const char *env_name = getenv("USERNAME"); + if (env_name != NULL && strlen(env_name) < size) { + strcpy(buffer, env_name); + return 0; + } + if (size > 4) { + strcpy(buffer, "user"); + return 0; + } + return ERANGE; +} + +static void get_current_home(char *buffer, size_t size) { + const char *profile = getenv("USERPROFILE"); + if (profile != NULL && strlen(profile) < size) { + strcpy(buffer, profile); + return; + } + const char *drive = getenv("HOMEDRIVE"); + const char *path = getenv("HOMEPATH"); + if (drive != NULL && path != NULL && strlen(drive) + strlen(path) < size) { + strcpy(buffer, drive); + strcat(buffer, path); + return; + } + if (size > 0) { + buffer[0] = '\0'; + } +} + +static int fill_current_pwd(uint64_t uid, char *buffer, int32_t bufferSize, uint64_t *output) { + char name[256]; + char dir[1024]; + const char *shell = getenv("COMSPEC"); + size_t offset = 0; + int result = get_current_username(name, sizeof(name)); + if (result != 0) { + return result; + } + get_current_home(dir, sizeof(dir)); + if (shell == NULL) { + shell = ""; + } + if ((result = copy_string_to_buffer(buffer, bufferSize, &offset, name, output, 0)) != 0) { + return result; + } + output[1] = uid; + output[2] = 0; + if ((result = copy_string_to_buffer(buffer, bufferSize, &offset, dir, output, 3)) != 0) { + return result; + } + return copy_string_to_buffer(buffer, bufferSize, &offset, shell, output, 4); +} + +static void stat_struct_to_longs(struct __stat64 *st, int64_t *out) { + out[0] = st->st_mode; + out[1] = st->st_ino; + out[2] = st->st_dev; + out[3] = st->st_nlink; + out[4] = st->st_uid; + out[5] = st->st_gid; + out[6] = st->st_size; + out[7] = st->st_atime; + out[8] = st->st_mtime; + out[9] = st->st_ctime; + out[10] = 0; + out[11] = 0; + out[12] = 0; +} + +static void stat_handle_to_longs(HANDLE handle, int64_t *out) { + BY_HANDLE_FILE_INFORMATION info; + if (GetFileInformationByHandle(handle, &info)) { + out[1] = ((int64_t) info.nFileIndexHigh << 32) | info.nFileIndexLow; + out[2] = info.dwVolumeSerialNumber; + out[3] = info.nNumberOfLinks; + out[7] = filetime_to_unix_time(info.ftLastAccessTime); + out[8] = filetime_to_unix_time(info.ftLastWriteTime); + out[9] = filetime_to_unix_time(info.ftCreationTime); + } +} + +static void stat_path_handle_to_longs(const wchar_t *path, int32_t followSymlinks, int64_t *out) { + DWORD flags = FILE_FLAG_BACKUP_SEMANTICS; + if (!followSymlinks) { + flags |= FILE_FLAG_OPEN_REPARSE_POINT; + } + HANDLE handle = CreateFileW(path, FILE_READ_ATTRIBUTES, FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, NULL, OPEN_EXISTING, flags, NULL); + if (handle != INVALID_HANDLE_VALUE) { + stat_handle_to_longs(handle, out); + CloseHandle(handle); + } +} + +static void stat_attributes_to_longs(DWORD attributes, DWORD sizeHigh, DWORD sizeLow, FILETIME accessTime, FILETIME writeTime, FILETIME creationTime, int64_t *out) { + memset(out, 0, 13 * sizeof(*out)); + if (attributes & FILE_ATTRIBUTE_DIRECTORY) { + out[0] = _S_IFDIR | 0111; + } else { + out[0] = _S_IFREG; + } + out[0] |= attributes & FILE_ATTRIBUTE_READONLY ? 0444 : 0666; + out[3] = 1; + out[6] = ((int64_t) sizeHigh << 32) | sizeLow; + out[7] = filetime_to_unix_time(accessTime); + out[8] = filetime_to_unix_time(writeTime); + out[9] = filetime_to_unix_time(creationTime); +} + +static int stat_path_attributes_to_longs(const wchar_t *path, int64_t *out) { + WIN32_FILE_ATTRIBUTE_DATA info; + if (GetFileAttributesExW(path, GetFileExInfoStandard, &info)) { + stat_attributes_to_longs(info.dwFileAttributes, info.nFileSizeHigh, info.nFileSizeLow, + info.ftLastAccessTime, info.ftLastWriteTime, info.ftCreationTime, out); + return 0; + } + DWORD error = GetLastError(); + if (error != ERROR_ACCESS_DENIED && error != ERROR_SHARING_VIOLATION) { + set_win_errno(error); + return -1; + } + + WIN32_FIND_DATAW findInfo; + HANDLE findHandle = FindFirstFileW(path, &findInfo); + if (findHandle == INVALID_HANDLE_VALUE) { + DWORD findError = GetLastError(); + switch (findError) { + case ERROR_FILE_NOT_FOUND: + case ERROR_PATH_NOT_FOUND: + case ERROR_NOT_READY: + case ERROR_BAD_NET_NAME: + error = findError; + break; + } + set_win_errno(error); + return -1; + } + stat_attributes_to_longs(findInfo.dwFileAttributes, findInfo.nFileSizeHigh, findInfo.nFileSizeLow, + findInfo.ftLastAccessTime, findInfo.ftLastWriteTime, findInfo.ftCreationTime, out); + FindClose(findHandle); + return 0; +} + +GP_EXPORT int64_t call_getpid(void) { + return _getpid(); +} + +GP_EXPORT int32_t call_umask(int32_t mask) { + return _umask(mask); +} + +GP_EXPORT int32_t get_inheritable(int32_t fd) { + int socket_index = win_socket_entry_index(fd); + if (socket_index >= 0) { + DWORD flags; + SOCKET s = win_socket_from_fd(fd); + if (!GetHandleInformation((HANDLE) s, &flags)) { + set_win_errno(GetLastError()); + return -1; + } + return (flags & HANDLE_FLAG_INHERIT) != 0; + } + intptr_t os_handle = get_osfhandle_noraise(fd); + if (os_handle == -1) { + return -1; + } + DWORD flags; + if (!GetHandleInformation((HANDLE) os_handle, &flags)) { + set_win_errno(GetLastError()); + return -1; + } + return (flags & HANDLE_FLAG_INHERIT) != 0; +} + +GP_EXPORT int32_t set_inheritable(int32_t fd, int32_t inheritable) { + int socket_index = win_socket_entry_index(fd); + if (socket_index >= 0) { + SOCKET s = win_socket_from_fd(fd); + if (!SetHandleInformation((HANDLE) s, HANDLE_FLAG_INHERIT, inheritable ? HANDLE_FLAG_INHERIT : 0)) { + set_win_errno(GetLastError()); + return -1; + } + intptr_t reserve_handle = get_osfhandle_noraise(fd); + if (reserve_handle != -1) { + SetHandleInformation((HANDLE) reserve_handle, HANDLE_FLAG_INHERIT, inheritable ? HANDLE_FLAG_INHERIT : 0); + } + return 0; + } + intptr_t os_handle = get_osfhandle_noraise(fd); + if (os_handle == -1) { + return -1; + } + if (!SetHandleInformation((HANDLE) os_handle, HANDLE_FLAG_INHERIT, inheritable ? HANDLE_FLAG_INHERIT : 0)) { + set_win_errno(GetLastError()); + return -1; + } + return 0; +} + +GP_EXPORT int32_t call_openat(int32_t dirFd, const wchar_t *pathname, int32_t flags, int32_t mode) { + if (!is_default_dir_fd(dirFd)) { + return unsupported(); + } + int open_flags = flags | _O_NOINHERIT; + if ((open_flags & _O_TEXT) == 0) { + open_flags |= _O_BINARY; + } + int result = _wopen(pathname, open_flags, mode); + // The following _setmode is defensive enforcement; _wopen(..., _O_BINARY, ...) should already set this mode. + if (result < 0) { + capture_errno(); + } else if ((open_flags & _O_TEXT) == 0 && _setmode(result, _O_BINARY) < 0) { + int error = errno; + close_noraise(result); + set_posix_errno(error); + return -1; + } + return result; +} + +GP_EXPORT int32_t call_close(int32_t fd) { + SOCKET s = win_remove_socket_fd(fd); + if (s != INVALID_SOCKET) { + int socket_result = closesocket(s); + int socket_error = socket_result == SOCKET_ERROR ? WSAGetLastError() : 0; + int fd_result = close_noraise(fd); + if (socket_result == SOCKET_ERROR) { + return set_wsa_errno_from_error(socket_error); + } + return fd_result; + } + return close_noraise(fd); +} + +GP_EXPORT int64_t call_read(int32_t fd, void *buf, uint64_t count) { + if (win_socket_entry_index(fd) >= 0) { + int result = recv(win_socket_from_fd(fd), buf, count > INT_MAX ? INT_MAX : (int) count, 0); + return result == SOCKET_ERROR ? set_wsa_errno() : result; + } + return read_noraise(fd, buf, count > INT_MAX ? INT_MAX : (unsigned int) count); +} + +GP_EXPORT int64_t call_write(int32_t fd, void *buf, uint64_t count) { + if (win_socket_entry_index(fd) >= 0) { + int result = send(win_socket_from_fd(fd), buf, count > INT_MAX ? INT_MAX : (int) count, 0); + return result == SOCKET_ERROR ? set_wsa_errno() : result; + } + return write_noraise(fd, buf, count > INT_MAX ? INT_MAX : (unsigned int) count); +} + +GP_EXPORT int32_t call_dup(int32_t fd) { + if (win_socket_entry_index(fd) >= 0) { + return win_socket_dup(fd); + } + int result = dup_noraise(fd); + if (result >= 0) { + set_inheritable(result, 0); + } + return result; +} + +GP_EXPORT int32_t call_dup2(int32_t oldfd, int32_t newfd, int32_t inheritable) { + if (oldfd == newfd) { + int result = dup2_noraise(oldfd, newfd); + if (result == 0 && !inheritable) { + result = set_inheritable(newfd, 0); + } + return result == 0 ? newfd : -1; + } + if (win_socket_entry_index(oldfd) >= 0) { + return win_socket_dup2(oldfd, newfd, inheritable) == 0 ? newfd : -1; + } + int result = dup2_noraise(oldfd, newfd); + if (result != 0) { + return -1; + } + SOCKET stale_socket = win_remove_socket_fd(newfd); + if (stale_socket != INVALID_SOCKET) { + closesocket(stale_socket); + } + if (!inheritable) { + result = set_inheritable(newfd, 0); + } + return result == 0 ? newfd : -1; +} + +GP_EXPORT int32_t call_get_osfhandle(int32_t fd, int64_t *out) { + intptr_t handle = get_osfhandle_noraise(fd); + if (handle == -1) { + return -1; + } + out[0] = (int64_t) handle; + return 0; +} + +GP_EXPORT int32_t graalpy_get_socket_handle(int32_t fd, int64_t *out) { + SOCKET s = win_socket_from_fd(fd); + if (s != INVALID_SOCKET) { + out[0] = (int64_t) s; + return 0; + } + if (get_osfhandle_noraise(fd) == -1) { + set_posix_errno(EBADF); + } else { + set_posix_errno(GP_ENOTSOCK); + } + return -1; +} + +GP_EXPORT int32_t call_open_osfhandle(int64_t handle, int32_t flags) { + if ((flags & _O_TEXT) == 0) { + flags |= _O_BINARY; + } + return open_osfhandle_noraise(handle, flags); +} + +GP_EXPORT int32_t call_pipe2(int32_t *pipefd) { + int result = _pipe(pipefd, 8192, _O_BINARY | _O_NOINHERIT); + if (result < 0) { + capture_errno(); + } + return result; +} + +static int win_fill_fd_set(fd_set *set, int32_t *fds, int32_t len) { + FD_ZERO(set); + for (int32_t i = 0; i < len; i++) { + SOCKET s = win_socket_from_fd(fds[i]); + int found = 0; + for (unsigned int j = 0; j < set->fd_count; j++) { + if (set->fd_array[j] == s) { + found = 1; + break; + } + } + if (!found) { + if (set->fd_count >= (unsigned int) FD_SETSIZE) { + return set_wsa_errno_from_error(WSAEINVAL); + } + set->fd_array[set->fd_count++] = s; + } + } + return 0; +} + +GP_EXPORT int32_t call_select(int32_t nfds, int32_t* readfds, int32_t readfdsLen, + int32_t* writefds, int32_t writefdsLen, int32_t* errfds, int32_t errfdsLen, + int64_t timeoutSec, int64_t timeoutUsec, int8_t* selected) { + fd_set read_set, write_set, err_set; + fd_set *read_ptr = readfdsLen ? &read_set : NULL; + fd_set *write_ptr = writefdsLen ? &write_set : NULL; + fd_set *err_ptr = errfdsLen ? &err_set : NULL; + int32_t selectedLen = readfdsLen + writefdsLen + errfdsLen; + if (ensure_winsock() < 0) { + return -1; + } + if (win_fill_fd_set(&read_set, readfds, readfdsLen) < 0) { + return -1; + } + if (win_fill_fd_set(&write_set, writefds, writefdsLen) < 0) { + return -1; + } + if (win_fill_fd_set(&err_set, errfds, errfdsLen) < 0) { + return -1; + } + if (selectedLen > 0) { + memset(selected, 0, selectedLen * sizeof(*selected)); + } + struct timeval timeout = {(long) timeoutSec, (long) timeoutUsec}; + int result = select(nfds, read_ptr, write_ptr, err_ptr, timeoutSec >= 0 ? &timeout : NULL); + if (result == SOCKET_ERROR) { + return set_wsa_errno(); + } + for (int32_t i = 0; i < readfdsLen; i++) { + selected[i] = FD_ISSET(win_socket_from_fd(readfds[i]), &read_set) != 0; + } + for (int32_t i = 0; i < writefdsLen; i++) { + selected[readfdsLen + i] = FD_ISSET(win_socket_from_fd(writefds[i]), &write_set) != 0; + } + for (int32_t i = 0; i < errfdsLen; i++) { + selected[readfdsLen + writefdsLen + i] = FD_ISSET(win_socket_from_fd(errfds[i]), &err_set) != 0; + } + return result; +} + +GP_EXPORT int32_t call_poll(int32_t fd, int32_t writing, int64_t timeoutSec, int64_t timeoutUsec) { + int8_t selected = 0; + return call_select(1, writing ? NULL : &fd, writing ? 0 : 1, writing ? &fd : NULL, writing ? 1 : 0, NULL, 0, timeoutSec, timeoutUsec, &selected); +} + +GP_EXPORT int64_t call_lseek(int32_t fd, int64_t offset, int32_t whence) { + return lseek_noraise(fd, offset, whence); +} + +GP_EXPORT int32_t call_ftruncate(int32_t fd, int64_t length) { + return chsize_noraise(fd, length) == 0 ? 0 : -1; +} + +GP_EXPORT int32_t call_truncate(const wchar_t *path, int64_t length) { + int fd = _wopen(path, _O_RDWR | _O_BINARY | _O_NOINHERIT); + if (fd < 0) { + capture_errno(); + return -1; + } + int result = call_ftruncate(fd, length); + int error = result != 0 ? errno_capture : 0; + close_noraise(fd); + if (result != 0) { + set_posix_errno(error); + } + return result; +} + +GP_EXPORT int32_t call_fsync(int32_t fd) { + return commit_noraise(fd); +} + +GP_EXPORT int32_t call_flock(int32_t fd, int32_t operation) { return unsupported(); } +GP_EXPORT int32_t call_fcntl_lock(int32_t fd, int32_t blocking, int32_t lockType, int32_t whence, int64_t start, int64_t length) { return unsupported(); } +GP_EXPORT int32_t get_blocking(int32_t fd) { + return win_socket_blocking_from_fd(fd); +} + +GP_EXPORT int32_t set_blocking(int32_t fd, int32_t blocking) { + int socket_index = win_socket_entry_index(fd); + if (socket_index >= 0) { + u_long nonblocking = blocking ? 0 : 1; + if (ioctlsocket(win_socket_from_fd(fd), FIONBIO, &nonblocking) == SOCKET_ERROR) { + return set_wsa_errno(); + } + ensure_socket_table(); + EnterCriticalSection(&win_socket_table_lock); + for (int i = 0; i < WIN_SOCKET_TABLE_SIZE; i++) { + if (win_socket_table[i].fd == fd) { + win_socket_table[i].blocking = blocking != 0; + break; + } + } + LeaveCriticalSection(&win_socket_table_lock); + return 0; + } + if (get_osfhandle_noraise(fd) == -1) { + return -1; + } + return 0; +} + +GP_EXPORT int32_t get_terminal_size(int32_t fd, int32_t *size) { + CONSOLE_SCREEN_BUFFER_INFO csbi; + intptr_t os_handle = get_osfhandle_noraise(fd); + if (os_handle == -1 || !GetConsoleScreenBufferInfo((HANDLE) os_handle, &csbi)) { + set_posix_errno(ENOTTY); + return -1; + } + size[0] = csbi.srWindow.Right - csbi.srWindow.Left + 1; + size[1] = csbi.srWindow.Bottom - csbi.srWindow.Top + 1; + return 0; +} + +GP_EXPORT int32_t call_fstatat(int32_t dirFd, const wchar_t *path, int32_t followSymlinks, int64_t *out) { + if (!is_default_dir_fd(dirFd)) { + return unsupported(); + } + struct __stat64 st; + int result = _wstat64(path, &st); + if (result == 0) { + stat_struct_to_longs(&st, out); + stat_path_handle_to_longs(path, followSymlinks, out); + return 0; + } + return stat_path_attributes_to_longs(path, out); +} + +GP_EXPORT int32_t call_fstat(int32_t fd, int64_t *out) { + if (get_osfhandle_noraise(fd) == -1) { + return -1; + } + struct __stat64 st; + int result; + BEGIN_SUPPRESS_IPH + result = _fstat64(fd, &st); + END_SUPPRESS_IPH + if (result == 0) { + stat_struct_to_longs(&st, out); + stat_handle_to_longs((HANDLE) get_osfhandle_noraise(fd), out); + } + if (result != 0) { + capture_errno(); + } + return result; +} + +GP_EXPORT int32_t call_statvfs(const wchar_t *path, int64_t *out) { + return statvfs_from_path(path, out); +} + +GP_EXPORT int32_t call_fstatvfs(int32_t fd, int64_t *out) { + intptr_t os_handle = get_osfhandle_noraise(fd); + if (os_handle == -1) { + return -1; + } + wchar_t path[32768]; + DWORD len = GetFinalPathNameByHandleW((HANDLE) os_handle, path, (DWORD) (sizeof(path) / sizeof(wchar_t)), FILE_NAME_NORMALIZED | VOLUME_NAME_DOS); + if (len == 0 || len >= sizeof(path) / sizeof(wchar_t)) { + set_win_errno(GetLastError()); + return -1; + } + const wchar_t *path_for_volume = path; + if (wcsncmp(path, L"\\\\?\\", 4) == 0) { + path_for_volume = path + 4; + } + return statvfs_from_path(path_for_volume, out); +} +GP_EXPORT int32_t call_uname(char *sysname, char *nodename, char *release, char *version, char *machine, int32_t size) { return unsupported(); } + +GP_EXPORT int32_t call_unlinkat(int32_t dirFd, const wchar_t *pathname, int32_t rmdir) { + if (!is_default_dir_fd(dirFd)) { + return unsupported(); + } + if (rmdir) { + int result = _wrmdir(pathname); + if (result != 0 && errno == EINVAL) { + DWORD attributes = GetFileAttributesW(pathname); + if (attributes != INVALID_FILE_ATTRIBUTES && !(attributes & FILE_ATTRIBUTE_DIRECTORY)) { + errno = ENOTDIR; + } + } + if (result != 0) { + capture_errno(); + } + return result; + } + int result = _wunlink(pathname); + if (result != 0) { + capture_errno(); + } + return result; +} + +GP_EXPORT int32_t call_linkat(int32_t oldDirFd, const wchar_t *oldPath, int32_t newDirFd, const wchar_t *newPath, int32_t flags) { return unsupported(); } +GP_EXPORT int32_t call_symlinkat(const wchar_t *target, int32_t dirFd, const wchar_t *linkpath) { return unsupported(); } + +GP_EXPORT int32_t call_mkdirat(int32_t dirFd, const wchar_t *pathname, int32_t mode) { + if (!is_default_dir_fd(dirFd)) { + return unsupported(); + } + int result = _wmkdir(pathname); + if (result != 0) { + capture_errno(); + } + return result; +} + +GP_EXPORT int32_t call_getcwd(wchar_t *buf, uint64_t size) { + DWORD len = GetCurrentDirectoryW(size > UINT32_MAX ? UINT32_MAX : (DWORD) size, buf); + if (len == 0) { + set_win_errno(GetLastError()); + return -1; + } + if (len >= size) { + set_posix_errno(ERANGE); + return -1; + } + return 0; +} +GP_EXPORT int32_t call_chdir(const wchar_t *path) { + int result = _wchdir(path); + if (result != 0) { + capture_errno(); + } + return result; +} +GP_EXPORT int32_t call_fchdir(int32_t fd) { + if (get_osfhandle_noraise(fd) == -1) { + return -1; + } + return unsupported(); +} +GP_EXPORT int32_t call_fchown(int32_t fd, int64_t owner, int64_t group) { return unsupported(); } +GP_EXPORT int32_t call_fchownat(int32_t dirfd, const wchar_t *pathname, int64_t owner, int64_t group, int32_t followSymlinks) { return unsupported(); } +GP_EXPORT int32_t call_isatty(int32_t fd) { return _isatty(fd); } + +GP_EXPORT intptr_t call_opendir(const wchar_t *name) { + if (name[0] == L'\0') { + set_posix_errno(ENOENT); + return 0; + } + size_t len = wcslen(name); + int needs_separator = len > 0 && name[len - 1] != L'\\' && name[len - 1] != L'/' && name[len - 1] != L':'; + size_t pattern_len = len + (needs_separator ? 4 : 3) + 1; + wchar_t *pattern = (wchar_t *) malloc(pattern_len * sizeof(wchar_t)); + if (!pattern) { + set_posix_errno(ENOMEM); + return 0; + } + wcscpy(pattern, name); + if (needs_separator) { + pattern[len++] = L'\\'; + } + wcscpy(pattern + len, L"*.*"); + + win_dir_t *dir = (win_dir_t *) calloc(1, sizeof(win_dir_t)); + if (!dir) { + free(pattern); + set_posix_errno(ENOMEM); + return 0; + } + dir->handle = FindFirstFileW(pattern, &dir->data); + dir->first = 1; + free(pattern); + if (dir->handle == INVALID_HANDLE_VALUE) { + DWORD error = GetLastError(); + if (error == ERROR_FILE_NOT_FOUND) { + DWORD attributes = GetFileAttributesW(name); + if (attributes != INVALID_FILE_ATTRIBUTES && (attributes & FILE_ATTRIBUTE_DIRECTORY)) { + dir->exhausted = 1; + return (intptr_t) dir; + } + if (attributes == INVALID_FILE_ATTRIBUTES) { + error = GetLastError(); + } + } + free(dir); + set_win_errno(error); + return 0; + } + return (intptr_t) dir; +} + +GP_EXPORT intptr_t call_fdopendir(int32_t fd) { + (void) fd; + set_posix_errno(ENOSYS); + return 0; +} + +GP_EXPORT int32_t call_closedir(intptr_t dirp) { + win_dir_t *dir = (win_dir_t *) dirp; + if (!dir) { + set_posix_errno(EBADF); + return -1; + } + BOOL ok = dir->handle == INVALID_HANDLE_VALUE ? TRUE : FindClose(dir->handle); + DWORD error = ok ? ERROR_SUCCESS : GetLastError(); + free(dir); + if (!ok) { + set_win_errno(error); + return -1; + } + return 0; +} + +GP_EXPORT int32_t call_readdir(intptr_t dirp, wchar_t *nameBuf, uint64_t nameBufSize, int64_t *out) { + win_dir_t *dir = (win_dir_t *) dirp; + if (!dir || nameBufSize == 0) { + set_posix_errno(EBADF); + return -1; + } + if (dir->exhausted) { + return 0; + } + if (!dir->first) { + if (!FindNextFileW(dir->handle, &dir->data)) { + DWORD error = GetLastError(); + if (error == ERROR_NO_MORE_FILES) { + return 0; + } else { + set_win_errno(error); + return -1; + } + } + } + dir->first = 0; + + size_t name_len = wcslen(dir->data.cFileName) + 1; + if (name_len > nameBufSize) { + set_posix_errno(ENAMETOOLONG); + return -1; + } + memcpy(nameBuf, dir->data.cFileName, name_len * sizeof(wchar_t)); + out[0] = 0; + out[1] = WIN_DT_UNKNOWN; + return 1; +} + +GP_EXPORT void call_rewinddir(intptr_t dirp) {} +GP_EXPORT int32_t call_utimensat(int32_t dirFd, const wchar_t *path, int64_t *timespec, int32_t followSymlinks) { + if (!is_default_dir_fd(dirFd)) { + return unsupported(); + } + return set_path_times(path, timespec, 1, followSymlinks); +} + +GP_EXPORT int32_t call_futimens(int32_t fd, int64_t *timespec) { + intptr_t os_handle = get_osfhandle_noraise(fd); + if (os_handle == -1) { + return -1; + } + return set_file_times((HANDLE) os_handle, timespec, 1); +} + +GP_EXPORT int32_t call_futimes(int32_t fd, int64_t *timeval) { + intptr_t os_handle = get_osfhandle_noraise(fd); + if (os_handle == -1) { + return -1; + } + return set_file_times((HANDLE) os_handle, timeval, 0); +} + +GP_EXPORT int32_t call_lutimes(const wchar_t *filename, int64_t *timeval) { + return set_path_times(filename, timeval, 0, 0); +} + +GP_EXPORT int32_t call_utimes(const wchar_t *filename, int64_t *timeval) { + return set_path_times(filename, timeval, 0, 1); +} + +GP_EXPORT int32_t call_renameat(int32_t oldDirFd, const wchar_t *oldPath, int32_t newDirFd, const wchar_t *newPath) { + if (!is_default_dir_fd(oldDirFd) || !is_default_dir_fd(newDirFd)) { + return unsupported(); + } + int result = _wrename(oldPath, newPath); + if (result != 0) { + capture_errno(); + } + return result; +} + +GP_EXPORT int32_t call_replaceat(int32_t oldDirFd, const wchar_t *oldPath, int32_t newDirFd, const wchar_t *newPath) { + if (!is_default_dir_fd(oldDirFd) || !is_default_dir_fd(newDirFd)) { + return unsupported(); + } + BOOL result = MoveFileExW(oldPath, newPath, MOVEFILE_REPLACE_EXISTING); + DWORD error = result ? ERROR_SUCCESS : GetLastError(); + if (!result) { + set_win_errno(error); + return -1; + } + return 0; +} + +GP_EXPORT int32_t call_faccessat(int32_t dirFd, const wchar_t *path, int32_t mode, int32_t effectiveIds, int32_t followSymlinks) { + if (!is_default_dir_fd(dirFd)) { + return unsupported(); + } + int result = _waccess(path, mode); + if (result != 0) { + capture_errno(); + } + return result; +} + +GP_EXPORT int32_t call_fchmodat(int32_t dirFd, const wchar_t *path, int32_t mode, int32_t followSymlinks) { + if (!is_default_dir_fd(dirFd)) { + return unsupported(); + } + int result = _wchmod(path, mode); + if (result != 0) { + capture_errno(); + } + return result; +} + +GP_EXPORT int32_t call_fchmod(int32_t fd, int32_t mode) { + (void) mode; + if (get_osfhandle_noraise(fd) == -1) { + return -1; + } + return unsupported(); +} +GP_EXPORT int64_t call_readlinkat(int32_t dirFd, const wchar_t *path, wchar_t *buf, uint64_t size) { + if (!is_default_dir_fd(dirFd)) { + return unsupported(); + } + DWORD attributes = GetFileAttributesW(path); + DWORD error = attributes == INVALID_FILE_ATTRIBUTES ? GetLastError() : ERROR_SUCCESS; + if (attributes == INVALID_FILE_ATTRIBUTES) { + set_win_errno(error); + return -1; + } + return unsupported(); +} +GP_EXPORT int64_t call_waitpid(int64_t pid, int32_t *status, int32_t options) { return unsupported(); } +GP_EXPORT int32_t call_wcoredump(int32_t status) { return 0; } +GP_EXPORT int32_t call_wifcontinued(int32_t status) { return 0; } +GP_EXPORT int32_t call_wifstopped(int32_t status) { return 0; } +GP_EXPORT int32_t call_wifsignaled(int32_t status) { return 0; } +GP_EXPORT int32_t call_wifexited(int32_t status) { return 0; } +GP_EXPORT int32_t call_wexitstatus(int32_t status) { return status; } +GP_EXPORT int32_t call_wtermsig(int32_t status) { return 0; } +GP_EXPORT int32_t call_wstopsig(int32_t status) { return 0; } +GP_EXPORT int32_t call_kill(int64_t pid, int32_t signal) { + if (signal == CTRL_C_EVENT || signal == CTRL_BREAK_EVENT) { + if (!GenerateConsoleCtrlEvent((DWORD) signal, (DWORD) pid)) { + set_win_errno(GetLastError()); + return -1; + } + return 0; + } + + HANDLE process = OpenProcess(PROCESS_ALL_ACCESS, FALSE, (DWORD) pid); + if (process == NULL) { + set_win_errno(GetLastError()); + return -1; + } + BOOL result = TerminateProcess(process, (UINT) signal); + DWORD error = GetLastError(); + CloseHandle(process); + if (!result) { + set_win_errno(error); + return -1; + } + return 0; +} +GP_EXPORT int32_t call_raise(int32_t signal) { return raise(signal); } +GP_EXPORT int32_t call_alarm(int32_t seconds) { return 0; } +GP_EXPORT int32_t call_getitimer(int32_t which, int64_t *current_value) { return unsupported(); } +GP_EXPORT int32_t call_setitimer(int32_t which, int64_t *new_value, int64_t *old_value) { return unsupported(); } +GP_EXPORT int32_t signal_self(int32_t signal) { + int result = raise(signal); + if (result != 0) { + capture_errno(); + } + return result; +} +GP_EXPORT int32_t call_killpg(int64_t pgid, int32_t signal) { return unsupported(); } +GP_EXPORT int64_t call_getuid(void) { return 0; } +GP_EXPORT int64_t call_geteuid(void) { return 1; } +GP_EXPORT int64_t call_getgid(void) { return 0; } +GP_EXPORT int64_t call_getegid(void) { return 0; } +GP_EXPORT int64_t call_getppid(void) { return 0; } +GP_EXPORT int64_t call_getpgid(int64_t pid) { return unsupported(); } +GP_EXPORT int32_t call_setpgid(int64_t pid, int64_t pgid) { return unsupported(); } +GP_EXPORT int64_t call_getpgrp(void) { return 0; } +GP_EXPORT int64_t call_getsid(int64_t pid) { return unsupported(); } +GP_EXPORT int64_t call_setsid(void) { return unsupported(); } +GP_EXPORT int32_t call_getgroups(int64_t size, int64_t* out) { return 0; } +GP_EXPORT int32_t call_getrusage(int32_t who, uint64_t* out) { return unsupported(); } +GP_EXPORT int32_t call_openpty(int32_t *outvars) { return unsupported(); } +GP_EXPORT int32_t call_ctermid(char *buf) { return unsupported(); } +GP_EXPORT int32_t call_setenv(wchar_t *name, wchar_t *value, int overwrite) { + (void) overwrite; + size_t name_len = wcslen(name); + size_t value_len = wcslen(value); + wchar_t *env = malloc((name_len + value_len + 2) * sizeof(wchar_t)); + if (env == NULL) { + set_posix_errno(ENOMEM); + return -1; + } + memcpy(env, name, name_len * sizeof(wchar_t)); + env[name_len] = L'='; + memcpy(env + name_len + 1, value, (value_len + 1) * sizeof(wchar_t)); + int result = _wputenv(env); + free(env); + if (result != 0) { + capture_errno(); + return -1; + } + return 0; +} +GP_EXPORT int32_t call_unsetenv(wchar_t *name) { + size_t name_len = wcslen(name); + wchar_t *env = malloc((name_len + 2) * sizeof(wchar_t)); + if (env == NULL) { + set_posix_errno(ENOMEM); + return -1; + } + memcpy(env, name, name_len * sizeof(wchar_t)); + env[name_len] = L'='; + env[name_len + 1] = L'\0'; + int result = _wputenv(env); + free(env); + if (result != 0) { + capture_errno(); + return -1; + } + return 0; +} +GP_EXPORT void call_execv(char *data, int64_t *offsets, int32_t offsetsLen) { set_posix_errno(ENOSYS); } +GP_EXPORT int32_t call_system(const wchar_t *command) { return _wsystem(command); } + +GP_EXPORT int64_t call_mmap(int64_t length, int32_t prot, int32_t flags, int32_t fd, int64_t offset, wchar_t *tagname) { + const int32_t win_prot_read = 1; + const int32_t win_prot_write = 2; + const int32_t win_map_shared = 1; + HANDLE file_handle = INVALID_HANDLE_VALUE; + DWORD protect; + DWORD desired_access; + uint64_t max_size = (uint64_t) (offset + length); + HANDLE mapping; + DWORD mapping_error; + void *view; + + if (length < 0 || offset < 0) { + set_posix_errno(EINVAL); + return 0; + } + + if (fd != -1) { + intptr_t os_handle = get_osfhandle_noraise(fd); + if (os_handle == -1) { + return 0; + } + file_handle = (HANDLE) os_handle; + } else { + max_size = (uint64_t) length; + } + + if ((prot & win_prot_write) != 0) { + if ((flags & win_map_shared) != 0) { + protect = PAGE_READWRITE; + desired_access = FILE_MAP_WRITE; + } else { + protect = PAGE_WRITECOPY; + desired_access = FILE_MAP_COPY; + } + } else if ((prot & win_prot_read) != 0) { + protect = PAGE_READONLY; + desired_access = FILE_MAP_READ; + } else { + protect = PAGE_NOACCESS; + desired_access = 0; + } + + SetLastError(ERROR_SUCCESS); + mapping = CreateFileMappingW(file_handle, NULL, protect, (DWORD) (max_size >> 32), (DWORD) max_size, tagname); + mapping_error = GetLastError(); + if (mapping == NULL) { + set_win_errno(mapping_error); + return 0; + } + + view = MapViewOfFile(mapping, desired_access, (DWORD) ((uint64_t) offset >> 32), (DWORD) offset, (SIZE_T) length); + if (view == NULL) { + set_win_errno(GetLastError()); + CloseHandle(mapping); + return 0; + } + + if (win_add_mmap(view, mapping) < 0) { + UnmapViewOfFile(view); + CloseHandle(mapping); + return 0; + } + SetLastError(mapping_error); + return (int64_t) view; +} + +GP_EXPORT int32_t call_munmap(int64_t address, int64_t length) { + (void) length; + void *view = (void *) address; + HANDLE mapping = win_remove_mmap(view); + int result = 0; + if (mapping == NULL) { + set_posix_errno(EINVAL); + return -1; + } + if (!UnmapViewOfFile(view)) { + set_win_errno(GetLastError()); + result = -1; + } + if (!CloseHandle(mapping) && result == 0) { + set_win_errno(GetLastError()); + result = -1; + } + return result; +} + +GP_EXPORT void call_msync(int64_t address, int64_t offset, int64_t length) { + FlushViewOfFile((void *) (address + offset), (SIZE_T) length); +} + +GP_EXPORT int32_t call_socket(int32_t family, int32_t type, int32_t protocol) { + if (ensure_winsock() < 0) { + return -1; + } + SOCKET s = socket(family, type, protocol); + if (s == INVALID_SOCKET) { + return set_wsa_errno(); + } + return win_alloc_socket_fd(s, 1); +} + +GP_EXPORT int32_t call_accept(int32_t sockfd, int8_t *addr, int32_t *addr_len) { + int len = sizeof(struct sockaddr_storage); + SOCKET s = accept(win_socket_from_fd(sockfd), (struct sockaddr *) addr, &len); + if (s == INVALID_SOCKET) { + return set_wsa_errno(); + } + *addr_len = len; + return win_alloc_socket_fd(s, 1); +} + +GP_EXPORT int32_t call_bind(int32_t sockfd, int8_t *addr, int32_t addr_len) { int r = bind(win_socket_from_fd(sockfd), (struct sockaddr *) addr, addr_len); return r == SOCKET_ERROR ? set_wsa_errno() : r; } +GP_EXPORT int32_t call_connect(int32_t sockfd, int8_t *addr, int32_t addr_len) { int r = connect(win_socket_from_fd(sockfd), (struct sockaddr *) addr, addr_len); return r == SOCKET_ERROR ? set_wsa_errno() : r; } +GP_EXPORT int32_t call_listen(int32_t sockfd, int32_t backlog) { int r = listen(win_socket_from_fd(sockfd), backlog); return r == SOCKET_ERROR ? set_wsa_errno() : r; } +GP_EXPORT int32_t call_getpeername(int32_t sockfd, int8_t *addr, int32_t *addr_len) { int len = sizeof(struct sockaddr_storage); int r = getpeername(win_socket_from_fd(sockfd), (struct sockaddr *) addr, &len); if (r == SOCKET_ERROR) return set_wsa_errno(); *addr_len = len; return r; } +GP_EXPORT int32_t call_getsockname(int32_t sockfd, int8_t *addr, int32_t *addr_len) { + SOCKET s = win_socket_from_fd(sockfd); + if (s == INVALID_SOCKET) { + set_posix_errno(get_osfhandle_noraise(sockfd) == -1 ? EBADF : GP_ENOTSOCK); + return -1; + } + int len = sizeof(struct sockaddr_storage); + int r = getsockname(s, (struct sockaddr *) addr, &len); + if (r == SOCKET_ERROR) { + return set_wsa_errno(); + } + *addr_len = len; + return r; +} +GP_EXPORT int32_t call_send(int32_t sockfd, void *buf, int32_t len, int32_t flags) { int r = send(win_socket_from_fd(sockfd), buf, len, flags); return r == SOCKET_ERROR ? set_wsa_errno() : r; } +GP_EXPORT int32_t call_sendto(int32_t sockfd, void *buf, int32_t offset, int32_t len, int32_t flags, int8_t *addr, int32_t addr_len) { int r = sendto(win_socket_from_fd(sockfd), ((char *) buf) + offset, len, flags, (struct sockaddr *) addr, addr_len); return r == SOCKET_ERROR ? set_wsa_errno() : r; } +GP_EXPORT int32_t call_recv(int32_t sockfd, void *buf, int32_t len, int32_t flags) { int r = recv(win_socket_from_fd(sockfd), buf, len, flags); return r == SOCKET_ERROR ? set_wsa_errno() : r; } +GP_EXPORT int32_t call_recvfrom(int32_t sockfd, void *buf, int32_t offset, int32_t len, int32_t flags, int8_t *src_addr, int32_t *addr_len) { int l = sizeof(struct sockaddr_storage); int r = recvfrom(win_socket_from_fd(sockfd), ((char *) buf) + offset, len, flags, (struct sockaddr *) src_addr, &l); if (r == SOCKET_ERROR) return set_wsa_errno(); *addr_len = l; return r; } +GP_EXPORT int32_t call_shutdown(int32_t sockfd, int32_t how) { int r = shutdown(win_socket_from_fd(sockfd), how); return r == SOCKET_ERROR ? set_wsa_errno() : r; } +GP_EXPORT int32_t call_getsockopt(int32_t sockfd, int32_t level, int32_t optname, void *buf, int32_t *bufLen) { int len = *bufLen; int r = getsockopt(win_socket_from_fd(sockfd), level, optname, buf, &len); if (r == SOCKET_ERROR) return set_wsa_errno(); *bufLen = len; return r; } +GP_EXPORT int32_t call_setsockopt(int32_t sockfd, int32_t level, int32_t optname, void *buf, int32_t bufLen) { + SOCKET s = win_socket_from_fd(sockfd); + uintptr_t socket_handle; + if (s == INVALID_SOCKET) { + return set_wsa_errno(); + } + if (level == SOL_SOCKET && optname == SO_UPDATE_ACCEPT_CONTEXT && buf != NULL && bufLen == sizeof(uintptr_t)) { + int32_t listen_fd = (int32_t) (*(uintptr_t *) buf); + SOCKET listen_socket = win_socket_from_fd(listen_fd); + if (listen_socket == INVALID_SOCKET) { + return set_wsa_errno(); + } + socket_handle = (uintptr_t) listen_socket; + buf = &socket_handle; + } + int r = setsockopt(s, level, optname, buf, bufLen); + return r == SOCKET_ERROR ? set_wsa_errno() : r; +} +GP_EXPORT int32_t call_inet_addr(const char *src) { return ntohl(inet_addr(src)); } +GP_EXPORT int64_t call_inet_aton(const char *src) { + unsigned long packed = inet_addr(src); + if (packed == INADDR_NONE && strcmp(src, "255.255.255.255") != 0) { + return -1; + } + return (int64_t) (uint32_t) ntohl(packed); +} +GP_EXPORT int32_t call_inet_ntoa(int32_t src, char *dst) { struct in_addr addr; addr.s_addr = htonl(src); const char *s = inet_ntop(AF_INET, &addr, dst, INET_ADDRSTRLEN); return s == NULL ? -1 : (int32_t) strlen(s); } +GP_EXPORT int32_t call_inet_pton(int32_t family, const char *src, void *dst) { int r = inet_pton(family, src, dst); return r == SOCKET_ERROR ? set_wsa_errno() : r; } +GP_EXPORT int32_t call_inet_ntop(int32_t family, void *src, char *dst, int32_t dstSize) { return inet_ntop(family, src, dst, dstSize) == NULL ? set_wsa_errno() : 0; } +GP_EXPORT int32_t call_gethostname(wchar_t *buf, int64_t bufLen) { + wchar_t stack_buf[MAX_COMPUTERNAME_LENGTH + 1]; + DWORD size = sizeof(stack_buf) / sizeof(stack_buf[0]); + wchar_t *name = stack_buf; + int result = 0; + + if (!GetComputerNameExW(ComputerNamePhysicalDnsHostname, stack_buf, &size)) { + if (GetLastError() != ERROR_MORE_DATA) { + set_win_errno(GetLastError()); + return -1; + } + if (size == 0) { + if (bufLen > 0) { + buf[0] = L'\0'; + return 0; + } + set_posix_errno(EFAULT); + return -1; + } + name = malloc(size * sizeof(wchar_t)); + if (name == NULL) { + set_posix_errno(ENOMEM); + return -1; + } + if (!GetComputerNameExW(ComputerNamePhysicalDnsHostname, name, &size)) { + set_win_errno(GetLastError()); + free(name); + return -1; + } + } + + if ((int64_t) size >= bufLen) { + set_posix_errno(ENAMETOOLONG); + result = -1; + } else { + memcpy(buf, name, size * sizeof(wchar_t)); + buf[size] = L'\0'; + } + + if (name != stack_buf) { + free(name); + } + return result; +} +GP_EXPORT int32_t call_getnameinfo(int8_t *addr, int32_t addr_len, char *hostBuf, int32_t hostBufLen, char *servBuf, int32_t servBufLen, int32_t flags) { return getnameinfo((struct sockaddr *) addr, addr_len, hostBuf, hostBufLen, servBuf, servBufLen, flags); } +GP_EXPORT int32_t call_getaddrinfo(const char *node, const char *service, int32_t family, int32_t sockType, int32_t protocol, int32_t flags, int64_t *ptr) { struct addrinfo hints = {0}; struct addrinfo *res = NULL; hints.ai_family = family; hints.ai_socktype = sockType; hints.ai_protocol = protocol; hints.ai_flags = flags; int ret = getaddrinfo(node, service, &hints, &res); if (ret == 0) { *ptr = (int64_t) res; } return ret; } +GP_EXPORT void call_freeaddrinfo(int64_t ptr) { freeaddrinfo((struct addrinfo *) ptr); } +GP_EXPORT void call_gai_strerror(int32_t error, char *buf, int32_t buflen) { snprintf(buf, buflen, "%d", error); } +GP_EXPORT int32_t get_addrinfo_members(int64_t ptr, int32_t *intData, int64_t *longData, int8_t *addr) { + struct addrinfo *ai = (struct addrinfo *) ptr; + if (!ai) { + return 0; + } + + memcpy(addr, ai->ai_addr, ai->ai_addrlen); + + longData[0] = (int64_t) ai->ai_canonname; + longData[1] = (int64_t) ai->ai_next; + + intData[0] = ai->ai_flags; + intData[1] = ai->ai_family; + intData[2] = ai->ai_socktype; + intData[3] = ai->ai_protocol; + intData[4] = (int32_t) ai->ai_addrlen; + intData[5] = ai->ai_addr->sa_family; + intData[6] = 0; + if (ai->ai_canonname != NULL) { + size_t len = strlen(ai->ai_canonname); + if (len >= 0x7fffffff) { + return -1; + } + intData[6] = (int32_t) len; + } + return 0; +} + +GP_EXPORT void *call_sem_open(const char *name, int32_t openFlags, int32_t mode, int32_t value) { + (void) mode; + HANDLE handle; + if ((openFlags & O_CREAT) != 0) { + handle = CreateSemaphoreA(NULL, value, LONG_MAX, NULL); + } else { + handle = OpenSemaphoreA(SEMAPHORE_ALL_ACCESS, FALSE, name); + } + if (handle == NULL) { + set_win_errno(GetLastError()); + } + return handle; +} +GP_EXPORT int32_t call_sem_close(void* handle) { + if (!CloseHandle((HANDLE) handle)) { + set_win_errno(GetLastError()); + return -1; + } + return 0; +} +GP_EXPORT int32_t call_sem_unlink(const char *name) { + (void) name; + return 0; +} +GP_EXPORT int32_t call_sem_getvalue(void* handle, int32_t *value) { + DWORD result = WaitForSingleObject((HANDLE) handle, 0); + if (result == WAIT_TIMEOUT) { + *value = 0; + return 0; + } + if (result != WAIT_OBJECT_0) { + set_win_errno(GetLastError()); + return -1; + } + LONG previousCount; + if (!ReleaseSemaphore((HANDLE) handle, 1, &previousCount)) { + set_win_errno(GetLastError()); + return -1; + } + *value = previousCount + 1; + return 0; +} +GP_EXPORT int32_t call_sem_post(void* handle) { + if (!ReleaseSemaphore((HANDLE) handle, 1, NULL)) { + DWORD error = GetLastError(); + set_win_errno(error); + return -1; + } + return 0; +} +GP_EXPORT int32_t call_sem_wait(void* handle) { + DWORD result = WaitForSingleObject((HANDLE) handle, INFINITE); + if (result == WAIT_OBJECT_0) { + return 0; + } + set_win_errno(GetLastError()); + return -1; +} +GP_EXPORT int32_t call_sem_trywait(void* handle) { + DWORD result = WaitForSingleObject((HANDLE) handle, 0); + if (result == WAIT_OBJECT_0) { + return 0; + } + if (result == WAIT_TIMEOUT) { + set_posix_errno(EAGAIN); + } else { + set_win_errno(GetLastError()); + } + return -1; +} +GP_EXPORT int32_t call_sem_timedwait(void* handle, int64_t deadlineNs) { return unsupported(); } +GP_EXPORT int64_t get_sysconf_getpw_r_size_max(void) { return -1; } +GP_EXPORT int32_t call_getpwuid_r(uint64_t uid, char *buffer, int32_t bufferSize, uint64_t *output) { + if (uid != 0) { + return -1; + } + return fill_current_pwd(uid, buffer, bufferSize, output); +} + +GP_EXPORT int32_t call_getpwname_r(const char *name, char *buffer, int32_t bufferSize, uint64_t *output) { + char current_name[256]; + int result = get_current_username(current_name, sizeof(current_name)); + if (result != 0) { + return result; + } + if (_stricmp(name, current_name) != 0) { + return -1; + } + return fill_current_pwd(0, buffer, bufferSize, output); +} +GP_EXPORT void call_setpwent(void) {} +GP_EXPORT void call_endpwent(void) {} +GP_EXPORT void *call_getpwent(int64_t *bufferSize) { return NULL; } +GP_EXPORT int32_t get_getpwent_data(void *p, char *buffer, int32_t bufferSize, uint64_t *output) { return ENOSYS; } +GP_EXPORT int32_t call_ioctl_bytes(int32_t fd, uint64_t request, char* buffer) { return unsupported(); } +GP_EXPORT int32_t call_ioctl_int(int32_t fd, uint64_t request, int32_t arg) { return unsupported(); } +GP_EXPORT int64_t call_sysconf(int32_t name) { set_posix_errno(EINVAL); return -1; } +GP_EXPORT int32_t get_errno(void) { return errno_capture; } +GP_EXPORT int32_t get_winerror(void) { return winerror_capture; } +GP_EXPORT int32_t get_wsaerror(void) { return wsaerror_capture; } +GP_EXPORT int32_t get_error_source(void) { return error_source_capture; } +GP_EXPORT void set_errno(int e) { errno = e; } + +GP_EXPORT void call_initialize(void) { + _setmode(0, _O_BINARY); + _setmode(1, _O_BINARY); + _setmode(2, _O_BINARY); +} + +GP_EXPORT int32_t init_constants(int64_t* out, int32_t len) { + if (len != 33) + return -1; + out[0] = sizeof(struct sockaddr); + out[1] = sizeof(((struct sockaddr*)0)->sa_family); + out[2] = offsetof(struct sockaddr, sa_family); + out[3] = sizeof(struct sockaddr_storage); + out[4] = sizeof(struct sockaddr_in); + out[5] = sizeof(((struct sockaddr_in*)0)->sin_family); + out[6] = offsetof(struct sockaddr_in, sin_family); + out[7] = sizeof(((struct sockaddr_in*)0)->sin_port); + out[8] = offsetof(struct sockaddr_in, sin_port); + out[9] = sizeof(((struct sockaddr_in*)0)->sin_addr); + out[10] = offsetof(struct sockaddr_in, sin_addr); + out[11] = sizeof(struct sockaddr_in6); + out[12] = sizeof(((struct sockaddr_in6*)0)->sin6_family); + out[13] = offsetof(struct sockaddr_in6, sin6_family); + out[14] = sizeof(((struct sockaddr_in6*)0)->sin6_port); + out[15] = offsetof(struct sockaddr_in6, sin6_port); + out[16] = sizeof(((struct sockaddr_in6*)0)->sin6_flowinfo); + out[17] = offsetof(struct sockaddr_in6, sin6_flowinfo); + out[18] = sizeof(((struct sockaddr_in6*)0)->sin6_addr); + out[19] = offsetof(struct sockaddr_in6, sin6_addr); + out[20] = sizeof(((struct sockaddr_in6*)0)->sin6_scope_id); + out[21] = offsetof(struct sockaddr_in6, sin6_scope_id); + out[22] = sizeof(struct in_addr); + out[23] = sizeof(((struct in_addr*)0)->s_addr); + out[24] = offsetof(struct in_addr, s_addr); + out[25] = sizeof(struct in6_addr); + out[26] = sizeof(((struct in6_addr*)0)->s6_addr); + out[27] = offsetof(struct in6_addr, s6_addr); + out[28] = sizeof(struct sockaddr_un); + out[29] = sizeof(((struct sockaddr_un*)0)->sun_family); + out[30] = offsetof(struct sockaddr_un, sun_family); + out[31] = sizeof(((struct sockaddr_un*)0)->sun_path); + out[32] = offsetof(struct sockaddr_un, sun_path); + return 0; +} + +#else + // This file uses GNU extensions. Functions that require non-GNU versions (e.g. strerror_r) // need to go to posix_no_gnu.c #if defined(__gnu_linux__) && !defined(_GNU_SOURCE) @@ -170,6 +2156,20 @@ int32_t call_dup2(int32_t oldfd, int32_t newfd, int32_t inheritable) { return res; } +int32_t call_get_osfhandle(int32_t fd, int64_t *out) { + (void) fd; + (void) out; + errno = ENOSYS; + return -1; +} + +int32_t call_open_osfhandle(int64_t handle, int32_t flags) { + (void) handle; + (void) flags; + errno = ENOSYS; + return -1; +} + int32_t call_pipe2(int32_t *pipefd) { #ifdef __gnu_linux__ CAPTURE_ERRNO_AND_RETURN(-1, pipe2(pipefd, O_CLOEXEC)); @@ -571,6 +2571,10 @@ int32_t call_renameat(int32_t oldDirFd, const char *oldPath, int32_t newDirFd, c CAPTURE_ERRNO_AND_RETURN(-1, renameat(oldDirFd, oldPath, newDirFd, newPath)); } +int32_t call_replaceat(int32_t oldDirFd, const char *oldPath, int32_t newDirFd, const char *newPath) { + return renameat(oldDirFd, oldPath, newDirFd, newPath); +} + int32_t call_faccessat(int32_t dirFd, const char *path, int32_t mode, int32_t effectiveIds, int32_t followSymlinks) { int flags = 0; if (!followSymlinks) { @@ -831,7 +2835,8 @@ int32_t call_system(const char *pathname) { return system(pathname); } -int64_t call_mmap(int64_t length, int32_t prot, int32_t flags, int32_t fd, int64_t offset) { +int64_t call_mmap(int64_t length, int32_t prot, int32_t flags, int32_t fd, int64_t offset, char *tagname) { + (void) tagname; void *result = mmap(NULL, length, prot, flags, fd, offset); if (result == MAP_FAILED) { capture_errno(); @@ -1256,11 +3261,21 @@ int32_t get_errno() { return errno_capture; } -#ifdef _WIN32 -#define unix_or_0(x) 0 -#else +void call_initialize(void) { +} + +int32_t get_winerror() { + return 0; +} + +int32_t get_wsaerror() { + return 0; +} + +int32_t get_error_source() { + return ERROR_CAPTURE_ERRNO; +} #define unix_or_0(x) x -#endif // start generated int32_t init_constants(int64_t* out, int32_t len) { @@ -1302,3 +3317,5 @@ int32_t init_constants(int64_t* out, int32_t len) { return 0; } // end generated + +#endif diff --git a/graalpython/python-libposix/src/posix_no_gnu.c b/graalpython/python-libposix/src/posix_no_gnu.c index 7b3c76ae3a..2eda881e43 100644 --- a/graalpython/python-libposix/src/posix_no_gnu.c +++ b/graalpython/python-libposix/src/posix_no_gnu.c @@ -46,19 +46,28 @@ #include #include -/* - There are two versions of strerror_r and we need the POSIX one. The following lines double-check - that we got it. First, we check that _GNU_SOURCE has not been defined by any of the included headers. - Then we explicitly declare the function with POSIX signature which should force the compiler to - report an error in case we got the GNU version somehow. -*/ +#ifdef _WIN32 +#define GP_EXPORT __declspec(dllexport) +#else +#define GP_EXPORT +#endif + +GP_EXPORT void call_strerror(int32_t error, char *buf, int32_t buflen) { +#ifdef _WIN32 + int err = strerror_s(buf, buflen, error); +#else + /* + There are two versions of strerror_r and we need the POSIX one. The following lines double-check + that we got it. First, we check that _GNU_SOURCE has not been defined by any of the included headers. + Then we explicitly declare the function with POSIX signature which should force the compiler to + report an error in case we got the GNU version somehow. + */ #ifdef _GNU_SOURCE #error "Someone defined _GNU_SOURCE" #endif -int strerror_r(int errnum, char *buf, size_t buflen); - -void call_strerror(int32_t error, char *buf, int32_t buflen) { + int strerror_r(int errnum, char *buf, size_t buflen); int err = strerror_r(error, buf, buflen); +#endif if (err) { snprintf(buf, buflen, "Unknown error %d", error); } diff --git a/graalpython/python-venvlauncher/src/venvlauncher.c b/graalpython/python-venvlauncher/src/venvlauncher.c index e69e0035f6..f5f909ce4e 100644 --- a/graalpython/python-venvlauncher/src/venvlauncher.c +++ b/graalpython/python-venvlauncher/src/venvlauncher.c @@ -201,6 +201,7 @@ launchEnvironment(wchar_t *env, wchar_t *exe) *** PROCESS CONTROLLER *** \******************************************************************************/ +#define GRAAL_PYTHON_ARGS L"GRAAL_PYTHON_ARGS=" #define GRAAL_PYTHON_VM_ARGS L"GRAAL_PYTHON_VM_ARGS=" #define GRAAL_PYTHON_EXE_ARG L"--python.Executable=" #define GRAAL_PYTHON_BASE_EXECUTABLE_ARG L"--python.BaseExecutable=" @@ -384,6 +385,10 @@ wmain(int argc, wchar_t ** argv) } envCur = env; for (int i = 0; i = wcslen(envCur); i) { + if (wcsncmp(envCur, GRAAL_PYTHON_ARGS, wcslen(GRAAL_PYTHON_ARGS)) == 0) { + envCur = envCur + i + 1; + continue; + } // env needs room for key=value and \0 envSize = envSize + i + 1; envCur = envCur + i + 1; @@ -416,6 +421,10 @@ wmain(int argc, wchar_t ** argv) envCur = env; newEnvCur = newEnv; for (int i = 0; i = wcslen(envCur); i) { + if (wcsncmp(envCur, GRAAL_PYTHON_ARGS, wcslen(GRAAL_PYTHON_ARGS)) == 0) { + envCur = envCur + i + 1; + continue; + } exitCode = wcscpy_s(newEnvCur, envSize, envCur); if (exitCode) { winerror(exitCode, L"Failed to copy envvar"); diff --git a/mx.graalpython/suite.py b/mx.graalpython/suite.py index 9a9bb1f2a2..9691f0fe28 100644 --- a/mx.graalpython/suite.py +++ b/mx.graalpython/suite.py @@ -813,12 +813,12 @@ "deliverable": "posix", "buildDependencies": [], "cflags": [ - "-DNDEBUG", "-O3", "-Wall", "-Werror", + "-DNDEBUG", "-O3", ], "os_arch": { "windows": { "": { - "defaultBuild": False, + "ldlibs": ["Ws2_32.lib"], "multitarget": { "libc": ["default"], }, @@ -826,7 +826,7 @@ }, "darwin": { "": { - "defaultBuild": True, + "cflags": ["-Wall", "-Werror"], "multitarget": { "libc": ["default"], }, @@ -834,8 +834,8 @@ }, "": { "": { + "cflags": ["-Wall", "-Werror"], "ldlibs": ["-lutil", "-lrt"], - "defaultBuild": True, "multitarget": [ {"libc": ["glibc", "default"]}, {"libc": ["musl"], "variant": ["swcfi"]}, @@ -1054,6 +1054,7 @@ "layout": { "//": [ "dependency:com.oracle.graal.python.cext/-//bin/*", + "dependency:python-libposix/-//*", "dependency:python-libbz2/-//bin/*", ] },