From 0bb0b041fa42cc38844ef093da6529146756f4f0 Mon Sep 17 00:00:00 2001 From: Tadeja Kadunc Date: Wed, 8 Jul 2026 18:11:12 +0200 Subject: [PATCH 01/15] Verify fix --- compose.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose.yaml b/compose.yaml index 6676a87bd57..6809d6994b6 100644 --- a/compose.yaml +++ b/compose.yaml @@ -982,7 +982,7 @@ services: clang_tools: ${CLANG_TOOLS} llvm: ${LLVM} pyodide_version: "0.26.0" - chrome_version: "148" + chrome_version: "latest" selenium_version: "4.41.0" required_python_min: "(3,12)" python: ${PYTHON} From 8c245cf21c1a2037987d7b3711f81390ee90c82b Mon Sep 17 00:00:00 2001 From: Tadeja Kadunc Date: Wed, 8 Jul 2026 22:08:35 +0200 Subject: [PATCH 02/15] Disable zlib --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 07054b4bb29..703368e92a6 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -2073,7 +2073,10 @@ function(build_protobuf) "-DCMAKE_C_FLAGS=" "-DCMAKE_INSTALL_PREFIX=${PROTOBUF_HOST_PREFIX}" -Dprotobuf_BUILD_TESTS=OFF - -Dprotobuf_DEBUG_POSTFIX=) + -Dprotobuf_DEBUG_POSTFIX= + # OFF so protobuf does not take conda's zlib and add its include dir + # globally, so vendored Abseil sources compile against env's old headers + -Dprotobuf_WITH_ZLIB=OFF) if(ABSL_VENDORED) # Force protobuf to reuse Arrow's already-extracted absl source # so we don't re-download and we don't have issues with multiple abseil. From c55e01dcf2002f711fea531a6861c297194ca774 Mon Sep 17 00:00:00 2001 From: Tadeja Kadunc Date: Wed, 8 Jul 2026 22:54:19 +0200 Subject: [PATCH 03/15] Install newer pytest veersion --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 2 +- python/scripts/run_emscripten_tests.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 703368e92a6..26b3e37a496 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -2075,7 +2075,7 @@ function(build_protobuf) -Dprotobuf_BUILD_TESTS=OFF -Dprotobuf_DEBUG_POSTFIX= # OFF so protobuf does not take conda's zlib and add its include dir - # globally, so vendored Abseil sources compile against env's old headers + # globally, causing vendored Abseil sources to include env's old headers -Dprotobuf_WITH_ZLIB=OFF) if(ABSL_VENDORED) # Force protobuf to reuse Arrow's already-extracted absl source diff --git a/python/scripts/run_emscripten_tests.py b/python/scripts/run_emscripten_tests.py index 3cd7d5fd67b..05557a4c465 100644 --- a/python/scripts/run_emscripten_tests.py +++ b/python/scripts/run_emscripten_tests.py @@ -283,7 +283,7 @@ def _load_pyarrow_in_runner(driver, wheel_name): """import sys import micropip if "pyarrow" not in sys.modules: - await micropip.install("hypothesis") + await micropip.install(["hypothesis", "pytest>=8.2"]) import pyodide_js as pjs await pjs.loadPackage("tzdata") await pjs.loadPackage("numpy") From 5374a13ba8f9c74e88c68befa7998deb9de64fe6 Mon Sep 17 00:00:00 2001 From: Tadeja Kadunc Date: Thu, 9 Jul 2026 07:54:06 +0200 Subject: [PATCH 04/15] Keep pytest8.1.1 plus patch for exec_type --- python/scripts/run_emscripten_tests.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/python/scripts/run_emscripten_tests.py b/python/scripts/run_emscripten_tests.py index 05557a4c465..002698a77e2 100644 --- a/python/scripts/run_emscripten_tests.py +++ b/python/scripts/run_emscripten_tests.py @@ -283,12 +283,20 @@ def _load_pyarrow_in_runner(driver, wheel_name): """import sys import micropip if "pyarrow" not in sys.modules: - await micropip.install(["hypothesis", "pytest>=8.2"]) + await micropip.install("hypothesis") import pyodide_js as pjs await pjs.loadPackage("tzdata") await pjs.loadPackage("numpy") await pjs.loadPackage("pandas") import pytest + import inspect + if "exc_type" not in inspect.signature(pytest.importorskip).parameters: + _original_importorskip = pytest.importorskip + def _importorskip(modname, minversion=None, reason=None, *, + exc_type=ImportError): + return _original_importorskip(modname, minversion=minversion, + reason=reason) + pytest.importorskip = _importorskip import pandas # import pandas after pyarrow package load for pandas/pyarrow # functions to work import pyarrow From b05e5b094cc01fdb4b7b2f46b45b5ff77c3239f5 Mon Sep 17 00:00:00 2001 From: Tadeja Kadunc Date: Thu, 9 Jul 2026 10:13:20 +0200 Subject: [PATCH 05/15] Fix script timeout chrome 150 --- python/scripts/run_emscripten_tests.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/python/scripts/run_emscripten_tests.py b/python/scripts/run_emscripten_tests.py index 002698a77e2..330b7d89cee 100644 --- a/python/scripts/run_emscripten_tests.py +++ b/python/scripts/run_emscripten_tests.py @@ -33,6 +33,9 @@ from selenium import webdriver +BROWSER_TIMEOUT_SECONDS = 3600 + + class TemplateOverrider(http.server.SimpleHTTPRequestHandler): def log_request(self, code="-", size="-"): # don't log successful requests but log errors @@ -203,7 +206,7 @@ def __init__(self, hostname, port, driver): self.driver = driver self.driver.get(f"http://{hostname}:{port}/test.html") # Chrome on CI takes longer than locally to compile. - self.driver.set_script_timeout(1200) + self.driver.set_script_timeout(BROWSER_TIMEOUT_SECONDS) def load_pyodide(self, dist_dir): pass @@ -262,8 +265,9 @@ def __init__(self, hostname, port): options = Options() options.add_argument("--headless") options.add_argument("--no-sandbox") + options.add_argument("--disable-dev-shm-usage") driver = webdriver.Chrome(options=options) - driver.command_executor._client_config.timeout = 1200 + driver.command_executor._client_config.timeout = BROWSER_TIMEOUT_SECONDS super().__init__(hostname, port, driver) From f4fc717b6b0477d0ec733a553cb36520853dfd1b Mon Sep 17 00:00:00 2001 From: Tadeja Kadunc Date: Thu, 9 Jul 2026 11:20:39 +0200 Subject: [PATCH 06/15] Add logging --- python/scripts/run_emscripten_tests.py | 74 +++++++++++++++----------- 1 file changed, 43 insertions(+), 31 deletions(-) diff --git a/python/scripts/run_emscripten_tests.py b/python/scripts/run_emscripten_tests.py index 330b7d89cee..a4a57ad9568 100644 --- a/python/scripts/run_emscripten_tests.py +++ b/python/scripts/run_emscripten_tests.py @@ -85,22 +85,36 @@ def do_GET(self) -> bytes | None: elif self.path.endswith("/worker.js"): body = b""" importScripts("./pyodide.js"); + function postLog(message) { + self.postMessage({print: Array.from(new TextEncoder().encode( + `[worker] ${new Date().toISOString()} ${message}\n`))}); + } onmessage = async function (e) { const data = e.data; - if (!self.pyodide) { - self.pyodide = await loadPyodide(); - } - function do_print(arg) { - let databytes = Array.from(arg); - self.postMessage({print:databytes}); - return databytes.length; + try { + if (!self.pyodide) { + postLog("loading Pyodide"); + self.pyodide = await loadPyodide(); + postLog("loaded Pyodide"); + } + function do_print(arg) { + let databytes = Array.from(arg); + self.postMessage({print:databytes}); + return databytes.length; + } + self.pyodide.setStdout({write:do_print,isatty:data.isatty}); + self.pyodide.setStderr({write:do_print,isatty:data.isatty}); + + postLog("loading packages from imports"); + await self.pyodide.loadPackagesFromImports(data.python); + postLog("running Python"); + let results = await self.pyodide.runPythonAsync(data.python); + postLog(`Python completed with result ${results}`); + self.postMessage({results}); + } catch (error) { + postLog(`Python failed: ${error && error.stack || error}`); + self.postMessage({results: 1}); } - self.pyodide.setStdout({write:do_print,isatty:data.isatty}); - self.pyodide.setStderr({write:do_print,isatty:data.isatty}); - - await self.pyodide.loadPackagesFromImports(data.python); - let results = await self.pyodide.runPythonAsync(data.python); - self.postMessage({results}); } """ self.send_response(200) @@ -218,30 +232,23 @@ def load_arrow(self): ) def execute_python(self, code, wait_for_terminate=True): + self.driver.execute_script( + f""" + let python = `{code}`; + delete window.python_script_done; + window.python_done_callback= (x) => {{window.python_script_done=x;}}; + window.pyworker.postMessage( + {{python,isatty:{'true' if sys.stdout.isatty() else 'false'}}}); + """ + ) if wait_for_terminate: - self.driver.execute_async_script( - f""" - let callback = arguments[arguments.length-1]; - python = `{code}`; - window.python_done_callback = callback; - window.pyworker.postMessage( - {{python, isatty: {'true' if sys.stdout.isatty() else 'false'}}}); - """ - ) - else: - self.driver.execute_script( - f""" - let python = `{code}`; - window.python_done_callback= (x) => {{window.python_script_done=x;}}; - window.pyworker.postMessage( - {{python,isatty:{'true' if sys.stdout.isatty() else 'false'}}}); - """ - ) + return self.wait_for_done() def clear_logs(self): self.driver.execute_script("window.python_logs = [];") def wait_for_done(self): + last_heartbeat = time.monotonic() while True: # poll for console.log messages from our webworker # which are the output of pytest @@ -250,11 +257,16 @@ def wait_for_done(self): ) if len(lines) > 0: sys.stdout.buffer.write(bytes(lines)) + sys.stdout.buffer.flush() done = self.driver.execute_script("return window.python_script_done;") if done is not None: value = done["result"] self.driver.execute_script("delete window.python_script_done;") return value + now = time.monotonic() + if now - last_heartbeat > 30: + print("Waiting for browser-side Python to finish...", flush=True) + last_heartbeat = now time.sleep(0.1) From 382801341d59fb46c8d98a4c22acc76f821da3ab Mon Sep 17 00:00:00 2001 From: Tadeja Kadunc Date: Thu, 9 Jul 2026 12:08:53 +0200 Subject: [PATCH 07/15] Revert browser timeout to 1200s --- python/scripts/run_emscripten_tests.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/python/scripts/run_emscripten_tests.py b/python/scripts/run_emscripten_tests.py index a4a57ad9568..c6dc12eec6d 100644 --- a/python/scripts/run_emscripten_tests.py +++ b/python/scripts/run_emscripten_tests.py @@ -33,9 +33,6 @@ from selenium import webdriver -BROWSER_TIMEOUT_SECONDS = 3600 - - class TemplateOverrider(http.server.SimpleHTTPRequestHandler): def log_request(self, code="-", size="-"): # don't log successful requests but log errors @@ -220,7 +217,7 @@ def __init__(self, hostname, port, driver): self.driver = driver self.driver.get(f"http://{hostname}:{port}/test.html") # Chrome on CI takes longer than locally to compile. - self.driver.set_script_timeout(BROWSER_TIMEOUT_SECONDS) + self.driver.set_script_timeout(1200) def load_pyodide(self, dist_dir): pass @@ -279,7 +276,7 @@ def __init__(self, hostname, port): options.add_argument("--no-sandbox") options.add_argument("--disable-dev-shm-usage") driver = webdriver.Chrome(options=options) - driver.command_executor._client_config.timeout = BROWSER_TIMEOUT_SECONDS + driver.command_executor._client_config.timeout = 1200 super().__init__(hostname, port, driver) From c616d1bb39ddf3045821a449d69258d465250103 Mon Sep 17 00:00:00 2001 From: Tadeja Kadunc Date: Thu, 9 Jul 2026 12:16:12 +0200 Subject: [PATCH 08/15] Further improve new logging --- python/scripts/run_emscripten_tests.py | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/python/scripts/run_emscripten_tests.py b/python/scripts/run_emscripten_tests.py index c6dc12eec6d..329c1b8f826 100644 --- a/python/scripts/run_emscripten_tests.py +++ b/python/scripts/run_emscripten_tests.py @@ -82,34 +82,26 @@ def do_GET(self) -> bytes | None: elif self.path.endswith("/worker.js"): body = b""" importScripts("./pyodide.js"); - function postLog(message) { - self.postMessage({print: Array.from(new TextEncoder().encode( - `[worker] ${new Date().toISOString()} ${message}\n`))}); + function do_print(arg) { + let databytes = Array.from(arg); + self.postMessage({print:databytes}); + return databytes.length; } onmessage = async function (e) { const data = e.data; try { if (!self.pyodide) { - postLog("loading Pyodide"); self.pyodide = await loadPyodide(); - postLog("loaded Pyodide"); - } - function do_print(arg) { - let databytes = Array.from(arg); - self.postMessage({print:databytes}); - return databytes.length; } self.pyodide.setStdout({write:do_print,isatty:data.isatty}); self.pyodide.setStderr({write:do_print,isatty:data.isatty}); - postLog("loading packages from imports"); await self.pyodide.loadPackagesFromImports(data.python); - postLog("running Python"); let results = await self.pyodide.runPythonAsync(data.python); - postLog(`Python completed with result ${results}`); self.postMessage({results}); } catch (error) { - postLog(`Python failed: ${error && error.stack || error}`); + do_print(new TextEncoder().encode( + `Worker failed: ${error && error.stack || error}\n`)); self.postMessage({results: 1}); } } @@ -274,7 +266,6 @@ def __init__(self, hostname, port): options = Options() options.add_argument("--headless") options.add_argument("--no-sandbox") - options.add_argument("--disable-dev-shm-usage") driver = webdriver.Chrome(options=options) driver.command_executor._client_config.timeout = 1200 super().__init__(hostname, port, driver) From 6c973266f01fe011173f78742fa505f5c6d0495d Mon Sep 17 00:00:00 2001 From: Tadeja Kadunc Date: Thu, 9 Jul 2026 13:23:40 +0200 Subject: [PATCH 09/15] Re-add logging to debug hang --- python/scripts/run_emscripten_tests.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/python/scripts/run_emscripten_tests.py b/python/scripts/run_emscripten_tests.py index 329c1b8f826..99e0e42ed9c 100644 --- a/python/scripts/run_emscripten_tests.py +++ b/python/scripts/run_emscripten_tests.py @@ -87,17 +87,25 @@ def do_GET(self) -> bytes | None: self.postMessage({print:databytes}); return databytes.length; } + function log(message) { + do_print(new TextEncoder().encode(`[worker] ${message}\n`)); + } onmessage = async function (e) { const data = e.data; try { if (!self.pyodide) { + log("loading Pyodide"); self.pyodide = await loadPyodide(); + log("loaded Pyodide"); } self.pyodide.setStdout({write:do_print,isatty:data.isatty}); self.pyodide.setStderr({write:do_print,isatty:data.isatty}); + log("loading packages from imports"); await self.pyodide.loadPackagesFromImports(data.python); + log("running Python"); let results = await self.pyodide.runPythonAsync(data.python); + log("Python completed"); self.postMessage({results}); } catch (error) { do_print(new TextEncoder().encode( From bee137c688a4008ed86452cb1591d4709a8fe160 Mon Sep 17 00:00:00 2001 From: Tadeja Kadunc Date: Thu, 9 Jul 2026 14:52:39 +0200 Subject: [PATCH 10/15] Re-add chrome opt --disable-dev-shm-usage --- python/scripts/run_emscripten_tests.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/scripts/run_emscripten_tests.py b/python/scripts/run_emscripten_tests.py index 99e0e42ed9c..188d6c26f01 100644 --- a/python/scripts/run_emscripten_tests.py +++ b/python/scripts/run_emscripten_tests.py @@ -274,6 +274,7 @@ def __init__(self, hostname, port): options = Options() options.add_argument("--headless") options.add_argument("--no-sandbox") + options.add_argument("--disable-dev-shm-usage") driver = webdriver.Chrome(options=options) driver.command_executor._client_config.timeout = 1200 super().__init__(hostname, port, driver) From e3c5c64132c666302ec2d500f8a7981a2c9a9828 Mon Sep 17 00:00:00 2001 From: Tadeja Kadunc Date: Thu, 9 Jul 2026 16:24:56 +0200 Subject: [PATCH 11/15] Test retries for Chrome worker/Pyodide init hang --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 3 +- python/scripts/run_emscripten_tests.py | 49 ++++++++++++++++----- 2 files changed, 39 insertions(+), 13 deletions(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 26b3e37a496..4eab2ab2086 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -2074,8 +2074,7 @@ function(build_protobuf) "-DCMAKE_INSTALL_PREFIX=${PROTOBUF_HOST_PREFIX}" -Dprotobuf_BUILD_TESTS=OFF -Dprotobuf_DEBUG_POSTFIX= - # OFF so protobuf does not take conda's zlib and add its include dir - # globally, causing vendored Abseil sources to include env's old headers + # OFF to avoid conda include dirs leaking into vendored Abseil -Dprotobuf_WITH_ZLIB=OFF) if(ABSL_VENDORED) # Force protobuf to reuse Arrow's already-extracted absl source diff --git a/python/scripts/run_emscripten_tests.py b/python/scripts/run_emscripten_tests.py index 188d6c26f01..45d4f34afdb 100644 --- a/python/scripts/run_emscripten_tests.py +++ b/python/scripts/run_emscripten_tests.py @@ -67,8 +67,20 @@ def do_GET(self) -> bytes | None: evt.data.print.forEach((x)=>{window.python_logs.push(x)}); } } - window.pyworker = new Worker("worker.js"); - window.pyworker.onmessage = capturelogs; + function startPyWorker() { + window.pyworker = new Worker("worker.js"); + window.pyworker.onmessage = capturelogs; + window.pyworker.onerror = (evt) => { + window.python_logs.push(...new TextEncoder().encode( + `Worker error: ${evt.message}\n`)); + if (window.python_done_callback) { + let callback = window.python_done_callback; + window.python_done_callback = undefined; + callback({result: 1}); + } + }; + } + startPyWorker(); @@ -223,12 +235,19 @@ def load_pyodide(self, dist_dir): pass def load_arrow(self): - self.execute_python( - f"import pyodide_js as pjs\n" - f"await pjs.loadPackage('{PYARROW_WHEEL_PATH.name}')\n" - ) - - def execute_python(self, code, wait_for_terminate=True): + code = (f"import pyodide_js as pjs\n" + f"await pjs.loadPackage('{PYARROW_WHEEL_PATH.name}')\n") + for attempt in range(3): + try: + return self.execute_python(code, timeout=300) + except TimeoutError: + if attempt == 2: + raise TimeoutError("Timed out loading PyArrow in browser") + print("Timed out loading PyArrow in browser; restarting worker", + flush=True) + self.restart_worker() + + def execute_python(self, code, wait_for_terminate=True, timeout=None): self.driver.execute_script( f""" let python = `{code}`; @@ -239,13 +258,19 @@ def execute_python(self, code, wait_for_terminate=True): """ ) if wait_for_terminate: - return self.wait_for_done() + return self.wait_for_done(timeout=timeout) + + def restart_worker(self): + self.driver.execute_script( + "window.pyworker.terminate(); startPyWorker(); " + "delete window.python_script_done; window.python_done_callback = undefined;" + ) def clear_logs(self): self.driver.execute_script("window.python_logs = [];") - def wait_for_done(self): - last_heartbeat = time.monotonic() + def wait_for_done(self, timeout=None): + start = last_heartbeat = time.monotonic() while True: # poll for console.log messages from our webworker # which are the output of pytest @@ -261,6 +286,8 @@ def wait_for_done(self): self.driver.execute_script("delete window.python_script_done;") return value now = time.monotonic() + if timeout is not None and now - start > timeout: + raise TimeoutError if now - last_heartbeat > 30: print("Waiting for browser-side Python to finish...", flush=True) last_heartbeat = now From bfed5af07926df192477f47c553003fb65f0a429 Mon Sep 17 00:00:00 2001 From: Tadeja Kadunc Date: Fri, 10 Jul 2026 12:27:12 +0200 Subject: [PATCH 12/15] Update retry message --- python/scripts/run_emscripten_tests.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/python/scripts/run_emscripten_tests.py b/python/scripts/run_emscripten_tests.py index 45d4f34afdb..7d458c8a8a5 100644 --- a/python/scripts/run_emscripten_tests.py +++ b/python/scripts/run_emscripten_tests.py @@ -242,8 +242,9 @@ def load_arrow(self): return self.execute_python(code, timeout=300) except TimeoutError: if attempt == 2: - raise TimeoutError("Timed out loading PyArrow in browser") - print("Timed out loading PyArrow in browser; restarting worker", + raise TimeoutError( + "Timed out initializing Pyodide/PyArrow in browser") + print("Timed out initializing Pyodide/loading PyArrow. Restarting", flush=True) self.restart_worker() From 4f88f852f9dded612699d081610959243d5fc2a2 Mon Sep 17 00:00:00 2001 From: Tadeja Kadunc Date: Fri, 10 Jul 2026 14:29:42 +0200 Subject: [PATCH 13/15] Improved fail-fast, poll and reduced debug --- python/scripts/run_emscripten_tests.py | 36 ++++++++++++++------------ 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/python/scripts/run_emscripten_tests.py b/python/scripts/run_emscripten_tests.py index 7d458c8a8a5..65abc24cf40 100644 --- a/python/scripts/run_emscripten_tests.py +++ b/python/scripts/run_emscripten_tests.py @@ -68,9 +68,11 @@ def do_GET(self) -> bytes | None: } } function startPyWorker() { + window.pyworker_failed = false; window.pyworker = new Worker("worker.js"); window.pyworker.onmessage = capturelogs; window.pyworker.onerror = (evt) => { + window.pyworker_failed = true; window.python_logs.push(...new TextEncoder().encode( `Worker error: ${evt.message}\n`)); if (window.python_done_callback) { @@ -99,25 +101,17 @@ def do_GET(self) -> bytes | None: self.postMessage({print:databytes}); return databytes.length; } - function log(message) { - do_print(new TextEncoder().encode(`[worker] ${message}\n`)); - } onmessage = async function (e) { const data = e.data; try { if (!self.pyodide) { - log("loading Pyodide"); self.pyodide = await loadPyodide(); - log("loaded Pyodide"); } self.pyodide.setStdout({write:do_print,isatty:data.isatty}); self.pyodide.setStderr({write:do_print,isatty:data.isatty}); - log("loading packages from imports"); await self.pyodide.loadPackagesFromImports(data.python); - log("running Python"); let results = await self.pyodide.runPythonAsync(data.python); - log("Python completed"); self.postMessage({results}); } catch (error) { do_print(new TextEncoder().encode( @@ -228,8 +222,6 @@ class BrowserDriver: def __init__(self, hostname, port, driver): self.driver = driver self.driver.get(f"http://{hostname}:{port}/test.html") - # Chrome on CI takes longer than locally to compile. - self.driver.set_script_timeout(1200) def load_pyodide(self, dist_dir): pass @@ -239,7 +231,12 @@ def load_arrow(self): f"await pjs.loadPackage('{PYARROW_WHEEL_PATH.name}')\n") for attempt in range(3): try: - return self.execute_python(code, timeout=300) + result = self.execute_python(code, timeout=300) + if result: + raise RuntimeError( + "PyArrow wheel failed to load in browser " + f"(exit status {result}), see log above") + return result except TimeoutError: if attempt == 2: raise TimeoutError( @@ -274,18 +271,23 @@ def wait_for_done(self, timeout=None): start = last_heartbeat = time.monotonic() while True: # poll for console.log messages from our webworker - # which are the output of pytest - lines = self.driver.execute_script( - "let temp = window.python_logs;window.python_logs=[];return temp;" + # (output of pytest), completion and worker death + state = self.driver.execute_script( + "let logs = window.python_logs; window.python_logs = [];" + "return {logs: logs, done: window.python_script_done," + " failed: window.pyworker_failed === true};" ) + lines = state.get("logs") or [] if len(lines) > 0: sys.stdout.buffer.write(bytes(lines)) sys.stdout.buffer.flush() - done = self.driver.execute_script("return window.python_script_done;") + done = state.get("done") if done is not None: - value = done["result"] self.driver.execute_script("delete window.python_script_done;") - return value + return done["result"] + if state.get("failed"): + raise RuntimeError( + "Browser-side worker died. See 'Worker error:' in log above") now = time.monotonic() if timeout is not None and now - start > timeout: raise TimeoutError From 3290d47ca4b4e386644a7a070a082a7f49b77c6f Mon Sep 17 00:00:00 2001 From: Tadeja Kadunc Date: Fri, 10 Jul 2026 18:53:34 +0200 Subject: [PATCH 14/15] Revert hang fixes --- python/scripts/run_emscripten_tests.py | 137 +++++++++---------------- 1 file changed, 47 insertions(+), 90 deletions(-) diff --git a/python/scripts/run_emscripten_tests.py b/python/scripts/run_emscripten_tests.py index 65abc24cf40..002698a77e2 100644 --- a/python/scripts/run_emscripten_tests.py +++ b/python/scripts/run_emscripten_tests.py @@ -67,22 +67,8 @@ def do_GET(self) -> bytes | None: evt.data.print.forEach((x)=>{window.python_logs.push(x)}); } } - function startPyWorker() { - window.pyworker_failed = false; - window.pyworker = new Worker("worker.js"); - window.pyworker.onmessage = capturelogs; - window.pyworker.onerror = (evt) => { - window.pyworker_failed = true; - window.python_logs.push(...new TextEncoder().encode( - `Worker error: ${evt.message}\n`)); - if (window.python_done_callback) { - let callback = window.python_done_callback; - window.python_done_callback = undefined; - callback({result: 1}); - } - }; - } - startPyWorker(); + window.pyworker = new Worker("worker.js"); + window.pyworker.onmessage = capturelogs; @@ -96,28 +82,22 @@ def do_GET(self) -> bytes | None: elif self.path.endswith("/worker.js"): body = b""" importScripts("./pyodide.js"); - function do_print(arg) { - let databytes = Array.from(arg); - self.postMessage({print:databytes}); - return databytes.length; - } onmessage = async function (e) { const data = e.data; - try { - if (!self.pyodide) { - self.pyodide = await loadPyodide(); - } - self.pyodide.setStdout({write:do_print,isatty:data.isatty}); - self.pyodide.setStderr({write:do_print,isatty:data.isatty}); - - await self.pyodide.loadPackagesFromImports(data.python); - let results = await self.pyodide.runPythonAsync(data.python); - self.postMessage({results}); - } catch (error) { - do_print(new TextEncoder().encode( - `Worker failed: ${error && error.stack || error}\n`)); - self.postMessage({results: 1}); + if (!self.pyodide) { + self.pyodide = await loadPyodide(); + } + function do_print(arg) { + let databytes = Array.from(arg); + self.postMessage({print:databytes}); + return databytes.length; } + self.pyodide.setStdout({write:do_print,isatty:data.isatty}); + self.pyodide.setStderr({write:do_print,isatty:data.isatty}); + + await self.pyodide.loadPackagesFromImports(data.python); + let results = await self.pyodide.runPythonAsync(data.python); + self.postMessage({results}); } """ self.send_response(200) @@ -222,78 +202,56 @@ class BrowserDriver: def __init__(self, hostname, port, driver): self.driver = driver self.driver.get(f"http://{hostname}:{port}/test.html") + # Chrome on CI takes longer than locally to compile. + self.driver.set_script_timeout(1200) def load_pyodide(self, dist_dir): pass def load_arrow(self): - code = (f"import pyodide_js as pjs\n" - f"await pjs.loadPackage('{PYARROW_WHEEL_PATH.name}')\n") - for attempt in range(3): - try: - result = self.execute_python(code, timeout=300) - if result: - raise RuntimeError( - "PyArrow wheel failed to load in browser " - f"(exit status {result}), see log above") - return result - except TimeoutError: - if attempt == 2: - raise TimeoutError( - "Timed out initializing Pyodide/PyArrow in browser") - print("Timed out initializing Pyodide/loading PyArrow. Restarting", - flush=True) - self.restart_worker() - - def execute_python(self, code, wait_for_terminate=True, timeout=None): - self.driver.execute_script( - f""" - let python = `{code}`; - delete window.python_script_done; - window.python_done_callback= (x) => {{window.python_script_done=x;}}; - window.pyworker.postMessage( - {{python,isatty:{'true' if sys.stdout.isatty() else 'false'}}}); - """ + self.execute_python( + f"import pyodide_js as pjs\n" + f"await pjs.loadPackage('{PYARROW_WHEEL_PATH.name}')\n" ) - if wait_for_terminate: - return self.wait_for_done(timeout=timeout) - def restart_worker(self): - self.driver.execute_script( - "window.pyworker.terminate(); startPyWorker(); " - "delete window.python_script_done; window.python_done_callback = undefined;" - ) + def execute_python(self, code, wait_for_terminate=True): + if wait_for_terminate: + self.driver.execute_async_script( + f""" + let callback = arguments[arguments.length-1]; + python = `{code}`; + window.python_done_callback = callback; + window.pyworker.postMessage( + {{python, isatty: {'true' if sys.stdout.isatty() else 'false'}}}); + """ + ) + else: + self.driver.execute_script( + f""" + let python = `{code}`; + window.python_done_callback= (x) => {{window.python_script_done=x;}}; + window.pyworker.postMessage( + {{python,isatty:{'true' if sys.stdout.isatty() else 'false'}}}); + """ + ) def clear_logs(self): self.driver.execute_script("window.python_logs = [];") - def wait_for_done(self, timeout=None): - start = last_heartbeat = time.monotonic() + def wait_for_done(self): while True: # poll for console.log messages from our webworker - # (output of pytest), completion and worker death - state = self.driver.execute_script( - "let logs = window.python_logs; window.python_logs = [];" - "return {logs: logs, done: window.python_script_done," - " failed: window.pyworker_failed === true};" + # which are the output of pytest + lines = self.driver.execute_script( + "let temp = window.python_logs;window.python_logs=[];return temp;" ) - lines = state.get("logs") or [] if len(lines) > 0: sys.stdout.buffer.write(bytes(lines)) - sys.stdout.buffer.flush() - done = state.get("done") + done = self.driver.execute_script("return window.python_script_done;") if done is not None: + value = done["result"] self.driver.execute_script("delete window.python_script_done;") - return done["result"] - if state.get("failed"): - raise RuntimeError( - "Browser-side worker died. See 'Worker error:' in log above") - now = time.monotonic() - if timeout is not None and now - start > timeout: - raise TimeoutError - if now - last_heartbeat > 30: - print("Waiting for browser-side Python to finish...", flush=True) - last_heartbeat = now + return value time.sleep(0.1) @@ -304,7 +262,6 @@ def __init__(self, hostname, port): options = Options() options.add_argument("--headless") options.add_argument("--no-sandbox") - options.add_argument("--disable-dev-shm-usage") driver = webdriver.Chrome(options=options) driver.command_executor._client_config.timeout = 1200 super().__init__(hostname, port, driver) From 04d63eed4574f1dfc3099a5132f63bf0467f843c Mon Sep 17 00:00:00 2001 From: Tadeja Kadunc Date: Fri, 10 Jul 2026 20:30:00 +0200 Subject: [PATCH 15/15] Revert to bfed5af plus shm-usage --- ci/scripts/python_test_emscripten.sh | 2 + python/scripts/run_emscripten_tests.py | 120 ++++++++++++++++--------- 2 files changed, 81 insertions(+), 41 deletions(-) diff --git a/ci/scripts/python_test_emscripten.sh b/ci/scripts/python_test_emscripten.sh index 4029722568b..866262923e8 100755 --- a/ci/scripts/python_test_emscripten.sh +++ b/ci/scripts/python_test_emscripten.sh @@ -34,5 +34,7 @@ echo "-------------- Running emscripten tests in Node ----------------------" python scripts/run_emscripten_tests.py ${pyodide_wheel} --dist-dir=${pyodide_dist_dir} --runtime=node echo "-------------- Running emscripten tests in Chrome --------------------" +# verify compose shm_size is applied inside the container (see compose.yaml) +df -h /dev/shm python scripts/run_emscripten_tests.py ${pyodide_wheel} --dist-dir=${pyodide_dist_dir} --runtime=chrome diff --git a/python/scripts/run_emscripten_tests.py b/python/scripts/run_emscripten_tests.py index 002698a77e2..7f97c8759c2 100644 --- a/python/scripts/run_emscripten_tests.py +++ b/python/scripts/run_emscripten_tests.py @@ -67,8 +67,20 @@ def do_GET(self) -> bytes | None: evt.data.print.forEach((x)=>{window.python_logs.push(x)}); } } - window.pyworker = new Worker("worker.js"); - window.pyworker.onmessage = capturelogs; + function startPyWorker() { + window.pyworker = new Worker("worker.js"); + window.pyworker.onmessage = capturelogs; + window.pyworker.onerror = (evt) => { + window.python_logs.push(...new TextEncoder().encode( + `Worker error: ${evt.message}\n`)); + if (window.python_done_callback) { + let callback = window.python_done_callback; + window.python_done_callback = undefined; + callback({result: 1}); + } + }; + } + startPyWorker(); @@ -82,22 +94,36 @@ def do_GET(self) -> bytes | None: elif self.path.endswith("/worker.js"): body = b""" importScripts("./pyodide.js"); + function do_print(arg) { + let databytes = Array.from(arg); + self.postMessage({print:databytes}); + return databytes.length; + } + function log(message) { + do_print(new TextEncoder().encode(`[worker] ${message}\n`)); + } onmessage = async function (e) { const data = e.data; - if (!self.pyodide) { - self.pyodide = await loadPyodide(); - } - function do_print(arg) { - let databytes = Array.from(arg); - self.postMessage({print:databytes}); - return databytes.length; + try { + if (!self.pyodide) { + log("loading Pyodide"); + self.pyodide = await loadPyodide(); + log("loaded Pyodide"); + } + self.pyodide.setStdout({write:do_print,isatty:data.isatty}); + self.pyodide.setStderr({write:do_print,isatty:data.isatty}); + + log("loading packages from imports"); + await self.pyodide.loadPackagesFromImports(data.python); + log("running Python"); + let results = await self.pyodide.runPythonAsync(data.python); + log("Python completed"); + self.postMessage({results}); + } catch (error) { + do_print(new TextEncoder().encode( + `Worker failed: ${error && error.stack || error}\n`)); + self.postMessage({results: 1}); } - self.pyodide.setStdout({write:do_print,isatty:data.isatty}); - self.pyodide.setStderr({write:do_print,isatty:data.isatty}); - - await self.pyodide.loadPackagesFromImports(data.python); - let results = await self.pyodide.runPythonAsync(data.python); - self.postMessage({results}); } """ self.send_response(200) @@ -202,43 +228,48 @@ class BrowserDriver: def __init__(self, hostname, port, driver): self.driver = driver self.driver.get(f"http://{hostname}:{port}/test.html") - # Chrome on CI takes longer than locally to compile. - self.driver.set_script_timeout(1200) def load_pyodide(self, dist_dir): pass def load_arrow(self): - self.execute_python( - f"import pyodide_js as pjs\n" - f"await pjs.loadPackage('{PYARROW_WHEEL_PATH.name}')\n" + code = (f"import pyodide_js as pjs\n" + f"await pjs.loadPackage('{PYARROW_WHEEL_PATH.name}')\n") + for attempt in range(3): + try: + return self.execute_python(code, timeout=300) + except TimeoutError: + if attempt == 2: + raise TimeoutError( + "Timed out initializing Pyodide/PyArrow in browser") + print("Timed out initializing Pyodide/loading PyArrow. Restarting", + flush=True) + self.restart_worker() + + def execute_python(self, code, wait_for_terminate=True, timeout=None): + self.driver.execute_script( + f""" + let python = `{code}`; + delete window.python_script_done; + window.python_done_callback= (x) => {{window.python_script_done=x;}}; + window.pyworker.postMessage( + {{python,isatty:{'true' if sys.stdout.isatty() else 'false'}}}); + """ ) - - def execute_python(self, code, wait_for_terminate=True): if wait_for_terminate: - self.driver.execute_async_script( - f""" - let callback = arguments[arguments.length-1]; - python = `{code}`; - window.python_done_callback = callback; - window.pyworker.postMessage( - {{python, isatty: {'true' if sys.stdout.isatty() else 'false'}}}); - """ - ) - else: - self.driver.execute_script( - f""" - let python = `{code}`; - window.python_done_callback= (x) => {{window.python_script_done=x;}}; - window.pyworker.postMessage( - {{python,isatty:{'true' if sys.stdout.isatty() else 'false'}}}); - """ - ) + return self.wait_for_done(timeout=timeout) + + def restart_worker(self): + self.driver.execute_script( + "window.pyworker.terminate(); startPyWorker(); " + "delete window.python_script_done; window.python_done_callback = undefined;" + ) def clear_logs(self): self.driver.execute_script("window.python_logs = [];") - def wait_for_done(self): + def wait_for_done(self, timeout=None): + start = last_heartbeat = time.monotonic() while True: # poll for console.log messages from our webworker # which are the output of pytest @@ -247,11 +278,18 @@ def wait_for_done(self): ) if len(lines) > 0: sys.stdout.buffer.write(bytes(lines)) + sys.stdout.buffer.flush() done = self.driver.execute_script("return window.python_script_done;") if done is not None: value = done["result"] self.driver.execute_script("delete window.python_script_done;") return value + now = time.monotonic() + if timeout is not None and now - start > timeout: + raise TimeoutError + if now - last_heartbeat > 30: + print("Waiting for browser-side Python to finish...", flush=True) + last_heartbeat = now time.sleep(0.1)