diff --git a/.circleci/config.yml b/.circleci/config.yml index 112e3eca1a3e8..e514efee8239b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -53,6 +53,12 @@ commands: bootstrap: description: "bootstrap" steps: + - run: + name: Install dependencies (Linux) + command: | + if command -v apt-get >/dev/null 2>&1; then + apt-get install -q -y cmake ninja-build + fi - run: "$EMSDK_PYTHON ./bootstrap.py" pip-install: description: "pip install" @@ -1182,30 +1188,6 @@ jobs: steps: - test-sockets-chrome - build-windows-launcher: - executor: - name: win/server-2022 - shell: bash.exe -eo pipefail - steps: - - checkout - - run: - name: "build pylauncher" - shell: cmd.exe - command: .circleci\setup_vs2022.bat && cd tools\pylauncher && call build.bat - - store_artifacts: - path: tools/pylauncher/pylauncher.exe - destination: pylauncher.exe - - install-emsdk - - pip-install: - python: "$EMSDK_PYTHON" - - run: - name: "create_entry_points" - command: $EMSDK_PYTHON tools/maint/create_entry_points.py --exe-files - - run: - name: "crossplatform tests" - command: test/runner.exe core0.test_hello_world - - # windows and mac do not have separate build and test jobs, as they only run # a limited set of tests; it is simpler and faster to do it all in one job. test-windows: @@ -1233,10 +1215,6 @@ jobs: EMTEST_BROWSER: "0" steps: - checkout - - run: - name: Build launcher - command: call .circleci\setup_vs2019.bat && cd tools\pylauncher && call build.bat - shell: cmd.exe - run: name: Install packages command: | @@ -1257,9 +1235,7 @@ jobs: - upload-test-results - run: name: "check clean" - command: | - git checkout tools/pylauncher - $EMSDK_PYTHON test/check_clean.py + command: $EMSDK_PYTHON test/check_clean.py test-mac-arm64: executor: mac-arm64 @@ -1357,7 +1333,6 @@ workflows: - test-node-compat - test-windows - test-windows-browser-firefox - - build-windows-launcher - test-mac-arm64: requires: - build-linux diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1fe44de52d93a..0ae178418b74e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -66,6 +66,9 @@ jobs: echo "Be sure that you have installed the current emsdk version. See test/emsdk_version.txt ($(cat test/emsdk_version.txt))." exit 1 fi + - name: Check emcc_native generated settings + run: | + ./tools/emcc_native/gen_settings.py --check clang-format-diff: # This job is disabled until we can make it more precise @@ -83,22 +86,3 @@ jobs: sudo apt-get install clang-format-19 sudo update-alternatives --install /usr/bin/git-clang-format git-clang-format /usr/bin/git-clang-format-19 100 - run: tools/maint/clang-format-diff.sh origin/$GITHUB_BASE_REF - - build-pylauncher-arm64: - name: Build pylauncher (ARM64) - runs-on: windows-11-arm - steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - - name: Setup MSVC - uses: ilammy/msvc-dev-cmd@0b201ec74fa43914dc39ae48a89fd1d8cb592756 # v1.13.0 - with: - arch: arm64 - - name: Build pylauncher - run: | - cd tools\pylauncher - call build.bat arm64 - shell: cmd - - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: pylauncher-arm64.exe - path: tools/pylauncher/pylauncher-arm64.exe diff --git a/Makefile b/Makefile index bf66dcae258fa..c6d7d2fb7822c 100644 --- a/Makefile +++ b/Makefile @@ -13,10 +13,15 @@ install: ./tools/install.py $(DESTDIR) npm install --omit=dev --prefix $(DESTDIR) +emcc_native: + cmake -B out/build_emcc_native -S tools/emcc_native -G Ninja -DCMAKE_BUILD_TYPE=Release + cmake --build out/build_emcc_native --config Release + cmake --install out/build_emcc_native --config Release + # Create an distributable archive of emscripten suitable for use # by end users. This archive excludes node_modules as it can include native # modules which can't be safely pre-packaged. $(DISTFILE): install tar cf $@ --exclude=node_modules -C `dirname $(DESTDIR)` `basename $(DESTDIR)` -.PHONY: dist install +.PHONY: dist install emcc_native diff --git a/README.md b/README.md index 656e778a1d936..d0f5a6ba50021 100644 --- a/README.md +++ b/README.md @@ -39,11 +39,11 @@ There are two primary ways to install Emscripten: The easiest way to get started is by using the Emscripten SDK. Follow the instructions on the [downloads page](https://emscripten.org/docs/getting_started/downloads.html) to install it. 2. **From a Git Checkout (Manual Installation)** - If you have cloned the repository from Git, you can install the dependencies manually and then run the bootstrap script: + If you have cloned the repository from Git, run `bootstrap.py` to build the native launcher (`emcc_native`) and set up dependencies: ```bash ./bootstrap.py ``` - For more details, see the [developer guide](https://emscripten.org/docs/contributing/developers_guide.html). + Building `emcc_native` requires CMake 3.20+ and a C++20 host compiler toolchain. Alternatively, setting `EMCC_NATIVE=0` in your environment before running `./bootstrap.py` will generate legacy Python launcher scripts (e.g. `.bat` / `.ps1` files on Windows) via `./tools/maint/create_entry_points.py`. For more details, see the [developer guide](https://emscripten.org/docs/contributing/developers_guide.html). ## Using the compiler diff --git a/bootstrap.py b/bootstrap.py index 7ed6823571fd8..431e2b832d8c6 100755 --- a/bootstrap.py +++ b/bootstrap.py @@ -95,18 +95,22 @@ def run_cmd(cmd): subprocess.run(cmd, check=True, text=True, encoding='utf-8', cwd=utils.path_from_root()) +def build_emcc_native(): + build_dir = utils.path_from_root('out/build_emcc_native') + source_dir = utils.path_from_root('tools/emcc_native') + cmd = ['cmake', '-B', build_dir, source_dir, '-DCMAKE_BUILD_TYPE=Release'] + if not utils.WINDOWS and shutil.which('ninja'): + cmd.extend(['-G', 'Ninja']) + run_cmd(cmd) + run_cmd(['cmake', '--build', build_dir, '--config', 'Release']) + run_cmd(['cmake', '--install', build_dir, '--config', 'Release']) + + actions = [ ('npm packages', [ 'package.json', 'package-lock.json', ], ['npm', 'ci']), - ('create entry points', [ - 'tools/maint/create_entry_points.py', - 'tools/pylauncher/pylauncher.exe', - 'tools/maint/run_python.bat', - 'tools/maint/run_python.sh', - 'tools/maint/run_python.ps1', - ], [sys.executable, 'tools/maint/create_entry_points.py']), ('git submodules', [ 'test/third_party/posixtestsuite/', 'test/third_party/googletest', @@ -118,6 +122,30 @@ def run_cmd(cmd): ], maybe_install_hooks), ] +if os.environ.get('EMCC_NATIVE') == '0': + actions.append(('legacy entry points', [ + 'tools/maint/create_entry_points.py', + 'tools/maint/run_python.sh', + 'tools/maint/run_python.bat', + 'tools/maint/run_python.ps1', + 'tools/maint/run_python_compiler.sh', + 'tools/maint/run_python_compiler.bat', + 'tools/maint/run_python_compiler.ps1', + ], [sys.executable, utils.path_from_root('tools/maint/create_entry_points.py')])) +else: + actions.append(('build emcc_native', [ + 'tools/emcc_native/CMakeLists.txt', + 'tools/emcc_native/main.cpp', + 'tools/emcc_native/diagnostics.cpp', + 'tools/emcc_native/diagnostics.h', + 'tools/emcc_native/driver.cpp', + 'tools/emcc_native/driver.h', + 'tools/emcc_native/exec.cpp', + 'tools/emcc_native/exec.h', + 'tools/emcc_native/config.cpp', + 'tools/emcc_native/config.h', + ], build_emcc_native)) + def main(args): parser = argparse.ArgumentParser(description=__doc__) diff --git a/docs/design/03-native-clang-frontend.md b/docs/design/03-native-clang-frontend.md index 7970c87c18033..efe2264257d97 100644 --- a/docs/design/03-native-clang-frontend.md +++ b/docs/design/03-native-clang-frontend.md @@ -1,6 +1,6 @@ # Design Doc: Native Launcher / Clang Frontend -- **Status**: Draft +- **Status**: Phase 1 Completed - **Bug**: https://github.com/emscripten-core/emscripten/issues/26453 ## Context diff --git a/site/source/docs/building_from_source/index.rst b/site/source/docs/building_from_source/index.rst index feffe360c1eb6..e50c7a728b9e0 100644 --- a/site/source/docs/building_from_source/index.rst +++ b/site/source/docs/building_from_source/index.rst @@ -7,11 +7,18 @@ Building Emscripten from Source Building Emscripten yourself is an alternative to getting binaries using the emsdk. -Emscripten itself is written in Python and JavaScript so it does not need to be -compiled. However, after checkout you will need to run the top level -``bootstrap.py`` script before the toolchain is usable. This performs -various steps including ``npm install`` and the creation of compiler entry -points (e.g. `.bat` files on windows). +Emscripten itself is primarily written in Python and JavaScript. However, +after checkout you will need to run the top-level ``bootstrap.py`` script +before the toolchain is usable. This performs various steps including ``npm +install`` and building the native compiler frontend launcher (``emcc_native``), +which provides high-performance ``emcc`` and ``em++`` binaries. + +Building ``emcc_native`` requires CMake 3.20+ and a C++20 host compiler +toolchain (which you already need for building LLVM and Binaryen). If you prefer +not to perform a native build, setting ``EMCC_NATIVE=0`` in your environment +before running ``./bootstrap.py`` instructs it to generate legacy Python launcher +scripts (e.g., ``.bat`` / ``.ps1`` files on Windows) via +``./tools/maint/create_entry_points.py``. Emscripten comes with its own versions of some C/C++ system libraries which ``emcc`` builds automatically as and when needed (in the emsdk builds, these are diff --git a/site/source/docs/building_from_source/toolchain_what_is_needed.rst b/site/source/docs/building_from_source/toolchain_what_is_needed.rst index 33575b220282e..85506a2461cc1 100644 --- a/site/source/docs/building_from_source/toolchain_what_is_needed.rst +++ b/site/source/docs/building_from_source/toolchain_what_is_needed.rst @@ -39,7 +39,9 @@ In general a complete Emscripten environment requires the following tools. First Compiler toolchain ------------------ -When building LLVM and Binaryen from source code, whether "manually" or using the SDK, you will need a *compiler toolchain*: +When building Emscripten from source (via ``bootstrap.py`` to build +``emcc_native``), or when building LLVM and Binaryen from source, you will need +a C++20 host *compiler toolchain* and CMake 3.20+: - Windows: You will need `Visual Studio `_ (2019 or above) and `cmake `_ (3.20 or above). diff --git a/site/source/docs/contributing/developers_guide.rst b/site/source/docs/contributing/developers_guide.rst index 60cdcd0109da4..ff3d9e224a317 100644 --- a/site/source/docs/contributing/developers_guide.rst +++ b/site/source/docs/contributing/developers_guide.rst @@ -16,10 +16,22 @@ interested in helping out! Setting up ========== -For contributing to core Emscripten code, such as ``emcc.py``, you don't need to -build any binaries as ``emcc.py`` is in Python, and the core JS generation is -in JavaScript. You do still need binaries for LLVM and Binaryen, which you can -get using the emsdk. +When setting up a Git checkout of Emscripten, run the top-level ``bootstrap.py`` +script to set up dependencies (such as ``npm install``) and build the native +compiler frontend launcher (``emcc_native``): + +:: + + ./bootstrap.py + +Building ``emcc_native`` requires CMake 3.20+ and a C++20 host compiler +toolchain. If you prefer not to build the native launcher, setting ``EMCC_NATIVE=0`` +in your environment before running ``./bootstrap.py`` will generate legacy Python +launcher scripts (e.g., ``.bat`` / ``.ps1`` files on Windows) via +``./tools/maint/create_entry_points.py``. + +For LLVM and Binaryen binaries, you don't need to build them from source if you +are only contributing to Emscripten; you can get them using the emsdk. If you want to contribute back to Emscripten, it is recommended that you install the precise version of the emsdk binaries that are used by Emscripten CI when diff --git a/test/test_other.py b/test/test_other.py index 4c30e88c20cfd..51e4f6d526a7c 100644 --- a/test/test_other.py +++ b/test/test_other.py @@ -12301,9 +12301,9 @@ def test_xclang_flag(self): self.run_process([EMCC, '-c', '-o', 'out.o', '-Xclang', '-include', '-Xclang', 'foo.h', test_file('hello_world.c')]) def test_emcc_size_parsing(self): - create_file('foo.h', ' ') - self.assert_fail([EMCC, '-sTOTAL_MEMORY=X', 'foo.h'], 'error: invalid byte size `X`. Valid suffixes are: kb, mb, gb, tb') - self.assert_fail([EMCC, '-sTOTAL_MEMORY=11PB', 'foo.h'], 'error: invalid byte size `11PB`. Valid suffixes are: kb, mb, gb, tb') + create_file('foo.c', ' ') + self.assert_fail([EMCC, '-sTOTAL_MEMORY=X', 'foo.c'], 'error: invalid byte size `X`. Valid suffixes are: kb, mb, gb, tb') + self.assert_fail([EMCC, '-sTOTAL_MEMORY=11PB', 'foo.c'], 'error: invalid byte size `11PB`. Valid suffixes are: kb, mb, gb, tb') def test_native_call_before_init(self): self.set_setting('ASSERTIONS') @@ -14432,12 +14432,9 @@ def test_cpp_module(self): self.run_process([EMXX, '-std=c++20', test_file('other/hello_world.cppm'), '--precompile', '-o', 'hello_world.pcm']) self.do_other_test('test_cpp_module.cpp', cflags=['-std=c++20', '-fprebuilt-module-path=.', 'hello_world.pcm']) - @crossplatform def test_pthreads_flag(self): - # We support just the singular form of `-pthread`, like gcc - # Clang supports the plural form too but I think just due to historical accident: - # See https://github.com/llvm/llvm-project/commit/c800391fb974cdaaa62bd74435f76408c2e5ceae - self.assert_fail([EMCC, '-pthreads', '-c', test_file('hello_world.c')], 'emcc: error: unrecognized command-line option `-pthreads`; did you mean `-pthread`?') + # Test support for plural `-pthreads` flag + self.do_runf_out_file('hello_world.c', cflags=['-pthreads']) def test_missing_struct_info(self): create_file('lib.js', ''' diff --git a/tools/cmdline.py b/tools/cmdline.py index 1b6d02be73115..e4692ecd56c8e 100644 --- a/tools/cmdline.py +++ b/tools/cmdline.py @@ -537,7 +537,7 @@ def consume_arg_file(): options.openmp = 1 settings.PTHREADS = 1 settings.USE_PTHREADS = 1 - elif arg == '-pthread': + elif arg in {'-pthread', '-pthreads'}: settings.PTHREADS = 1 # Also set the legacy setting name, in case use JS code depends on it. settings.USE_PTHREADS = 1 @@ -545,8 +545,6 @@ def consume_arg_file(): settings.PTHREADS = 0 # Also set the legacy setting name, in case use JS code depends on it. settings.USE_PTHREADS = 0 - elif arg == '-pthreads': - exit_with_error('unrecognized command-line option `-pthreads`; did you mean `-pthread`?') elif arg == '-fno-rtti': settings.USE_RTTI = 0 elif arg == '-frtti': diff --git a/tools/compile.py b/tools/compile.py index 01bc6c83b13f3..e96c5d5937a12 100644 --- a/tools/compile.py +++ b/tools/compile.py @@ -16,6 +16,9 @@ get_cflags(): In addition to compiler flags this function also returns pre-processor flags. For example, include paths and macro definitions. + +NOTE: Default compiler flag construction logic here is also implemented natively +in tools/emcc_native/driver.cpp. Keep changes in sync between both places! """ import os diff --git a/tools/emcc_native/CMakeLists.txt b/tools/emcc_native/CMakeLists.txt new file mode 100644 index 0000000000000..2a198666312fd --- /dev/null +++ b/tools/emcc_native/CMakeLists.txt @@ -0,0 +1,122 @@ +cmake_minimum_required(VERSION 3.20) +project(emcc_native CXX) + +if(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT) + set(CMAKE_INSTALL_PREFIX "${CMAKE_CURRENT_SOURCE_DIR}/../.." CACHE PATH "Default install prefix to Emscripten root" FORCE) +endif() + +set(CMAKE_CXX_STANDARD 20) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_CXX_EXTENSIONS OFF) + +if(MSVC) + add_compile_options(/W4 /WX /UNDEBUG) + add_compile_definitions(_CRT_SECURE_NO_WARNINGS) +else() + add_compile_options(-Wall -Wextra -Werror -UNDEBUG) +endif() + +add_library(native_launcher_lib OBJECT + config.cpp + diagnostics.cpp + driver.cpp + exec.cpp +) + +include_directories(${CMAKE_CURRENT_SOURCE_DIR}) + +# Build `emcc` native launcher +add_executable(emcc main.cpp) +target_link_libraries(emcc PRIVATE native_launcher_lib) + +set(TOP_LEVEL_ENTRY_POINTS + em++ + emar + embuilder + emcmake + em-config + emconfigure + emmake + emranlib + emrun + emscons + emsize + emprofile + emdwp + emnm + emstrip + emsymbolizer + emscan-deps + empath-split +) + +set(TOOLS_ENTRY_POINTS + file_packager + webidl_binder +) + +set(TEST_ENTRY_POINTS + runner +) + +if(WIN32) + foreach(entry ${TOP_LEVEL_ENTRY_POINTS}) + add_custom_command( + TARGET emcc POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy_if_different $ $/${entry}$ + COMMENT "Creating ${entry} launcher executable" + ) + install(PROGRAMS $/${entry}$ DESTINATION .) + endforeach() + + foreach(entry ${TOOLS_ENTRY_POINTS}) + add_custom_command( + TARGET emcc POST_BUILD + COMMAND ${CMAKE_COMMAND} -E make_directory $/tools + COMMAND ${CMAKE_COMMAND} -E copy_if_different $ $/tools/${entry}$ + COMMENT "Creating tools/${entry} launcher executable" + ) + install(PROGRAMS $/tools/${entry}$ DESTINATION tools) + endforeach() + + foreach(entry ${TEST_ENTRY_POINTS}) + add_custom_command( + TARGET emcc POST_BUILD + COMMAND ${CMAKE_COMMAND} -E make_directory $/test + COMMAND ${CMAKE_COMMAND} -E copy_if_different $ $/test/${entry}$ + COMMENT "Creating test/${entry} launcher executable" + ) + install(PROGRAMS $/test/${entry}$ DESTINATION test) + endforeach() +else() + foreach(entry ${TOP_LEVEL_ENTRY_POINTS}) + add_custom_command( + TARGET emcc POST_BUILD + COMMAND ${CMAKE_COMMAND} -E create_symlink $ $/${entry}$ + COMMENT "Creating ${entry} launcher executable" + ) + install(PROGRAMS $/${entry}$ DESTINATION .) + endforeach() + + foreach(entry ${TOOLS_ENTRY_POINTS}) + add_custom_command( + TARGET emcc POST_BUILD + COMMAND ${CMAKE_COMMAND} -E make_directory $/tools + COMMAND ${CMAKE_COMMAND} -E create_symlink ../$ $/tools/${entry}$ + COMMENT "Creating tools/${entry} launcher executable" + ) + install(PROGRAMS $/tools/${entry}$ DESTINATION tools) + endforeach() + + foreach(entry ${TEST_ENTRY_POINTS}) + add_custom_command( + TARGET emcc POST_BUILD + COMMAND ${CMAKE_COMMAND} -E make_directory $/test + COMMAND ${CMAKE_COMMAND} -E create_symlink ../$ $/test/${entry}$ + COMMENT "Creating test/${entry} launcher executable" + ) + install(PROGRAMS $/test/${entry}$ DESTINATION test) + endforeach() +endif() + +install(TARGETS emcc DESTINATION .) diff --git a/tools/emcc_native/README.md b/tools/emcc_native/README.md new file mode 100644 index 0000000000000..147110b4737fb --- /dev/null +++ b/tools/emcc_native/README.md @@ -0,0 +1,117 @@ +# Native Clang Frontend Launcher (`emcc_native`) + +`emcc_native` is a C++ launcher for Emscripten's Python-based tools. In its +basic function it acts as a replacement for the Window-only `pylauncher` and +the legacy `run_python` launcher scripts (`.sh`, `.bat`, `.ps1`). + +However, it goes one step beyond previous tools by bypassing Python completely +for many common invocations of the compiler, and directly running `clang` or +`clang++`. + +## Overview & Architecture + +When executing large builds, `emcc` can be run hundreds or thousands of times. +Launching the Python interpreter for each translation unit can add significant +overhead (especially on Windows starting Python can be more expensive). + +`emcc_native` provides native executables (`emcc`, `em++`, etc) that: +1. **Directly invoke Clang** for pure compilation steps (`-c`, `-S`, `-E`, + `-M`, `-MM`), injecting: + - Target triple (`-target wasm32-unknown-emscripten` or + `wasm64-unknown-emscripten`) + - Frontend exceptions flag (`-fignore-exceptions`) + - Default LLVM backend flags (e.g. `-mllvm -enable-emscripten-sjlj`) + - Emscripten sysroot (`--sysroot=/sysroot`) + - Clang sysroot include paths (e.g. `-Xclang -iwithsysroot/include/compat`) + - SIMD/SSE/NEON preprocessor macros (`-D__SSE__=1`, `-D__SSE2__=1`, + `-D__ARM_NEON__=1`, etc.) when architecture flags are specified + - Visibility flag (`-fvisibility=default` when `-fPIC` is passed without + `-fvisibility`) +2. **Ignore compile-unused linker flags**: Link-only flags (`--js-library`, + `--embed-file`, etc.) and linker settings (`-sEXPORTED_FUNCTIONS`, etc.) + are ignored during compilation (with diagnostic warnings matching + `emcc.py`), allowing compile steps with link flags to run natively. +3. **Fall back to Python** (`emcc.py` / `em++.py`) when link-phase invocations + are run, or when compile-time `-s` settings or system flags + (`--clear-cache`, `--build`, `--tracing`, etc.) are present. +4. Directly run Python for non-compiler tools (e.g. `embuilder`, `emsymbolizer`) + +## Building + +Building requires CMake 3.20+ and a C++20 compiler. + +Using CMake directly: + +```bash +cmake -B out/build_emcc_native -S tools/emcc_native +cmake --build out/build_emcc_native +cmake --install out/build_emcc_native +``` + +Alternatively, you can build using `bootstrap.py` or `make`: + +```bash +./bootstrap.py +# or +make emcc_native +``` + +The output executable (`emcc`, `em++`, etc) will be installed directly +alongside their python counterparts. + +## Code Generation + +Compile-time settings, link-only flags, and Emscripten warning options are +generated in `generated_settings.h`. To update this header from Python +definitions, run: + +```bash +./tools/emcc_native/gen_settings.py +``` + +To verify whether `generated_settings.h` is up to date: + +```bash +./tools/emcc_native/gen_settings.py --check +``` + +## Configuration & Environment Variables + +- `EMCC_NATIVE`: + - Set to `0` to disable the native driver and unconditionally fall back to + `emcc.py`. When set during `./bootstrap.py`, it instructs `bootstrap.py` to + generate legacy Python launcher scripts instead of building `emcc_native`. + - Set to `1` to force strict native mode; if an invocation requires falling + back to Python, `emcc_native` will print the fallback reason and exit with + an error (useful for debugging). +- `EMCC_NATIVE_DEBUG`: When set (e.g. `EMCC_NATIVE_DEBUG=1`), logs launcher + decision details (whether direct Clang execution or Python fallback was + selected, reason, target binary, and command arguments) specifically for the + native launcher without enabling Python driver debug output. +- `EMCC_DEBUG`: When set (e.g. `EMCC_DEBUG=1`), logs launcher decision details + along with Python driver debug output. +- `EMSDK_PYTHON`: Path to the Python executable (defaults to `python3` or + `python.exe` on Windows). +- `EM_CACHE`: Path to Emscripten cache directory (defaults to + `/cache`). +- `EM_CONFIG`: Path to `.emscripten` configuration file (reads `LLVM_ROOT` and + `CACHE`). +- `EM_LLVM_ROOT`: Environment variable override for the directory containing + LLVM binaries (`clang`, `clang++`). + +## CI Benchmark Results + +Compile-time performance was benchmarked on CI across Linux, macOS, and Windows +(`embuilder build libc --force` compiling 1,075 files sequentially with +`EMCC_CORES=1`, `EMCC_USE_NINJA=0`, and `EMCC_BATCH_BUILD=0`). + +| Platform | Before (Python Baseline) | After (Native Launcher) | Improvement | Speedup | +| :---------: | :----------------------: | :---------------------: | :------------------------: | :-------: | +| **Linux** | 181.96 s (169.3 ms/file) | 64.82 s (60.3 ms/file) | -117.14 s (-109.0 ms/file) | **2.81x** | +| **Windows** | 343.90 s (319.9 ms/file) | 105.12 s (97.8 ms/file) | -238.78 s (-222.1 ms/file) | **3.27x** | +| **macOS** | 162.08 s (150.6 ms/file) | 64.73 s (60.1 ms/file) | -97.35 s (-90.5 ms/file) | **2.50x** | + +As expected, because process creation and `python.exe` startup carry +significantly higher overhead on Windows than on POSIX systems, the speedup on +Windows CI (**3.27x**, saving over 222 ms per invocation) is even larger than +on Linux and macOS. diff --git a/tools/emcc_native/benchmark.py b/tools/emcc_native/benchmark.py new file mode 100755 index 0000000000000..4fda4ceed7b31 --- /dev/null +++ b/tools/emcc_native/benchmark.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 +# Copyright 2026 The Emscripten Authors. All rights reserved. +# Emscripten is available under two separate licenses, the MIT license and the +# University of Illinois/NCSA Open Source License. Both these licenses can be +# found in the LICENSE file. + +"""Benchmark Emscripten compiler invocation speed across CI platforms. + +Measures the elapsed time required to compile small source files (by default, +building libc via embuilder with batching and Ninja disabled so each C file is +invoked individually). Running this script before compiling the native launcher +benchmarks the Python driver baseline; running it after compiling benchmarks +the native C++ launcher. +""" + +import argparse +import os +import subprocess +import sys +import time + +script_dir = os.path.dirname(os.path.abspath(__file__)) +root_dir = os.path.dirname(os.path.dirname(script_dir)) +sys.path.insert(0, root_dir) + +from tools.utils import WINDOWS + + +def find_native_launcher(): + ext = '.exe' if WINDOWS else '' + native_bin = os.path.join(root_dir, 'emcc' + ext) + if os.path.exists(native_bin): + try: + with open(native_bin, 'rb') as f: + data = f.read(4) + if data.startswith((b'#!', b'@echo', b'rem')): + return None + return native_bin + except OSError: + pass + return None + + +def run_benchmark(target='libc', cores=1, iterations=1): + native_launcher = find_native_launcher() + if native_launcher: + mode = f'Native Launcher ({os.path.relpath(native_launcher, root_dir)})' + else: + mode = 'Python Launcher (baseline)' + + env = os.environ.copy() + env['EMCC_CORES'] = str(cores) + env['EMCC_USE_NINJA'] = '0' + env['EMCC_BATCH_BUILD'] = '0' + env.pop('EM_COMPILER_WRAPPER', None) + if native_launcher: + if 'EMCC_NATIVE' not in env: + env['EMCC_NATIVE'] = '1' + else: + env.pop('EMCC_NATIVE', None) + + embuilder_py = os.path.join(root_dir, 'embuilder.py') + cmd = [sys.executable, embuilder_py, 'build', target, '--force'] + + print('=' * 60) + print('Emscripten Compiler Benchmark') + print('=' * 60) + print(f'Mode: {mode}') + print(f'Target: {target}') + print(f'Iterations: {iterations}') + print(f'Settings: EMCC_CORES={cores}, EMCC_USE_NINJA=0, EMCC_BATCH_BUILD=0') + print('=' * 60) + + times = [] + for i in range(1, iterations + 1): + if iterations > 1: + print(f'\n--- Iteration {i} of {iterations} ---') + start_time = time.perf_counter() + res = subprocess.run(cmd, env=env, check=False) + elapsed = time.perf_counter() - start_time + if res.returncode != 0: + print(f'Error: benchmark command failed with exit code {res.returncode}') + return res.returncode + times.append(elapsed) + print(f'Iteration {i} took: {elapsed:.3f} s') + + print('\n' + '=' * 60) + print('Benchmark Summary') + print('=' * 60) + print(f'Mode: {mode}') + if iterations == 1: + print(f'Total Time: {times[0]:.3f} s') + else: + avg_time = sum(times) / len(times) + min_time = min(times) + max_time = max(times) + print(f'Average Time: {avg_time:.3f} s') + print(f'Min Time: {min_time:.3f} s') + print(f'Max Time: {max_time:.3f} s') + print('=' * 60) + return 0 + + +def main(): + parser = argparse.ArgumentParser( + description='Benchmark Emscripten compiler invocation speed.', + ) + parser.add_argument( + 'target', + nargs='?', + default='libc', + help='Library target to build (default: libc)', + ) + parser.add_argument( + '--cores', + type=int, + default=1, + help='Number of cores for EMCC_CORES (default: 1)', + ) + parser.add_argument( + '-n', + '--iterations', + type=int, + default=1, + help='Number of benchmark iterations to run (default: 1)', + ) + args = parser.parse_args() + + return run_benchmark( + target=args.target, cores=args.cores, iterations=args.iterations, + ) + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/tools/emcc_native/config.cpp b/tools/emcc_native/config.cpp new file mode 100644 index 0000000000000..5593471dbe7e7 --- /dev/null +++ b/tools/emcc_native/config.cpp @@ -0,0 +1,219 @@ +/* + * Copyright 2026 The Emscripten Authors. All rights reserved. + * Emscripten is available under two separate licenses, the MIT license and the + * University of Illinois/NCSA Open Source License. Both these licenses can be + * found in the LICENSE file. + */ + +#include "config.h" + +#include +#include +#include +#include + +#include + +namespace fs = std::filesystem; + +namespace emscripten { + +using std::string_view; + +namespace { + +std::string get_env(const char* name) { + const char* val = std::getenv(name); + return val ? std::string(val) : std::string(); +} + +string_view trim(string_view str) { + size_t first = str.find_first_not_of(" \t\r\n"); + if (first == string_view::npos) + return ""; + size_t last = str.find_last_not_of(" \t\r\n"); + return str.substr(first, (last - first + 1)); +} + +string_view strip_quotes(string_view str) { + string_view s = trim(str); + if (s.size() >= 2 && ((s.starts_with('\'') && s.ends_with('\'')) || + (s.starts_with('"') && s.ends_with('"')))) { + return s.substr(1, s.size() - 2); + } + return s; +} + +void set_env_var(const std::string& key, const std::string& val) { +#ifdef _WIN32 + _putenv_s(key.c_str(), val.c_str()); +#else + setenv(key.c_str(), val.c_str(), 1); +#endif +} + +std::string expand_user(string_view path) { + if (path.starts_with('~')) { + std::string home = get_env("HOME"); + if (home.empty()) { + home = get_env("USERPROFILE"); + } + if (!home.empty()) { + return home + std::string(path.substr(1)); + } + } + return std::string(path); +} + +std::string expand_vars(string_view input) { + std::string s = expand_user(input); + if (s.empty()) + return s; + + std::string result; + size_t i = 0; + while (i < s.size()) { + if (s[i] == '$') { + if (i + 1 < s.size() && s[i + 1] == '{') { + size_t end = s.find('}', i + 2); + if (end != std::string::npos) { + std::string var_name = s.substr(i + 2, end - (i + 2)); + result += get_env(var_name.c_str()); + i = end + 1; + continue; + } + } else { + size_t start = i + 1; + size_t end = start; + while (end < s.size() && + (std::isalnum(static_cast(s[end])) || s[end] == '_')) { + ++end; + } + if (end > start) { + std::string var_name = s.substr(start, end - start); + result += get_env(var_name.c_str()); + i = end; + continue; + } + } + } +#ifdef _WIN32 + else if (s[i] == '%') { + size_t end = s.find('%', i + 1); + if (end != std::string::npos && end > i + 1) { + std::string var_name = s.substr(i + 1, end - (i + 1)); + result += get_env(var_name.c_str()); + i = end + 1; + continue; + } + } +#endif + + result += s[i]; + ++i; + } + + return result; +} + +} // namespace + +// Search order for the config file (must match find_config_file() in tools/config.py): +// 1. Specified via EM_CONFIG environment variable +// 2. Local .emscripten file in emscripten_root (/.emscripten) +// 3. Embedded config file two levels above emscripten_root, as used by +// `emsdk --embedded` (/../../.emscripten) +// 4. User home directory config (~/.emscripten) +fs::path find_config_file(const fs::path& emscripten_root) { + std::string env_config = get_env("EM_CONFIG"); + if (!env_config.empty() && fs::exists(env_config)) { + return fs::path(env_config); + } + + fs::path root_config = emscripten_root / ".emscripten"; + if (fs::exists(root_config)) { + return root_config; + } + + // Look two levels up for emsdk --embedded compatibility + // (e.g. emsdk/upstream/emscripten or emsdk/emscripten/x.y.z -> emsdk) + fs::path emsdk_embedded_config = + emscripten_root.parent_path().parent_path() / ".emscripten"; + if (fs::exists(emsdk_embedded_config)) { + return emsdk_embedded_config; + } + + std::string home = get_env("HOME"); + if (home.empty()) + home = get_env("USERPROFILE"); + if (!home.empty()) { + fs::path home_config = fs::path(home) / ".emscripten"; + if (fs::exists(home_config)) { + return home_config; + } + } + + return ""; +} + +Config load_config(const fs::path& emscripten_root) { + Config config; + + fs::path config_file = find_config_file(emscripten_root); + if (!config_file.empty() && fs::exists(config_file)) { + set_env_var("CFGDIR", config_file.parent_path().string()); + std::ifstream in(config_file); + std::string line; + while (std::getline(in, line)) { + size_t comment = line.find('#'); + if (comment != std::string::npos) { + line = line.substr(0, comment); + } + string_view tline = trim(line); + if (tline.empty()) + continue; + + size_t eq = tline.find('='); + if (eq != string_view::npos) { + string_view key = trim(tline.substr(0, eq)); + if (key == "LLVM_ROOT" || key == "CACHE") { + string_view raw_val = trim(tline.substr(eq + 1)); + if (raw_val.find('+') != string_view::npos || + (!raw_val.empty() && raw_val.front() != '\'' && + raw_val.front() != '"' && raw_val.front() != '$')) { + config.failure = true; + config.failure_reason = + "Complex expression in config file for " + std::string(key) + ": " + std::string(raw_val); + continue; + } + std::string val = expand_vars(strip_quotes(raw_val)); + if (key == "LLVM_ROOT") { + config.llvm_root = val; + } else if (key == "CACHE") { + config.em_cache = val; + } + } + } + } + } + + // Override with environment variables if present + std::string env_llvm = get_env("EM_LLVM_ROOT"); + if (!env_llvm.empty()) { + config.llvm_root = env_llvm; + } + + std::string env_cache = get_env("EM_CACHE"); + if (!env_cache.empty()) { + config.em_cache = env_cache; + } + + // Apply defaults + if (config.em_cache.empty()) { + config.em_cache = (emscripten_root / "cache").string(); + } + + return config; +} + +} // namespace emscripten diff --git a/tools/emcc_native/config.h b/tools/emcc_native/config.h new file mode 100644 index 0000000000000..929aac067ac57 --- /dev/null +++ b/tools/emcc_native/config.h @@ -0,0 +1,36 @@ +/* + * Copyright 2026 The Emscripten Authors. All rights reserved. + * Emscripten is available under two separate licenses, the MIT license and the + * University of Illinois/NCSA Open Source License. Both these licenses can be + * found in the LICENSE file. + */ + +#ifndef EMCC_NATIVE_CONFIG_H +#define EMCC_NATIVE_CONFIG_H + +#include +#include +#include + +namespace emscripten { + +namespace fs = std::filesystem; + +struct Config { + // There are more possible settings in an emscripten config + // but the native launcher only cares about these two. + std::string llvm_root; + std::string em_cache; + bool failure = false; + std::string failure_reason; +}; + +// Find the config file (.emscripten) location. +fs::path find_config_file(const fs::path& emscripten_root); + +// Parse configuration file and environment variables. +Config load_config(const fs::path& emscripten_root); + +} // namespace emscripten + +#endif // EMCC_NATIVE_CONFIG_H diff --git a/tools/emcc_native/diagnostics.cpp b/tools/emcc_native/diagnostics.cpp new file mode 100644 index 0000000000000..bcf2eba070a5e --- /dev/null +++ b/tools/emcc_native/diagnostics.cpp @@ -0,0 +1,76 @@ +/* + * Copyright 2026 The Emscripten Authors. All rights reserved. + * Emscripten is available under two separate licenses, the MIT license and the + * University of Illinois/NCSA Open Source License. Both these licenses can be + * found in the LICENSE file. + */ + +#include "diagnostics.h" +#include "generated_settings.h" + +#include +#include + +namespace emscripten { + +namespace { + +// Warning state for diagnostics +bool g_warn_unused = true; +bool g_error_unused = false; + +} // namespace + +void parse_warning_flags(const std::vector& user_args) { + g_warn_unused = true; + g_error_unused = false; + for (std::string_view arg : user_args) { + if (arg == "-w") { + g_warn_unused = false; + } else if (arg == "-Werror") { + g_error_unused = true; + } else if (arg == "-Wno-error") { + g_error_unused = false; + } else if (arg == "-Wunused-command-line-argument") { + g_warn_unused = true; + } else if (arg == "-Wno-unused-command-line-argument") { + g_warn_unused = false; + } else if (arg == "-Werror=unused-command-line-argument") { + g_warn_unused = true; + g_error_unused = true; + } else if (arg == "-Wno-error=unused-command-line-argument") { + g_error_unused = false; + } + } +} + +void emit_unused_warning(std::string_view msg) { + if (!g_warn_unused) { + return; + } + if (g_error_unused) { + std::cerr << "emcc: error: " << msg + << " [-Wunused-command-line-argument] [-Werror]" << std::endl; + std::exit(1); + } else { + std::cerr << "emcc: warning: " << msg << " [-Wunused-command-line-argument]" + << std::endl; + } +} + +bool is_emscripten_only_warning(std::string_view arg) { + if (!arg.starts_with("-W")) { + return false; + } + std::string_view name = arg.substr(2); + if (name.starts_with("error=")) { + name = name.substr(6); + } else if (name.starts_with("no-error=")) { + name = name.substr(9); + } else if (name.starts_with("no-")) { + name = name.substr(3); + } + return EMSCRIPTEN_ONLY_WARNINGS.contains(name); +} + +} // namespace emscripten diff --git a/tools/emcc_native/diagnostics.h b/tools/emcc_native/diagnostics.h new file mode 100644 index 0000000000000..5d6b8b669f948 --- /dev/null +++ b/tools/emcc_native/diagnostics.h @@ -0,0 +1,23 @@ +/* + * Copyright 2026 The Emscripten Authors. All rights reserved. + * Emscripten is available under two separate licenses, the MIT license and the + * University of Illinois/NCSA Open Source License. Both these licenses can be + * found in the LICENSE file. + */ + +#ifndef EMCC_NATIVE_DIAGNOSTICS_H +#define EMCC_NATIVE_DIAGNOSTICS_H + +#include +#include +#include + +namespace emscripten { + +void parse_warning_flags(const std::vector& user_args); +void emit_unused_warning(std::string_view msg); +bool is_emscripten_only_warning(std::string_view arg); + +} // namespace emscripten + +#endif // EMCC_NATIVE_DIAGNOSTICS_H diff --git a/tools/emcc_native/driver.cpp b/tools/emcc_native/driver.cpp new file mode 100644 index 0000000000000..6cb10fb669b48 --- /dev/null +++ b/tools/emcc_native/driver.cpp @@ -0,0 +1,700 @@ +/* + * Copyright 2026 The Emscripten Authors. All rights reserved. + * Emscripten is available under two separate licenses, the MIT license and the + * University of Illinois/NCSA Open Source License. Both these licenses can be + * found in the LICENSE file. + */ + +#include "diagnostics.h" +#include "driver.h" +#include "exec.h" +#include "generated_settings.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; + +namespace emscripten { + +using std::string_view; + +namespace { + +constexpr string_view get_extension(string_view path) { + size_t idx = path.rfind('.'); + if (idx == string_view::npos || idx == 0) { + return {}; + } + size_t slash = path.find_last_of("/\\"); + if (slash != string_view::npos && slash > idx) { + return {}; + } + return path.substr(idx); +} + +// Flags that require fallback to the Python driver because they represent +// complex operations or standing system commands. +// NOTE: Keep in sync with system options in tools/cmdline.py and complex +// options in tools/compile.py. +const std::unordered_set COMPLEX_OR_SYSTEM_FLAGS = { + "--clear-cache", + "--clear-ports", + "--build", + "--generate-config", + "--compiler-wrapper", + "--tracing", + "--memoryprofiler", + "--em-config", +}; + +// Subset of LINK_ONLY_FLAGS that take a value argument when passed without '='. +const std::unordered_set LINK_ONLY_FLAGS_WITH_ARGS = { + "--closure", + "--embed-file", + "--exclude-file", + "--extern-post-js", + "--extern-pre-js", + "--js-library", + "--js-transform", + "--oformat", + "--output-eol", + "--output_eol", + "--post-js", + "--pre-js", + "--preload-file", + "--shell-file", + "--source-map-base", +}; + +// Default LLVM backend arguments injected during compilation. +// NOTE: Keep in sync with llvm_backend_args() in tools/building.py. +const std::vector DEFAULT_LLVM_BACKEND_FLAGS = { + "-mllvm", + "-combiner-global-alias-analysis=false", + "-mllvm", + "-enable-emscripten-sjlj", + "-mllvm", + "-disable-lsr", +}; + +void create_python_command(const fs::path& script_path, + const std::vector& user_args, + DriverDecision& decision) { + decision.target_binary = get_python_executable(); + decision.target_args.push_back("-E"); +#ifdef _WIN32 + decision.target_args.push_back("-X"); + decision.target_args.push_back("utf8"); +#endif + decision.target_args.push_back(script_path.generic_string()); + for (const auto& arg : user_args) { + decision.target_args.push_back(arg); + } +} + +// Process SIMD/SSE/NEON feature flags and inject corresponding macro +// definitions. NOTE: Keep in sync with get_cflags() in tools/compile.py and +// SIMD_INTEL_FEATURE_TOWER / SIMD_NEON_FLAGS in tools/cmdline.py. +void handle_simd_flags(const std::vector& filtered_user_args, + DriverDecision& decision) { + bool has_simd = false; + bool has_sse = false, has_sse2 = false, has_sse3 = false, has_ssse3 = false; + bool has_sse4_1 = false, has_sse4_2 = false, has_avx = false, + has_avx2 = false; + bool has_fma = false, has_neon = false; + bool has_intel_simd = false; + + for (string_view arg : filtered_user_args) { + if (arg == "-msimd128" || arg == "-mrelaxed-simd") { + has_simd = true; + } else if (arg == "-msse") { + has_sse = true; + has_intel_simd = true; + } else if (arg == "-msse2") { + has_sse = has_sse2 = true; + has_intel_simd = true; + } else if (arg == "-msse3") { + has_sse = has_sse2 = has_sse3 = true; + has_intel_simd = true; + } else if (arg == "-mssse3") { + has_sse = has_sse2 = has_sse3 = has_ssse3 = true; + has_intel_simd = true; + } else if (arg == "-msse4.1") { + has_sse = has_sse2 = has_sse3 = has_ssse3 = has_sse4_1 = true; + has_intel_simd = true; + } else if (arg == "-msse4.2" || arg == "-msse4") { + has_sse = has_sse2 = has_sse3 = has_ssse3 = has_sse4_1 = has_sse4_2 = + true; + has_intel_simd = true; + } else if (arg == "-mavx") { + has_sse = has_sse2 = has_sse3 = has_ssse3 = has_sse4_1 = has_sse4_2 = + has_avx = true; + has_intel_simd = true; + } else if (arg == "-mavx2") { + has_sse = has_sse2 = has_sse3 = has_ssse3 = has_sse4_1 = has_sse4_2 = + has_avx = has_avx2 = true; + has_intel_simd = true; + } else if (arg == "-mfma") { + has_sse = has_sse2 = has_sse3 = has_ssse3 = has_sse4_1 = has_sse4_2 = + has_avx = has_avx2 = has_fma = true; + has_intel_simd = true; + } else if (arg == "-mfpu=neon" || arg == "-mneon") { + has_neon = true; + } + } + + if ((has_intel_simd || has_neon) && !has_simd) { + std::cerr << "emcc: error: passing any of -msse, -msse2, -msse3, -mssse3, " + "-msse4.1, -msse4.2, -msse4, -mavx, -mavx2, -mfma, -mfpu=neon " + "flags also requires passing -msimd128 (or -mrelaxed-simd)!" + << std::endl; + std::exit(1); + } + + if (has_sse || has_neon) decision.target_args.push_back("-D__SSE__=1"); + if (has_sse2) decision.target_args.push_back("-D__SSE2__=1"); + if (has_sse3) decision.target_args.push_back("-D__SSE3__=1"); + if (has_ssse3) decision.target_args.push_back("-D__SSSE3__=1"); + if (has_sse4_1) decision.target_args.push_back("-D__SSE4_1__=1"); + if (has_sse4_2) decision.target_args.push_back("-D__SSE4_2__=1"); + if (has_avx) decision.target_args.push_back("-D__AVX__=1"); + if (has_avx2) decision.target_args.push_back("-D__AVX2__=1"); + if (has_fma) decision.target_args.push_back("-D__FMA__=1"); + if (has_neon) decision.target_args.push_back("-D__ARM_NEON__=1"); +} + +// Construct the native Clang/Clang++ binary path and compiler argument vector. +// NOTE: Keep in sync with get_clang_flags() and get_cflags() in +// tools/compile.py. +void create_clang_command(bool is_cxx, + bool is_wasm64, + bool is_asm_only, + const std::vector& filtered_user_args, + const Config& config, + DriverDecision& decision) { + std::string clang_name = is_cxx ? "clang++" : "clang"; +#ifdef _WIN32 + clang_name += ".exe"; +#endif + + if (!config.llvm_root.empty()) { + decision.target_binary = (fs::path(config.llvm_root) / clang_name).generic_string(); + } else { + decision.target_binary = clang_name; + } + + // Target flags + std::string target_triple = + is_wasm64 ? "wasm64-unknown-emscripten" : "wasm32-unknown-emscripten"; + decision.target_args.push_back("-target"); + decision.target_args.push_back(target_triple); + + if (!is_asm_only) { + // Frontend exceptions flag + bool has_exceptions = false; + for (string_view arg : filtered_user_args) { + if (arg == "-fexceptions" || arg == "-fwasm-exceptions" || + arg == "-fno-ignore-exceptions") { + has_exceptions = true; + break; + } + } + if (!has_exceptions) { + decision.target_args.push_back("-fignore-exceptions"); + } else { + decision.target_args.push_back("-mllvm"); + decision.target_args.push_back("-enable-emscripten-cxx-exceptions"); + } + + // Backend flags + for (const auto& flag : DEFAULT_LLVM_BACKEND_FLAGS) { + decision.target_args.push_back(flag); + } + + // Sysroot + fs::path sysroot = fs::path(config.em_cache) / "sysroot"; + decision.target_args.push_back("--sysroot=" + sysroot.generic_string()); + + // Handle SIMD flags + handle_simd_flags(filtered_user_args, decision); + + // Check user args for special flags + bool nostdinc = false; + bool has_fpic = false; + bool has_fvisibility = false; + bool has_pthread = false; + bool has_explicit_pthread = false; + + for (string_view arg : filtered_user_args) { + if (arg == "-nostdinc") { + nostdinc = true; + } else if (arg == "-fPIC") { + has_fpic = true; + } else if (arg.starts_with("-fvisibility")) { + has_fvisibility = true; + } else if (arg == "-pthread") { + has_pthread = true; + has_explicit_pthread = true; + } else if (arg == "-fopenmp" || arg == "-fopenmp=libomp") { + has_pthread = true; + } + } + + if (has_pthread) { + decision.target_args.push_back("-D__EMSCRIPTEN_SHARED_MEMORY__=1"); + if (!has_explicit_pthread) { + decision.target_args.push_back("-pthread"); + } + } + + if (has_fpic && !has_fvisibility) { + decision.target_args.push_back("-fvisibility=default"); + } + + if (!nostdinc) { + decision.target_args.push_back("-Xclang"); + decision.target_args.push_back("-iwithsysroot/include/fakesdl"); + decision.target_args.push_back("-Xclang"); + decision.target_args.push_back("-iwithsysroot/include/compat"); + } + } + + for (string_view arg : filtered_user_args) { + decision.target_args.push_back(std::string(arg)); + } +} + +bool is_upper_identifier(string_view s) { + if (s.empty() || std::isdigit(static_cast(s[0]))) { + return false; + } + bool has_upper = false; + for (char c : s) { + unsigned char uc = static_cast(c); + if (std::isupper(uc)) { + has_upper = true; + } else if (!std::isdigit(uc) && uc != '_') { + return false; + } + } + return has_upper; +} + +bool is_dash_s_setting(const std::vector& user_args, + size_t i, + string_view& setting_key, + bool& ate_next) { + string_view arg = user_args[i]; + ate_next = false; + string_view val; + if (arg == "-s") { + if (i + 1 >= user_args.size()) + return false; + val = user_args[i + 1]; + ate_next = true; + } else if (arg.starts_with("-s")) { + val = arg.substr(2); + } else { + return false; + } + + size_t eq = val.find('='); + if (eq != string_view::npos) { + setting_key = val.substr(0, eq); + } else { + setting_key = val; + } + return is_upper_identifier(setting_key); +} + +bool is_assembly_only(const std::vector& user_args) { + static const std::unordered_set ASM_EXTS = {".s", ".S"}; + static const std::unordered_set C_EXTS = { + ".c", ".i", ".cppm", ".pcm", ".cpp", ".cxx", ".cc", ".c++", + ".CPP", ".CXX", ".C", ".CC", ".C++", ".ii", ".m", ".mi", ".mm", ".mii", + ".bc", ".ll" + }; + + bool has_asm = false; + bool has_c_source = false; + + for (size_t i = 0; i < user_args.size(); ++i) { + const std::string& arg = user_args[i]; + if (arg.empty() || arg[0] == '-') { + if ((arg == "-o" || arg == "-I" || arg == "-L" || arg == "-include" || + arg == "-isystem" || arg == "-MF" || arg == "-MT" || arg == "-MQ" || + arg == "-x") && i + 1 < user_args.size()) { + ++i; + } + continue; + } + string_view ext = get_extension(arg); + if (ASM_EXTS.contains(ext)) { + has_asm = true; + } else if (C_EXTS.contains(ext)) { + has_c_source = true; + } + } + + return has_asm && !has_c_source; +} + +// Check if any input argument is a header file (via extension) or if an explicit +// header language flag (e.g. -xc++-header) is specified. Compiling header inputs +// is a compile-only operation that generates precompiled headers (.pch / .gch). +// NOTE: Keep in sync with HEADER_EXTENSIONS and phase_setup() in emcc.py. +bool has_header_inputs(const std::vector& user_args) { + static const std::unordered_set HEADER_EXTS = { + ".h", ".hxx", ".hpp", ".hh", ".H", ".HXX", ".HPP", ".HH" + }; + + for (size_t i = 0; i < user_args.size(); ++i) { + const std::string& arg = user_args[i]; + if (arg.empty()) { + continue; + } + + if (arg == "-x") { + if (i + 1 < user_args.size() && user_args[i + 1].find("header") != std::string::npos) { + return true; + } + if (i + 1 < user_args.size()) { + ++i; + } + continue; + } + if (arg.starts_with("-x") && arg.find("header") != std::string::npos) { + return true; + } + + if (arg[0] == '-') { + if ((arg == "-o" || arg == "-I" || arg == "-L" || arg == "-include" || + arg == "-isystem" || arg == "-MF" || arg == "-MT" || arg == "-MQ") && + i + 1 < user_args.size()) { + ++i; + } + continue; + } + + std::string_view ext = get_extension(arg); + if (HEADER_EXTS.contains(ext)) { + return true; + } + } + + return false; +} + +std::string get_tool_name(std::string_view driver_arg0) { + if (driver_arg0.empty()) { + std::cerr << "emcc_native: error: empty command name (argv[0])" << std::endl; + std::exit(1); + } + std::string tool = fs::path(driver_arg0).stem().string(); + if (tool.empty()) { + std::cerr << "emcc_native: error: unable to determine tool name from command '" + << driver_arg0 << "'" << std::endl; + std::exit(1); + } + return tool; +} + +fs::path find_script_for_tool(const fs::path& emscripten_root, + const fs::path& exe_path, + string_view tool) { + static const char* const SEARCH_SUBDIRS[] = {"", "tools", "test"}; + + for (const char* subdir : SEARCH_SUBDIRS) { + fs::path candidate = emscripten_root / subdir / (std::string(tool) + ".py"); + if (fs::exists(candidate)) { + return candidate; + } + } + + if (exe_path.has_parent_path()) { + fs::path exe_dir = exe_path.parent_path(); + for (const char* subdir : SEARCH_SUBDIRS) { + fs::path candidate = exe_dir / subdir / (std::string(tool) + ".py"); + if (fs::exists(candidate)) { + return candidate; + } + } + } + + std::cerr << "emcc_native: error: python script for tool '" << tool << "' not found" << std::endl; + std::exit(1); +} + +DriverDecision make_fallback_decision(string_view tool, + const fs::path& emscripten_root, + const fs::path& exe_path, + const std::vector& user_args, + std::string reason) { + fs::path script_path = find_script_for_tool(emscripten_root, exe_path, tool); + + DriverDecision decision; + decision.use_fallback = true; + decision.reason = std::move(reason); + create_python_command(script_path, user_args, decision); + return decision; +} + +std::optional check_system_environment(const Config& config) { + const char* native_env = std::getenv("EMCC_NATIVE"); + if (native_env && std::string(native_env) == "0") { + return "EMCC_NATIVE set to disable native launcher"; + } + + const char* compiler_wrapper = std::getenv("EM_COMPILER_WRAPPER"); + if (compiler_wrapper && compiler_wrapper[0] != '\0') { + return "EM_COMPILER_WRAPPER configured"; + } + + if (config.failure) { + return config.failure_reason; + } + + std::error_code ec; + fs::path sysroot = fs::path(config.em_cache) / "sysroot"; + fs::path sysroot_stamp = fs::path(config.em_cache) / "sysroot_install.stamp"; + if (!fs::exists(sysroot, ec) || ec || !fs::exists(sysroot_stamp, ec) || ec) { + return "Emscripten sysroot not installed in cache: " + sysroot.string(); + } + + return std::nullopt; +} + +struct FilterArgsResult { + std::vector args; + std::string failure_reason; +}; + +FilterArgsResult filter_compiler_args(const std::vector& user_args) { + FilterArgsResult result; + + for (size_t i = 0; i < user_args.size(); ++i) { + string_view arg = user_args[i]; + + string_view arg_base = arg; + size_t eq_pos = arg_base.find('='); + if (eq_pos != string_view::npos) { + arg_base = arg_base.substr(0, eq_pos); + } + + if (COMPLEX_OR_SYSTEM_FLAGS.contains(arg_base)) { + result.failure_reason = + "Contains Emscripten system or complex flag: " + std::string(arg); + return result; + } + + if (is_emscripten_only_warning(arg)) { + continue; + } + + string_view setting_key; + bool ate_next = false; + if (is_dash_s_setting(user_args, i, setting_key, ate_next)) { + if (setting_key == "STRICT") { + if (ate_next) { + ++i; + } + continue; + } + if (COMPILE_TIME_SETTINGS.contains(setting_key)) { + result.failure_reason = + "Contains Emscripten compile-time setting: -s" + std::string(setting_key); + return result; + } else { + // Linker-only setting: warn and ignore during compilation + emit_unused_warning("linker setting ignored during compilation: '" + + std::string(setting_key) + "'"); + if (ate_next) { + ++i; + } + continue; + } + } + + // Check for .bc output file suffix without -flto or -emit-llvm + if (arg == "-o" && i + 1 < user_args.size()) { + string_view out_path = user_args[i + 1]; + if (get_extension(out_path) == ".bc") { + bool has_lto_or_emit_llvm = false; + for (const auto& a : user_args) { + string_view sv_a(a); + if (sv_a.starts_with("-flto") || sv_a == "-emit-llvm") { + has_lto_or_emit_llvm = true; + break; + } + } + if (!has_lto_or_emit_llvm) { + result.failure_reason = + ".bc output file suffix used without -flto or -emit-llvm"; + return result; + } + } + } + + if (arg == "-g4") { + result.failure_reason = "Contains deprecated debug flag: -g4"; + return result; + } + + // Check if arg is a debug flag that Clang doesn't accept directly + if (arg == "-g1" || arg == "-g2") { + result.args.push_back("-g0"); + continue; + } + if (arg == "-gsource-map" || arg == "-gsource-map=inline" || + arg.starts_with("-gseparate-dwarf")) { + result.args.push_back("-g"); + continue; + } + + // Check if arg is a link-only flag (e.g. --js-library or + // --js-library=lib.js) + string_view flag_name = arg; + size_t eq = flag_name.find('='); + bool has_eq = (eq != string_view::npos); + if (has_eq) { + flag_name = flag_name.substr(0, eq); + } + + if (LINK_ONLY_FLAGS.contains(flag_name)) { + emit_unused_warning("linker flag ignored during compilation: '" + + std::string(arg) + "'"); + if (!has_eq && LINK_ONLY_FLAGS_WITH_ARGS.contains(flag_name) && + i + 1 < user_args.size() && !user_args[i + 1].starts_with("-")) { + ++i; + } + continue; + } + + result.args.push_back(arg); + } + + return result; +} + +} // namespace + +std::string get_python_executable() { + const char* env_python = std::getenv("EMSDK_PYTHON"); + if (env_python && env_python[0] != '\0') { + return env_python; + } +#ifdef _WIN32 + return "python.exe"; +#else + return "python3"; +#endif +} + +// Analyze user arguments to determine if native compilation is supported or if +// fallback to Python is required. +DriverDecision analyze_request(string_view driver_arg0, + const fs::path& emscripten_root, + const fs::path& exe_path, + const std::vector& user_args, + const Config& config) { + std::string tool = get_tool_name(driver_arg0); + + auto fallback = [&](string_view reason) { + return make_fallback_decision(tool, emscripten_root, exe_path, user_args, std::string(reason)); + }; + + bool is_compiler = (tool == "emcc" || tool == "em++"); + if (!is_compiler) { + return fallback("Tool " + tool + " runs via Python script"); + } + + if (auto reason = check_system_environment(config)) { + return fallback(*reason); + } + + parse_warning_flags(user_args); + + bool compile_only = has_header_inputs(user_args); + bool is_wasm64 = false; + + // Response files (@file) require complex tokenization (handling shell quoting, + // escaping, character encodings like UTF-8 with BOM, and recursive response + // file expansion). In LLVM/Clang, this is handled by llvm::cl::ExpandResponseFiles + // and llvm::cl::TokenizeGNUCommandLine / TokenizeWindowsCommandLine. Because + // emcc_native is a standalone executable without LLVM library dependencies, + // we fall back to Python (which uses shlex.split() in response_file.py) + // rather than maintaining a custom cross-platform tokenizer and encoding parser. + // TODO: Implement native response file expansion if we add LLVM dependencies + // or a robust lightweight tokenizer. + for (const auto& arg : user_args) { + if (arg.starts_with("@")) { + return fallback("Response files (@file) not yet supported by native launcher"); + } + if (arg == "-c" || arg == "-S" || arg == "-E" || arg == "-M" || + arg == "-MM" || arg == "-fsyntax-only") { + compile_only = true; + } else if (arg == "-m64" || arg == "-sMEMORY64" || + arg == "-sMEMORY64=1" || arg == "-sMEMORY64=2") { + is_wasm64 = true; + } + } + + // Pure compile step requires -c, -S, -E, -M, -MM, -fsyntax-only, or header compilation + if (!compile_only) { + return fallback("No compile-only flag (-c, -S, -E) or header input found; defaulting to link phase"); + } + + FilterArgsResult filter_result = filter_compiler_args(user_args); + if (!filter_result.failure_reason.empty()) { + return fallback(filter_result.failure_reason); + } + + DriverDecision decision; + bool is_cxx = (tool == "em++"); + bool is_asm_only = is_assembly_only(user_args); + create_clang_command(is_cxx, is_wasm64, is_asm_only, + filter_result.args, config, decision); + + // Fall back if total command line length exceeds platform limits + size_t total_cmd_len = decision.target_binary.size(); + for (const auto& a : decision.target_args) { + total_cmd_len += a.size() + 1; + } +#ifdef _WIN32 + constexpr size_t MAX_CMD_LEN = 8192; +#else + constexpr size_t MAX_CMD_LEN = 32768; +#endif + if (total_cmd_len > MAX_CMD_LEN) { + std::string reason = "Command line length (" + std::to_string(total_cmd_len) + + " chars) exceeds limit (" + std::to_string(MAX_CMD_LEN) + + "); falling back to Python driver for response file handling"; + return fallback(reason); + } + + return decision; +} + +void apply_ccache_wrapper(DriverDecision& decision) { + const char* emcc_ccache = std::getenv("_EMCC_CCACHE"); + if (!emcc_ccache || emcc_ccache[0] == '\0') { + return; + } + unsetenv("_EMCC_CCACHE"); + + decision.target_args.insert(decision.target_args.begin(), decision.target_binary); +#ifdef _WIN32 + decision.target_binary = "ccache.exe"; +#else + decision.target_binary = "ccache"; +#endif +} + +} // namespace emscripten diff --git a/tools/emcc_native/driver.h b/tools/emcc_native/driver.h new file mode 100644 index 0000000000000..49a02b676a9d8 --- /dev/null +++ b/tools/emcc_native/driver.h @@ -0,0 +1,38 @@ +/* + * Copyright 2026 The Emscripten Authors. All rights reserved. + * Emscripten is available under two separate licenses, the MIT license and the + * University of Illinois/NCSA Open Source License. Both these licenses can be + * found in the LICENSE file. + */ + +#ifndef EMCC_NATIVE_DRIVER_H +#define EMCC_NATIVE_DRIVER_H + +#include "config.h" +#include +#include + +namespace emscripten { + +struct DriverDecision { + bool use_fallback = false; + std::string target_binary; + std::vector target_args; + std::string reason; +}; + +// Get the Python executable path (from EMSDK_PYTHON or default). +std::string get_python_executable(); + +DriverDecision analyze_request(std::string_view driver_arg0, + const fs::path& emscripten_root, + const fs::path& exe_path, + const std::vector& user_args, + const Config& config); + +// If _EMCC_CCACHE is set, unsets it and transforms target_binary to ccache. +void apply_ccache_wrapper(DriverDecision& decision); + +} // namespace emscripten + +#endif // EMCC_NATIVE_DRIVER_H diff --git a/tools/emcc_native/exec.cpp b/tools/emcc_native/exec.cpp new file mode 100644 index 0000000000000..d6e54efe4bdd1 --- /dev/null +++ b/tools/emcc_native/exec.cpp @@ -0,0 +1,118 @@ +/* + * Copyright 2026 The Emscripten Authors. All rights reserved. + * Emscripten is available under two separate licenses, the MIT license and the + * University of Illinois/NCSA Open Source License. Both these licenses can be + * found in the LICENSE file. + */ + +#include "exec.h" + +#include +#include +#include +#include + +#ifdef _WIN32 +#include +#else +#include +#endif + +namespace emscripten { + +// Quotes a command line argument for Windows CreateProcess / CommandLineToArgvW. +// +// Arguments that are empty or contain spaces, tabs, or double quotes must be +// wrapped in double quotes. According to standard Windows command-line parsing +// rules (CommandLineToArgvW): +// - 2N backslashes followed by a double quote produce N literal backslashes and +// a string quote delimiter (start/end of quote). +// - (2N + 1) backslashes followed by a double quote produce N literal +// backslashes and a literal double quote character ("). +// - Backslashes not followed by a double quote are literal and are not doubled. +std::string quote_for_windows(string_view arg) { + if (!arg.empty() && arg.find_first_of(" \t\"") == string_view::npos) { + return std::string(arg); + } + std::string quoted = "\""; + for (size_t i = 0; i < arg.size(); ++i) { + size_t num_backslashes = 0; + while (i < arg.size() && arg[i] == '\\') { + num_backslashes++; + i++; + } + if (i == arg.size()) { + quoted.append(num_backslashes * 2, '\\'); + break; + } + if (arg[i] == '\"') { + quoted.append(num_backslashes * 2 + 1, '\\'); + quoted.push_back('\"'); + } else { + quoted.append(num_backslashes, '\\'); + quoted.push_back(arg[i]); + } + } + quoted += "\""; + return quoted; +} + +[[noreturn]] void exec_process(const std::string& binary, + const std::vector& args) { + unsetenv("_PYTHON_SYSCONFIGDATA_NAME"); + +#ifdef _WIN32 + if (GetEnvironmentVariableW(L"EM_WORKAROUND_PYTHON_BUG_34780", nullptr, 0) > 0) { + CloseHandle(GetStdHandle(STD_INPUT_HANDLE)); + } + + std::string cmdline = quote_for_windows(binary); + for (const auto& arg : args) { + cmdline += " " + quote_for_windows(arg); + } + + int wlen = MultiByteToWideChar(CP_UTF8, 0, cmdline.c_str(), -1, nullptr, 0); + if (wlen == 0) { + std::cerr << "emcc_native: error converting command line to UTF-16" << std::endl; + std::exit(1); + } + std::vector wcmdline(wlen); + MultiByteToWideChar(CP_UTF8, 0, cmdline.c_str(), -1, wcmdline.data(), wlen); + + STARTUPINFOW si; + ZeroMemory(&si, sizeof(si)); + si.cb = sizeof(si); + PROCESS_INFORMATION pi; + ZeroMemory(&pi, sizeof(pi)); + + if (!CreateProcessW(nullptr, wcmdline.data(), nullptr, nullptr, TRUE, 0, nullptr, nullptr, &si, &pi)) { + DWORD err = GetLastError(); + std::cerr << "emcc_native: error executing " << binary + << " (CreateProcessW failed: " << err << ")" << std::endl; + std::exit(1); + } + + WaitForSingleObject(pi.hProcess, INFINITE); + DWORD exit_code = 0; + GetExitCodeProcess(pi.hProcess, &exit_code); + CloseHandle(pi.hProcess); + CloseHandle(pi.hThread); + std::exit(static_cast(exit_code)); +#else + std::vector argv; + argv.reserve(args.size() + 2); + argv.push_back(binary.c_str()); + for (const auto& arg : args) { + argv.push_back(arg.c_str()); + } + argv.push_back(nullptr); + + execvp(binary.c_str(), const_cast(argv.data())); + + std::cerr << "emcc_native: error executing " << binary << ": " + << std::strerror(errno) << std::endl; + std::exit(1); +#endif +} + +} // namespace emscripten diff --git a/tools/emcc_native/exec.h b/tools/emcc_native/exec.h new file mode 100644 index 0000000000000..cd08a6cc76dfb --- /dev/null +++ b/tools/emcc_native/exec.h @@ -0,0 +1,36 @@ +/* + * Copyright 2026 The Emscripten Authors. All rights reserved. + * Emscripten is available under two separate licenses, the MIT license and the + * University of Illinois/NCSA Open Source License. Both these licenses can be + * found in the LICENSE file. + */ + +#ifndef EMCC_NATIVE_EXEC_H +#define EMCC_NATIVE_EXEC_H + +#include +#include +#include +#include + +#ifdef _WIN32 +inline int unsetenv(const char* name) { + return _putenv_s(name, ""); +} +#endif + +namespace emscripten { + +using std::string_view; + +// Quote a command line argument for Windows _spawnvp / CreateProcess. +std::string quote_for_windows(string_view arg); + +// Execute the specified binary with args, replacing the current process or +// exiting with the child's return code. Does not return. +[[noreturn]] void exec_process(const std::string& binary, + const std::vector& args); + +} // namespace emscripten + +#endif // EMCC_NATIVE_EXEC_H diff --git a/tools/emcc_native/gen_settings.py b/tools/emcc_native/gen_settings.py new file mode 100755 index 0000000000000..d3fdfde361a20 --- /dev/null +++ b/tools/emcc_native/gen_settings.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +# Copyright 2026 The Emscripten Authors. All rights reserved. +# Emscripten is available under two separate licenses, the MIT license and the +# University of Illinois/NCSA Open Source License. Both these licenses can be +# found in the LICENSE file. + +"""Generate C++ header tools/emcc_native/generated_settings.h from Python flag definitions.""" + +import os +import sys + +script_dir = os.path.dirname(os.path.abspath(__file__)) +root_dir = os.path.dirname(os.path.dirname(script_dir)) +sys.path.insert(0, root_dir) + +from emcc import LINK_ONLY_FLAGS +from tools import diagnostics +from tools.settings import COMPILE_TIME_SETTINGS +from tools.utils import path_from_root, read_file, write_file + +HEADER_PATH = path_from_root('tools/emcc_native/generated_settings.h') + + +def generate(check_only=False): + link_flags = sorted(LINK_ONLY_FLAGS) + compile_settings = sorted(COMPILE_TIME_SETTINGS) + ems_warnings = sorted(name for name, info in diagnostics.manager.warnings.items() if not info['shared']) + + lines = [ + '/*', + ' * Copyright 2026 The Emscripten Authors. All rights reserved.', + ' * Emscripten is available under two separate licenses, the MIT license and the', + ' * University of Illinois/NCSA Open Source License. Both these licenses can be', + ' * found in the LICENSE file.', + ' *', + ' * Auto-generated by tools/emcc_native/gen_settings.py. DO NOT EDIT.', + ' */', + '', + '#ifndef EMCC_NATIVE_GENERATED_SETTINGS_H', + '#define EMCC_NATIVE_GENERATED_SETTINGS_H', + '', + '#include ', + '#include ', + '', + 'namespace emscripten {', + '', + 'inline const std::unordered_set LINK_ONLY_FLAGS = {', + ] + + for flag in link_flags: + lines.append(f' "{flag}",') + lines.extend([ + '};', + '', + 'inline const std::unordered_set COMPILE_TIME_SETTINGS = {', + ]) + + for setting in compile_settings: + lines.append(f' "{setting}",') + lines.extend([ + '};', + '', + 'inline const std::unordered_set EMSCRIPTEN_ONLY_WARNINGS = {', + ]) + + for warning in ems_warnings: + lines.append(f' "{warning}",') + lines.extend([ + '};', + '', + '} // namespace emscripten', + '', + '#endif // EMCC_NATIVE_GENERATED_SETTINGS_H', + ]) + + content = '\n'.join(lines) + '\n' + + if check_only: + existing = read_file(HEADER_PATH) + if existing != content: + print(f'Error: {HEADER_PATH} is out of date.', file=sys.stderr) + print('Run tools/emcc_native/gen_settings.py to update it.', file=sys.stderr) + sys.exit(1) + print(f'{HEADER_PATH} is up to date.') + else: + write_file(HEADER_PATH, content) + print(f'Wrote {HEADER_PATH}') + + +if __name__ == '__main__': + check = '--check' in sys.argv + generate(check_only=check) diff --git a/tools/emcc_native/generated_settings.h b/tools/emcc_native/generated_settings.h new file mode 100644 index 0000000000000..abead2d118ad3 --- /dev/null +++ b/tools/emcc_native/generated_settings.h @@ -0,0 +1,117 @@ +/* + * Copyright 2026 The Emscripten Authors. All rights reserved. + * Emscripten is available under two separate licenses, the MIT license and the + * University of Illinois/NCSA Open Source License. Both these licenses can be + * found in the LICENSE file. + * + * Auto-generated by tools/emcc_native/gen_settings.py. DO NOT EDIT. + */ + +#ifndef EMCC_NATIVE_GENERATED_SETTINGS_H +#define EMCC_NATIVE_GENERATED_SETTINGS_H + +#include +#include + +namespace emscripten { + +inline const std::unordered_set LINK_ONLY_FLAGS = { + "--bind", + "--closure", + "--cpuprofiler", + "--embed-file", + "--emit-symbol-map", + "--emrun", + "--exclude-file", + "--extern-post-js", + "--extern-pre-js", + "--ignore-dynamic-linking", + "--js-library", + "--js-transform", + "--oformat", + "--output-eol", + "--output_eol", + "--post-js", + "--pre-js", + "--preload-file", + "--profiling-funcs", + "--proxy-to-worker", + "--shell-file", + "--source-map-base", + "--threadprofiler", + "--use-preload-plugins", +}; + +inline const std::unordered_set COMPILE_TIME_SETTINGS = { + "DEBUG_LEVEL", + "DISABLE_EXCEPTION_CATCHING", + "DISABLE_EXCEPTION_THROWING", + "EMSCRIPTEN_TRACING", + "EXCEPTION_CATCHING_ALLOWED", + "INLINING_LIMIT", + "LINKABLE", + "LTO", + "MAIN_MODULE", + "MEMORY64", + "OPT_LEVEL", + "PTHREADS", + "SDL2_IMAGE_FORMATS", + "SDL2_MIXER_FORMATS", + "SHARED_MEMORY", + "SIDE_MODULE", + "STRICT", + "SUPPORT_LONGJMP", + "USE_BOOST_HEADERS", + "USE_BULLET", + "USE_BZIP2", + "USE_COCOS2D", + "USE_FREETYPE", + "USE_GIFLIB", + "USE_HARFBUZZ", + "USE_ICU", + "USE_LIBJPEG", + "USE_LIBPNG", + "USE_MODPLUG", + "USE_MPG123", + "USE_OGG", + "USE_PTHREADS", + "USE_REGAL", + "USE_SDL", + "USE_SDL_GFX", + "USE_SDL_IMAGE", + "USE_SDL_MIXER", + "USE_SDL_NET", + "USE_SDL_TTF", + "USE_SQLITE3", + "USE_VORBIS", + "USE_ZLIB", + "WASM_EXCEPTIONS", + "WASM_LEGACY_EXCEPTIONS", + "WASM_OBJECT_FILES", + "WASM_WORKERS", +}; + +inline const std::unordered_set EMSCRIPTEN_ONLY_WARNINGS = { + "absolute-paths", + "almost-asm", + "closure", + "compatibility", + "em-js-i64", + "emcc", + "experimental", + "export-main", + "js-compiler", + "legacy-settings", + "limited-postlink-optimizations", + "linkflags", + "map-unrecognized-libraries", + "pthreads-mem-growth", + "undefined", + "unsupported", + "unused-main", + "version-check", +}; + +} // namespace emscripten + +#endif // EMCC_NATIVE_GENERATED_SETTINGS_H diff --git a/tools/emcc_native/main.cpp b/tools/emcc_native/main.cpp new file mode 100644 index 0000000000000..c8467d7d9bffe --- /dev/null +++ b/tools/emcc_native/main.cpp @@ -0,0 +1,146 @@ +/* + * Copyright 2026 The Emscripten Authors. All rights reserved. + * Emscripten is available under two separate licenses, the MIT license and the + * University of Illinois/NCSA Open Source License. Both these licenses can be + * found in the LICENSE file. + */ + +#include "config.h" +#include "driver.h" +#include "exec.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef _WIN32 +#include +#elif defined(__APPLE__) +#include +#include +#else +#include +#endif + +#if defined(_MSC_VER) && !defined(__clang__) +#define UNREACHABLE() __assume(0) +#else +#define UNREACHABLE() __builtin_unreachable() +#endif + +namespace fs = std::filesystem; + +using namespace emscripten; + +namespace { + +template void errlog(Args&&... args) { + (std::cerr << ... << std::forward(args)) << std::endl; +} + + +fs::path get_self_executable_path() { +#if defined(_WIN32) + std::vector buf(MAX_PATH); + DWORD len = GetModuleFileNameW(NULL, buf.data(), static_cast(buf.size())); + while (len >= buf.size()) { + buf.resize(buf.size() * 2); + len = GetModuleFileNameW(NULL, buf.data(), static_cast(buf.size())); + } + if (len == 0) { + errlog("emcc_native: error: GetModuleFileNameW failed"); + std::exit(1); + } + return fs::path(buf.data()); +#elif defined(__linux__) + std::error_code ec; + fs::path proc_path = fs::read_symlink("/proc/self/exe", ec); + if (ec || proc_path.empty()) { + errlog("emcc_native: error: reading /proc/self/exe failed: ", ec.message()); + std::exit(1); + } + return proc_path; +#elif defined(__APPLE__) + uint32_t size = 1024; + std::vector buf(size); + if (_NSGetExecutablePath(buf.data(), &size) != 0) { + buf.resize(size); + if (_NSGetExecutablePath(buf.data(), &size) != 0) { + errlog("emcc_native: error: _NSGetExecutablePath failed"); + std::exit(1); + } + } + return fs::path(buf.data()); +#else +#error "Unsupported platform for get_self_executable_path" +#endif +} + +fs::path find_emscripten_root(const fs::path& exe_path) { + fs::path dir = fs::weakly_canonical(exe_path).parent_path(); + while (!dir.empty() && dir != dir.root_path()) { + if (fs::exists(dir / "emcc.py")) { + return dir; + } + dir = dir.parent_path(); + } + + errlog("emcc_native: error: could not locate Emscripten root directory " + "(emcc.py not found relative to launcher binary at ", + exe_path.generic_string(), + ")"); + std::exit(1); +} + +void log_decision(const DriverDecision& decision) { + if (decision.use_fallback) { + errlog("emcc_native: falling back to python driver (", decision.reason, ")"); + } else { + errlog("emcc_native: executing clang directly"); + } + + std::string full_cmd = decision.target_binary; + for (const auto& arg : decision.target_args) { + full_cmd += " " + quote_for_windows(arg); + } + + errlog("emcc_native: exec: ", full_cmd); +} + +} // namespace + +int main(int argc, char** argv) { + assert(argc >= 1); + + fs::path exe_path = get_self_executable_path(); + fs::path emscripten_root = find_emscripten_root(exe_path); + Config config = load_config(emscripten_root); + + std::vector user_args(argv + 1, argv + argc); + + auto decision = analyze_request(argv[0], emscripten_root, exe_path, user_args, config); + + const char* native_env = std::getenv("EMCC_NATIVE"); + if (decision.use_fallback && native_env && std::string(native_env) == "1") { + errlog("emcc_native: error: falling back to python driver with EMCC_NATIVE=1 (", + decision.reason, + ")"); + return 1; + } + + apply_ccache_wrapper(decision); + + const char* emcc_debug = std::getenv("EMCC_DEBUG"); + const char* native_debug = std::getenv("EMCC_NATIVE_DEBUG"); + if ((emcc_debug && emcc_debug[0] != '\0') || (native_debug && native_debug[0] != '\0')) { + log_decision(decision); + } + + exec_process(decision.target_binary, decision.target_args); + UNREACHABLE(); +} diff --git a/tools/maint/create_entry_points.py b/tools/maint/create_entry_points.py index ead15389b2eda..ded8450d7b580 100755 --- a/tools/maint/create_entry_points.py +++ b/tools/maint/create_entry_points.py @@ -6,15 +6,15 @@ """Tool for creating/maintaining the python launcher scripts for emscripten tools. -This tool makes copies or `run_python.sh/.bat` and `run_python_compiler.sh/.bat` +Note: This tool is deprecated and being replaced by emcc_native (built via `bootstrap.py`). + +This tool makes copies of `run_python.sh/.bat` and `run_python_compiler.sh/.bat` script for each entry point. On UNIX we previously used symbolic links for simplicity but this breaks MINGW users on windows who want to use the shell script launcher but don't have symlink support. """ import os -import platform -import shutil import stat import sys @@ -61,11 +61,6 @@ } -windows_exe = os.path.join(__rootdir__, 'tools/pylauncher/pylauncher.exe') -if platform.machine().lower() in {'arm64', 'aarch64'}: - windows_exe = os.path.join(__rootdir__, 'tools/pylauncher/pylauncher-arm64.exe') - - def make_executable(filename): old_mode = stat.S_IMODE(os.stat(filename).st_mode) os.chmod(filename, old_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) @@ -87,9 +82,8 @@ def write_file(filename, content): f.write(content) -def main(all_platforms, use_bat_file): - if 'EM_USE_BAT_FILES' in os.environ: - use_bat_file = True +def main(all_platforms): + print('warning: create_entry_points.py is deprecated and being replaced by emcc_native', file=sys.stderr) is_windows = sys.platform.startswith('win') is_msys2 = 'MSYSTEM' in os.environ do_unix = all_platforms or not is_windows or is_msys2 @@ -118,17 +112,12 @@ def generate_entry_points(cmd, path): make_executable(launcher) if do_windows: - maybe_remove(launcher + '.ps1') - maybe_remove(launcher + '.exe') - if use_bat_file: - write_file(launcher + '.bat', bat_data) - write_file(launcher + '.ps1', ps1_data) - else: - shutil.copyfile(windows_exe, launcher + '.exe') + write_file(launcher + '.bat', bat_data) + write_file(launcher + '.ps1', ps1_data) generate_entry_points(entry_points, os.path.join(__scriptdir__, 'run_python')) generate_entry_points(compiler_entry_points, os.path.join(__scriptdir__, 'run_python_compiler')) if __name__ == '__main__': - sys.exit(main('--all' in sys.argv, '--bat-files' in sys.argv)) + sys.exit(main('--all' in sys.argv)) diff --git a/tools/pylauncher/README.md b/tools/pylauncher/README.md deleted file mode 100644 index 5a5d5fa27c1e8..0000000000000 --- a/tools/pylauncher/README.md +++ /dev/null @@ -1,67 +0,0 @@ -# Windows Python Script Launcher - -This directory contains a simple launcher program for Windows which is used to -execute the Emscripten compiler entry points using the python interpreter. It -uses the its own name (the name of the currently running executable) to -determine which python script to run and serves the same purpose as the shell -scripts (see `run_python.sh` ) do on non-Windows platforms. - -## Building - -The primary `build.bat` script uses MSVC and links against `ucrtbase.dll` which -ships as part of the OS since Windows 10 (2015) and is available via -Windows Update for Vista/7/8/8.1. - -`build.sh` cross-compiles with MinGW and links against `msvcrt.dll` (present -on all Windows versions). This script is mostly useful for debugging on Linux. -The .exe file that is checked in here, and used in by emsdk is currently always -built with the `build.bat` script above. - -## Related projects - -### posy-trampoline - -The posy-trampoline project does something similar: - - https://github.com/njsmith/posy/tree/main/src/trampolines/windows-trampolines/posy-trampoline - -However, IIRC it also embeds the target python file onto the executable itself -which is not some Emscripten needs (or wants) to do. Its also written in rust -(which is not something Emscripten has in any of its dependencies yet). -The `uv` tool also embeds a version of this trampiline: - - https://github.com/astral-sh/uv/tree/main/crates/uv-trampoline - -There is also PyInstaller (https://pyinstaller.org/en/stable/), but that seems -to want to wrap up the whole application into the executable too. - -Both these projects seem to change the way the python code itself is delivered -to Windows users. The difference with Emscripten's pylauncher is that -it leaves the layout of the python files unchanged. i.e. Its just a launcher, -not any kind of bundler. This means that Windows users can still see, and even -modify in place, all the python files just like non-Windows users. All the -launcher does is allows the entry point to be an `.exe`. - -The Emscripten launcher is also very small, coming it at just 5.5k at time of -writing. - -The Emscripten launcher also doesn't require any modification of the `.exe` to -deploy it. All one needs to do is copy the unmodified executable alongside a -python file of the same name, and the launcher will find it and run it based -purely on the name of the launcher itself (i.e. argv0). - -### setuptools - -There is windows launcher that is part of setuptools: - - https://github.com/pypa/setuptools/blob/main/launcher.c - -The code is very similar to Emscripten's launcher. In fact, perhaps we should -consider switch it this in the future? - -Unlike Emscripten's laucnher this launcher seems to look for -`-script.py` rather than just `.py`, which is what -the Emscripten laucnher uses. - -This laucnher also seem to examine the `#!` line of the target script and -locate the python executable somehow based on this. diff --git a/tools/pylauncher/build.bat b/tools/pylauncher/build.bat deleted file mode 100644 index e204c642c7a33..0000000000000 --- a/tools/pylauncher/build.bat +++ /dev/null @@ -1,22 +0,0 @@ -:: Build pylauncher.exe using MSVC. -:: -:: This links against ucrtbase.dll (via ucrt.lib) which ships as part of the OS -:: since Windows 10 (2015) and via Windows Update for Vista/7/8/8.1. -:: -:: /O1 : Favor small code (optimization for size) -:: /GS- : Disable buffer security checks (requires vc runtime and not necessary for our tiny command line wrapper) -:: /NODEFAULTLIB : Do not link the default libraries -:: /ENTRY:launcher_main : Use launcher_main() as entry point directly (no CRT startup) -:: /SUBSYSTEM:CONSOLE : Designate as a console app instead of WINDOWS. Needed explicitly because of custom entrypoint. -:: /Brepro : Deterministic (reproducible) output -:: ucrt.lib : Link only against Universal CRT (no vcruntime dependency) - -set OUT=pylauncher.exe -set MACHINE=X64 - -if /i "%~1"=="arm64" ( - set OUT=pylauncher-arm64.exe - set MACHINE=ARM64 -) - -cl pylauncher.c /Fe:%OUT% /O1 /GS- /link /NODEFAULTLIB /ENTRY:launcher_main /SUBSYSTEM:CONSOLE /MACHINE:%MACHINE% /Brepro ucrt.lib kernel32.lib diff --git a/tools/pylauncher/pylauncher-arm64.exe b/tools/pylauncher/pylauncher-arm64.exe deleted file mode 100644 index 88f4b2b87dafd..0000000000000 Binary files a/tools/pylauncher/pylauncher-arm64.exe and /dev/null differ diff --git a/tools/pylauncher/pylauncher.c b/tools/pylauncher/pylauncher.c deleted file mode 100644 index 5a0dcdef9f166..0000000000000 --- a/tools/pylauncher/pylauncher.c +++ /dev/null @@ -1,230 +0,0 @@ -/* - * Copyright 2026 The Emscripten Authors. All rights reserved. - * Emscripten is available under two separate licenses, the MIT license and the - * University of Illinois/NCSA Open Source License. Both these licenses can be - * found in the LICENSE file. - * - * Small win32 application that is used to launcher emscripten via python.exe. - * On non-windows platforms this is done via the run_python.sh shell script. - * - * The binary will look for a python script that matches its own name and run - * that using python.exe. - * - * Built with /NODEFAULTLIB linking only against ucrt.lib (ucrtbase.dll) to - * avoid any dependency on a specific Visual C++ Redistributable version. - */ - -// Define _WIN32_WINNT to Windows 7 for max portability -#define _WIN32_WINNT 0x0601 - -#include -#include -#include -#include -#include -#include - -// ZeroMemory expands to memset which lives in vcruntime, not ucrt. -// SecureZeroMemory is an inline in with no runtime dependency. -#undef ZeroMemory -#define ZeroMemory SecureZeroMemory - -#define WLEN(lit) (sizeof(lit) / sizeof(wchar_t) - 1) - -static bool launcher_debug = false; - -static void dbg(const char* format, ...) { - if (launcher_debug) { - va_list args; - va_start(args, format); - vfprintf(stderr, format, args); - va_end(args); - } -} - -static const wchar_t* get_python_executable() { - const wchar_t* python_exe_w = _wgetenv(L"EMSDK_PYTHON"); - if (!python_exe_w) { - return L"python.exe"; - } - return python_exe_w; -} - -// Get the name of the currently running executable (module) -static wchar_t* get_module_path() { - DWORD buffer_size = MAX_PATH; - wchar_t* module_path_w = malloc(sizeof(wchar_t) * buffer_size); - if (!module_path_w) - abort(); - - DWORD path_len = GetModuleFileNameW(NULL, module_path_w, buffer_size); - // Keep doubling buffer size until GetModuleFileNameW returns something - // less than the full buffer size - while (path_len == buffer_size) { - buffer_size *= 2; - module_path_w = realloc(module_path_w, sizeof(wchar_t) * buffer_size); - if (!module_path_w) - abort(); - path_len = GetModuleFileNameW(NULL, module_path_w, buffer_size); - } - - if (path_len == 0) - abort(); - - return module_path_w; -} - -/** - * A custom replacement for PathGetArgsW that is safe for command lines - * longer than MAX_PATH. - */ -static const wchar_t* find_args(const wchar_t* command_line) { - const wchar_t* p = command_line; - - // Skip past the executable name, which can be quoted. - if (*p == L'"') { - // The path is quoted, find the closing quote. - p++; - while (*p) { - if (*p == L'"') { - p++; - break; - } - p++; - } - } else { - // The path is not quoted, find the first space. - while (*p && *p != L' ' && *p != L'\t') { - p++; - } - } - - // Skip any whitespace between the executable and the first argument. - while (*p && (*p == L' ' || *p == L'\t')) { - p++; - } - - return p; -} - -static bool path_exists(const wchar_t* path) { - return GetFileAttributesW(path) != INVALID_FILE_ATTRIBUTES; -} - -/** - * Create the script path by finding the launcher path and replacing the - * extension with .py. For example `C:\path\to\emcc.exe` becomes - * `C:\path\to\emcc.py`. - * - * If the corresponging .py file does not exist then also look it in the tools - * subdirectory. e.g. `C:\path\to\tools\emcc.py` - */ -static wchar_t* get_script_path() { - wchar_t* script_path = get_module_path(); - if (!script_path) - abort(); - - size_t path_len = wcslen(script_path); - if (path_len < WLEN(L".exe") || _wcsicmp(script_path + path_len - WLEN(L".exe"), L".exe") != 0) - abort(); - // Strip .exe - path_len -= WLEN(L".exe"); - // Append .py (no need to realloc since ".py" is shorter than ".exe") - wcscpy(script_path + path_len, L".py"); - path_len += WLEN(L".py"); - - if (path_exists(script_path)) { - return script_path; - } - - // Python file not found alongside launcher; try under tools - // C:\path\to\emcc.py` => C:\path\to\tools\emcc.py` - size_t dir_len = 0; - for (size_t i = path_len; i > 0; i--) { - if (script_path[i - 1] == L'\\') { - dir_len = i; - break; - } - } - size_t tools_path_size = path_len + WLEN(L"tools\\") + 1; - wchar_t* script_path_tools = malloc(tools_path_size * sizeof(wchar_t)); - swprintf(script_path_tools, tools_path_size, L"%.*lstools\\%ls", (int)dir_len, script_path, script_path + dir_len); - - if (!path_exists(script_path_tools)) { - fprintf(stderr, "pylauncher: target python file not found: %ls / %ls\n", script_path, script_path_tools); - abort(); - } - free(script_path); - - return script_path_tools; -} - -// This gets a name other than main() as a reminder that its return value is not sent anywhere -// (because this file is compiled without a CRT). -void launcher_main() { - // Setting EMCC_LAUNCHER_DEBUG enabled debug output for the launcher itself. - launcher_debug = GetEnvironmentVariableW(L"EMCC_LAUNCHER_DEBUG", NULL, 0); - - dbg("pylauncher: launcher_main\n"); - - const wchar_t* ccache_prefix = L""; - DWORD env_len = GetEnvironmentVariableW(L"_EMCC_CCACHE", NULL, 0); - if (env_len) { - dbg("pylauncher: running via ccache.exe\n"); - ccache_prefix = L"ccache.exe "; - SetEnvironmentVariableW(L"_EMCC_CCACHE", NULL); - } - - const wchar_t* application_name = get_python_executable(); - wchar_t* script_path_w = get_script_path(); - size_t command_line_len = wcslen(ccache_prefix) + wcslen(application_name) + wcslen(script_path_w) + 17; - wchar_t* command_line = malloc(sizeof(wchar_t) * command_line_len); - swprintf(command_line, command_line_len, L"%ls\"%ls\" -E -X utf8 \"%ls\"", ccache_prefix, application_name, script_path_w); - free(script_path_w); - - // -E will not ignore _PYTHON_SYSCONFIGDATA_NAME an internal - // of cpython used in cross compilation via setup.py. - SetEnvironmentVariableW(L"_PYTHON_SYSCONFIGDATA_NAME", NULL); - - // Build the final command line by appending the original arguments - const wchar_t* all_args = find_args(GetCommandLineW()); - if (all_args && *all_args) { - size_t current_len = wcslen(command_line); - size_t args_len = wcslen(all_args); - // +2 for the space and the null terminator - command_line = realloc(command_line, (current_len + args_len + 2) * sizeof(wchar_t)); - if (!command_line) - abort(); - wcscat_s(command_line, current_len + args_len + 2, L" "); - wcscat_s(command_line, current_len + args_len + 2, all_args); - } - - // Work around python bug 34780 by closing stdin, so that it is not inherited - // by the python subprocess. - env_len = GetEnvironmentVariableW(L"EM_WORKAROUND_PYTHON_BUG_34780", NULL, 0); - if (env_len) { - dbg("pylauncher: using EM_WORKAROUND_PYTHON_BUG_34780\n"); - CloseHandle(GetStdHandle(STD_INPUT_HANDLE)); - } - - STARTUPINFOW si; - PROCESS_INFORMATION pi; - ZeroMemory(&si, sizeof(si)); - si.cb = sizeof(si); - ZeroMemory(&pi, sizeof(pi)); - - dbg("pylauncher: running: %ls\n", command_line); - if (!CreateProcessW(NULL, command_line, NULL, NULL, FALSE, 0, NULL, NULL, &si, &pi)) { - fprintf(stderr, "pylauncher: CreateProcess failed (%lu): %ls\n", GetLastError(), command_line); - abort(); - } - WaitForSingleObject(pi.hProcess, INFINITE); - - DWORD exit_code; - GetExitCodeProcess(pi.hProcess, &exit_code); - CloseHandle(pi.hProcess); - CloseHandle(pi.hThread); - - dbg("pylauncher: done: %d\n", exit_code); - ExitProcess(exit_code); -} diff --git a/tools/pylauncher/pylauncher.exe b/tools/pylauncher/pylauncher.exe deleted file mode 100644 index 616cd5f44a34f..0000000000000 Binary files a/tools/pylauncher/pylauncher.exe and /dev/null differ