Skip to content

[NFC] Move fuzzer VMs out of CompareVMs #8587

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
kripken merged 2 commits into from
Apr 9, 2026
+174 −166
Merged

[NFC] Move fuzzer VMs out of CompareVMs #8587

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
c50d2f5
move
kripken Apr 9, 2026
3f0dc3d
format
kripken Apr 9, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
340 changes: 174 additions & 166 deletions scripts/fuzz_opt.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -786,179 +786,187 @@ class FuzzExec(TestCaseHandler):
def handle_pair(self, input, before_wasm, after_wasm, opts):
run([in_bin('wasm-opt'), before_wasm] + opts + ['--fuzz-exec'])

# VMs


class BinaryenInterpreter:
name = 'binaryen interpreter'

def run(self, wasm):
output = run_bynterp(wasm, ['--fuzz-exec-before'])
if output != IGNORE:
calls = output.count(FUZZ_EXEC_EXPORT_PREFIX)
errors = output.count(TRAP_PREFIX) + output.count(HOST_LIMIT_PREFIX)
if errors > calls / 2:
# A significant amount of execution on this testcase
# simply trapped, and was not very useful, so mark it
# as ignored. Ideally the fuzzer testcases would be
# improved to reduce this number.
#
# Note that we don't change output=IGNORE as there may
# still be useful testing here (up to 50%), so we only
# note that this is a mostly-ignored run, but we do not
# ignore the parts that are useful.
#
# Note that we set amount to 0.5 because we are run both
# on the before wasm and the after wasm. Those will be
# in sync (because the optimizer does not remove traps)
# and so by setting 0.5 we only increment by 1 for the
# entire iteration.
note_ignored_vm_run('too many errors vs calls',
extra_text=f' ({calls} calls, {errors} errors)',
amount=0.5)
return output

def can_run(self, wasm):
return True

def can_compare_to_self(self):
return True

def can_compare_to_other(self, other):
return True


class D8:
name = 'd8'

def run(self, wasm, extra_d8_flags=[]):
return run_vm([shared.V8, get_fuzz_shell_js()] + shared.V8_OPTS + get_v8_extra_flags() + extra_d8_flags + ['--', wasm])

def can_run(self, wasm):
return all_disallowed(DISALLOWED_FEATURES_IN_V8)

def can_compare_to_self(self):
# With nans, VM differences can confuse us, so only very simple VMs
# can compare to themselves after opts in that case.
return not NANS

def can_compare_to_other(self, other):
# Relaxed SIMD allows different behavior between VMs, so only
# allow comparisons to other d8 variants if it is enabled.
if not all_disallowed(['relaxed-simd']) and not other.name.startswith('d8'):
return False

# If not legalized, the JS will fail immediately, so no point to
# compare to others.
return self.can_compare_to_self() and LEGALIZE


class D8Liftoff(D8):
name = 'd8_liftoff'

def run(self, wasm):
return super().run(wasm, extra_d8_flags=V8_LIFTOFF_ARGS)


class D8Turboshaft(D8):
name = 'd8_turboshaft'

def run(self, wasm):
flags = ['--no-liftoff']
return super().run(wasm, extra_d8_flags=flags)


class Wasm2C:
name = 'wasm2c'

def __init__(self):
# look for wabt in the path. if it's not here, don't run wasm2c
try:
wabt_bin = shared.which('wasm2c')
wabt_root = os.path.dirname(os.path.dirname(wabt_bin))
self.wasm2c_dir = os.path.join(wabt_root, 'wasm2c')
if not os.path.isdir(self.wasm2c_dir):
print('wabt found, but not wasm2c support dir')
self.wasm2c_dir = None
except Exception as e:
print('warning: no wabt found:', e)
self.wasm2c_dir = None

def can_run(self, wasm):
if self.wasm2c_dir is None:
return False
# if we legalize for JS, the ABI is not what C wants
if LEGALIZE:
return False
# relatively slow, so run it less frequently
if random.random() < 0.5:
return False
# wasm2c doesn't support most features
return all_disallowed(['exception-handling', 'simd', 'threads', 'bulk-memory', 'nontrapping-float-to-int', 'tail-call', 'sign-ext', 'reference-types', 'multivalue', 'gc', 'custom-descriptors', 'relaxed-atomics'])

def run(self, wasm):
run([in_bin('wasm-opt'), wasm, '--emit-wasm2c-wrapper=main.c'] + FEATURE_OPTS)
run(['wasm2c', wasm, '-o', 'wasm.c'])
compile_cmd = ['clang', 'main.c', 'wasm.c', os.path.join(self.wasm2c_dir, 'wasm-rt-impl.c'), '-I' + self.wasm2c_dir, '-lm', '-Werror']
run(compile_cmd)
return run_vm(['./a.out'])

def can_compare_to_self(self):
# The binaryen optimizer changes NaNs in the ways that wasm
# expects, but that's not quite what C has
return not NANS

def can_compare_to_other(self, other):
# C won't trap on OOB, and NaNs can differ from wasm VMs
return not OOB and not NANS


class Wasm2C2Wasm(Wasm2C):
name = 'wasm2c2wasm'

def __init__(self):
super().__init__()

self.has_emcc = shared.which('emcc') is not None

def run(self, wasm):
run([in_bin('wasm-opt'), wasm, '--emit-wasm2c-wrapper=main.c'] + FEATURE_OPTS)
run(['wasm2c', wasm, '-o', 'wasm.c'])
compile_cmd = ['emcc', 'main.c', 'wasm.c',
os.path.join(self.wasm2c_dir, 'wasm-rt-impl.c'),
'-I' + self.wasm2c_dir,
'-lm',
'-s', 'ENVIRONMENT=shell',
'-s', 'ALLOW_MEMORY_GROWTH']
# disable the signal handler: emcc looks like unix, but wasm has
# no signals
compile_cmd += ['-DWASM_RT_MEMCHECK_SIGNAL_HANDLER=0']
if random.random() < 0.5:
compile_cmd += ['-O' + str(random.randint(1, 3))]
elif random.random() < 0.5:
if random.random() < 0.5:
compile_cmd += ['-Os']
else:
compile_cmd += ['-Oz']
# avoid pass-debug on the emcc invocation itself (which runs
# binaryen to optimize the wasm), as the wasm here can be very
# large and it isn't what we are focused on testing here
with no_pass_debug():
run(compile_cmd)
return run_d8_js(abspath('a.out.js'))

def can_run(self, wasm):
# quite slow (more steps), so run it less frequently
if random.random() < 0.8:
return False
# prefer not to run if the wasm is very large, as it can OOM
# the JS engine.
return super().can_run(wasm) and self.has_emcc and \
os.path.getsize(wasm) <= INPUT_SIZE_MEAN

def can_compare_to_other(self, other):
# NaNs can differ from wasm VMs
return not NANS


class CompareVMs(TestCaseHandler):
frequency = 1

def __init__(self):
super().__init__()

class BinaryenInterpreter:
name = 'binaryen interpreter'

def run(self, wasm):
output = run_bynterp(wasm, ['--fuzz-exec-before'])
if output != IGNORE:
calls = output.count(FUZZ_EXEC_EXPORT_PREFIX)
errors = output.count(TRAP_PREFIX) + output.count(HOST_LIMIT_PREFIX)
if errors > calls / 2:
# A significant amount of execution on this testcase
# simply trapped, and was not very useful, so mark it
# as ignored. Ideally the fuzzer testcases would be
# improved to reduce this number.
#
# Note that we don't change output=IGNORE as there may
# still be useful testing here (up to 50%), so we only
# note that this is a mostly-ignored run, but we do not
# ignore the parts that are useful.
#
# Note that we set amount to 0.5 because we are run both
# on the before wasm and the after wasm. Those will be
# in sync (because the optimizer does not remove traps)
# and so by setting 0.5 we only increment by 1 for the
# entire iteration.
note_ignored_vm_run('too many errors vs calls',
extra_text=f' ({calls} calls, {errors} errors)',
amount=0.5)
return output

def can_run(self, wasm):
return True

def can_compare_to_self(self):
return True

def can_compare_to_other(self, other):
return True

class D8:
name = 'd8'

def run(self, wasm, extra_d8_flags=[]):
return run_vm([shared.V8, get_fuzz_shell_js()] + shared.V8_OPTS + get_v8_extra_flags() + extra_d8_flags + ['--', wasm])

def can_run(self, wasm):
return all_disallowed(DISALLOWED_FEATURES_IN_V8)

def can_compare_to_self(self):
# With nans, VM differences can confuse us, so only very simple VMs
# can compare to themselves after opts in that case.
return not NANS

def can_compare_to_other(self, other):
# Relaxed SIMD allows different behavior between VMs, so only
# allow comparisons to other d8 variants if it is enabled.
if not all_disallowed(['relaxed-simd']) and not other.name.startswith('d8'):
return False

# If not legalized, the JS will fail immediately, so no point to
# compare to others.
return self.can_compare_to_self() and LEGALIZE

class D8Liftoff(D8):
name = 'd8_liftoff'

def run(self, wasm):
return super().run(wasm, extra_d8_flags=V8_LIFTOFF_ARGS)

class D8Turboshaft(D8):
name = 'd8_turboshaft'

def run(self, wasm):
flags = ['--no-liftoff']
return super().run(wasm, extra_d8_flags=flags)

class Wasm2C:
name = 'wasm2c'

def __init__(self):
# look for wabt in the path. if it's not here, don't run wasm2c
try:
wabt_bin = shared.which('wasm2c')
wabt_root = os.path.dirname(os.path.dirname(wabt_bin))
self.wasm2c_dir = os.path.join(wabt_root, 'wasm2c')
if not os.path.isdir(self.wasm2c_dir):
print('wabt found, but not wasm2c support dir')
self.wasm2c_dir = None
except Exception as e:
print('warning: no wabt found:', e)
self.wasm2c_dir = None

def can_run(self, wasm):
if self.wasm2c_dir is None:
return False
# if we legalize for JS, the ABI is not what C wants
if LEGALIZE:
return False
# relatively slow, so run it less frequently
if random.random() < 0.5:
return False
# wasm2c doesn't support most features
return all_disallowed(['exception-handling', 'simd', 'threads', 'bulk-memory', 'nontrapping-float-to-int', 'tail-call', 'sign-ext', 'reference-types', 'multivalue', 'gc', 'custom-descriptors', 'relaxed-atomics'])

def run(self, wasm):
run([in_bin('wasm-opt'), wasm, '--emit-wasm2c-wrapper=main.c'] + FEATURE_OPTS)
run(['wasm2c', wasm, '-o', 'wasm.c'])
compile_cmd = ['clang', 'main.c', 'wasm.c', os.path.join(self.wasm2c_dir, 'wasm-rt-impl.c'), '-I' + self.wasm2c_dir, '-lm', '-Werror']
run(compile_cmd)
return run_vm(['./a.out'])

def can_compare_to_self(self):
# The binaryen optimizer changes NaNs in the ways that wasm
# expects, but that's not quite what C has
return not NANS

def can_compare_to_other(self, other):
# C won't trap on OOB, and NaNs can differ from wasm VMs
return not OOB and not NANS

class Wasm2C2Wasm(Wasm2C):
name = 'wasm2c2wasm'

def __init__(self):
super().__init__()

self.has_emcc = shared.which('emcc') is not None

def run(self, wasm):
run([in_bin('wasm-opt'), wasm, '--emit-wasm2c-wrapper=main.c'] + FEATURE_OPTS)
run(['wasm2c', wasm, '-o', 'wasm.c'])
compile_cmd = ['emcc', 'main.c', 'wasm.c',
os.path.join(self.wasm2c_dir, 'wasm-rt-impl.c'),
'-I' + self.wasm2c_dir,
'-lm',
'-s', 'ENVIRONMENT=shell',
'-s', 'ALLOW_MEMORY_GROWTH']
# disable the signal handler: emcc looks like unix, but wasm has
# no signals
compile_cmd += ['-DWASM_RT_MEMCHECK_SIGNAL_HANDLER=0']
if random.random() < 0.5:
compile_cmd += ['-O' + str(random.randint(1, 3))]
elif random.random() < 0.5:
if random.random() < 0.5:
compile_cmd += ['-Os']
else:
compile_cmd += ['-Oz']
# avoid pass-debug on the emcc invocation itself (which runs
# binaryen to optimize the wasm), as the wasm here can be very
# large and it isn't what we are focused on testing here
with no_pass_debug():
run(compile_cmd)
return run_d8_js(abspath('a.out.js'))

def can_run(self, wasm):
# quite slow (more steps), so run it less frequently
if random.random() < 0.8:
return False
# prefer not to run if the wasm is very large, as it can OOM
# the JS engine.
return super().can_run(wasm) and self.has_emcc and \
os.path.getsize(wasm) <= INPUT_SIZE_MEAN

def can_compare_to_other(self, other):
# NaNs can differ from wasm VMs
return not NANS

# the binaryen interpreter is specifically useful for various things
self.bynterpreter = BinaryenInterpreter()

Expand Down
Loading