Skip to content

Commit f7b08ed

Browse files
kripkenkripken
authored
[NFC] Move fuzzer VMs out of CompareVMs (#8587)
Diff without whitespace is trivial.
1 parent d918f9c commit f7b08ed

File tree

1 file changed

+174
-166
lines changed

1 file changed

+174
-166
lines changed

‎scripts/fuzz_opt.py‎

Lines changed: 174 additions & 166 deletions
Original file line numberDiff line numberDiff line change
@@ -786,179 +786,187 @@ class FuzzExec(TestCaseHandler):
786786
def handle_pair(self, input, before_wasm, after_wasm, opts):
787787
run([in_bin('wasm-opt'), before_wasm] + opts + ['--fuzz-exec'])
788788

789+
# VMs
790+
791+
792+
class BinaryenInterpreter:
793+
name = 'binaryen interpreter'
794+
795+
def run(self, wasm):
796+
output = run_bynterp(wasm, ['--fuzz-exec-before'])
797+
if output != IGNORE:
798+
calls = output.count(FUZZ_EXEC_EXPORT_PREFIX)
799+
errors = output.count(TRAP_PREFIX) + output.count(HOST_LIMIT_PREFIX)
800+
if errors > calls / 2:
801+
# A significant amount of execution on this testcase
802+
# simply trapped, and was not very useful, so mark it
803+
# as ignored. Ideally the fuzzer testcases would be
804+
# improved to reduce this number.
805+
#
806+
# Note that we don't change output=IGNORE as there may
807+
# still be useful testing here (up to 50%), so we only
808+
# note that this is a mostly-ignored run, but we do not
809+
# ignore the parts that are useful.
810+
#
811+
# Note that we set amount to 0.5 because we are run both
812+
# on the before wasm and the after wasm. Those will be
813+
# in sync (because the optimizer does not remove traps)
814+
# and so by setting 0.5 we only increment by 1 for the
815+
# entire iteration.
816+
note_ignored_vm_run('too many errors vs calls',
817+
extra_text=f' ({calls} calls, {errors} errors)',
818+
amount=0.5)
819+
return output
820+
821+
def can_run(self, wasm):
822+
return True
823+
824+
def can_compare_to_self(self):
825+
return True
826+
827+
def can_compare_to_other(self, other):
828+
return True
829+
830+
831+
class D8:
832+
name = 'd8'
833+
834+
def run(self, wasm, extra_d8_flags=[]):
835+
return run_vm([shared.V8, get_fuzz_shell_js()] + shared.V8_OPTS + get_v8_extra_flags() + extra_d8_flags + ['--', wasm])
836+
837+
def can_run(self, wasm):
838+
return all_disallowed(DISALLOWED_FEATURES_IN_V8)
839+
840+
def can_compare_to_self(self):
841+
# With nans, VM differences can confuse us, so only very simple VMs
842+
# can compare to themselves after opts in that case.
843+
return not NANS
844+
845+
def can_compare_to_other(self, other):
846+
# Relaxed SIMD allows different behavior between VMs, so only
847+
# allow comparisons to other d8 variants if it is enabled.
848+
if not all_disallowed(['relaxed-simd']) and not other.name.startswith('d8'):
849+
return False
850+
851+
# If not legalized, the JS will fail immediately, so no point to
852+
# compare to others.
853+
return self.can_compare_to_self() and LEGALIZE
854+
855+
856+
class D8Liftoff(D8):
857+
name = 'd8_liftoff'
858+
859+
def run(self, wasm):
860+
return super().run(wasm, extra_d8_flags=V8_LIFTOFF_ARGS)
861+
862+
863+
class D8Turboshaft(D8):
864+
name = 'd8_turboshaft'
865+
866+
def run(self, wasm):
867+
flags = ['--no-liftoff']
868+
return super().run(wasm, extra_d8_flags=flags)
869+
870+
871+
class Wasm2C:
872+
name = 'wasm2c'
873+
874+
def __init__(self):
875+
# look for wabt in the path. if it's not here, don't run wasm2c
876+
try:
877+
wabt_bin = shared.which('wasm2c')
878+
wabt_root = os.path.dirname(os.path.dirname(wabt_bin))
879+
self.wasm2c_dir = os.path.join(wabt_root, 'wasm2c')
880+
if not os.path.isdir(self.wasm2c_dir):
881+
print('wabt found, but not wasm2c support dir')
882+
self.wasm2c_dir = None
883+
except Exception as e:
884+
print('warning: no wabt found:', e)
885+
self.wasm2c_dir = None
886+
887+
def can_run(self, wasm):
888+
if self.wasm2c_dir is None:
889+
return False
890+
# if we legalize for JS, the ABI is not what C wants
891+
if LEGALIZE:
892+
return False
893+
# relatively slow, so run it less frequently
894+
if random.random() < 0.5:
895+
return False
896+
# wasm2c doesn't support most features
897+
return all_disallowed(['exception-handling', 'simd', 'threads', 'bulk-memory', 'nontrapping-float-to-int', 'tail-call', 'sign-ext', 'reference-types', 'multivalue', 'gc', 'custom-descriptors', 'relaxed-atomics'])
898+
899+
def run(self, wasm):
900+
run([in_bin('wasm-opt'), wasm, '--emit-wasm2c-wrapper=main.c'] + FEATURE_OPTS)
901+
run(['wasm2c', wasm, '-o', 'wasm.c'])
902+
compile_cmd = ['clang', 'main.c', 'wasm.c', os.path.join(self.wasm2c_dir, 'wasm-rt-impl.c'), '-I' + self.wasm2c_dir, '-lm', '-Werror']
903+
run(compile_cmd)
904+
return run_vm(['./a.out'])
905+
906+
def can_compare_to_self(self):
907+
# The binaryen optimizer changes NaNs in the ways that wasm
908+
# expects, but that's not quite what C has
909+
return not NANS
910+
911+
def can_compare_to_other(self, other):
912+
# C won't trap on OOB, and NaNs can differ from wasm VMs
913+
return not OOB and not NANS
914+
915+
916+
class Wasm2C2Wasm(Wasm2C):
917+
name = 'wasm2c2wasm'
918+
919+
def __init__(self):
920+
super().__init__()
921+
922+
self.has_emcc = shared.which('emcc') is not None
923+
924+
def run(self, wasm):
925+
run([in_bin('wasm-opt'), wasm, '--emit-wasm2c-wrapper=main.c'] + FEATURE_OPTS)
926+
run(['wasm2c', wasm, '-o', 'wasm.c'])
927+
compile_cmd = ['emcc', 'main.c', 'wasm.c',
928+
os.path.join(self.wasm2c_dir, 'wasm-rt-impl.c'),
929+
'-I' + self.wasm2c_dir,
930+
'-lm',
931+
'-s', 'ENVIRONMENT=shell',
932+
'-s', 'ALLOW_MEMORY_GROWTH']
933+
# disable the signal handler: emcc looks like unix, but wasm has
934+
# no signals
935+
compile_cmd += ['-DWASM_RT_MEMCHECK_SIGNAL_HANDLER=0']
936+
if random.random() < 0.5:
937+
compile_cmd += ['-O' + str(random.randint(1, 3))]
938+
elif random.random() < 0.5:
939+
if random.random() < 0.5:
940+
compile_cmd += ['-Os']
941+
else:
942+
compile_cmd += ['-Oz']
943+
# avoid pass-debug on the emcc invocation itself (which runs
944+
# binaryen to optimize the wasm), as the wasm here can be very
945+
# large and it isn't what we are focused on testing here
946+
with no_pass_debug():
947+
run(compile_cmd)
948+
return run_d8_js(abspath('a.out.js'))
949+
950+
def can_run(self, wasm):
951+
# quite slow (more steps), so run it less frequently
952+
if random.random() < 0.8:
953+
return False
954+
# prefer not to run if the wasm is very large, as it can OOM
955+
# the JS engine.
956+
return super().can_run(wasm) and self.has_emcc and \
957+
os.path.getsize(wasm) <= INPUT_SIZE_MEAN
958+
959+
def can_compare_to_other(self, other):
960+
# NaNs can differ from wasm VMs
961+
return not NANS
962+
789963

790964
class CompareVMs(TestCaseHandler):
791965
frequency = 1
792966

793967
def __init__(self):
794968
super().__init__()
795969

796-
class BinaryenInterpreter:
797-
name = 'binaryen interpreter'
798-
799-
def run(self, wasm):
800-
output = run_bynterp(wasm, ['--fuzz-exec-before'])
801-
if output != IGNORE:
802-
calls = output.count(FUZZ_EXEC_EXPORT_PREFIX)
803-
errors = output.count(TRAP_PREFIX) + output.count(HOST_LIMIT_PREFIX)
804-
if errors > calls / 2:
805-
# A significant amount of execution on this testcase
806-
# simply trapped, and was not very useful, so mark it
807-
# as ignored. Ideally the fuzzer testcases would be
808-
# improved to reduce this number.
809-
#
810-
# Note that we don't change output=IGNORE as there may
811-
# still be useful testing here (up to 50%), so we only
812-
# note that this is a mostly-ignored run, but we do not
813-
# ignore the parts that are useful.
814-
#
815-
# Note that we set amount to 0.5 because we are run both
816-
# on the before wasm and the after wasm. Those will be
817-
# in sync (because the optimizer does not remove traps)
818-
# and so by setting 0.5 we only increment by 1 for the
819-
# entire iteration.
820-
note_ignored_vm_run('too many errors vs calls',
821-
extra_text=f' ({calls} calls, {errors} errors)',
822-
amount=0.5)
823-
return output
824-
825-
def can_run(self, wasm):
826-
return True
827-
828-
def can_compare_to_self(self):
829-
return True
830-
831-
def can_compare_to_other(self, other):
832-
return True
833-
834-
class D8:
835-
name = 'd8'
836-
837-
def run(self, wasm, extra_d8_flags=[]):
838-
return run_vm([shared.V8, get_fuzz_shell_js()] + shared.V8_OPTS + get_v8_extra_flags() + extra_d8_flags + ['--', wasm])
839-
840-
def can_run(self, wasm):
841-
return all_disallowed(DISALLOWED_FEATURES_IN_V8)
842-
843-
def can_compare_to_self(self):
844-
# With nans, VM differences can confuse us, so only very simple VMs
845-
# can compare to themselves after opts in that case.
846-
return not NANS
847-
848-
def can_compare_to_other(self, other):
849-
# Relaxed SIMD allows different behavior between VMs, so only
850-
# allow comparisons to other d8 variants if it is enabled.
851-
if not all_disallowed(['relaxed-simd']) and not other.name.startswith('d8'):
852-
return False
853-
854-
# If not legalized, the JS will fail immediately, so no point to
855-
# compare to others.
856-
return self.can_compare_to_self() and LEGALIZE
857-
858-
class D8Liftoff(D8):
859-
name = 'd8_liftoff'
860-
861-
def run(self, wasm):
862-
return super().run(wasm, extra_d8_flags=V8_LIFTOFF_ARGS)
863-
864-
class D8Turboshaft(D8):
865-
name = 'd8_turboshaft'
866-
867-
def run(self, wasm):
868-
flags = ['--no-liftoff']
869-
return super().run(wasm, extra_d8_flags=flags)
870-
871-
class Wasm2C:
872-
name = 'wasm2c'
873-
874-
def __init__(self):
875-
# look for wabt in the path. if it's not here, don't run wasm2c
876-
try:
877-
wabt_bin = shared.which('wasm2c')
878-
wabt_root = os.path.dirname(os.path.dirname(wabt_bin))
879-
self.wasm2c_dir = os.path.join(wabt_root, 'wasm2c')
880-
if not os.path.isdir(self.wasm2c_dir):
881-
print('wabt found, but not wasm2c support dir')
882-
self.wasm2c_dir = None
883-
except Exception as e:
884-
print('warning: no wabt found:', e)
885-
self.wasm2c_dir = None
886-
887-
def can_run(self, wasm):
888-
if self.wasm2c_dir is None:
889-
return False
890-
# if we legalize for JS, the ABI is not what C wants
891-
if LEGALIZE:
892-
return False
893-
# relatively slow, so run it less frequently
894-
if random.random() < 0.5:
895-
return False
896-
# wasm2c doesn't support most features
897-
return all_disallowed(['exception-handling', 'simd', 'threads', 'bulk-memory', 'nontrapping-float-to-int', 'tail-call', 'sign-ext', 'reference-types', 'multivalue', 'gc', 'custom-descriptors', 'relaxed-atomics'])
898-
899-
def run(self, wasm):
900-
run([in_bin('wasm-opt'), wasm, '--emit-wasm2c-wrapper=main.c'] + FEATURE_OPTS)
901-
run(['wasm2c', wasm, '-o', 'wasm.c'])
902-
compile_cmd = ['clang', 'main.c', 'wasm.c', os.path.join(self.wasm2c_dir, 'wasm-rt-impl.c'), '-I' + self.wasm2c_dir, '-lm', '-Werror']
903-
run(compile_cmd)
904-
return run_vm(['./a.out'])
905-
906-
def can_compare_to_self(self):
907-
# The binaryen optimizer changes NaNs in the ways that wasm
908-
# expects, but that's not quite what C has
909-
return not NANS
910-
911-
def can_compare_to_other(self, other):
912-
# C won't trap on OOB, and NaNs can differ from wasm VMs
913-
return not OOB and not NANS
914-
915-
class Wasm2C2Wasm(Wasm2C):
916-
name = 'wasm2c2wasm'
917-
918-
def __init__(self):
919-
super().__init__()
920-
921-
self.has_emcc = shared.which('emcc') is not None
922-
923-
def run(self, wasm):
924-
run([in_bin('wasm-opt'), wasm, '--emit-wasm2c-wrapper=main.c'] + FEATURE_OPTS)
925-
run(['wasm2c', wasm, '-o', 'wasm.c'])
926-
compile_cmd = ['emcc', 'main.c', 'wasm.c',
927-
os.path.join(self.wasm2c_dir, 'wasm-rt-impl.c'),
928-
'-I' + self.wasm2c_dir,
929-
'-lm',
930-
'-s', 'ENVIRONMENT=shell',
931-
'-s', 'ALLOW_MEMORY_GROWTH']
932-
# disable the signal handler: emcc looks like unix, but wasm has
933-
# no signals
934-
compile_cmd += ['-DWASM_RT_MEMCHECK_SIGNAL_HANDLER=0']
935-
if random.random() < 0.5:
936-
compile_cmd += ['-O' + str(random.randint(1, 3))]
937-
elif random.random() < 0.5:
938-
if random.random() < 0.5:
939-
compile_cmd += ['-Os']
940-
else:
941-
compile_cmd += ['-Oz']
942-
# avoid pass-debug on the emcc invocation itself (which runs
943-
# binaryen to optimize the wasm), as the wasm here can be very
944-
# large and it isn't what we are focused on testing here
945-
with no_pass_debug():
946-
run(compile_cmd)
947-
return run_d8_js(abspath('a.out.js'))
948-
949-
def can_run(self, wasm):
950-
# quite slow (more steps), so run it less frequently
951-
if random.random() < 0.8:
952-
return False
953-
# prefer not to run if the wasm is very large, as it can OOM
954-
# the JS engine.
955-
return super().can_run(wasm) and self.has_emcc and \
956-
os.path.getsize(wasm) <= INPUT_SIZE_MEAN
957-
958-
def can_compare_to_other(self, other):
959-
# NaNs can differ from wasm VMs
960-
return not NANS
961-
962970
# the binaryen interpreter is specifically useful for various things
963971
self.bynterpreter = BinaryenInterpreter()
964972

0 commit comments

Comments
 (0)
⚡