diff --git a/.circleci/config.yml b/.circleci/config.yml index 773eb53b542a7..dd8523979302f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -77,6 +77,7 @@ commands: curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y export PATH=${HOME}/.cargo/bin:${PATH} rustup target add wasm32-unknown-emscripten + cargo install wasm-bindgen-cli echo "export PATH=\"\$HOME/.cargo/bin:\$PATH\"" >> $BASH_ENV install-node-version: description: "install a specific version of node" @@ -159,6 +160,7 @@ commands: # We use an out-of-tree cache directory so it can be part of the # persisted workspace (see below). echo "CACHE = os.path.expanduser('~/cache')" >> .emscripten + echo "WASM_BINDGEN = os.path.expanduser('~/.cargo/bin/wasm-bindgen')" >> .emscripten # Refer to commit 0bc3640 if we need to pin V8 version. echo "V8_ENGINE = [os.path.expanduser('~/.jsvu/bin/v8')]" >> .emscripten echo "JS_ENGINES = [NODE_JS]" >> .emscripten diff --git a/emcc.py b/emcc.py index 9d3d45d0686d6..080b67ea557a1 100644 --- a/emcc.py +++ b/emcc.py @@ -27,7 +27,6 @@ import shutil import sys import tarfile -from dataclasses import dataclass from enum import Enum, auto, unique # This assert needs to happen early, before any too-recent python syntax is used. @@ -51,7 +50,7 @@ from tools.settings import COMPILE_TIME_SETTINGS, default_setting, settings, user_settings from tools.shared import DEBUG, DYLIB_EXTENSIONS, in_temp from tools.toolchain_profiler import ToolchainProfiler -from tools.utils import exit_with_error, get_file_suffix, read_file, unsuffixed_basename +from tools.utils import LinkFlag, exit_with_error, get_file_suffix, read_file, unsuffixed_basename logger = logging.getLogger('emcc') @@ -96,20 +95,6 @@ class Mode(Enum): COMPILE_AND_LINK = auto() -@dataclass -class LinkFlag: - """Used to represent a linker flag. - - The flag value is stored along with a bool that distinguishes input - files from non-files. - - A list of these is returned by separate_linker_flags. - """ - - value: str - is_file: int - - class EmccState: def __init__(self, args): self.mode = Mode.COMPILE_AND_LINK @@ -578,7 +563,7 @@ def compile_source_file(input_file): # Default to assuming the inputs are object files and pass them to the linker pass - return [f.value for f in linker_args] + return linker_args if __name__ == '__main__': diff --git a/site/source/docs/tools_reference/settings_reference.rst b/site/source/docs/tools_reference/settings_reference.rst index 986c25c5d0bc4..7745d334af7de 100644 --- a/site/source/docs/tools_reference/settings_reference.rst +++ b/site/source/docs/tools_reference/settings_reference.rst @@ -3284,6 +3284,15 @@ Example use ``-sSIGNATURE_CONVERSIONS=someFunction:_p,anotherFunction:p`` Default value: [] +.. _wasm_bindgen: + +WASM_BINDGEN +============ + +Run wasm-bindgen and integrate the rust-exported symbols into the rest of Emscripten's JS output. + +Default value: 0 + .. _source_phase_imports: SOURCE_PHASE_IMPORTS diff --git a/src/runtime_common.js b/src/runtime_common.js index 80d504ffa7139..b22a567ef303f 100644 --- a/src/runtime_common.js +++ b/src/runtime_common.js @@ -79,7 +79,7 @@ if (ENVIRONMENT_IS_NODE && {{{ ENVIRONMENT_IS_WORKER_THREAD() }}}) { // Memory management -#if SUPPORT_BIG_ENDIAN +#if SUPPORT_BIG_ENDIAN || WASM_BINDGEN /** @type {!DataView} */ var HEAP_DATA_VIEW; #endif @@ -130,8 +130,10 @@ function updateMemoryViews() { {{{ maybeExportHeap('HEAP64') }}}HEAP64 = new BigInt64Array(b); {{{ maybeExportHeap('HEAPU64') }}}HEAPU64 = new BigUint64Array(b); #endif +#if SUPPORT_BIG_ENDIAN || WASM_BINDGEN + {{{ maybeExportHeap('HEAP_DATA_VIEW') }}}HEAP_DATA_VIEW = new DataView(b); +#endif #if SUPPORT_BIG_ENDIAN - {{{ maybeExportHeap('HEAP_DATA_VIEW') }}} HEAP_DATA_VIEW = new DataView(b); LE_HEAP_UPDATE(); #endif } diff --git a/src/settings.js b/src/settings.js index 1141fcd9807f2..4dbab9aaca843 100644 --- a/src/settings.js +++ b/src/settings.js @@ -2164,6 +2164,10 @@ var LEGACY_RUNTIME = false; // [link] var SIGNATURE_CONVERSIONS = []; +// Run wasm-bindgen and integrate the rust-exported symbols into the rest of Emscripten's JS output. +// [link] +var WASM_BINDGEN = 0; + // Experimental support for wasm source phase imports. // This is only currently implemented in the pre-release/nightly version of // node, and not yet supported by browsers. diff --git a/test/common.py b/test/common.py index 8f2ba6bdb312c..7591a1bd53b99 100644 --- a/test/common.py +++ b/test/common.py @@ -1374,8 +1374,7 @@ def ccshared(src, linkto=None): cfunc_ptr(); return 0; } - ''' % locals(), - 'a: loaded\na: b (prev: (null))\na: c (prev: b)\n', cflags=extra_args) + ''', 'a: loaded\na: b (prev: (null))\na: c (prev: b)\n', cflags=extra_args) def do_run(self, src, expected_output=None, force_c=False, **kwargs): if 'no_build' in kwargs: diff --git a/test/rust/basics/.cargo/config.toml b/test/rust/basics/.cargo/config.toml new file mode 100644 index 0000000000000..3301c205541b8 --- /dev/null +++ b/test/rust/basics/.cargo/config.toml @@ -0,0 +1,2 @@ +[build] +target = "wasm32-unknown-emscripten" diff --git a/test/rust/bindgen_integration/.cargo/config.toml b/test/rust/bindgen_integration/.cargo/config.toml new file mode 100644 index 0000000000000..c9be0d51ea09a --- /dev/null +++ b/test/rust/bindgen_integration/.cargo/config.toml @@ -0,0 +1,7 @@ +[build] +target = "wasm32-unknown-emscripten" +rustflags = [ + "-Cllvm-args=-enable-emscripten-cxx-exceptions=0", + "-Cpanic=abort", + "-Crelocation-model=static", +] diff --git a/test/rust/bindgen_integration/Cargo.toml b/test/rust/bindgen_integration/Cargo.toml new file mode 100644 index 0000000000000..044293686a9e6 --- /dev/null +++ b/test/rust/bindgen_integration/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "bindgen_integration" +edition = "2021" + +[lib] +crate-type = ["staticlib"] + +[dependencies] diff --git a/test/rust/bindgen_integration/src/lib.rs b/test/rust/bindgen_integration/src/lib.rs new file mode 100644 index 0000000000000..541bc882468cb --- /dev/null +++ b/test/rust/bindgen_integration/src/lib.rs @@ -0,0 +1,6 @@ +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +pub fn rs_add(a: i32, b: i32) -> i32 { + return a + b; +} diff --git a/test/test_other.py b/test/test_other.py index d1c489530eba7..6e3530a1d355e 100644 --- a/test/test_other.py +++ b/test/test_other.py @@ -14845,7 +14845,7 @@ def test_embool(self): @requires_rust def test_rust_integration_basics(self): copytree(test_file('rust/basics'), '.') - self.run_process(['cargo', 'build', '--target=wasm32-unknown-emscripten']) + self.run_process(['cargo', 'build']) lib = 'target/wasm32-unknown-emscripten/debug/libbasics.a' self.assertExists(lib) @@ -14857,6 +14857,31 @@ def test_rust_integration_basics(self): }''') self.do_runf('main.cpp', 'Hello from rust!', cflags=[lib]) + @requires_rust + def test_wasm_bindgen_integration(self): + copytree(test_file('rust/bindgen_integration'), '.') + self.run_process(['cargo', 'add', 'wasm-bindgen']) + self.run_process(['cargo', 'build']) + lib = 'target/wasm32-unknown-emscripten/debug/libbindgen_integration.a' + self.assertExists(lib) + self.assertExists(os.path.abspath(config.WASM_BINDGEN[0])) + + create_file('main.cpp', '') + create_file('post.js', ''' + Module.onRuntimeInitialized = () => { + out(Module.rs_add(17, 25)); + }; + ''') + emcc_args = [ + lib, + '-sWASM_BINDGEN', + '--post-js', + 'post.js', + '-lexports.js', + ] + + self.do_runf('main.cpp', '42', cflags=emcc_args) + def test_relative_em_cache(self): with env_modify({'EM_CACHE': 'foo'}): self.assert_fail([EMCC, '-c', test_file('hello_world.c')], 'emcc: error: environment variable EM_CACHE must be an absolute path: foo') diff --git a/tools/building.py b/tools/building.py index f05b2f628c99d..712f0306bb45b 100644 --- a/tools/building.py +++ b/tools/building.py @@ -273,12 +273,43 @@ def lld_flags_for_executable(external_symbols): return cmd -def lld_flags(args): +def get_wasm_bindgen_exported_symbols(input_files): + if not os.path.exists(LLVM_NM): + exit_with_error('llvm-nm not found in LLVM directory: %s', LLVM_NM) + + nm_args = [ + LLVM_NM, + '--defined-only', + '--extern-only', + '--format=just-symbols', + '--print-file-name', + '--quiet', + ] + if input_files is not None: + nm_args += input_files + + result = run_process(nm_args, stdout=subprocess.PIPE) + symbols = [] + for line in result.stdout.splitlines(): + (path, symbol) = line.split() + # Skip mangled (non-C) symbols + if symbol.startswith(('_Z', '_R', 'anon.')): + continue + symbols.append(symbol) + + return symbols + + +def lld_flags(args, linker_inputs=None): # lld doesn't currently support --start-group/--end-group since the # semantics are more like the windows linker where there is no need for # grouping. args = [a for a in args if a not in {'--start-group', '--end-group'}] + if settings.WASM_BINDGEN: + exported_symbols = get_wasm_bindgen_exported_symbols(linker_inputs) + args.extend(f'--export-if-defined={e}' for e in exported_symbols) + # Emscripten currently expects linkable output (SIDE_MODULE/MAIN_MODULE) to # include all archive contents. if settings.LINKABLE: @@ -312,7 +343,7 @@ def lld_flags(args): return args -def link_lld(args, target, external_symbols=None): +def link_lld(args, target, external_symbols=None, linker_inputs=None): # runs lld to link things. if not os.path.exists(WASM_LD): exit_with_error('linker binary not found in LLVM directory: %s', WASM_LD) @@ -321,7 +352,7 @@ def link_lld(args, target, external_symbols=None): # normal linker flags that are used when building and executable if '--relocatable' not in args and '-r' not in args: cmd += lld_flags_for_executable(external_symbols) - cmd += lld_flags(args) + cmd += lld_flags(args, linker_inputs) cmd = get_command_with_possible_response_file(cmd) check_call(cmd) @@ -1254,6 +1285,28 @@ def run_wasm_opt(infile, outfile=None, args=[], **kwargs): # noqa return run_binaryen_command('wasm-opt', infile, outfile, args=args, **kwargs) +def run_wasm_bindgen(infile, outfile=None, args=[], **kwargs): # noqa + bindgen_out_dir = get_emscripten_temp_dir() + '/bindgen_out/' + + cmd = config.WASM_BINDGEN + [ + infile, + '--keep-lld-exports', + '--keep-debug', + '--out-dir', + bindgen_out_dir, + ] + check_call(cmd) + + # Don't try to predict the .wasm filename that wasm-bindgen outputs. Instead + # just grab the .wasm file itself. + all_output_files = os.listdir(bindgen_out_dir) + new_wasm_file = list(filter(lambda x: x.endswith('.wasm'), all_output_files))[0] + if outfile is None: + outfile = infile + + shutil.copyfile(bindgen_out_dir + new_wasm_file, outfile) + + intermediate_counter = 0 diff --git a/tools/config.py b/tools/config.py index 147623070bf4a..e7f51e0310707 100644 --- a/tools/config.py +++ b/tools/config.py @@ -27,6 +27,7 @@ CACHE = None PORTS = None COMPILER_WRAPPER = None +WASM_BINDGEN = None # Set by init() EM_CONFIG = None @@ -60,6 +61,7 @@ def fix_js_engine(old, new): def normalize_config_settings(): global CACHE, PORTS, LLVM_ADD_VERSION, CLANG_ADD_VERSION, CLOSURE_COMPILER global NODE_JS, NODE_JS_TEST, V8_ENGINE, JS_ENGINES, SPIDERMONKEY_ENGINE, WASM_ENGINES + global WASM_BINDGEN SPIDERMONKEY_ENGINE = fix_js_engine(SPIDERMONKEY_ENGINE, listify(SPIDERMONKEY_ENGINE)) NODE_JS = fix_js_engine(NODE_JS, listify(NODE_JS)) @@ -68,6 +70,7 @@ def normalize_config_settings(): JS_ENGINES = [listify(engine) for engine in JS_ENGINES] WASM_ENGINES = [listify(engine) for engine in WASM_ENGINES] CLOSURE_COMPILER = listify(CLOSURE_COMPILER) + WASM_BINDGEN = listify(WASM_BINDGEN) if not CACHE: CACHE = path_from_root('cache') if not PORTS: @@ -131,6 +134,7 @@ def parse_config_file(): 'CACHE', 'PORTS', 'COMPILER_WRAPPER', + 'WASM_BINDGEN', ) # Only propagate certain settings from the config file. diff --git a/tools/emscripten.py b/tools/emscripten.py index c801a7465a6e0..8c6fe3c88dd1b 100644 --- a/tools/emscripten.py +++ b/tools/emscripten.py @@ -680,7 +680,7 @@ def create_tsd_exported_runtime_methods(metadata): return utils.read_file(in_temp(f'{file}.d.ts')) -def create_tsd(metadata, embind_tsd): +def create_tsd(metadata, embind_tsd, bindgen_tsd=None): out = '// TypeScript bindings for emscripten-generated code. Automatically generated at compile time.\n' if settings.EXPORTED_RUNTIME_METHODS: out += create_tsd_exported_runtime_methods(metadata) @@ -712,6 +712,12 @@ def create_tsd(metadata, embind_tsd): # Add in embind definitions. if embind_tsd: export_interfaces += ' & EmbindModule' + if settings.WASM_BINDGEN and bindgen_tsd: + for file_path in bindgen_tsd: + with open(file_path, encoding='utf-8') as file: + for line in file: + out += f'{line}' + export_interfaces += ' & BindgenModule' out += f'export type MainModule = {export_interfaces};\n' if settings.MODULARIZE: return_type = 'MainModule' diff --git a/tools/link.py b/tools/link.py index 70fdfe43a2ce8..d1b0905c102f7 100644 --- a/tools/link.py +++ b/tools/link.py @@ -4,6 +4,7 @@ # found in the LICENSE file. import base64 +import glob import json import logging import os @@ -41,10 +42,11 @@ settings, user_settings, ) -from .shared import DEBUG, DYLIB_EXTENSIONS, do_replace, in_temp +from .shared import DEBUG, DYLIB_EXTENSIONS, do_replace, get_emscripten_temp_dir, in_temp from .toolchain_profiler import ToolchainProfiler from .utils import ( WINDOWS, + LinkFlag, delete_file, exit_with_error, get_file_suffix, @@ -1839,7 +1841,7 @@ def phase_calculate_system_libraries(options): @ToolchainProfiler.profile_block('link') -def phase_link(linker_args, wasm_target, js_syms): +def phase_link(linker_args, linker_inputs, wasm_target, js_syms): logger.debug(f'linking: {linker_args}') # Make a final pass over settings.EXPORTED_FUNCTIONS to remove any @@ -1861,11 +1863,12 @@ def phase_link(linker_args, wasm_target, js_syms): # TODO(sbc): Remove this double execution of wasm-ld if we ever find a way to # distinguish EMSCRIPTEN_KEEPALIVE exports from `--export-dynamic` exports. settings.LINKABLE = False - building.link_lld(linker_args, wasm_target, external_symbols=js_syms) + building.link_lld(linker_args, wasm_target, external_symbols=js_syms, + linker_inputs=linker_inputs) settings.LINKABLE = True rtn = extract_metadata.extract_metadata(wasm_target) - building.link_lld(linker_args, wasm_target, external_symbols=js_syms) + building.link_lld(linker_args, wasm_target, external_symbols=js_syms, linker_inputs=linker_inputs) return rtn @@ -1887,6 +1890,10 @@ def phase_post_link(options, in_wasm, wasm_target, target, js_syms, base_metadat settings.TARGET_JS_NAME = os.path.basename(js_target) + if settings.WASM_BINDGEN: + building.run_wasm_bindgen(in_wasm) + settings.JS_LIBRARIES += [get_emscripten_temp_dir() + '/bindgen_out/library_bindgen.js'] + metadata = phase_emscript(in_wasm, wasm_target, js_syms, base_metadata) if settings.EMBIND_AOT: @@ -2018,7 +2025,10 @@ def phase_emit_tsd(options, wasm_target, js_target, js_syms, metadata): embind_tsd = '' if settings.EMBIND: embind_tsd = run_embind_gen(options, wasm_target, js_syms, {'EMBIND_AOT': False}) - all_tsd = emscripten.create_tsd(metadata, embind_tsd) + bindgen_ts_files = glob.glob(get_emscripten_temp_dir() + "/bindgen_out/*.d.ts", recursive=False) + # This list comprehension then filters out any files that end with .wasm.d.ts. + bindgen_ts_files = [file for file in bindgen_ts_files if not file.endswith('.wasm.d.ts')] + all_tsd = emscripten.create_tsd(metadata, embind_tsd, bindgen_ts_files) out_file = os.path.join(os.path.dirname(js_target), filename) write_file(out_file, all_tsd) @@ -3060,11 +3070,14 @@ def run(options, linker_args): settings.limit_settings(None) if settings.RUNTIME_LINKED_LIBS: - linker_args += settings.RUNTIME_LINKED_LIBS + linker_args += [LinkFlag(f, False) for f in settings.RUNTIME_LINKED_LIBS] if not linker_args: exit_with_error('no input files') + linker_inputs = [f.value for f in linker_args if f.is_file] + linker_args = [f.value for f in linker_args] + if options.output_file and options.output_file.startswith('-'): exit_with_error(f'invalid output filename: `{options.output_file}`') @@ -3122,7 +3135,7 @@ def add_js_deps(sym): settings.ASYNCIFY_IMPORTS_EXCEPT_JS_LIBS = settings.ASYNCIFY_IMPORTS[:] settings.ASYNCIFY_IMPORTS += ['*.' + x for x in js_info['asyncFuncs']] - base_metadata = phase_link(linker_args, wasm_target, js_syms) + base_metadata = phase_link(linker_args, linker_inputs, wasm_target, js_syms) # Special handling for when the user passed '-Wl,--version'. In this case the linker # does not create the output file, but just prints its version and exits with 0. diff --git a/tools/utils.py b/tools/utils.py index 9e80c400d7522..e29695be80157 100644 --- a/tools/utils.py +++ b/tools/utils.py @@ -17,6 +17,7 @@ import stat import subprocess import sys +from dataclasses import dataclass from pathlib import Path from . import diagnostics @@ -234,3 +235,17 @@ def set_version_globals(): EMSCRIPTEN_VERSION = read_file(filename).strip().strip('"') parts = [int(x) for x in EMSCRIPTEN_VERSION.split('-')[0].split('.')] EMSCRIPTEN_VERSION_MAJOR, EMSCRIPTEN_VERSION_MINOR, EMSCRIPTEN_VERSION_TINY = parts + + +@dataclass +class LinkFlag: + """Used to represent a linker flag. + + The flag value is stored along with a bool that distinguishes input + files from non-files. + + A list of these is returned by separate_linker_flags. + """ + + value: str + is_file: int