diff --git a/.github/workflows/test-and-publish.yaml b/.github/workflows/test-and-publish.yaml index 9927ebdb..fa998c6a 100644 --- a/.github/workflows/test-and-publish.yaml +++ b/.github/workflows/test-and-publish.yaml @@ -100,10 +100,10 @@ jobs: apt-get install -y lsb-release wget software-properties-common gnupg wget https://apt.llvm.org/llvm.sh chmod +x llvm.sh - ./llvm.sh 18 # install LLVM version 18 - update-alternatives --install /usr/bin/llvm-config llvm-config /usr/bin/llvm-config-18 18 - update-alternatives --install /usr/bin/clang clang /usr/bin/clang-18 18 - update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-18 18 + ./llvm.sh 19 # SpiderMonkey at the current mozcentral.version pin requires clang/llvm >= 19 (confirmed via its own configure error) + update-alternatives --install /usr/bin/llvm-config llvm-config /usr/bin/llvm-config-19 19 + update-alternatives --install /usr/bin/clang clang /usr/bin/clang-19 19 + update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-19 19 clang --version clang++ --version - name: Setup Python @@ -238,9 +238,30 @@ jobs: if [[ "$OSTYPE" == "linux-gnu"* ]]; then # Linux sudo apt-get update -y sudo apt-get install -y cmake llvm + # SpiderMonkey's headers now require clang/llvm >= 19 and + # libstdc++ >= 10 to compile against (same requirement as the + # build-spidermonkey job) -- this job's default toolchain + # (ubuntu:20.04's stock gcc-9) doesn't meet it. + sudo apt-get install -y lsb-release wget software-properties-common gnupg + wget https://apt.llvm.org/llvm.sh + chmod +x llvm.sh + sudo ./llvm.sh 19 + sudo update-alternatives --install /usr/bin/llvm-config llvm-config /usr/bin/llvm-config-19 19 + sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-19 19 + sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-19 19 + sudo apt-get install -y libstdc++-10-dev + echo "CC=clang" >> $GITHUB_ENV + echo "CXX=clang++" >> $GITHUB_ENV elif [[ "$OSTYPE" == "darwin"* ]]; then # macOS brew update || true # allow failure brew install cmake pkg-config wget unzip coreutils # `coreutils` installs the `realpath` command + # Xcode's bundled clang is older than SpiderMonkey's own + # minimum (>=19) -- same fix as setup.sh's macOS branch. Pinned + # to llvm@19: the unversioned `llvm` formula (currently 23.x) + # has no bottle for Intel macOS or macOS 14, so it silently + # falls back to a multi-hour from-source build here. + brew install llvm@19 + echo "PATH=$(brew --prefix llvm@19)/bin:$PATH" >> $GITHUB_ENV fi echo "Installing python deps" poetry self add "poetry-dynamic-versioning[plugin]" @@ -255,10 +276,10 @@ jobs: run: | sudo apt-get install -y graphviz # the newest version in Ubuntu 20.04 repository is 1.8.17, but we need Doxygen 1.9 series - wget -c -q https://www.doxygen.nl/files/doxygen-1.9.7.linux.bin.tar.gz - tar xf doxygen-1.9.7.linux.bin.tar.gz - cd doxygen-1.9.7 && sudo make install && cd - - rm -rf doxygen-1.9.7 doxygen-1.9.7.linux.bin.tar.gz + wget -c -q https://www.doxygen.nl/files/doxygen-1.15.0.linux.bin.tar.gz + tar xf doxygen-1.15.0.linux.bin.tar.gz + cd doxygen-1.15.0 && sudo make install && cd - + rm -rf doxygen-1.15.0 doxygen-1.15.0.linux.bin.tar.gz BUILD_DOCS=1 BUILD_TYPE=None poetry install - name: Upload Doxygen-generated docs as CI artifacts if: ${{ matrix.os == 'ubuntu-22.04' && matrix.python_version == '3.11' }} diff --git a/CMakeLists.txt b/CMakeLists.txt index 1577c299..3921f368 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -30,7 +30,9 @@ if(CMAKE_PROJECT_NAME STREQUAL PROJECT_NAME) include(FetchContent) if (WIN32) - SET(COMPILE_FLAGS "/GR- /W0") + # This build bypasses moz.build, which normally defines XP_WIN; without + # it SpiderMonkey headers (e.g. PlatformMutex.h) take a POSIX path. + SET(COMPILE_FLAGS "/GR- /W0 /DXP_WIN") SET(OPTIMIZED "/O2") SET(UNOPTIMIZED "/Od") @@ -39,7 +41,9 @@ if(CMAKE_PROJECT_NAME STREQUAL PROJECT_NAME) SET(PROFILE "/PROFILE") SET(ADDRESS_SANITIZE "/fsanitize=address /Oy-") else() - SET(COMPILE_FLAGS "-fno-rtti -Wno-invalid-offsetof") + # Same as -DXP_WIN above: without XP_UNIX, headers such as + # mfbt/UniquePtrExtensions.h hit their "Unsupported OS" #error. + SET(COMPILE_FLAGS "-fno-rtti -Wno-invalid-offsetof -DXP_UNIX") SET(OPTIMIZED "-Ofast -DNDEBUG") SET(UNOPTIMIZED "-O0") diff --git a/include/JobQueue.hh b/include/JobQueue.hh index 36734f92..a83089dd 100644 --- a/include/JobQueue.hh +++ b/include/JobQueue.hh @@ -49,45 +49,37 @@ bool init(JSContext *cx); * If any error happens while generating the host defined data, this method * should set a pending exception to `cx` and return `false`. */ -bool getHostDefinedData(JSContext *cx, JS::MutableHandle data) const override; +bool getHostDefinedData(JSContext *cx, JS::MutableHandle incumbentGlobal, JS::MutableHandle data) const override; /** - * @brief Enqueue a reaction job `job` for `promise`, which was allocated at - * `allocationSite`. Provide `incumbentGlobal` as the incumbent global for - * the reaction job's execution. + * @brief Ask the embedding for the host defined global to use when running + * a JS microtask. * - * `promise` can be null if the promise is optimized out. - * `promise` is guaranteed not to be optimized out if the promise has - * non-default user-interaction flag. + * Not tracked by this embedding; null falls back to SpiderMonkey's default, + * as its own InternalJobQueue does. */ -bool enqueuePromiseJob(JSContext *cx, JS::HandleObject promise, - JS::HandleObject job, JS::HandleObject allocationSite, - JS::HandleObject incumbentGlobal) override; +bool getHostDefinedGlobal(JSContext *cx, JS::MutableHandle out) const override; /** - * @brief Run all jobs in the queue. Running one job may enqueue others; continue to - * run jobs until the queue is empty. + * @brief Pull every job SpiderMonkey has queued internally and forward each + * one to the Python event-loop. + * + * SpiderMonkey no longer pushes promise jobs to the embedding as they are + * created; it queues them and expects the embedder to drain them at microtask + * checkpoints via js::RunJobs(cx) (jsfriendapi.h), which calls this method. + * Every place PythonMonkey enters JS from Python must checkpoint afterwards, + * or promises settled there never run their reactions. * * Calling this method at the wrong time can break the web. The HTML spec * indicates exactly when the job queue should be drained (in HTML jargon, * when it should "perform a microtask checkpoint"), and doing so at other * times can incompatibly change the semantics of programs that use promises * or other microtask-based features. - * - * This method is called only via AutoDebuggerJobQueueInterruption, used by - * the Debugger API implementation to ensure that the debuggee's job queue is - * protected from the debugger's own activity. See the comments on - * AutoDebuggerJobQueueInterruption. */ void runJobs(JSContext *cx) override; /** - * @return true if the job queue is empty, false otherwise. - */ -bool empty() const override; - -/** - * @return true if the job queue stopped draining, which results in `empty()` being false after `runJobs()`. + * @return true if the job queue stopped draining before it was empty. */ bool isDrainingStopped() const override; @@ -127,11 +119,31 @@ js::UniquePtr saveJobQueue(JSContext *) override; * @brief The callback for dispatching an off-thread promise to the event loop * see https://hg.mozilla.org/releases/mozilla-esr102/file/tip/js/public/Promise.h#l580 * https://hg.mozilla.org/releases/mozilla-esr102/file/tip/js/src/vm/OffThreadPromiseRuntimeState.cpp#l160 + * + * Takes ownership of the Dispatchable (run via the public static + * Dispatchable::Run, since Dispatchable::run() is protected). + * * @param closure - closure, currently the javascript context - * @param dispatchable - Pointer to the Dispatchable to be called + * @param dispatchable - the Dispatchable to be called; ownership transferred to this callback * @return not shutting down */ -static bool dispatchToEventLoop(void *closure, JS::Dispatchable *dispatchable); +static bool dispatchToEventLoop(void *closure, js::UniquePtr &&dispatchable); + +/** + * @brief The callback for dispatching an off-thread promise to the event + * loop after a delay. + * + * Always declines (returns false), which js/public/Promise.h permits when the + * embedding has no timeout manager. Only engine features needing a delayed + * off-thread callback (e.g. Atomics.waitAsync timeouts) are affected; + * setTimeout/setInterval use PyEventLoop::enqueueWithDelay instead. + * + * @param closure - closure, currently the javascript context + * @param dispatchable - the Dispatchable that would be called; ownership transferred to this callback + * @param delay - requested delay in milliseconds + * @return false (no timeout manager available) + */ +static bool delayedDispatchToEventLoop(void *closure, js::UniquePtr &&dispatchable, uint32_t delay); /** * @brief The callback that gets invoked whenever a Promise is rejected without a rejection handler (uncaught/unhandled exception) diff --git a/mozcentral.version b/mozcentral.version index 55aeecbf..2373436c 100644 --- a/mozcentral.version +++ b/mozcentral.version @@ -1 +1 @@ -6bca861985ba51920c1cacc21986af01c51bd690 +1704651e7d6c706fcb753adab577e0954d61cee0 diff --git a/setup.sh b/setup.sh index 68560ade..3b21a196 100755 --- a/setup.sh +++ b/setup.sh @@ -15,41 +15,71 @@ if [[ "$OSTYPE" == "linux-gnu"* ]]; then # Linux echo "Installing apt packages" $SUDO apt-get install --yes cmake llvm clang pkg-config m4 unzip \ wget curl python3-dev + # SpiderMonkey's configure requires libstdc++ >= 10 (toolchain.configure, + # minimum_gcc_version). Ubuntu 20.04, which CI builds in for wheel + # compatibility, defaults to gcc-9's; the -10 headers are in its own repos + # and don't change which libstdc++.so the wheel links against. + $SUDO apt-get install --yes libstdc++-10-dev elif [[ "$OSTYPE" == "darwin"* ]]; then # macOS brew update || true # allow failure brew install cmake pkg-config wget unzip coreutils # `coreutils` installs the `realpath` command brew install lld -elif [[ "$OSTYPE" == "msys"* ]]; then # Windows + # SpiderMonkey requires clang >= 19; Xcode's bundled clang is older. + # Pinned to llvm@19 because the unversioned `llvm` formula has no bottle for + # Intel macOS or macOS 14 and would build from source for hours. Homebrew + # doesn't put llvm on PATH by default. + brew install llvm@19 + export PATH="$(brew --prefix llvm@19)/bin:$PATH" +elif [[ "$OSTYPE" == "msys"* || "$OSTYPE" == "cygwin"* ]]; then # Windows echo "Dependencies are not going to be installed automatically on Windows." else echo "Unsupported OS" exit 1 fi -# Install rust compiler -echo "Installing rust compiler" -unset HOST_ABI_FLAGS -if [[ "$OSTYPE" == "msys"* ]]; then # Windows - HOST_ABI_FLAGS=("--default-host" "$(clang --print-target-triple)") +# Install rust compiler, skipping if the pinned toolchain is already the +# default so this script can be re-run without re-downloading the installer. +if command -v rustup >/dev/null && rustup default 2>/dev/null | grep -q '^1\.90'; then + echo "Rust 1.90 toolchain already installed, skipping rustup-init" +else + echo "Installing rust compiler" + unset HOST_ABI_FLAGS + if [[ "$OSTYPE" == "msys"* || "$OSTYPE" == "cygwin"* ]]; then # Windows + HOST_ABI_FLAGS=("--default-host" "$(clang --print-target-triple)") + fi + # SpiderMonkey's configure requires rustc >= 1.90.0 + curl --proto '=https' --tlsv1.2 https://raw.githubusercontent.com/rust-lang/rustup/refs/tags/1.28.2/rustup-init.sh -sSf | sh -s -- -y ${HOST_ABI_FLAGS+"${HOST_ABI_FLAGS[@]}"} --default-toolchain 1.90.0 fi -curl --proto '=https' --tlsv1.2 https://raw.githubusercontent.com/rust-lang/rustup/refs/tags/1.28.2/rustup-init.sh -sSf | sh -s -- -y ${HOST_ABI_FLAGS+"${HOST_ABI_FLAGS[@]}"} --default-toolchain 1.85 CARGO_BIN="$HOME/.cargo/bin/cargo" # also works for Windows. On Windows this equals to %USERPROFILE%\.cargo\bin\cargo -$CARGO_BIN install cbindgen +command -v cbindgen >/dev/null || $CARGO_BIN install cbindgen # Setup Poetry -echo "Installing poetry" -curl -sSL https://install.python-poetry.org | python3 - --version "1.7.1" -if [[ "$OSTYPE" == "msys"* ]]; then # Windows +if [[ "$OSTYPE" == "msys"* || "$OSTYPE" == "cygwin"* ]]; then # Windows POETRY_BIN="$APPDATA/Python/Scripts/poetry" else POETRY_BIN="$HOME/.local/bin/poetry" fi -$POETRY_BIN self add 'poetry-dynamic-versioning[plugin]' +# Skip if already installed, for the same re-run reason as rustup above. +if [ -x "$POETRY_BIN" ]; then + echo "Poetry already installed, skipping" +else + echo "Installing poetry" + # Windows Python installs often provide `python` but not `python3` + PYTHON_FOR_POETRY=$(command -v python3 || command -v python) + curl -sSL https://install.python-poetry.org | "$PYTHON_FOR_POETRY" - --version "1.7.1" + "$POETRY_BIN" self add 'poetry-dynamic-versioning[plugin]' +fi echo "Done installing dependencies" echo "Downloading spidermonkey source code" # Read the commit hash for mozilla-central from the `mozcentral.version` file MOZCENTRAL_VERSION=$(cat mozcentral.version) -wget -c -q -O firefox-source-${MOZCENTRAL_VERSION}.zip https://github.com/mozilla-firefox/firefox/archive/${MOZCENTRAL_VERSION}.zip -unzip -q firefox-source-${MOZCENTRAL_VERSION}.zip && mv firefox-${MOZCENTRAL_VERSION} firefox-source +# Skip if already extracted so this script can be re-run after a later failure. +if [ ! -d firefox-source ]; then + # curl rather than wget: it ships with Windows, macOS and most Linux distros + curl -fsSL -o firefox-source-${MOZCENTRAL_VERSION}.zip https://github.com/mozilla-firefox/firefox/archive/${MOZCENTRAL_VERSION}.zip + unzip -q firefox-source-${MOZCENTRAL_VERSION}.zip && mv firefox-${MOZCENTRAL_VERSION} firefox-source +else + echo "firefox-source already exists, skipping download+extract" +fi echo "Done downloading spidermonkey source code" echo "Building spidermonkey" @@ -69,6 +99,7 @@ sed -i'' -e '/MOZ_CRASH_UNSAFE_PRINTF/,/__PRETTY_FUNCTION__);/d' ./mfbt/LinkedLi sed -i'' -e '/MOZ_ASSERT(stackRootPtr == nullptr);/d' ./js/src/vm/JSContext.cpp # would assert false in Debug Build since we extensively use `new JS::Rooted` sed -i'' -e 's/"-fuse-ld=ld"/"-ld64" if c_compiler.version > "14.0.0" else "-fuse-ld=ld"/' ./build/moz.configure/toolchain.configure # XCode 15 changed the linker behaviour. See https://developer.apple.com/documentation/xcode-release-notes/xcode-15-release-notes#Linking sed -i'' -e 's/defined(XP_WIN)/defined(_WIN32)/' ./mozglue/baseprofiler/public/BaseProfilerUtils.h # this header file is introduced to js/Debug.h in https://phabricator.services.mozilla.com/D221102, but it would be compiled without XP_WIN in this building configuration +sed -i'' -e 's/os\.environ\["MOZILLABUILD"\]/os.environ.get("MOZILLABUILD", "")/g' ./python/mozbuild/mozbuild/backend/visualstudio.py # avoid KeyError: we don't use the official Mozilla Build package, so this is never set cd js/src mkdir -p _build @@ -77,16 +108,14 @@ mkdir -p ../../../../_spidermonkey_install/ ../configure --target=$(clang --print-target-triple) \ --prefix=$(realpath $PWD/../../../../_spidermonkey_install) \ --with-intl-api \ - $(if [[ "$OSTYPE" != "msys"* ]]; then echo "--without-system-zlib"; fi) \ + $(if [[ "$OSTYPE" != "msys"* && "$OSTYPE" != "cygwin"* ]]; then echo "--without-system-zlib"; fi) \ --disable-debug-symbols \ --disable-jemalloc \ --disable-tests \ $(if [[ "$OSTYPE" == "darwin"* ]]; then echo "--enable-linker=ld64"; fi) \ - --enable-optimize \ - --disable-explicit-resource-management -# disable-explicit-resource-management: Disable the `using` syntax that is enabled by default in SpiderMonkey nightly, otherwise the header files will disagree with the compiled lib .so file -# when it's using a `IF_EXPLICIT_RESOURCE_MANAGEMENT` macro, e.g., the `enum JSProtoKey` index would be off by 1 (header `JSProto_Uint8Array` 27 will be interpreted as `JSProto_Int8Array` in lib as lib has an extra element) -# https://bugzilla.mozilla.org/show_bug.cgi?id=1940342 + --enable-optimize +# --disable-explicit-resource-management (Bugzilla 1940342 workaround) is no +# longer a recognized flag; the feature it gated has shipped. make -j$CPUS echo "Done building spidermonkey" @@ -120,7 +149,7 @@ if test -f .git/hooks/pre-commit; then cd uncrustify-source mkdir -p build cd build - if [[ "$OSTYPE" == "msys"* ]]; then # Windows + if [[ "$OSTYPE" == "msys"* || "$OSTYPE" == "cygwin"* ]]; then # Windows cmake ../ cmake --build . -j$CPUS --config Release cp Release/uncrustify.exe ../../uncrustify.exe diff --git a/src/BufferType.cc b/src/BufferType.cc index f0726bce..6fced027 100644 --- a/src/BufferType.cc +++ b/src/BufferType.cc @@ -14,6 +14,7 @@ #include #include #include +#include #include #include @@ -80,17 +81,15 @@ PyObject *BufferType::fromJsTypedArray(JSContext *cx, JS::HandleObject typedArra JS::Scalar::Type subtype = JS_GetArrayBufferViewType(typedArray); auto byteLength = JS_GetTypedArrayByteLength(typedArray); - // Retrieve/Create the underlying ArrayBuffer object for side-effect. - // - // If byte length is less than `JS_MaxMovableTypedArraySize()`, - // the ArrayBuffer object would be created lazily and the data is stored inline in the TypedArray. - // We don't want inline data because the data pointer would be invalidated during a GC as the TypedArray object is moved. - bool isSharedMemory; - if (!JS_GetArrayBufferViewBuffer(cx, typedArray, &isSharedMemory)) return nullptr; + // Python keeps the returned pointer (as a memoryview) long after this call, + // so the data must not live inline in a GC-movable object. This moves both + // TypedArray-inline and small-ArrayBuffer-inline data out of line. + if (!JS::EnsureNonInlineArrayBufferOrView(cx, typedArray)) return nullptr; - uint8_t __destBuf[0] = {}; // we don't care about its value as it's used only if the TypedArray still having inline data - uint8_t *data = JS_GetArrayBufferViewFixedData(typedArray, __destBuf, 0 /* making sure we don't copy inline data */); - if (data == nullptr) { // shared memory or still having inline data + JS::AutoCheckCannotGC nogc(cx); + bool isSharedMemory; + uint8_t *data = static_cast(JS_GetArrayBufferViewData(typedArray, &isSharedMemory, nogc)); + if (isSharedMemory || data == nullptr) { PyErr_SetString(PyExc_TypeError, "PythonMonkey cannot coerce TypedArrays backed by shared memory."); return nullptr; } diff --git a/src/JSFunctionProxy.cc b/src/JSFunctionProxy.cc index 99a32552..ad11b19f 100644 --- a/src/JSFunctionProxy.cc +++ b/src/JSFunctionProxy.cc @@ -16,6 +16,7 @@ #include "include/setSpiderMonkeyException.hh" #include +#include #include @@ -59,6 +60,10 @@ PyObject *JSFunctionProxyMethodDefinitions::JSFunctionProxy_call(PyObject *self, return NULL; } + // Python->JS callbacks (e.g. setTimeout handlers) can settle promises, and + // nothing else drains those reaction jobs. See JobQueue::runJobs. + js::RunJobs(cx); + if (PyErr_Occurred()) { return NULL; } diff --git a/src/JSMethodProxy.cc b/src/JSMethodProxy.cc index 78e1189b..ad0ada61 100644 --- a/src/JSMethodProxy.cc +++ b/src/JSMethodProxy.cc @@ -16,6 +16,7 @@ #include "include/setSpiderMonkeyException.hh" #include +#include #include @@ -70,6 +71,9 @@ PyObject *JSMethodProxyMethodDefinitions::JSMethodProxy_call(PyObject *self, PyO return NULL; } + // Same checkpoint as JSFunctionProxy_call, for bound methods. + js::RunJobs(cx); + if (PyErr_Occurred()) { return NULL; } diff --git a/src/JobQueue.cc b/src/JobQueue.cc index 928746fd..a896c28d 100644 --- a/src/JobQueue.cc +++ b/src/JobQueue.cc @@ -14,11 +14,12 @@ #include "include/PyEventLoop.hh" #include "include/pyTypeFactory.hh" #include "include/PromiseType.hh" +#include "include/setSpiderMonkeyException.hh" #include #include -#include +#include #include @@ -26,41 +27,87 @@ JobQueue::JobQueue(JSContext *cx) { finalizationRegistryCallbacks = new JS::PersistentRooted(cx); // Leaks but it's OK since freed at process exit } -bool JobQueue::getHostDefinedData(JSContext *cx, JS::MutableHandle data) const { +bool JobQueue::getHostDefinedData(JSContext *cx, JS::MutableHandle incumbentGlobal, JS::MutableHandle data) const { + incumbentGlobal.set(nullptr); // We don't need the incumbent global data.set(nullptr); // We don't need the host defined data return true; // `true` indicates no error } -bool JobQueue::enqueuePromiseJob(JSContext *cx, - [[maybe_unused]] JS::HandleObject promise, - JS::HandleObject job, - [[maybe_unused]] JS::HandleObject allocationSite, - JS::HandleObject incumbentGlobal) { - - // Convert the `job` JS function to a Python function for event-loop callback - JS::RootedValue jobv(cx, JS::ObjectValue(*job)); - PyObject *callback = pyTypeFactory(cx, jobv); - - // Send job to the running Python event-loop - PyEventLoop loop = PyEventLoop::getRunningLoop(); - if (!loop.initialized()) return false; +bool JobQueue::getHostDefinedGlobal(JSContext *cx, JS::MutableHandle out) const { + out.set(nullptr); + return true; +} - // Inform the JS runtime that the job queue is no longer empty - JS::JobQueueMayNotBeEmpty(cx); +// Runs one JS microtask once the Python event-loop gets to it. `closure` is +// (JSContext*, JS::PersistentRooted*) smuggled through as PyLongs, +// the same way callDispatchFunc below handles JS::Dispatchable. +static PyObject *runMicroTaskCallback(PyObject *closure, PyObject *Py_UNUSED(unused)) { + JSContext *cx = (JSContext *)PyLong_AsVoidPtr(PyTuple_GetItem(closure, 0)); + auto *rootedJob = (JS::PersistentRooted *)PyLong_AsVoidPtr(PyTuple_GetItem(closure, 1)); + + JS::Rooted job(cx, rootedJob->get()); + delete rootedJob; // the PersistentRooted was only needed to keep `job` alive until now + + bool ok = true; + JSObject *global = JS::GetExecutionGlobalFromJSMicroTask(job); + if (global) { + JSAutoRealm ar(cx, global); + ok = JS::RunJSMicroTask(cx, job); + } - loop.enqueue(callback); + // Running this microtask may enqueue the next one in an await chain. + js::RunJobs(cx); - Py_DECREF(callback); - return true; + if (!ok) { + setSpiderMonkeyException(cx); + return NULL; // surfaces through the event-loop's exception handler + } + Py_RETURN_NONE; } -void JobQueue::runJobs(JSContext *cx) { - // Do nothing -} +static PyMethodDef runMicroTaskCallbackDef = {"JsMicroTaskCallable", runMicroTaskCallback, METH_NOARGS, NULL}; -bool JobQueue::empty() const { - // TODO (Tom Tang): implement using `get_running_loop` and getting job count on loop??? - return true; // see https://hg.mozilla.org/releases/mozilla-esr128/file/tip/js/src/builtin/Promise.cpp#l6946 +// SpiderMonkey queues promise jobs internally and expects the embedder to pull +// them at microtask checkpoints (js::RunJobs). Each job is handed to the Python +// event-loop rather than run here, preserving the ordering the old push-based +// enqueuePromiseJob gave relative to Python callbacks. +void JobQueue::runJobs(JSContext *cx) { + while (JS::HasAnyMicroTasks(cx)) { + JS::RootedValue entry(cx, JS::DequeueNextMicroTask(cx)); + if (entry.isNull()) { + break; + } + + JS::Rooted job(cx, JS::ToMaybeWrappedJSMicroTask(entry)); + if (!job) { + continue; // not a JS microtask; nothing we support runs these + } + + // JSMicroTask is a JSObject, so a PersistentRooted keeps it alive and + // traced until the event-loop runs it, possibly long after we return. + auto *rootedJob = new JS::PersistentRooted(cx, job); + + PyObject *cxArg = PyLong_FromVoidPtr(cx); + PyObject *jobArg = PyLong_FromVoidPtr(rootedJob); + PyObject *closure = PyTuple_Pack(2, cxArg, jobArg); + Py_DECREF(cxArg); + Py_DECREF(jobArg); + PyObject *callback = PyCFunction_New(&runMicroTaskCallbackDef, closure); + Py_DECREF(closure); + + PyEventLoop loop = PyEventLoop::getRunningLoop(); + if (!loop.initialized()) { + delete rootedJob; + Py_DECREF(callback); + return; + } + + // Inform the JS runtime that the job queue is no longer empty + JS::JobQueueMayNotBeEmpty(cx); + + loop.enqueue(callback); + Py_DECREF(callback); + } } bool JobQueue::isDrainingStopped() const { @@ -79,7 +126,9 @@ js::UniquePtr JobQueue::saveJobQueue(JSContext *cx) bool JobQueue::init(JSContext *cx) { JS::SetJobQueue(cx, this); - JS::InitDispatchToEventLoop(cx, dispatchToEventLoop, cx); + // Last two args (asyncTaskStarted/FinishedCallback) are optional; this + // embedding doesn't need to track background-task liveness. + JS::InitAsyncTaskCallbacks(cx, dispatchToEventLoop, delayedDispatchToEventLoop, nullptr, nullptr, cx); JS::SetPromiseRejectionTrackerCallback(cx, promiseRejectionTracker); return true; } @@ -87,13 +136,20 @@ bool JobQueue::init(JSContext *cx) { static PyObject *callDispatchFunc(PyObject *dispatchFuncTuple, PyObject *Py_UNUSED(unused)) { JSContext *cx = (JSContext *)PyLong_AsVoidPtr(PyTuple_GetItem(dispatchFuncTuple, 0)); JS::Dispatchable *dispatchable = (JS::Dispatchable *)PyLong_AsVoidPtr(PyTuple_GetItem(dispatchFuncTuple, 1)); - dispatchable->run(cx, JS::Dispatchable::NotShuttingDown); + // Dispatchable::run() is protected; reconstruct the UniquePtr released + // into raw form by dispatchToEventLoop() below and run it via Run(). + JS::Dispatchable::Run(cx, js::UniquePtr(dispatchable), JS::Dispatchable::NotShuttingDown); + + // Running the dispatchable resumes JS (e.g. an off-thread wasm compile + // finishing), which can settle promises; nothing else drains those jobs. + js::RunJobs(cx); + Py_RETURN_NONE; } static PyMethodDef callDispatchFuncDef = {"JsDispatchCallable", callDispatchFunc, METH_NOARGS, NULL}; -bool JobQueue::dispatchToEventLoop(void *closure, JS::Dispatchable *dispatchable) { +bool JobQueue::dispatchToEventLoop(void *closure, js::UniquePtr &&dispatchable) { JSContext *cx = (JSContext *)closure; // The `dispatchToEventLoop` function is running in a helper thread, so @@ -101,7 +157,10 @@ bool JobQueue::dispatchToEventLoop(void *closure, JS::Dispatchable *dispatchable // see https://docs.python.org/3/c-api/init.html#non-python-created-threads PyGILState_STATE gstate = PyGILState_Ensure(); - PyObject *dispatchFuncTuple = PyTuple_Pack(2, PyLong_FromVoidPtr(cx), PyLong_FromVoidPtr(dispatchable)); + // Release ownership into a raw pointer to smuggle it through the Python + // closure; reclaimed by callDispatchFunc via Dispatchable::Run above. + JS::Dispatchable *raw = dispatchable.release(); + PyObject *dispatchFuncTuple = PyTuple_Pack(2, PyLong_FromVoidPtr(cx), PyLong_FromVoidPtr(raw)); PyObject *pyFunc = PyCFunction_New(&callDispatchFuncDef, dispatchFuncTuple); // Avoid using the current, JS helper thread to send jobs to event-loop as it may cause deadlock @@ -111,6 +170,13 @@ bool JobQueue::dispatchToEventLoop(void *closure, JS::Dispatchable *dispatchable return true; } +bool JobQueue::delayedDispatchToEventLoop(void *closure, js::UniquePtr &&dispatchable, uint32_t delay) { + // No thread-safe delayed dispatch exists here (see JobQueue.hh); + // ReleaseFailedTask is the public way to hand a declined task back. + JS::Dispatchable::ReleaseFailedTask(std::move(dispatchable)); + return false; +} + bool sendJobToMainLoop(PyObject *pyFunc) { PyGILState_STATE gstate = PyGILState_Ensure(); @@ -165,7 +231,7 @@ void JobQueue::promiseRejectionTracker(JSContext *cx, } void JobQueue::queueFinalizationRegistryCallback(JSFunction *callback) { - mozilla::Unused << finalizationRegistryCallbacks->append(callback); + (void)finalizationRegistryCallbacks->append(callback); } bool JobQueue::runFinalizationRegistryCallbacks(JSContext *cx) { @@ -179,7 +245,7 @@ bool JobQueue::runFinalizationRegistryCallbacks(JSContext *cx) { JS::RootedFunction func(cx, f); JS::RootedValue unused_rval(cx); // we don't raise an exception here because there is nowhere to catch it - mozilla::Unused << JS_CallFunction(cx, NULL, func, JS::HandleValueArray::empty(), &unused_rval); + (void)JS_CallFunction(cx, NULL, func, JS::HandleValueArray::empty(), &unused_rval); ranCallbacks = true; } diff --git a/src/PromiseType.cc b/src/PromiseType.cc index 5a3f94b3..1c23acdd 100644 --- a/src/PromiseType.cc +++ b/src/PromiseType.cc @@ -78,6 +78,10 @@ PyObject *PromiseType::getPyObject(JSContext *cx, JS::HandleObject promise) { js::SetFunctionNativeReserved(onResolved, PROMISE_OBJ_SLOT, JS::ObjectValue(*promise)); JS::AddPromiseReactions(cx, promise, onResolved, onResolved); + // If `promise` was already settled, AddPromiseReactions just queued a job + // that nothing else will drain. See JobQueue::runJobs. + js::RunJobs(cx); + return future.getFutureObject(); // must be a new reference, ref count == 3 // Here the ref count for the `future` object is 3, but will immediately decrease to 2 in `PyEventLoop::Future`'s destructor when the `PromiseType::getPyObject` function ends // Leaving one reference for the returned Python object, and another one for the `onResolved` callback function @@ -109,6 +113,10 @@ static PyObject *futureOnDoneCallback(PyObject *futureCallbackTuple, PyObject *a } else { // having exception set, to reject the promise JS::RejectPromise(cx, promise, JS::RootedValue(cx, jsTypeFactorySafe(cx, exception))); } + + // Same as getPyObject above: settling the promise queues reaction jobs. + js::RunJobs(cx); + Py_XDECREF(exception); // cleanup delete rootedPtr; // no longer needed to be rooted, clean it up diff --git a/src/modules/pythonmonkey/pythonmonkey.cc b/src/modules/pythonmonkey/pythonmonkey.cc index 8408b594..37b402e0 100644 --- a/src/modules/pythonmonkey/pythonmonkey.cc +++ b/src/modules/pythonmonkey/pythonmonkey.cc @@ -34,6 +34,7 @@ #include #include #include +#include #include #include #include @@ -85,6 +86,19 @@ void nurseryCollectionCallback(JSContext *cx, JS::GCNurseryProgress progress, JS } } +// pythonmonkey doesn't load ES modules, but `import(...)` still parses. With +// no hook registered, HostLoadImportedModule (js/src/vm/Modules.cpp) reports +// an error without ever settling the import promise, so `await import(...)` +// hangs. A hook that fails every load makes the engine reject it instead. +static bool pythonmonkeyModuleLoadHook( + JSContext *cx, JS::Handle referrer, JS::Handle moduleRequest, + JS::Handle hostDefined, JS::Handle payload, + uint32_t lineNumber, JS::ColumnNumberOneOrigin columnNumber +) { + JS_ReportErrorASCII(cx, "Dynamic module import is disabled or not supported in this context"); + return false; +} + bool functionRegistryCallback(JSContext *cx, unsigned int argc, JS::Value *vp) { JS::CallArgs callargs = JS::CallArgsFromVp(argc, vp); Py_DECREF((PyObject *)callargs[0].toPrivate()); @@ -488,6 +502,10 @@ static PyObject *eval(PyObject *self, PyObject *args) { return NULL; } + // Microtask checkpoint (the HTML spec's "clean up after running script"); + // see JobQueue::runJobs. + js::RunJobs(GLOBAL_CX); + // translate to the proper python type PyObject *returnValue = pyTypeFactory(GLOBAL_CX, rval); if (PyErr_Occurred()) { @@ -573,7 +591,6 @@ PyMODINIT_FUNC PyInit_pythonmonkey(void) JS::ContextOptionsRef(GLOBAL_CX) .setWasm(true) - .setAsmJS(true) .setAsyncStack(true) .setSourcePragmas(true); @@ -583,6 +600,8 @@ PyMODINIT_FUNC PyInit_pythonmonkey(void) return NULL; } + JS::SetModuleLoadHook(JS_GetRuntime(GLOBAL_CX), pythonmonkeyModuleLoadHook); + if (!JS::InitSelfHostedCode(GLOBAL_CX)) { PyErr_SetString(SpiderMonkeyError, "Spidermonkey could not initialize self-hosted code."); return NULL; @@ -594,6 +613,9 @@ PyMODINIT_FUNC PyInit_pythonmonkey(void) JS::AddGCNurseryCollectionCallback(GLOBAL_CX, nurseryCollectionCallback, NULL); JS::RealmCreationOptions creationOptions = JS::RealmCreationOptions(); + // Off by default as a Spectre mitigation for untrusted web content, which + // doesn't apply here; threaded wasm builds (e.g. Pyodide) need it to link. + creationOptions.setSharedMemoryAndAtomicsEnabled(true); JS::RealmBehaviors behaviours = JS::RealmBehaviors(); JS::RealmOptions options = JS::RealmOptions(creationOptions, behaviours); static JSClass globalClass = {"global", JSCLASS_GLOBAL_FLAGS, &JS::DefaultGlobalClassOps};