diff --git a/.ci/check-matrix-lists.sh b/.ci/check-matrix-lists.sh index 2f49d8a9..0d602abb 100755 --- a/.ci/check-matrix-lists.sh +++ b/.ci/check-matrix-lists.sh @@ -18,6 +18,13 @@ # 2. A label in both lists at once. The test is then skipped under every # runner the matrix has, so it never executes anywhere while still looking # registered. +# 3. A tests/manifest.txt binary that no test_* call registers at all. Same +# cost, arrived at from the other side: "make check" runs it, the matrix +# never does, and the reference kernel therefore never adjudicates it. Six +# tests reached that state before this check existed, four of them added +# in the same branch that claimed they encoded Linux behaviour. Only the +# matrix can substantiate such a claim, so a test that skips it is a claim +# nobody checked. # # The pass counts themselves are not checked here. test-matrix.sh already holds # each lane to its EXPECTED_BASELINES floor at runtime, which is a stronger @@ -47,9 +54,61 @@ registered_labels() | sed -E 's/.*"\$runner" +"([^"]+)"$/\1/' | sort -u } +# Manifest binaries the matrix deliberately does not run. Each asserts an +# elfuse-internal implementation detail with no counterpart on a real kernel, so +# the reference lane has nothing to say about it. The matrix's own comment above +# its suite list is the long form; this is the machine-readable copy. +MATRIX_EXEMPT=" +test-oom-proc +test-shim-identity +test-shim-identity-attention +test-shim-verbose-trace +test-shim-data-el1 +test-shim-urandom-smp +test-shim-urandom-toctou +test-shim-urandom-wrap +test-shim-cred-race +test-mremap-infra +test-mremap-fork-tracking +test-dev-shm-paths +" + +# The binary name from each manifest line: the first field, with any trailing +# arguments and any "#" marker dropped. +# +# Matching the whole line instead missed 17 of the 84 entries, because a +# manifest line carries arguments (test-argc a b c) and markers (test-thread # +# diff=skip) beside the name. Those were exactly the entries a coverage guard +# most wants to see, and it reported success while checking two thirds of the +# list. +manifest_tests() +{ + local manifest + manifest="$(dirname "$0")/../tests/manifest.txt" + if [ ! -r "$manifest" ]; then + echo "Error: cannot read $manifest; the coverage check needs it" >&2 + return 1 + fi + sed -E 's/#.*//' "$manifest" | awk '{print $1}' \ + | grep -E '^test-[A-Za-z0-9._-]+$' | sort -u +} + ret=0 registered="$(registered_labels)" +manifest_list="$(manifest_tests)" || exit 1 + +while IFS= read -r test; do + [ -n "$test" ] || continue + printf '%s\n' "$MATRIX_EXEMPT" | grep -qxF "$test" && continue + if ! printf '%s\n' "$registered" | grep -qxF "$test"; then + echo "Error: tests/manifest.txt has '$test', which the matrix never runs." >&2 + echo " Add a test_* call for it, or name it in MATRIX_EXEMPT here" >&2 + echo " with the reason the reference kernel cannot adjudicate it." >&2 + ret=1 + fi +done <<< "$manifest_list" + for list in QEMU_SKIP ELFUSE_SKIP; do while IFS= read -r label; do [ -n "$label" ] || continue diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index efe76efa..3ce99c32 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -123,6 +123,31 @@ jobs: # empty and the macOS job does not start at all. extra=$(comm -23 <(printf '%s\n' "$targets" | sort -u) \ <(printf '%s\n' "$mutants" | sort -u)) + # Write the split to the run summary. Someone reading a skipped + # "Frama-C WP proofs" job needs to see that its targets moved to the + # mutation legs rather than nowhere, without reading this file. + { + echo "### Proof scope" + echo + if [ -n "$mutants" ]; then + echo "Proved and mutated, in the mutation legs (each leg proves" + echo "its target unmutated first, as the control):" + echo + printf '%s\n' "$mutants" | sed 's/^/- /' + else + echo "No target's mutation verdict can change in this diff." + fi + echo + if [ -n "$extra" ]; then + echo "Proved in the standalone job, which nothing else covers:" + echo + printf '%s\n' "$extra" | sed 's/^/- /' + else + echo "The standalone proof job has nothing left to prove and" + echo "skips; every target above is proved in its mutation leg." + fi + } >> "$GITHUB_STEP_SUMMARY" + # rules is already prefixed, so the prove job needs no shell of its # own; mutants is JSON because it is a matrix. The guard matters: # printf runs its format once even with no arguments, so an empty @@ -185,7 +210,11 @@ jobs: # a proof-only job would quietly stop enforcing half of what anyone requiring # it expected. The combined verdict keeps that name; see the last job here. verify-proofs: - name: Frama-C WP proofs + # The name says which targets, because the skip is the common case and a + # bare "Frama-C WP proofs: skipped" on a diff that edits a proof reads as + # though the proofs did not run. They did, inside the mutation legs, each + # of which proves its target unmutated before it mutates anything. + name: Frama-C WP proofs (targets no mutation leg covers) needs: proof-targets # Nothing to prove, so do not boot a macOS runner for it. if: ${{ needs.proof-targets.outputs.empty != 'true' }} diff --git a/Makefile b/Makefile index 230bfc6d..5de3b4f5 100644 --- a/Makefile +++ b/Makefile @@ -277,6 +277,14 @@ $(BUILD_DIR)/test-guest-env-host: $(BUILD_DIR)/test-guest-env-host.o \ @echo " LD $@" $(Q)$(CC) $(CFLAGS) -o $@ $^ +# test-stdio-nonblock-host launches elfuse with a pipe as stdin and checks the +# flags on its own end of that pipe afterwards, so it is a host binary. It sits +# outside the guest-binary guard below because check requires it through +# CHECK_HOST_UNIT_BINS whether or not the guest binaries are pre-built. +$(BUILD_DIR)/test-stdio-nonblock-host: tests/test-stdio-nonblock-host.c | $(BUILD_DIR) + @echo " CC $<" + $(Q)$(CC) $(CFLAGS) -Itests -o $@ $< + # Guest test binaries (cross-compiled, aarch64-linux) # Only used when GUEST_TEST_BINARIES is not set. @@ -298,6 +306,29 @@ $(BUILD_DIR)/%: tests/%.c | $(BUILD_DIR) @echo " CROSS $<" $(Q)$(CROSS_COMPILE)gcc $(CROSS_TEST_CFLAGS) -o $@ $< +# test-eventfd-semaphore-contended races two blocking readers on one eventfd. +$(BUILD_DIR)/test-eventfd-semaphore-contended: \ + tests/test-eventfd-semaphore-contended.c | $(BUILD_DIR) + @echo " CROSS $< (with -lpthread)" + $(Q)$(CROSS_COMPILE)gcc $(CROSS_TEST_CFLAGS) -o $@ $< -lpthread + +# test-socket-accept-contended parks two threads on one listener. +$(BUILD_DIR)/test-socket-accept-contended: \ + tests/test-socket-accept-contended.c | $(BUILD_DIR) + @echo " CROSS $< (with -lpthread)" + $(Q)$(CROSS_COMPILE)gcc $(CROSS_TEST_CFLAGS) -o $@ $< -lpthread + +# test-socket-waitall drips the tail of a MSG_WAITALL request from a second +# thread. +$(BUILD_DIR)/test-socket-waitall: tests/test-socket-waitall.c | $(BUILD_DIR) + @echo " CROSS $< (with -lpthread)" + $(Q)$(CROSS_COMPILE)gcc $(CROSS_TEST_CFLAGS) -o $@ $< -lpthread + +# test-dup-setfl-race races a dup against an F_SETFL sweep from a second thread. +$(BUILD_DIR)/test-dup-setfl-race: tests/test-dup-setfl-race.c | $(BUILD_DIR) + @echo " CROSS $< (with -lpthread)" + $(Q)$(CROSS_COMPILE)gcc $(CROSS_TEST_CFLAGS) -o $@ $< -lpthread + # test-pthread needs -lpthread $(BUILD_DIR)/test-pthread: tests/test-pthread.c | $(BUILD_DIR) @echo " CROSS $< (with -lpthread)" @@ -329,6 +360,16 @@ $(BUILD_DIR)/test-threaded-exec: tests/test-threaded-exec.c | $(BUILD_DIR) @echo " CROSS $< (with -lpthread)" $(Q)$(CROSS_COMPILE)gcc $(CROSS_TEST_CFLAGS) -o $@ $< -lpthread +# test-sigpipe needs a thread to close the reader mid-write. +$(BUILD_DIR)/test-sigpipe: tests/test-sigpipe.c | $(BUILD_DIR) + @echo " CROSS $< (with -lpthread)" + $(Q)$(CROSS_COMPILE)gcc $(CROSS_TEST_CFLAGS) -Itests -o $@ $< -lpthread + +# test-pipe-steal contends several readers for one byte, then execs on top. +$(BUILD_DIR)/test-pipe-steal: tests/test-pipe-steal.c | $(BUILD_DIR) + @echo " CROSS $< (with -lpthread)" + $(Q)$(CROSS_COMPILE)gcc $(CROSS_TEST_CFLAGS) -Itests -o $@ $< -lpthread + # test-exec-handoff parks the leader while a worker hands it a failing execve. $(BUILD_DIR)/test-exec-handoff: tests/test-exec-handoff.c | $(BUILD_DIR) @echo " CROSS $< (with -lpthread)" @@ -444,6 +485,12 @@ $(BUILD_DIR)/test-lowbase-mem-300000: tests/test-lowbase-mem.c | $(BUILD_DIR) $(Q)$(CROSS_COMPILE)gcc $(CROSS_TEST_CFLAGS) -no-pie \ -Wl,-Ttext-segment=0x300000 -o $@ $< +# bench-hot-guard grew a bulk lane with a draining thread and a lane that runs +# with a sibling alive, so it needs -lpthread; the pattern rule does not link it. +$(BUILD_DIR)/bench-hot-guard: tests/bench-hot-guard.c | $(BUILD_DIR) + @echo " CROSS $< (with -lpthread)" + $(Q)$(CROSS_COMPILE)gcc $(CROSS_TEST_CFLAGS) -o $@ $< -lpthread + # bench-hot-guard-glibc is the dynamic-glibc twin of bench-hot-guard. # Built only when the cross-glibc toolchain ships its own sysroot # (so a host without that toolchain can still run the rest of the @@ -460,7 +507,7 @@ ifneq ($(wildcard $(LINUX_TOOLCHAIN)/aarch64-unknown-linux-gnu/sysroot/.),) $(BUILD_DIR)/bench-hot-guard-glibc: tests/bench-hot-guard.c | $(BUILD_DIR) @echo " CROSS $< (dynamic glibc)" $(Q)$(CROSS_COMPILE)gcc -D_GNU_SOURCE -DGUARD_USE_LIBC_CG=1 -O2 \ - -o $@ $< + -o $@ $< -lpthread endif endif diff --git a/mk/common.mk b/mk/common.mk index b9334fc8..13c2f19b 100644 --- a/mk/common.mk +++ b/mk/common.mk @@ -118,8 +118,25 @@ ifdef BUILD_FLAVOR_STALE # having succeeded, and a removal that failed silently is invisible to stderr. # Stopping is the point. Carrying on writes a stamp claiming a flavor the # leftover objects do not have. -BUILD_FLAVOR_RM := $(shell find $(BUILD_DIR) \( -name '*.o' -o -name '*.d' \) -delete 2>/dev/null; \ - find $(BUILD_DIR) \( -name '*.o' -o -name '*.d' \) 2>/dev/null | head -3) +# Only the host objects and the dependency files that belong to them. A find +# over the whole tree also takes the .d files of the cross-compiled guest +# binaries, which are built with CROSS_TEST_CFLAGS and so have no flavor: the +# binary survives the wipe while the record of which headers it depends on does +# not, and editing tests/test-harness.h then stops rebuilding any of them. That +# was invisible while the sanitizer lanes still depended on "clean", which +# removed binary and .d together; dropping that prerequisite made the mismatch +# permanent, so the wipe's unit has to match the flavor's unit. +# +# A .d does not sit beside its object. DEPFLAGS above writes it flat under +# build/ with the path separators replaced by underscores, so the object +# build/syscall/casefold.o is described by build/syscall_casefold.d. Deriving +# the name by suffix substitution alone names a file that has never existed on +# this tree, which is a wipe that silently keeps every stale record it claims +# to remove. +BUILD_FLAVOR_DEPS := $(foreach o,$(BUILD_FLAVOR_OBJS),\ + $(BUILD_DIR)/$(subst /,_,$(patsubst $(BUILD_DIR)/%,%,$(basename $(o)))).d) +BUILD_FLAVOR_RM := $(shell rm -f $(BUILD_FLAVOR_OBJS) $(BUILD_FLAVOR_DEPS) \ + 2>/dev/null; ls $(BUILD_FLAVOR_OBJS) $(BUILD_FLAVOR_DEPS) 2>/dev/null | head -3) ifneq ($(BUILD_FLAVOR_RM),) $(error FLAVOR: stale objects under $(BUILD_DIR) survived removal: $(BUILD_FLAVOR_RM)) endif diff --git a/mk/config.mk b/mk/config.mk index 4e0f2943..3abeb1b1 100644 --- a/mk/config.mk +++ b/mk/config.mk @@ -29,6 +29,7 @@ NATIVE_TESTS := tests/test-multi-vcpu.c tests/test-rwx.c \ tests/test-dynamic-array-host.c \ tests/test-string-builder-host.c \ tests/test-wakeup-pipe-host.c \ + tests/test-stdio-nonblock-host.c \ tests/test-guest-env-host.c SPECIAL_TEST_SRCS := tests/test-lowbase-mem.c SPECIAL_TEST_BINS := $(BUILD_DIR)/test-lowbase-mem-200000 $(BUILD_DIR)/test-lowbase-mem-300000 diff --git a/mk/format.mk b/mk/format.mk index 840a90c8..ad89cd98 100644 --- a/mk/format.mk +++ b/mk/format.mk @@ -31,10 +31,16 @@ check-format: check-syscall-dispatch @echo " MATRIX skip lists" $(Q)bash .ci/check-matrix-lists.sh $(call require-tool,shellcheck,brew install shellcheck) + @# -x follows the "# shellcheck source=..." directives the scripts already + @# carry. Without it those directives are inert, every variable a sourced + @# lib sets reads as unassigned, and the counters in tests/lib/report.sh + @# had to be duplicated into each of its twelve callers to keep the gate + @# quiet -- which then failed the other way, as twelve assignments nobody + @# in that file uses. @printf " SHCHK %d scripts\n" $(words $(SHELL_SCRIPTS)) @fail=0; \ for f in $(SHELL_SCRIPTS); do \ - if shellcheck --severity=warning "$$f" 2>&1; then \ + if shellcheck -x --severity=warning "$$f" 2>&1; then \ printf " $(GREEN)OK$(RESET) %s\n" "$$f"; \ else \ printf " $(RED)FAIL$(RESET) %s\n" "$$f"; \ diff --git a/mk/tests.mk b/mk/tests.mk index 84dfaf2d..abd37668 100644 --- a/mk/tests.mk +++ b/mk/tests.mk @@ -107,16 +107,28 @@ endef # spurious TIMEOUT. TEST_TIMEOUT is only overridden if the caller has not # already set one. +# No "clean" prerequisite. These lanes used to depend on it, which removed the +# whole build tree including the 186 cross-compiled guest binaries -- built with +# CROSS_TEST_CFLAGS, and so untouched by anything EXTRA_CFLAGS says. The FLAVOR +# stamp in mk/common.mk was added later to solve the same problem exactly: +# it removes the *.o and *.d that a CFLAGS change actually invalidates, and +# leaves the rest. Measured on this tree, the clean cost 186 needless +# cross-compiles per sanitizer run, most of the lane's wall time. +# +# What still protects the link is the stamp, not the clean: the sub-make below +# re-reads common.mk with the sanitizer CFLAGS, sees a different flavor, and +# drops every host object before anything is compiled or linked. + ## Run the sanitizer subset with AddressSanitizer (ASAN) -check-asan: clean +check-asan: ASAN_OPTIONS="abort_on_error=1:detect_leaks=0" TEST_TIMEOUT="$${TEST_TIMEOUT:-30}" $(MAKE) EXTRA_CFLAGS="-fsanitize=address -fno-omit-frame-pointer" check-sanitizer ## Run the sanitizer subset with UndefinedBehaviorSanitizer (UBSAN) -check-ubsan: clean +check-ubsan: UBSAN_OPTIONS="halt_on_error=1:print_stacktrace=1" TEST_TIMEOUT="$${TEST_TIMEOUT:-30}" $(MAKE) EXTRA_CFLAGS="-fsanitize=undefined -fno-sanitize-recover=undefined -fno-omit-frame-pointer" check-sanitizer ## Run the sanitizer subset with ThreadSanitizer (TSAN) -check-tsan: clean +check-tsan: TSAN_OPTIONS="halt_on_error=1" TEST_TIMEOUT="$${TEST_TIMEOUT:-60}" $(MAKE) EXTRA_CFLAGS="-fsanitize=thread -fno-omit-frame-pointer" check-sanitizer # Manifest sections that exercise elfuse-internal concurrency, memory, fork, @@ -180,7 +192,7 @@ endef CHECK_HOST_UNIT_BINS := $(addprefix $(BUILD_DIR)/, \ test-tlbi-encoder-host test-fork-ipc-protocol-host \ test-vcpu-run-hooks-host test-identity-override-host \ - test-teardown-live-vcpu-host test-casefold-host \ + test-teardown-live-vcpu-host test-stdio-nonblock-host test-casefold-host \ test-casefold-walk-host test-absock-names-host \ test-dynamic-array-host test-string-builder-host \ test-wakeup-pipe-host test-guest-env-host) @@ -201,6 +213,7 @@ $(call run-host-unit,test-absock-names-host,absock derived-name unit test) $(call run-host-unit,test-dynamic-array-host,dynamic array unit test) $(call run-host-unit,test-string-builder-host,string builder unit test) $(call run-host-unit,test-wakeup-pipe-host,wakeup pipe concurrency unit test) +$(call run-host-unit,test-stdio-nonblock-host,launcher stdio flags across a guest) $(call run-host-unit,test-guest-env-host,guest environment merge cross product) $(call run-lane,test-sysroot-name-unique,one on-disk name per guest name) $(call run-lane,test-sysroot-name-relative,relative and dirfd-relative names) diff --git a/mk/verify.mk b/mk/verify.mk index be780133..3f10fa16 100644 --- a/mk/verify.mk +++ b/mk/verify.mk @@ -272,13 +272,21 @@ VERIFY_DIRENT_CLAIM := for ANY name length a host or FUSE directory can present VERIFY_DIRENT_UNPROVED := the readdir walk and the name translation stay test-covered VERIFY_IOV_SRC := src/proved/iov.h -VERIFY_IOV_FCTS := iov_count_ok iov_total_add -VERIFY_IOV_MIN_GOALS ?= 17 +VERIFY_IOV_FCTS := iov_count_ok iov_total_add iov_advance_index +VERIFY_IOV_MIN_GOALS ?= 40 VERIFY_IOV_MODEL := typed VERIFY_IOV_SCAN := src/proved/iov.h VERIFY_IOV_CLAIM := for ANY iovec array a guest can write VERIFY_IOV_UNPROVED := the per-entry guest_ptr bounds stay test-covered +VERIFY_ASYNCUDATA_SRC := src/proved/asyncudata.h +VERIFY_ASYNCUDATA_FCTS := async_udata_fd async_udata_gen async_udata_pack +VERIFY_ASYNCUDATA_MIN_GOALS ?= 15 +VERIFY_ASYNCUDATA_MODEL := typed +VERIFY_ASYNCUDATA_SCAN := src/proved/asyncudata.h +VERIFY_ASYNCUDATA_CLAIM := for ANY guest fd and slot generation the watcher can arm +VERIFY_ASYNCUDATA_UNPROVED := the kevent registration and the delivery-side owner checks stay test-covered + VERIFY_FDSET_SRC := src/proved/fdset.h VERIFY_FDSET_FCTS := fdset_words fdset_fd_index fdset_slot VERIFY_FDSET_MIN_GOALS ?= 43 @@ -336,7 +344,11 @@ commafy = $(subst $(verify_space),$(verify_comma),$(strip $(1))) # target name is enough and stays readable; a new target using a letter not # listed here shows up immediately as a literal upper-case character in the # rule name rather than silently misbehaving. -lc = $(subst A,a,$(subst B,b,$(subst C,c,$(subst D,d,$(subst E,e,$(subst F,f,$(subst G,g,$(subst H,h,$(subst I,i,$(subst K,k,$(subst L,l,$(subst M,m,$(subst N,n,$(subst O,o,$(subst P,p,$(subst Q,q,$(subst R,r,$(subst S,s,$(subst T,t,$(subst U,u,$(subst V,v,$(subst W,w,$(subst X,x,$(1)))))))))))))))))))))))) +# Lowercase a target name. Spelled out per letter because make has no case +# function; J, Y and Z were missing from this chain, so a proof target whose +# name contained one produced a rule nobody could invoke -- silently, since the +# .PHONY list and the rule name were wrong in the same way. Keep all 26. +lc = $(subst A,a,$(subst B,b,$(subst C,c,$(subst D,d,$(subst E,e,$(subst F,f,$(subst G,g,$(subst H,h,$(subst I,i,$(subst J,j,$(subst K,k,$(subst L,l,$(subst M,m,$(subst N,n,$(subst O,o,$(subst P,p,$(subst Q,q,$(subst R,r,$(subst S,s,$(subst T,t,$(subst U,u,$(subst V,v,$(subst W,w,$(subst X,x,$(subst Y,y,$(subst Z,z,$(1))))))))))))))))))))))))))) # The proof targets, derived rather than listed. Make knows every variable it # has read, so the set of VERIFY__SRC assignments above IS the target list; diff --git a/scripts/check-eintr-contract.py b/scripts/check-eintr-contract.py index e41ae4f8..dcc0f912 100644 --- a/scripts/check-eintr-contract.py +++ b/scripts/check-eintr-contract.py @@ -95,6 +95,18 @@ "drain (tty_io.c, no ERESTARTSYS), and part of the output has already " "drained, so a restart would wait the interval again.", ), + "syscall/io.c::copy_fd_range": ( + "forbids", + "sendfile and copy_file_range read a chunk before waiting to write it; " + "a pipe input cannot be rewound, so a restart would re-run the call " + "over an input missing that chunk.", + ), + "syscall/io.c::splice_drain_chunk": ( + "forbids", + "Same shape as copy_fd_range, and a pipe is the usual splice input. " + "This is the drain sys_splice runs per chunk; the forbid lives here " + "because only this loop knows how much of the chunk never left.", + ), "syscall/fuse.c::fuse_request_locked": ( "forbids", "FUSE_INTERRUPT is on the wire and the request is detached, so a " @@ -107,13 +119,79 @@ ), "syscall/io.c::io_wait_fd_or_interrupted": ( "restartable", - "Reports readiness or EINTR before any transfer; the caller has not " - "touched the fd yet.", + "Reports readiness or EINTR without transferring anything itself. It " + "no longer follows that its callers have not: io_xfer waits again " + "after a partial write, and copy_fd_range and splice_drain_chunk wait " + "to write a chunk they have already read. Those decide for themselves " + "and are listed above.", ), "syscall/fs.c::open_nonblocking_writer": ( "restartable", "FIFO open retry; nothing is opened until it succeeds.", ), + "syscall/fd.c::eventfd_read": ( + "restartable", + "The wait precedes the read of the counter pipe, so an interrupted " + "wait has not drained anything and the counter still holds what the " + "guest came for.", + ), + "syscall/fd.c::signalfd_read": ( + "restartable", + "The wait precedes the dequeue; the pending mask is untouched when it " + "returns EINTR.", + ), + "syscall/fd.c::timerfd_read": ( + "restartable", + "The wait precedes the zero-timeout kevent that collects the " + "expirations, so an interrupted wait leaves the count with the timer. " + "Restarting re-collects the same expirations.", + ), + "syscall/io.c::io_xfer": ( + "restartable", + "EINTR reaches the guest only on the total == 0 exit. Once any byte " + "has moved the short count is returned instead, which is what Linux " + "does and what makes the restart safe: there is nothing to redo.", + ), + "syscall/net.c::net_wait_or_interrupted": ( + "restartable", + "A wait, nothing more. Its callers own the question of what they had " + "already consumed before reaching it.", + ), + "syscall/net.c::net_recv_zero_payload_gate": ( + "restartable", + "Waits for readability before any recv runs, so no datagram has been " + "taken off the socket.", + ), + "syscall/net.c::connect_nonblock_wait": ( + "restartable", + "The SYN is already out, which is exactly why the restart needs its " + "precondition rather than a ban: sys_connect treats EALREADY as " + "in-flight always and EISCONN as in-flight only under " + "syscall_is_restarted(), so the retry waits the same connection out " + "instead of starting a second one.", + ), + "syscall/net.c::sys_sendto": ( + "restartable", + "POLLOUT wait, then send. An interrupted wait has sent nothing, and a " + "send that moved bytes returns the count.", + ), + "syscall/net-msg.c::sys_sendmsg": ( + "restartable", + "Both wait sites sit before their send in the EAGAIN retry loop, so " + "EINTR means nothing left the socket.", + ), + "syscall/net-msg.c::sys_sendmmsg": ( + "restartable", + "Interrupting message i reports i as the count when i > 0, so the " + "restart never re-sends a delivered message; only an interrupt before " + "the first send reaches the guest as EINTR.", + ), + "syscall/netlink.c::nl_wait_readable_locked": ( + "restartable", + "Waits for a reply to a request an earlier sendmsg put on the wire. " + "This call consumed nothing, and the restart waits for the same " + "reply.", + ), "syscall/inotify.c::inotify_read": ( "restartable", "Reached only when kevent returned no event, so nothing is consumed.", @@ -180,9 +258,49 @@ SELF = {"syscall/syscall.c::syscall_restart_forbid"} EINTR_RE = re.compile(r"(return\s+-LINUX_EINTR|=\s*-LINUX_EINTR|errno\s*=\s*EINTR)\b") + +# A function can also decide the restart question without naming EINTR itself: +# it forwards a wait helper's errno and calls syscall_restart_forbid() over the +# top. Those are deciders by the definition above and have to be classified, or +# the one class of caller the helper's own entry cannot describe -- the caller +# that consumed something before the wait -- stays invisible to this gate. +FORBID_MARKER = "syscall_restart_forbid()" + +# A wait routed through a shared helper reports the helper's EINTR without ever +# naming it. That is precisely the refactor this tree keeps making -- four +# synthetic readers moved onto io_wait_fd_or_interrupted, and each one silently +# left this gate's view the moment it stopped writing the literal. Treat calling +# one of these as reporting EINTR, so the classification survives the cleanup +# that hides the constant. +# +# A name belongs here when it can return -LINUX_EINTR to its caller. Callers of +# a helper listed here are still free to be 'restartable': the entry states the +# decision, it does not presume one. +EINTR_HELPERS = ("io_wait_fd_or_interrupted",) +HELPER_RE = re.compile(r"\b(" + "|".join(EINTR_HELPERS) + r")\s*\(") + FUNC_START_RE = re.compile(r"^(\w[\w \t\*]*?)\b(\w+)\s*\([^;]*$") +CODE_TOKEN_RE = re.compile(r'"(?:[^"\\]|\\.)*"|/\*.*?\*/|//[^\n]*', re.S) + + +def code_only(body): + """The body with comments and string literals blanked out. + + Every marker this gate looks for is a call, and a call is code. Matching + the raw text instead lets a function that merely *mentions* + io_wait_fd_or_interrupted in a comment read as one that calls it, which is + a gate that answers questions about prose. Newlines are preserved so line + numbers in any future diagnostic still line up. + """ + + def blank(m): + return re.sub(r"[^\n]", " ", m.group(0)) + + return CODE_TOKEN_RE.sub(blank, body) + + def functions(path): """Yield (name, first_line, last_line) for each top-level function.""" lines = path.read_text(errors="ignore").split("\n") @@ -217,11 +335,11 @@ def scan(): rel = path.relative_to(SRC).as_posix() text = path.read_text(errors="ignore").split("\n") for name, first, last in functions(path): - body = "\n".join(text[first - 1 : last]) - if not EINTR_RE.search(body): + body = code_only("\n".join(text[first - 1 : last])) + forbids = FORBID_MARKER in body + if not EINTR_RE.search(body) and not forbids and not HELPER_RE.search(body): continue - key = f"{rel}::{name}" - found[key] = "syscall_restart_forbid()" in body + found[f"{rel}::{name}"] = forbids return found diff --git a/scripts/check-mutants.py b/scripts/check-mutants.py index 00d7bfd5..d79d0502 100755 --- a/scripts/check-mutants.py +++ b/scripts/check-mutants.py @@ -728,6 +728,46 @@ def _load(stem, name): " *pad_start = DIRENT64_HDR_BYTES + name_len + 1;", " *pad_start = DIRENT64_HDR_BYTES + name_len;", ), + # ---- verify-asyncudata ------------------------------------------------- + # Not "drop the reduction": (g % 2^48) * 2^16 and g * 2^16 are the same + # value in 64-bit arithmetic, since the multiply discards the high bits the + # reduction would have. That mutant is equivalent, and a target that + # "catches" it is only reporting that the prover could not see the identity. + # Reduce by the wrong span instead, which really does lose generation bits. + ( + "asyncudata", + "src/proved/asyncudata.h", + "async_udata_pack", + "reduce the generation by the fd span, truncating it to 16 bits", + " return (generation % ASYNC_UDATA_GEN_SPAN) * ASYNC_UDATA_FD_SPAN +", + " return (generation % ASYNC_UDATA_FD_SPAN) * ASYNC_UDATA_FD_SPAN +", + ), + ( + "asyncudata", + "src/proved/asyncudata.h", + "async_udata_pack", + "scale the fd instead of the generation (the two fields swap places)", + " return (generation % ASYNC_UDATA_GEN_SPAN) * ASYNC_UDATA_FD_SPAN +\n" + " (uint64_t) guest_fd;", + " return (generation % ASYNC_UDATA_GEN_SPAN) +\n" + " (uint64_t) guest_fd * ASYNC_UDATA_FD_SPAN;", + ), + ( + "asyncudata", + "src/proved/asyncudata.h", + "async_udata_fd", + "read the fd out of the generation field", + " return (int) (v % ASYNC_UDATA_FD_SPAN);", + " return (int) (v / ASYNC_UDATA_FD_SPAN);", + ), + ( + "asyncudata", + "src/proved/asyncudata.h", + "async_udata_gen", + "drop the shift, so the generation carries the fd bits with it", + " return (v / ASYNC_UDATA_FD_SPAN) % ASYNC_UDATA_GEN_SPAN;", + " return v % ASYNC_UDATA_GEN_SPAN;", + ), # ---- verify-iov -------------------------------------------------------- ( "iov", @@ -753,6 +793,24 @@ def _load(stem, name): " return iovcnt >= 1 && iovcnt <= IOV_COUNT_MAX;", " return iovcnt >= 0 && iovcnt <= IOV_COUNT_MAX;", ), + ( + "iov", + "src/proved/iov.h", + "iov_advance_index", + "consume an entry the transfer only partly filled (the remainder is " + "no longer strictly inside the survivor)", + " while (spent < iovcnt && rem >= iov[spent].iov_len) {", + " while (spent < iovcnt && rem >= iov[spent].iov_len - 1) {", + ), + ( + "iov", + "src/proved/iov.h", + "iov_advance_index", + "stop subtracting the entries already spent, so the remainder can " + "exceed the entry it indexes", + " rem -= iov[spent].iov_len;\n", + "", + ), ( "iov", "src/proved/iov.h", diff --git a/scripts/install-git-hooks.sh b/scripts/install-git-hooks.sh index 44e61016..8b8aac4c 100755 --- a/scripts/install-git-hooks.sh +++ b/scripts/install-git-hooks.sh @@ -75,7 +75,7 @@ if [ "$mode" = uninstall ]; then say RM "$name" break fi - done < /dev/null; then diff --git a/scripts/test-git-hooks.sh b/scripts/test-git-hooks.sh index a22acc2e..0d869ad2 100755 --- a/scripts/test-git-hooks.sh +++ b/scripts/test-git-hooks.sh @@ -521,10 +521,14 @@ if ! ( # leaves the hook writing into a closed pipe, and the 141 that comes back # would fail this check on the run where the note was actually printed. out=$(bash scripts/git-pre-commit.sh 2>&1) + + # Ending the subshell on the case, rather than on a plain command, is what + # shfmt 3.13 mis-prints: it drops the ";" before "then" and leaves a script + # that no longer parses. Spelling the verdict out keeps the reformat stable. case "$out" in - *"format and API checks skipped"*) ;; - *) exit 1 ;; + *"format and API checks skipped"*) exit 0 ;; esac + exit 1 ) > /dev/null 2>&1; then echo 'FAIL: unreachable .ci checkers are skipped without a word' >&2 failures=$((failures + 1)) diff --git a/src/core/guest.c b/src/core/guest.c index 1a3ef353..50b6c250 100644 --- a/src/core/guest.c +++ b/src/core/guest.c @@ -519,11 +519,9 @@ int guest_init(guest_t *g, uint64_t size, uint32_t ipa_bits) * silently to MAP_ANON; fork will then use the IPC region-copy path * instead of SCM_RIGHTS fd passing. */ - char tmppath[] = "/tmp/elfuse-XXXXXX"; t0 = startup_trace_now_ns(); - int sfd = mkstemp(tmppath); + int sfd = tmpfile_anon("slab"); if (sfd >= 0) { - unlink(tmppath); /* Unlink immediately; fd keeps file alive */ if (ftruncate(sfd, (off_t) try_size) == 0) { void *p = mmap(g->host_base, try_size, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, sfd, 0); diff --git a/src/core/rosetta.c b/src/core/rosetta.c index 311dd8ab..4c36a837 100644 --- a/src/core/rosetta.c +++ b/src/core/rosetta.c @@ -639,6 +639,11 @@ static int aot_materialize_input_fd(int bin_fd, char out_path[PATH_MAX]) if (aot_cache_path("input.XXXXXX", out_path, PATH_MAX) < 0) return -1; + /* Not tmpfile_anon: the translate subprocess opens this scratch file by + * pathname, since in_path is handed to it in argv, so it cannot be + * anonymous. It is unlinked once the translation is done; the file that + * gets renamed into the cache is the translator's output, not this one. + */ int out_fd = mkstemp(out_path); if (out_fd < 0) return -1; diff --git a/src/main.c b/src/main.c index 1303dfab..32395079 100644 --- a/src/main.c +++ b/src/main.c @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -524,6 +525,20 @@ int main(int argc, char **argv) if (host_nofile_ensure_capacity() < 0) goto cleanup; + /* A host write to a pipe or socket whose reader is gone raises SIGPIPE in + * this process, and the default action is to die. Every such write is made + * on the guest's behalf, so the guest's own SIGPIPE is what the failure + * means: ignoring it here lets the write return EPIPE, which the transfer + * paths turn into a queued guest SIGPIPE (io_write_result). + * + * Without this a guest that writes to a pipe nobody is reading -- an + * ordinary shell pipeline whose reader exits first, `yes | head` -- takes + * the whole VM down with it, before the guest's own handler can run. + * Sockets were already covered one at a time by SO_NOSIGPIPE; pipes and + * fifos have no such option, so the disposition has to carry it. + */ + signal(SIGPIPE, SIG_IGN); + /* Block the vCPU-preemption signals and start the sigwait thread before any * vCPU thread exists, so both the normal path and the fork-child path below * inherit the block on every thread they spawn. diff --git a/src/proved/asyncudata.h b/src/proved/asyncudata.h new file mode 100644 index 00000000..35390e52 --- /dev/null +++ b/src/proved/asyncudata.h @@ -0,0 +1,128 @@ +/* + * SIGIO knote identity: the packing a proof can reach + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + * + * asyncio.c hands kqueue a udata word that has to name the fd a SIGIO belongs + * to, and it cannot be a bare guest fd number: a slot closed and reopened + * between arming the knote and the event firing would deliver the signal to + * whatever occupies that number now. The word therefore carries the fd in its + * low 16 bits and the slot generation stamped at arm time above it, and the + * delivery path drops any event whose generation no longer matches. + * + * That makes the packing an ABA guard, and the guard is only as good as the + * arithmetic: a fd that did not survive the round trip would deliver to the + * wrong slot, and a generation that aliased would defeat the check the whole + * scheme exists for. Both were asserted in a comment ("FD_TABLE_SIZE is 1024 + * (fits 16 bits) and the generation counter is monotonic, so 48 bits will not + * wrap") and checked by nothing. + * + * Split into a header because asyncio.c cannot be given to Frama-C: it includes + * sys/event.h for kqueue, which the analyzer's libc does not model and no stub + * of ours can honestly supply. This header needs nothing but stdint.h, so make + * verify-asyncudata proves it directly. + */ + +#pragma once + +#include + +#include "elfuse-limits.h" + +/* The fd occupies the low 16 bits, the generation the remaining 48. + * + * Spelled as a span to divide and modulo by rather than as a shift and a mask. + * The two are the same operation on an unsigned value and compile to the same + * instructions, but a symbolic shift is a postcondition no prover here + * discharges: written with & and >>, all three of these time out at 120s under + * both Alt-Ergo and Z3, and written this way they close in milliseconds. + * src/proved/fdset.h made the same choice for the same reason. + */ +#define ASYNC_UDATA_FD_BITS 16 +#define ASYNC_UDATA_GEN_BITS 48 + +/* Stated as bit widths and derived spans, so the layout invariant is one the + * compiler can actually check. An earlier spelling asserted GEN_SPAN * FD_SPAN + * == 0, which is true through unsigned wraparound and + * therefore asserted nothing: the || short-circuited and the real claim was + * never evaluated. + */ +_Static_assert(ASYNC_UDATA_FD_BITS + ASYNC_UDATA_GEN_BITS == 64, + "the two fields must span exactly 64 bits"); + +#define ASYNC_UDATA_FD_SPAN (1ULL << ASYNC_UDATA_FD_BITS) +#define ASYNC_UDATA_GEN_SPAN (1ULL << ASYNC_UDATA_GEN_BITS) + +/* Tied to the table's own bound rather than repeating 1024: a table that grows + * past the low field would otherwise keep passing this while packing aliased + * two fds onto one word, and SIGIO would reach the wrong slot. + */ +_Static_assert(FD_TABLE_SIZE <= ASYNC_UDATA_FD_SPAN, + "every guest fd must fit the low field"); + +/* The fd a packed udata word names. + * + * The second ensures pins which field is read, not just how big the answer is. + * A range alone is satisfied by an accessor that reads the wrong field, and + * pack's round-trip is stated as arithmetic rather than as calls to these (ACSL + * cannot call a C function), so nothing else here would catch that. make + * verify-mutants does: it flips each accessor onto the other field. + */ +/*@ + assigns \nothing; + ensures 0 <= \result < ASYNC_UDATA_FD_SPAN; + ensures \result == v % ASYNC_UDATA_FD_SPAN; + */ +static inline int async_udata_fd(uint64_t v) +{ + return (int) (v % ASYNC_UDATA_FD_SPAN); +} + +/* The slot generation a packed udata word names. Pinned the same way and for + * the same reason as async_udata_fd above. + */ +/*@ + assigns \nothing; + ensures \result < ASYNC_UDATA_GEN_SPAN; + ensures \result == (v / ASYNC_UDATA_FD_SPAN) % ASYNC_UDATA_GEN_SPAN; + */ +static inline uint64_t async_udata_gen(uint64_t v) +{ + return (v / ASYNC_UDATA_FD_SPAN) % ASYNC_UDATA_GEN_SPAN; +} + +/* Pack an fd and the generation of its slot into one word. + * + * The two ensures are the ABA guard stated as arithmetic: whatever goes in + * comes back out, so a delivery that compares the unpacked generation against + * the slot's own is comparing what was armed, and an fd that survives the round + * trip cannot name a different slot. They are written as the field expressions + * rather than as calls to the accessors above, because ACSL cannot call a C + * function from a specification; the accessors compute exactly these. + * + * The generation is reduced rather than required to be small, and the + * postcondition says so. Requiring it below 2^48 would have been an obligation + * on a caller that cannot meet it: asyncio.c passes fd_table's generation + * straight through, and nothing bounds that counter. What the reduction costs + * is stated rather than hidden: the delivery side reduces the slot's own + * generation the same way, so a stale event is accepted only if its generation + * and the current one are congruent modulo 2^48 -- 2^48 reuses of that one slot + * between arming the knote and the event firing. + * + * The fd is a genuine precondition: FD_TABLE_SIZE is 1024, every caller has + * already range-checked, and a fd that did not fit would name a different slot + * rather than be caught. + */ +/*@ + requires 0 <= guest_fd < ASYNC_UDATA_FD_SPAN; + assigns \nothing; + ensures \result % ASYNC_UDATA_FD_SPAN == (uint64_t) guest_fd; + ensures (\result / ASYNC_UDATA_FD_SPAN) % ASYNC_UDATA_GEN_SPAN + == generation % ASYNC_UDATA_GEN_SPAN; + */ +static inline uint64_t async_udata_pack(int guest_fd, uint64_t generation) +{ + return (generation % ASYNC_UDATA_GEN_SPAN) * ASYNC_UDATA_FD_SPAN + + (uint64_t) guest_fd; +} diff --git a/src/proved/iov.h b/src/proved/iov.h index 3c270160..3c5b627e 100644 --- a/src/proved/iov.h +++ b/src/proved/iov.h @@ -21,13 +21,14 @@ * * Split into a header because io.c and net-msg.c cannot be given to Frama-C: * they include the macOS uio and socket headers, which the analyzer's libc does - * not model. This header needs nothing but stdint.h, so make verify-iov proves - * it directly. + * not model. This header needs only stdint.h and sys/uio.h, and the analyzer + * supplies its own modeled sys/uio.h, so make verify-iov proves it directly. */ #pragma once #include +#include /* Linux UIO_MAXIOV: the cap on iovcnt every one of these syscalls enforces. */ #define IOV_COUNT_MAX 1024LL @@ -85,3 +86,57 @@ static inline int iov_total_add(uint64_t total, uint64_t len, uint64_t *out) *out = total + len; return 1; } + +/* How many iovec entries a partial transfer of `moved` bytes has fully spent, + * and how far into the first survivor it landed. + * + * The caller resumes at iov + result with iovcnt - result entries, after + * trimming that survivor by *rem_out. Splitting the index arithmetic out from + * the pointer bump is what makes the useful half provable: the two facts a + * caller needs before touching the survivor are that the index is in range and + * that the remainder is strictly inside it, and both are postconditions here. + * The bump itself stays in io.c, since iov_base points into guest memory whose + * extent no contract in this tree can name. + * + * The last postcondition is what ties the remainder back to the bytes moved. + * Without it the contract is satisfied by a loop that never subtracts anything, + * since the exit condition alone already puts rem below the entry it indexes; + * make verify-mutants found exactly that hole by deleting the subtraction. + * + * The separation precondition is not ceremony: without it *rem_out and the + * array may alias, the store can change the length the second postcondition + * talks about, and the proof fails. Every caller passes a local. + */ +/*@ + requires 0 <= iovcnt; + requires \valid_read(iov + (0 .. iovcnt - 1)); + requires \valid(rem_out); + requires \separated(rem_out, iov + (0 .. iovcnt - 1)); + assigns *rem_out; + ensures 0 <= \result <= iovcnt; + ensures \result < iovcnt ==> *rem_out < iov[\result].iov_len; + ensures *rem_out <= moved; + ensures \result > 0 ==> *rem_out + iov[\result - 1].iov_len <= moved; + */ +static inline int iov_advance_index(const struct iovec *iov, + int iovcnt, + size_t moved, + size_t *rem_out) +{ + int spent = 0; + size_t rem = moved; + + /*@ + loop invariant 0 <= spent <= iovcnt; + loop invariant rem <= moved; + loop invariant spent > 0 ==> rem + iov[spent - 1].iov_len <= moved; + loop assigns spent, rem; + loop variant iovcnt - spent; + */ + while (spent < iovcnt && rem >= iov[spent].iov_len) { + rem -= iov[spent].iov_len; + spent++; + } + *rem_out = rem; + return spent; +} diff --git a/src/runtime/fork-state.c b/src/runtime/fork-state.c index e60db22d..195aec7d 100644 --- a/src/runtime/fork-state.c +++ b/src/runtime/fork-state.c @@ -317,6 +317,9 @@ int fork_ipc_send_fd_table(int ipc_sock) fd_entries[num_fds].guest_fd = i; fd_entries[num_fds].type = fd_table[i].type; fd_entries[num_fds].linux_flags = fd_table[i].linux_flags; + fd_entries[num_fds].foreign_description = + fd_table[i].foreign_description; + fd_entries[num_fds].nonblock_owned = fd_table[i].nonblock_owned; fd_entries[num_fds].seals = fd_table[i].seals; fd_entries[num_fds].ofd_id = fd_table[i].ofd_id; fd_entries[num_fds].fasync_owner_type = fd_table[i].fasync_owner_type; @@ -469,7 +472,17 @@ int fork_ipc_recv_fd_table(int ipc_fd, guest_t *g) continue; } else { void (*cleanup)(int) = fd_cleanup_for_type(fd_entries[i].type); - fd_alloc_at(gfd, fd_entries[i].type, host_fds[i], cleanup, NULL); + + /* Every descriptor here aliases a description the parent already + * had, so the allocator takes the parent's answers rather than + * probing: a slot that aliases the launcher's stdio must not have + * O_NONBLOCK set on it here any more than it did there. + */ + fd_alias_spec_t spec = + fd_alias_carried(fd_entries[i].foreign_description != 0, + fd_entries[i].nonblock_owned != 0); + fd_alloc_alias_at(&spec, gfd, fd_entries[i].type, host_fds[i], + cleanup, NULL); fd_table[gfd].linux_flags = fd_entries[i].linux_flags; fd_refresh_urandom_bitmap(gfd); memcpy(fd_table[gfd].proc_path, fd_entries[i].proc_path, diff --git a/src/runtime/fork-state.h b/src/runtime/fork-state.h index c221afe9..cedfdd6e 100644 --- a/src/runtime/fork-state.h +++ b/src/runtime/fork-state.h @@ -19,7 +19,7 @@ /* Fork IPC protocol identity. Bump this whenever the header layout or ordered * fork payload changes incompatibly. */ -#define FORK_IPC_PROTOCOL_MAGIC 0x454C464FU /* "ELFO" */ +#define FORK_IPC_PROTOCOL_MAGIC 0x454C4650U /* "ELFP" */ #define IPC_MAGIC_HEADER FORK_IPC_PROTOCOL_MAGIC #define IPC_MAGIC_SENTINEL 0x454C4F4BU /* "ELOK" */ @@ -110,6 +110,14 @@ typedef struct { int32_t guest_fd, type, linux_flags, seals; uint64_t ofd_id; int32_t fasync_owner_type, fasync_owner; + + /* The child rebuilds its table from descriptions this process already + * holds, so it inherits the status-flag answers rather than probing them + * again: whether the description came from outside elfuse (the launcher's + * stdio, or an alias of it) and whether elfuse owns its O_NONBLOCK. See + * fd_alias_carried. + */ + int32_t foreign_description, nonblock_owned; char proc_path[FD_VIRTUAL_PATH_MAX]; } ipc_fd_entry_t; diff --git a/src/runtime/procemu.c b/src/runtime/procemu.c index 6610361d..76cc109f 100644 --- a/src/runtime/procemu.c +++ b/src/runtime/procemu.c @@ -664,11 +664,9 @@ const char *proc_get_shm_dir(void) */ static int proc_synthetic_fd(const void *data, size_t len) { - char template[] = "/tmp/elfuse-proc-XXXXXX"; - int fd = mkstemp(template); + int fd = tmpfile_anon("proc"); if (fd < 0) return -1; - unlink(template); /* Delete on close; fd keeps it alive */ const uint8_t *p = data; size_t remaining = len; diff --git a/src/syscall/asyncio.c b/src/syscall/asyncio.c index d3059e04..5c97124b 100644 --- a/src/syscall/asyncio.c +++ b/src/syscall/asyncio.c @@ -19,10 +19,13 @@ #include #include #include +#include #include #include "utils.h" +#include "proved/asyncudata.h" + #include "runtime/thread.h" #include "syscall/linux-wire.h" #include "syscall/internal.h" @@ -69,24 +72,28 @@ static bool async_owner_is_local(int owner_type, int owner) } } -/* kqueue udata packs the guest fd (low 16 bits) plus the fd slot generation at - * arm time (upper 48 bits). The generation guards against ABA: if the slot was - * closed and reused between arm and a stale event firing, the generation no - * longer matches and the event is dropped, so a SIGIO cannot land on a later, - * unrelated occupant of the same guest fd number. FD_TABLE_SIZE is 1024 (fits - * 16 bits) and the generation counter is monotonic, so 48 bits will not wrap. +/* kqueue udata carries the guest fd and the fd slot generation at arm time. The + * generation guards against ABA: if the slot was closed and reused between arm + * and a stale event firing, the generation no longer matches and the event is + * dropped, so a SIGIO cannot land on a later, unrelated occupant of the same + * guest fd number. + * + * The packing itself is proved in proved/asyncudata.h -- that whatever goes in + * comes back out is the whole of the guard, and it used to rest on a comment + * asserting that 1024 fds fit 16 bits and that 48 bits of generation will not + * wrap. make verify-asyncudata now discharges both, and that the multiply + * cannot overflow. */ static void *async_pack(int guest_fd, uint64_t generation) { - return (void *) (uintptr_t) (((generation & 0xFFFFFFFFFFFFULL) << 16) | - (uint32_t) (guest_fd & 0xFFFF)); + return (void *) (uintptr_t) async_udata_pack(guest_fd, generation); } static void async_unpack(void *udata, int *guest_fd, uint64_t *generation) { uint64_t v = (uint64_t) (uintptr_t) udata; - *guest_fd = (int) (v & 0xFFFF); - *generation = (v >> 16) & 0xFFFFFFFFFFFFULL; + *guest_fd = async_udata_fd(v); + *generation = async_udata_gen(v); } static void async_deliver(void *udata, int signum) @@ -98,7 +105,7 @@ static void async_deliver(void *udata, int signum) fd_entry_t snap; if (!fd_snapshot(guest_fd, &snap)) return; /* slot closed since the knote fired */ - if ((snap.generation & 0xFFFFFFFFFFFFULL) != generation) + if (snap.generation % ASYNC_UDATA_GEN_SPAN != generation) return; /* slot reused (ABA): this event belongs to the prior open */ if (signum != LINUX_SIGURG && !(snap.linux_flags & LINUX_O_ASYNC)) return; /* disarmed between fire and here (EV_CLEAR raced a disarm) */ @@ -214,6 +221,87 @@ static void async_reeval_slot_locked(int i) asyncio_disarm(fd_table[i].host_fd); } +typedef struct { + int32_t owner_type, owner; +} owner_set_ctx_t; + +/* Both sweeps below run under fd_lock inside fd_for_each_alias_locked, and both + * re-evaluate the slot's kevent registration after the change: arming inside + * the lock is what closes the close+reuse race, since a sibling cannot retire + * and reopen a host fd number while the scan holds it. + */ +static void owner_set_slot(int guest_fd, void *ctx) +{ + const owner_set_ctx_t *o = ctx; + fd_table[guest_fd].fasync_owner_type = o->owner_type; + fd_table[guest_fd].fasync_owner = o->owner; + async_reeval_slot_locked(guest_fd); +} + +/* True when F_SETFL(O_ASYNC) sticks, so F_GETFL reports it afterwards. + * + * Linux does not carry FASYNC in SETFL_MASK: setfl() lands the bit only by + * calling file_operations->fasync, so an object whose fops lack one keeps + * O_ASYNC clear however often the guest sets it. The set was measured against + * qemu-aarch64, not read off the kernel source, which reads as though the bit + * sticks everywhere: + * + * keeps it: pipe, fifo, socket, netlink, inotify, tty, /dev/urandom + * drops it: timerfd, eventfd, signalfd, epoll, pidfd, regular file, + * directory, /dev/null, /dev/zero + * + * The type alone cannot answer for FD_REGULAR and FD_STDIO, which may be a + * fifo, a socket, a tty, another character device or a plain file, so those two + * ask the host object what it is. can_block was the first answer here and was + * wrong in both directions: it takes in every character device, which put + * O_ASYNC on /dev/null, and it says nothing about /dev/urandom, which Linux + * does let the flag stick on (random_fasync). This runs on F_SETFL of O_ASYNC + * and nowhere else. + */ +static bool fd_keeps_fasync(int type, int host_fd) +{ + switch (type) { + case FD_PIPE: + case FD_SOCKET: + case FD_NETLINK: + case FD_INOTIFY: + case FD_FUSE_DEV: + case FD_URANDOM: + return true; + case FD_REGULAR: + case FD_STDIO: + break; + default: + return false; + } + + struct stat st; + if (host_fd < 0 || fstat(host_fd, &st) != 0) + return false; + if (S_ISFIFO(st.st_mode) || S_ISSOCK(st.st_mode)) + return true; + if (S_ISCHR(st.st_mode)) + return isatty(host_fd) == 1; + return false; +} + +static void async_flag_slot(int guest_fd, void *ctx) +{ + /* Setting the bit is conditional, clearing it never is: Linux lands FASYNC + * only through file_operations->fasync, so on an object without one the + * flag stays clear and F_GETFL keeps reporting 0. elfuse used to record the + * request for every type, which made a timerfd, an eventfd and a plain file + * all claim O_ASYNC they would never deliver. + */ + bool on = *(bool *) ctx && fd_keeps_fasync(fd_table[guest_fd].type, + fd_table[guest_fd].host_fd); + if (on) + fd_table[guest_fd].linux_flags |= LINUX_O_ASYNC; + else + fd_table[guest_fd].linux_flags &= ~LINUX_O_ASYNC; + async_reeval_slot_locked(guest_fd); +} + void fasync_owner_set(int guest_fd, uint64_t expect_gen, int owner_type, @@ -230,24 +318,9 @@ void fasync_owner_set(int guest_fd, * registration does not block, and the watcher thread never holds fd_lock * while parked in kevent(), so there is no deadlock. */ + owner_set_ctx_t ctx = {.owner_type = owner_type, .owner = owner}; pthread_mutex_lock(&fd_lock); - uint64_t ofd_id = 0; - if (fd_table[guest_fd].type != FD_CLOSED && - fd_table[guest_fd].generation == expect_gen) - ofd_id = fd_table[guest_fd].ofd_id; - if (ofd_id) { - /* An O(FD_TABLE_SIZE) scan reaches every alias sharing this - * open-file-description. Cold path (F_SETOWN only); an ofd_id to fd - * index is the upgrade if it ever shows up in a profile. - */ - for (int i = 0; i < FD_TABLE_SIZE; i++) { - if (fd_table[i].type == FD_CLOSED || fd_table[i].ofd_id != ofd_id) - continue; - fd_table[i].fasync_owner_type = owner_type; - fd_table[i].fasync_owner = owner; - async_reeval_slot_locked(i); - } - } + fd_for_each_alias_locked(guest_fd, expect_gen, owner_set_slot, &ctx); pthread_mutex_unlock(&fd_lock); } @@ -281,24 +354,6 @@ void asyncio_apply(int guest_fd, uint64_t expect_gen, bool on) * fasync_owner_set for the deadlock argument. */ pthread_mutex_lock(&fd_lock); - uint64_t ofd_id = 0; - if (fd_table[guest_fd].type != FD_CLOSED && - fd_table[guest_fd].generation == expect_gen) - ofd_id = fd_table[guest_fd].ofd_id; - if (ofd_id) { - /* An O(FD_TABLE_SIZE) scan reaches every alias sharing this - * open-file-description. Cold path (O_ASYNC toggles only); an ofd_id to - * fd index is the upgrade if it ever shows up in a profile. - */ - for (int i = 0; i < FD_TABLE_SIZE; i++) { - if (fd_table[i].type == FD_CLOSED || fd_table[i].ofd_id != ofd_id) - continue; - if (on) - fd_table[i].linux_flags |= LINUX_O_ASYNC; - else - fd_table[i].linux_flags &= ~LINUX_O_ASYNC; - async_reeval_slot_locked(i); - } - } + fd_for_each_alias_locked(guest_fd, expect_gen, async_flag_slot, &on); pthread_mutex_unlock(&fd_lock); } diff --git a/src/syscall/fd.c b/src/syscall/fd.c index 6d49d611..a0209174 100644 --- a/src/syscall/fd.c +++ b/src/syscall/fd.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -30,6 +31,7 @@ #include "syscall/linux-wire.h" #include "syscall/fd.h" #include "syscall/internal.h" +#include "syscall/io.h" #include "syscall/proc.h" #include "syscall/signal.h" @@ -222,8 +224,7 @@ int64_t sys_timerfd_create(int clockid, int flags) * access mode without re-deriving it. */ fd_publish_linux_flags( - gfd, LINUX_O_RDWR | - ((flags & LINUX_TFD_CLOEXEC) ? LINUX_O_CLOEXEC : 0) | + gfd, ((flags & LINUX_TFD_CLOEXEC) ? LINUX_O_CLOEXEC : 0) | ((flags & LINUX_TFD_NONBLOCK) ? LINUX_O_NONBLOCK : 0)); return gfd; } @@ -407,6 +408,7 @@ int64_t sys_timerfd_gettime(guest_t *g, int fd, uint64_t curr_value_gva) return 0; } + /* Read from timerfd: collect pending timer events from the kqueue, return * accumulated expiration count as uint64_t. Resets count to 0 after read (Linux * timerfd semantics). @@ -416,14 +418,7 @@ int64_t timerfd_read(int guest_fd, guest_t *g, uint64_t buf_gva, uint64_t count) if (count < 8) return -LINUX_EINVAL; - /* Snapshot the NONBLOCK status under fd_lock before sfd_lock to match the - * documented lock order (fd_lock=3 < sfd_lock=5a). The kqueue host fd - * rejects fcntl(F_SETFL, O_NONBLOCK) on macOS, so the flag lives in - * fd_table[guest_fd].linux_flags rather than on the host fd. - */ - pthread_mutex_lock(&fd_lock); - bool nonblock = fd_table[guest_fd].linux_flags & LINUX_O_NONBLOCK; - pthread_mutex_unlock(&fd_lock); + bool nonblock = fd_guest_nonblock(guest_fd); pthread_mutex_lock(&sfd_lock); int slot = timerfd_find(guest_fd); @@ -453,28 +448,44 @@ int64_t timerfd_read(int guest_fd, guest_t *g, uint64_t buf_gva, uint64_t count) return -LINUX_EAGAIN; } - /* Blocking: release lock, wait for the timer, re-lock. Another thread - * may close the fd while the timerfd wait is active -- kevent() returns - * EBADF in that case, and the code re-validates the slot. + /* Blocking: release the lock and wait for the timer, interruptibly. A + * kevent() with a NULL timeout parks this vCPU thread where neither + * hv_vcpus_exit nor the wakeup pipe reaches it, so an execve teardown + * counts it as a sibling that will not leave; a kqueue descriptor is + * pollable, so the wait can watch it alongside the wakeup pipe and + * collect with a zero timeout once it says there is something. Another + * thread may close the fd meanwhile, which the re-validation below + * catches. Loop on the expiration count rather than on one wait: the + * collect below runs with a zero timeout, so a sibling reading this + * same timerfd can take the event the wait reported and leave nothing + * here. A blocking read owes the guest a wait, not a spurious EAGAIN, + * which is what the NULL-timeout kevent this replaced could not + * produce. */ - struct kevent kev; - pthread_mutex_unlock(&sfd_lock); - int nev = kevent(kq, NULL, 0, &kev, 1, NULL); - pthread_mutex_lock(&sfd_lock); - /* Re-validate: slot may have been freed by timerfd_close() */ - if (timerfd_state[slot].guest_fd != guest_fd) { + while (timerfd_state[slot].expirations == 0) { + struct kevent kev; pthread_mutex_unlock(&sfd_lock); - return -LINUX_EBADF; - } - if (nev > 0) { - uint64_t fires = (uint64_t) kev.data; - if (fires == 0) - fires = 1; - timerfd_state[slot].expirations += fires; - } - if (timerfd_state[slot].expirations == 0) { - pthread_mutex_unlock(&sfd_lock); - return -LINUX_EAGAIN; + int64_t waited = io_wait_fd_or_interrupted(kq, POLLIN); + if (waited < 0) + return waited; + struct timespec collect = {0, 0}; + int nev = kevent(kq, NULL, 0, &kev, 1, &collect); + pthread_mutex_lock(&sfd_lock); + /* Re-validate: slot may have been freed by timerfd_close() */ + if (timerfd_state[slot].guest_fd != guest_fd) { + pthread_mutex_unlock(&sfd_lock); + return -LINUX_EBADF; + } + if (nev < 0 && errno != EINTR) { + pthread_mutex_unlock(&sfd_lock); + return linux_errno(); + } + if (nev > 0) { + uint64_t fires = (uint64_t) kev.data; + if (fires == 0) + fires = 1; + timerfd_state[slot].expirations += fires; + } } } @@ -560,7 +571,6 @@ static struct { int pipe_wr; /* Write end of self-pipe */ uint64_t counter; /* Accumulated event counter */ int semaphore; /* EFD_SEMAPHORE mode */ - int nonblock; /* O_NONBLOCK */ } eventfd_state[EVENTFD_MAX]; static int eventfd_owner[FD_TABLE_SIZE]; /* guest_fd -> slot, or -1 */ @@ -659,12 +669,16 @@ int64_t sys_eventfd2(unsigned int initval, int flags) eventfd_state[slot].pipe_wr = pipefd[1]; eventfd_state[slot].counter = (uint64_t) initval; eventfd_state[slot].semaphore = (flags & LINUX_EFD_SEMAPHORE) ? 1 : 0; - eventfd_state[slot].nonblock = (flags & LINUX_EFD_NONBLOCK) ? 1 : 0; eventfd_owner[gfd] = slot; pthread_mutex_unlock(&sfd_lock); - fd_publish_linux_flags(gfd, - (flags & LINUX_EFD_CLOEXEC) ? LINUX_O_CLOEXEC : 0); + /* Linux opens the eventfd inode O_RDWR (anon_inode_getfd in fs/eventfd.c), + * and O_NONBLOCK lives here rather than on the internal pipe, which stays + * nonblocking so the emulation can do its own waiting. + */ + fd_publish_linux_flags( + gfd, ((flags & LINUX_EFD_CLOEXEC) ? LINUX_O_CLOEXEC : 0) | + ((flags & LINUX_EFD_NONBLOCK) ? LINUX_O_NONBLOCK : 0)); /* If initial counter > 0, make the pipe readable so poll sees it */ if (initval > 0) { @@ -734,6 +748,16 @@ int eventfd_dup_fd(int src_fd, return -1; } eventfd_state[slot].refcount++; + + /* dup(2) hands back a second name for one open file description, so the + * alias has to carry the source's per-description state: the status flags + * (an eventfd keeps O_NONBLOCK and its access mode in the shadow, because + * the host fd behind it is elfuse's own pipe) and the ofd_id every alias + * sweep matches on. Rebuilding linux_flags from the dup argument alone left + * a dup of an EFD_NONBLOCK eventfd blocking forever on an empty read. + */ + int src_flags = fd_table[src_fd].linux_flags & FD_DESCRIPTION_FLAGS; + uint64_t src_ofd_id = fd_table[src_fd].ofd_id; int new_host_fd = dup(eventfd_state[slot].pipe_rd); int original_pipe_rd = eventfd_state[slot].pipe_rd; if (new_host_fd < 0) @@ -745,13 +769,15 @@ int eventfd_dup_fd(int src_fd, /* Publish the destination fd with eventfd_close as cleanup. The * eventfd_owner mapping is still -1, so a racing close here observes owner - * == -1 and does nothing; we detect that below. + * == -1 and does nothing; we detect that below. Aliases the source's + * description, so the allocator installs its identity with the slot rather + * than this path patching it on afterwards. */ - int new_guest_fd = - fixed_slot ? fd_alloc_at_relaxed(fixed_guest_fd, FD_EVENTFD, - new_host_fd, eventfd_close, NULL) - : fd_alloc_from_relaxed(min_guest_fd, FD_EVENTFD, - new_host_fd, eventfd_close, NULL); + fd_alias_spec_t spec = + fd_alias_identity(src_ofd_id, src_flags | linux_flags); + int new_guest_fd = fd_alloc_alias_relaxed( + &spec, fixed_slot ? fixed_guest_fd : -1, min_guest_fd, FD_EVENTFD, + new_host_fd, eventfd_close, NULL); if (new_guest_fd < 0) { close(new_host_fd); pthread_mutex_lock(&sfd_lock); @@ -791,7 +817,6 @@ int eventfd_dup_fd(int src_fd, return -1; } eventfd_owner[new_guest_fd] = slot; - fd_table[new_guest_fd].linux_flags = linux_flags; pthread_mutex_unlock(&sfd_lock); pthread_mutex_unlock(&fd_lock); return new_guest_fd; @@ -805,6 +830,8 @@ int64_t eventfd_read(int guest_fd, guest_t *g, uint64_t buf_gva, uint64_t count) if (count < 8) return -LINUX_EINVAL; + bool nonblock = fd_guest_nonblock(guest_fd); + pthread_mutex_lock(&sfd_lock); int slot = eventfd_find(guest_fd); if (slot < 0) { @@ -813,43 +840,43 @@ int64_t eventfd_read(int guest_fd, guest_t *g, uint64_t buf_gva, uint64_t count) } if (eventfd_state[slot].counter == 0) { - if (eventfd_state[slot].nonblock) { + if (nonblock) { pthread_mutex_unlock(&sfd_lock); return -LINUX_EAGAIN; } - /* Blocking mode: release lock, block on pipe, re-lock. The pipe is - * O_NONBLOCK, so temporarily make it blocking so read() actually waits - * for the writer. Matches signalfd_read. Another thread may close the - * fd while the eventfd wait is active; read() returns EBADF in that - * case, and the code re-validates the slot. + /* Blocking mode: release the lock and wait for the writer, without + * parking. The pipe stays nonblocking and the wait polls it alongside + * the wakeup pipe, so an execve teardown or a guest signal ends it; + * flipping the pipe to blocking for a read left this vCPU thread + * somewhere neither could reach. Another thread may close the fd + * meanwhile, which the slot re-validation below catches. */ - int rd_fd = eventfd_state[slot].pipe_rd; - pthread_mutex_unlock(&sfd_lock); + while (eventfd_state[slot].counter == 0) { + int rd_fd = eventfd_state[slot].pipe_rd; + pthread_mutex_unlock(&sfd_lock); - uint8_t byte; - fd_update_status_flag(rd_fd, O_NONBLOCK, - false); /* Make temporarily blocking */ - ssize_t r = read(rd_fd, &byte, 1); - fd_set_nonblock(rd_fd); /* Restore non-blocking */ - if (r < 0) - return linux_errno(); + int64_t waited = io_wait_fd_or_interrupted(rd_fd, POLLIN); + if (waited < 0) + return waited; - pthread_mutex_lock(&sfd_lock); + uint8_t byte; + ssize_t r = read(rd_fd, &byte, 1); + if (r < 0 && errno != EAGAIN) + return linux_errno(); - /* Re-validate via the owner table, not eventfd_state[slot].guest_fd: - * dup'd aliases bind multiple guest_fds to the same slot, so a - * legitimate caller's guest_fd may not equal the primary owner. - */ - if (eventfd_owner[guest_fd] != slot || - eventfd_state[slot].refcount <= 0) { - pthread_mutex_unlock(&sfd_lock); - return -LINUX_EBADF; - } - /* Counter was updated by the writer; re-check */ - if (eventfd_state[slot].counter == 0) { - pthread_mutex_unlock(&sfd_lock); - return -LINUX_EAGAIN; + pthread_mutex_lock(&sfd_lock); + + /* Re-validate via the owner table, not + * eventfd_state[slot].guest_fd: dup'd aliases bind multiple + * guest_fds to the same slot, so a legitimate caller's guest_fd may + * not equal the primary owner. + */ + if (eventfd_owner[guest_fd] != slot || + eventfd_state[slot].refcount <= 0) { + pthread_mutex_unlock(&sfd_lock); + return -LINUX_EBADF; + } } } @@ -862,17 +889,45 @@ int64_t eventfd_read(int guest_fd, guest_t *g, uint64_t buf_gva, uint64_t count) eventfd_state[slot].counter = 0; } - /* Drain pipe readability if counter is now 0. The pipe is O_NONBLOCK (see - * sys_eventfd2), so the loop returns once the pipe drains. readv (single - * iovec) is functionally identical to read here but bypasses clang's - * unix.BlockInCriticalSection checker, which flags read() while a pthread - * mutex is held. + /* The pipe is what poll, epoll and the blocking read above all watch, so + * its readability has to mean the same thing as counter > 0 rather than + * merely record the moment the counter left zero. + * + * Draining at zero is the whole story for a plain eventfd, whose read takes + * the counter to zero every time, and only half of it for EFD_SEMAPHORE, + * whose read decrements by one. A counter of 2 read once stays readable + * while the pipe goes empty -- the reader consumed the single byte + * eventfd_write posts on the 0-to-nonzero edge, and that edge will not come + * again until the counter returns to zero. Anything waiting on the pipe + * then sleeps through a count it was entitled to. Measured: two threads + * reading one EFD_SEMAPHORE eventfd hang here in three runs out of five and + * complete five out of five on the reference kernel. + * + * The pipe is O_NONBLOCK (see sys_eventfd2), so the drain loop ends once it + * is empty. readv with a single iovec is functionally identical to read but + * bypasses clang's unix.BlockInCriticalSection checker, which flags read() + * while a pthread mutex is held. */ + uint8_t drain; + struct iovec iov = {.iov_base = &drain, .iov_len = 1}; if (eventfd_state[slot].counter == 0) { - uint8_t drain; - struct iovec iov = {.iov_base = &drain, .iov_len = 1}; while (readv(eventfd_state[slot].pipe_rd, &iov, 1) > 0) ; + } else if (eventfd_state[slot].semaphore) { + /* Semaphore mode only: the plain path cannot reach here, so nothing + * pays for this but the case that needs it. Drain first so the re-arm + * cannot pile bytes up on a pipe that already had one. + */ + while (readv(eventfd_state[slot].pipe_rd, &iov, 1) > 0) + ; + uint8_t byte = 1; + ssize_t wr; + do { + wr = write(eventfd_state[slot].pipe_wr, &byte, 1); + } while (wr < 0 && errno == EINTR); + if (wr < 0) + log_error("eventfd_read: re-arm failed: %s (gfd=%d pipe_wr=%d)", + strerror(errno), guest_fd, eventfd_state[slot].pipe_wr); } pthread_mutex_unlock(&sfd_lock); @@ -987,7 +1042,6 @@ static struct { int pipe_rd; /* Read end for poll/epoll readiness */ int pipe_wr; /* Write end for signaling */ uint64_t mask; /* Signal mask (bitmask of signals to accept) */ - int nonblock; /* O_NONBLOCK */ } signalfd_state[SIGNALFD_MAX]; void signalfd_init(void) @@ -1089,11 +1143,14 @@ int64_t sys_signalfd4(guest_t *g, signalfd_state[slot].pipe_rd = pipefd[0]; signalfd_state[slot].pipe_wr = pipefd[1]; signalfd_state[slot].mask = mask; - signalfd_state[slot].nonblock = (flags & LINUX_SFD_NONBLOCK) ? 1 : 0; pthread_mutex_unlock(&sfd_lock); - fd_publish_linux_flags(gfd, - (flags & LINUX_SFD_CLOEXEC) ? LINUX_O_CLOEXEC : 0); + /* Linux opens the signalfd inode O_RDWR (anon_inode_getfd in + * fs/signalfd.c); same reasoning as eventfd for O_NONBLOCK. + */ + fd_publish_linux_flags( + gfd, ((flags & LINUX_SFD_CLOEXEC) ? LINUX_O_CLOEXEC : 0) | + ((flags & LINUX_SFD_NONBLOCK) ? LINUX_O_NONBLOCK : 0)); return gfd; } @@ -1122,11 +1179,17 @@ int64_t signalfd_read(int guest_fd, uint64_t buf_gva, uint64_t count) { + int nonblock; retry: /* Capture slot state under sfd_lock, then release BEFORE calling * signal_get_state() which acquires sig_lock(4). Holding sfd_lock(5a) while * taking sig_lock(4) would violate lock ordering. + * + * Re-read per attempt: a sibling can change the flag while this one is + * parked, and the block-or-report decision below is made fresh each round. */ + nonblock = fd_guest_nonblock(guest_fd); + pthread_mutex_lock(&sfd_lock); int slot = signalfd_find(guest_fd); if (slot < 0) { @@ -1135,7 +1198,6 @@ int64_t signalfd_read(int guest_fd, } uint64_t mask = signalfd_state[slot].mask; - int nonblock = signalfd_state[slot].nonblock; int pipe_rd = signalfd_state[slot].pipe_rd; size_t max_signals = count / sizeof(linux_signalfd_siginfo_t); if (max_signals == 0) { @@ -1179,16 +1241,40 @@ int64_t signalfd_read(int guest_fd, if (nonblock) goto no_pending; - /* Blocking mode: wait for signalfd_notify() to write to the pipe. - * Re-validate slot after wake. + /* Blocking mode: wait for signalfd_notify() to write to the pipe, and + * wait interruptibly. Making the pipe blocking for the duration of a + * read parks this vCPU thread in a host call that neither hv_vcpus_exit + * nor the wakeup pipe reaches, which is the failure this tree removed + * from every transfer path; a synthetic reader is no different. The + * pipe stays nonblocking and the wait polls it with the wakeup pipe, so + * teardown and guest signals both end it. */ + int64_t waited = io_wait_fd_or_interrupted(pipe_rd, POLLIN); + if (waited < 0) { + /* Both allocations, like every other exit here: they are made + * together when the guest asks for more signals than the stack + * arrays hold, and this is the one path that was added after the + * rest of the function agreed on that. + */ + free(heap); + free(src_heap); + return waited; + } + uint8_t byte; + ssize_t r = read(pipe_rd, &byte, 1); - /* pipe_rd is O_NONBLOCK, so temporarily make it blocking for the wait + /* The pipe stays nonblocking, so a sibling reading this same signalfd + * can take the byte the wait reported and leave EAGAIN here. A blocking + * read owes the guest a wait, not a spurious EAGAIN -- the same rule + * the drained-signal recheck below applies, and the reason the read + * used to be run with the pipe flipped blocking. */ - fcntl(pipe_rd, F_SETFL, 0); - ssize_t r = read(pipe_rd, &byte, 1); - fcntl(pipe_rd, F_SETFL, O_NONBLOCK); + if (r < 0 && errno == EAGAIN) { + free(heap); + free(src_heap); + goto retry; + } if (r <= 0) goto no_pending; diff --git a/src/syscall/fdtable.c b/src/syscall/fdtable.c index 554d3c68..76734ef3 100644 --- a/src/syscall/fdtable.c +++ b/src/syscall/fdtable.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -108,8 +109,15 @@ static bool host_fd_may_block(int host_fd) * and sockets always can; regular-file slots may actually be a fifo or char * device (opened_fd_type does not split those out) and stdio may be a tty, so * both need an fstat; everything else (dir, path, urandom, fuse, synthetic) - * never reaches the blocking wait path. Avoiding the fstat for the common - * pipe/socket/synthetic allocations keeps it off the fd-creation lock hold. + * never reaches the blocking wait path. + * + * Resolving the common pipe/socket/synthetic cases from the type keeps an fstat + * off the fd-creation lock hold, but does not empty it: fd_init_entry still + * takes O_NONBLOCK ownership with two fcntls in the same window. Measured + * against a build that skips them, the whole cost of both is 4.4% of a + * pipe()+close pair and 2.7% of an open()+close, so hoisting them out of the + * lock was not done -- it would not remove the syscalls, only the lock hold, + * and nothing measures a contended fd-creation path today. */ static bool type_may_block(int type, int host_fd) { @@ -125,27 +133,249 @@ static bool type_may_block(int type, int host_fd) } } +/* The pending alias inheritance, handed from an fd_alloc_alias_* wrapper to the + * fd_init_entry call it makes. Thread-local because the allocators take fd_lock + * themselves and cannot take a parameter through it without changing all eight + * signatures; private to this file because a caller that could set it directly + * could also forget to clear it. Every wrapper below clears it on the way out, + * including when the allocation fails. + */ +static _Thread_local bool fd_alias_pending; +static _Thread_local fd_alias_spec_t fd_alias_spec; + +static void fd_alias_begin(const fd_alias_spec_t *spec) +{ + fd_alias_pending = spec != NULL; + if (spec) + fd_alias_spec = *spec; +} + +static int fd_alias_end(int fd) +{ + fd_alias_pending = false; + return fd; +} + +/* The access mode Linux reports for an fd elfuse serves out of its own host + * description. A synthetic fd is backed by a pipe or a kqueue elfuse opened, so + * F_GETFL cannot ask the host what the guest opened; the answer is a property + * of the type and belongs in one table rather than at each creation site. Three + * types were missed when it was the creators' job, and each reported O_RDONLY + * where Linux reports O_RDWR. + * + * A type absent from here answers from the host description, which is right for + * regular files, directories, sockets and inherited stdio. + */ +static int fd_type_accmode(int type) +{ + switch (type) { + case FD_EVENTFD: /* anon_inode_getfd(O_RDWR), fs/eventfd.c */ + case FD_SIGNALFD: /* fs/signalfd.c */ + case FD_TIMERFD: /* fs/timerfd.c */ + case FD_EPOLL: /* fs/eventpoll.c */ + case FD_PIDFD: /* kernel/pid.c */ + case FD_NETLINK: /* a socket: O_RDWR */ + return LINUX_O_RDWR; + case FD_INOTIFY: /* anon_inode_getfd(O_RDONLY), inotify_user.c */ + return LINUX_O_RDONLY; + default: + return -1; + } +} + +/* Status flags a creator wants to publish, with the type's access mode forced + * back in. Publishing overwrites the field, so a creator that passes only its + * CLOEXEC/NONBLOCK bits would otherwise erase the mode fd_init_entry seeded -- + * which is how three synthetic types came to report O_RDONLY. + */ +static int fd_flags_with_accmode(int type, int linux_flags) +{ + int accmode = fd_type_accmode(type); + if (accmode < 0) + return linux_flags; + return (linux_flags & ~LINUX_O_ACCMODE) | accmode; +} + +/* The two answers fd_init_entry needs from the host descriptor, taken before + * fd_lock rather than under it. + * + * Both are host syscalls: type_may_block fstats a regular or stdio fd, and + * fd_set_nonblock is an F_GETFL/F_SETFL pair. Holding the table lock across + * them puts up to three host calls in front of every read, write and close that + * wants the table, and the design note above fd_alloc says the lock is held for + * table mutation only. + * + * Nothing races: the host fd is not published until fd_init_entry writes the + * slot, so this thread is the only one that can see it. The alias state is + * thread-local for the same reason, and an alias needs no probe at all -- it + * shares a description whose flag the source already answers for. + */ +typedef struct { + bool can_block; + bool nonblock_owned; + + /* The host status flags to put back if no slot is published after all, or + * -1 when the probe changed nothing. Taking the answers before the lock + * means taking them before the allocation can fail, and the probe is not a + * pure question: it sets O_NONBLOCK. A call that then returns EMFILE would + * leave the caller's descriptor mutated by a function that did nothing + * else, so the mutation is undone on the way out. + */ + int restore_flags; +} fd_host_probe_t; + +/* Put back what fd_probe_host changed, for a caller whose allocation failed. */ +static void fd_probe_rollback(const fd_host_probe_t *probe, int host_fd) +{ + if (probe->restore_flags < 0) + return; + int saved_errno = errno; + (void) fcntl(host_fd, F_SETFL, probe->restore_flags); + errno = saved_errno; +} + +static fd_host_probe_t fd_probe_host(int type, int host_fd) +{ + fd_host_probe_t probe = {.can_block = type_may_block(type, host_fd), + .restore_flags = -1}; + + if (fd_alias_pending) { + probe.nonblock_owned = fd_alias_spec.nonblock_owned; + return probe; + } + + bool foreign = (type == FD_STDIO); + if (!probe.can_block || foreign) + return probe; + + /* Remember the flags only when this call is the one adding O_NONBLOCK. A + * descriptor that already carried it is left alone on rollback, since + * putting back what was already there is not this function's to undo. + */ + int old_flags = fcntl(host_fd, F_GETFL); + probe.nonblock_owned = fd_set_nonblock(host_fd) >= 0; + if (probe.nonblock_owned && old_flags >= 0 && !(old_flags & O_NONBLOCK)) + probe.restore_flags = old_flags; + return probe; +} + static inline void fd_init_entry(int fd, int type, int host_fd, - void (*cleanup)(int)) + void (*cleanup)(int), + const fd_host_probe_t *probe) { fd_bitmap_set_used(fd); fd_table[fd].type = type; fd_table[fd].host_fd = host_fd; - fd_table[fd].ofd_id = fd_next_ofd_id++; + + /* Take the description state again, here, from the live source. The spec + * carries a snapshot the caller made before this lock was held, and an + * F_SETFL that lands in between sweeps the aliases that exist at that + * moment -- which cannot include the one being built. Publishing from the + * snapshot would leave the new name holding flags the rest of the + * description has already moved past, for the life of the description, with + * no generation change for a later check to notice. + * + * The generation is what makes this safe to do at all: a close+reopen in + * the same window puts a different description behind the same number, and + * copying from it would be worse than the staleness. When it has moved the + * snapshot stands, which is exactly the behaviour this replaces. + */ + if (fd_alias_pending && fd_alias_spec.src_guest_fd >= 0 && + RANGE_CHECK(fd_alias_spec.src_guest_fd, 0, FD_TABLE_SIZE)) { + const fd_entry_t *src = &fd_table[fd_alias_spec.src_guest_fd]; + if (src->type != FD_CLOSED && + src->generation == fd_alias_spec.src_generation) { + fd_alias_spec.ofd_id = src->ofd_id; + + /* Only the description's own bits. The caller may have ORed its own + * on top of the snapshot -- a dup3 asking for CLOEXEC -- and those + * belong to the new descriptor, not to the description, so a + * wholesale overwrite would drop them. + */ + fd_alias_spec.linux_flags = + (fd_alias_spec.linux_flags & ~FD_DESCRIPTION_FLAGS) | + (src->linux_flags & FD_DESCRIPTION_FLAGS); + fd_alias_spec.foreign_description = src->foreign_description; + fd_alias_spec.nonblock_owned = src->nonblock_owned; + } + } + + /* An alias names the description it was made from. Installed here, inside + * the same fd_lock window that publishes the slot, so there is no gap in + * which the slot is visible with a fresh identity: a close+reopen in such a + * gap would take the alias's ofd_id and be swept as though it shared a + * description it never saw. + */ + fd_table[fd].ofd_id = (fd_alias_pending && fd_alias_spec.ofd_id) + ? fd_alias_spec.ofd_id + : fd_next_ofd_id++; fd_table[fd].generation = fd_next_generation++; - fd_table[fd].linux_flags = 0; + + /* Seed the guest-visible flags with what the type alone decides. Creators + * OR in their own CLOEXEC/NONBLOCK afterwards; none of them has to know the + * access mode. + * + * An alias says its flags up front instead, and they land here rather than + * in a second window after the slot is published. That ordering is not + * cosmetic: the new slot joins an alias set other threads sweep by ofd_id, + * so a concurrent F_SETFL on the source can reach it in the gap and have + * its write clobbered by a publish that follows. + */ + int accmode = fd_type_accmode(type); + if (fd_alias_pending && fd_alias_spec.linux_flags) + fd_table[fd].linux_flags = + fd_flags_with_accmode(type, fd_alias_spec.linux_flags); + else + fd_table[fd].linux_flags = accmode < 0 ? 0 : accmode; fd_table[fd].dir = NULL; fd_table[fd].proc_path[0] = '\0'; fd_table[fd].seals = 0; - /* Cache whether a host read/write can block so the fast-path and slow-path + /* Whether a host read/write can block, so the fast-path and slow-path * readers can decide whether to divert into the interruptible wait without - * re-stating on every call. Only regular-file and stdio slots pay an fstat - * here; pipes/sockets/synthetic fds resolve from the type alone. + * re-stating it on every call. Taken by fd_probe_host before the lock. */ - fd_table[fd].can_block = type_may_block(type, host_fd); + fd_table[fd].can_block = probe->can_block; + + /* Own O_NONBLOCK on every fd whose host transfer could otherwise park a + * vCPU thread, and emulate the guest's blocking semantics on top of it + * (io_xfer). A readiness poll reserves nothing: the bytes it promised can + * be taken by a sibling thread or a forked process before the transfer + * runs, and a transfer that blocks there is reachable by neither + * hv_vcpus_exit nor the wakeup pipe. From here on linux_flags carries what + * the guest asked for, and sys_fcntl reports it from there. + * + * Sockets are owned like everything else that can block. They used to be + * excluded on the grounds that a per-call MSG_DONTWAIT made ownership + * unnecessary, and macOS does not honour that flag on AF_UNIX: a send on a + * full stream socket writes what fits and then blocks in the kernel for the + * rest, flag set, so the parked vCPU this whole mechanism exists to prevent + * was still reachable through every socket write. + * + * The inherited stdio descriptors are still left alone, because their open + * file description belongs to whoever launched elfuse -- a shell handing + * over its terminal would keep the flag long after elfuse exits. A dup of + * one of those descriptors is typed FD_REGULAR and so escapes the type + * test; an fd_alias_spec_t is how the dup path, and the fork restore, say + * the description is one that already exists. A socket that arrived over + * SCM_RIGHTS comes in the same way and stays unowned for the same reason: + * elfuse did not open it. + */ + fd_table[fd].foreign_description = + fd_alias_pending ? fd_alias_spec.foreign_description : type == FD_STDIO; + + /* An alias takes it from the spec, which the refresh above may have moved + * on from what fd_probe_host saw: the probe runs before this lock and for + * an alias only copies the caller's snapshot, so publishing the probe's + * copy here would quietly undo the one field the refresh had to work for. + * The two agree in every case measured so far -- ownership is decided when + * a description is created and F_SETFL does not change it -- which is + * exactly why it would have sat here unnoticed. + */ + fd_table[fd].nonblock_owned = + fd_alias_pending ? fd_alias_spec.nonblock_owned : probe->nonblock_owned; fd_table[fd].fasync_owner_type = FASYNC_OWNER_NONE; fd_table[fd].fasync_owner = 0; sock_opt_clear(&fd_table[fd]); @@ -239,8 +469,13 @@ void fdtable_init(void) .generation = fd_next_generation++}; /* The compound literals above zero can_block; recover the real value so a - * terminal stdin still routes reads through the interruptible wait. + * terminal stdin still routes reads through the interruptible wait. Same + * for foreign_description, which fd_init_entry derives from the type and + * nothing derives here: these three descriptions belong to whoever launched + * elfuse, and every alias of them reads the answer from this entry. */ + for (int i = 0; i < 3; i++) + fd_table[i].foreign_description = true; fd_table[0].can_block = host_fd_may_block(STDIN_FILENO); fd_table[1].can_block = host_fd_may_block(STDOUT_FILENO); fd_table[2].can_block = host_fd_may_block(STDERR_FILENO); @@ -259,16 +494,24 @@ void fdtable_init(void) static int fd_alloc_locked(int minfd, int type, int host_fd, - void (*cleanup)(int)) + void (*cleanup)(int), + const fd_host_probe_t *probe) { int fd = fd_bitmap_find_free(minfd); if (fd >= 0 && fd >= rlimit_nofile_cur) fd = -1; /* RLIMIT_NOFILE exceeded */ if (fd < 0) { + /* No slot, so nothing is published and the probe's O_NONBLOCK has to + * come back off: the caller asked for an allocation, got EMFILE, and + * should not also find its descriptor changed. This runs an fcntl under + * fd_lock, which the probe exists to avoid, but only on the path that + * is failing anyway. + */ + fd_probe_rollback(probe, host_fd); errno = EMFILE; return -1; } - fd_init_entry(fd, type, host_fd, cleanup); + fd_init_entry(fd, type, host_fd, cleanup, probe); return fd; } @@ -279,8 +522,9 @@ static int fd_alloc_locked(int minfd, */ int fd_alloc(int type, int host_fd, void (*cleanup)(int)) { + fd_host_probe_t probe = fd_probe_host(type, host_fd); pthread_mutex_lock(&fd_lock); - int fd = fd_alloc_locked(0, type, host_fd, cleanup); + int fd = fd_alloc_locked(0, type, host_fd, cleanup, &probe); pthread_mutex_unlock(&fd_lock); return fd; } @@ -291,11 +535,13 @@ int fd_alloc_dir(int type, void *dir, int linux_flags) { + fd_host_probe_t probe = fd_probe_host(type, host_fd); pthread_mutex_lock(&fd_lock); - int fd = fd_alloc_locked(0, type, host_fd, cleanup); + int fd = fd_alloc_locked(0, type, host_fd, cleanup, &probe); if (fd >= 0) { fd_table[fd].dir = dir; - fd_table[fd].linux_flags = linux_flags; + fd_table[fd].linux_flags = + fd_flags_with_accmode(fd_table[fd].type, linux_flags); } pthread_mutex_unlock(&fd_lock); return fd; @@ -314,11 +560,13 @@ int fd_alloc_dir_from(int minfd, void *dir, int linux_flags) { + fd_host_probe_t probe = fd_probe_host(type, host_fd); pthread_mutex_lock(&fd_lock); - int fd = fd_alloc_locked(minfd, type, host_fd, cleanup); + int fd = fd_alloc_locked(minfd, type, host_fd, cleanup, &probe); if (fd >= 0) { fd_table[fd].dir = dir; - fd_table[fd].linux_flags = linux_flags; + fd_table[fd].linux_flags = + fd_flags_with_accmode(fd_table[fd].type, linux_flags); } pthread_mutex_unlock(&fd_lock); return fd; @@ -341,14 +589,16 @@ int fd_alloc_dir_at(int fd, return -1; fd_entry_t old = {.type = FD_CLOSED}; + fd_host_probe_t probe = fd_probe_host(type, host_fd); pthread_mutex_lock(&fd_lock); if (fd_table[fd].type != FD_CLOSED) { old = fd_table[fd]; epoll_note_fd_closed(fd, old.ofd_id); } - fd_init_entry(fd, type, host_fd, cleanup); + fd_init_entry(fd, type, host_fd, cleanup, &probe); fd_table[fd].dir = dir; - fd_table[fd].linux_flags = linux_flags; + fd_table[fd].linux_flags = + fd_flags_with_accmode(fd_table[fd].type, linux_flags); pthread_mutex_unlock(&fd_lock); if (old.type != FD_CLOSED) @@ -367,8 +617,9 @@ int fd_alloc_from(int minfd, void (*cleanup)(int), uint64_t *out_gen) { + fd_host_probe_t probe = fd_probe_host(type, host_fd); pthread_mutex_lock(&fd_lock); - int fd = fd_alloc_locked(minfd, type, host_fd, cleanup); + int fd = fd_alloc_locked(minfd, type, host_fd, cleanup, &probe); /* Capture the freshly-stamped generation inside the allocating critical * section. Callers (dup) later revalidate it under fd_lock to prove the @@ -381,6 +632,63 @@ int fd_alloc_from(int minfd, return fd; } +/* The alias-aware entry points. Each is the plain allocator with the + * inheritance bracketed around it, so a caller states what it is claiming and + * cannot leave the claim set behind. + */ +int fd_alloc_alias(const fd_alias_spec_t *spec, + int type, + int host_fd, + void (*cleanup)(int)) +{ + fd_alias_begin(spec); + return fd_alias_end(fd_alloc(type, host_fd, cleanup)); +} + +int fd_alloc_alias_at(const fd_alias_spec_t *spec, + int fd, + int type, + int host_fd, + void (*cleanup)(int), + uint64_t *out_gen) +{ + fd_alias_begin(spec); + return fd_alias_end(fd_alloc_at(fd, type, host_fd, cleanup, out_gen)); +} + +int fd_alloc_alias_relaxed(const fd_alias_spec_t *spec, + int fixed_fd, + int minfd, + int type, + int host_fd, + void (*cleanup)(int), + uint64_t *out_gen) +{ + fd_alias_begin(spec); + int fd = + fixed_fd >= 0 + ? fd_alloc_at_relaxed(fixed_fd, type, host_fd, cleanup, out_gen) + : fd_alloc_from_relaxed(minfd, type, host_fd, cleanup, out_gen); + return fd_alias_end(fd); +} + +int fd_alloc_alias_dir(const fd_alias_spec_t *spec, + int fixed_fd, + int minfd, + int type, + int host_fd, + void (*cleanup)(int), + void *dir, + int linux_flags) +{ + fd_alias_begin(spec); + int fd = fixed_fd >= 0 ? fd_alloc_dir_at(fixed_fd, type, host_fd, cleanup, + dir, linux_flags) + : fd_alloc_dir_from(minfd, type, host_fd, cleanup, + dir, linux_flags); + return fd_alias_end(fd); +} + int fd_alloc_from_relaxed(int minfd, int type, int host_fd, @@ -393,7 +701,8 @@ int fd_alloc_from_relaxed(int minfd, /* Single active thread: no sibling can race the slot, so the unlocked * generation read is safe. */ - int fd = fd_alloc_locked(minfd, type, host_fd, cleanup); + fd_host_probe_t probe = fd_probe_host(type, host_fd); + int fd = fd_alloc_locked(minfd, type, host_fd, cleanup, &probe); if (out_gen && fd >= 0) *out_gen = fd_table[fd].generation; return fd; @@ -455,6 +764,7 @@ int fd_alloc_at(int fd, */ fd_entry_t old = {.type = FD_CLOSED}; + fd_host_probe_t probe = fd_probe_host(type, host_fd); pthread_mutex_lock(&fd_lock); if (fd_table[fd].type != FD_CLOSED) { old = fd_table[fd]; @@ -465,7 +775,7 @@ int fd_alloc_at(int fd, */ epoll_note_fd_closed(fd, old.ofd_id); } - fd_init_entry(fd, type, host_fd, cleanup); + fd_init_entry(fd, type, host_fd, cleanup, &probe); if (out_gen) *out_gen = fd_table[fd].generation; pthread_mutex_unlock(&fd_lock); @@ -493,10 +803,17 @@ int fd_alloc_at_relaxed(int fd, if (fd_table[fd].type != FD_CLOSED) return fd_alloc_at(fd, type, host_fd, cleanup, out_gen); + /* After the early returns, not before: every one of them either rejects the + * request or hands it to a variant that probes for itself, and the probe + * sets O_NONBLOCK on the host fd. Probing first would run that on an fd the + * call is about to refuse, and run it twice on the delegating path. + */ + fd_host_probe_t probe = fd_probe_host(type, host_fd); + /* Single active thread: no sibling can race the slot, so the unlocked init * and generation read are safe. */ - fd_init_entry(fd, type, host_fd, cleanup); + fd_init_entry(fd, type, host_fd, cleanup, &probe); if (out_gen) *out_gen = fd_table[fd].generation; return fd; @@ -666,14 +983,112 @@ int fd_get_type(int guest_fd) return type; } -bool fd_can_block(int guest_fd) +fd_block_state_t fd_block_state(int guest_fd) +{ + fd_block_state_t st = {.type = FD_CLOSED}; + if (!RANGE_CHECK(guest_fd, 0, FD_TABLE_SIZE)) + return st; + + /* Same relaxed rule the allocation paths use: with one active thread no + * sibling can mutate the slot, so the lock buys nothing. This runs on every + * read and write of a fd that can block, and the whole-entry fd_snapshot it + * replaces copied a few hundred bytes under a global lock to read three + * fields. + */ + bool locked = !thread_is_single_active(); + if (locked) + pthread_mutex_lock(&fd_lock); + st = fd_block_state_of(&fd_table[guest_fd]); + if (locked) + pthread_mutex_unlock(&fd_lock); + return st; +} + +bool fd_guest_nonblock(int guest_fd) +{ + /* Same field, same relaxed rule as fd_block_state: with one active thread + * no sibling can mutate the slot, so the lock buys nothing. This runs on + * every eventfd, signalfd, timerfd and inotify read, which is once per + * event-loop wakeup. + */ + return fd_block_state(guest_fd).guest_nonblock; +} + +void fd_for_each_alias_locked(int guest_fd, + uint64_t generation, + void (*fn)(int guest_fd, void *ctx), + void *ctx) +{ + if (!RANGE_CHECK(guest_fd, 0, FD_TABLE_SIZE)) + return; + + /* A close+reopen in the window makes ofd_id name a description the caller + * never asked about, so verify the slot first and change nothing if it + * moved. Generation is the discriminator: a reopen can reuse the same guest + * fd number, host fd number and type, and only the monotonic generation + * tells the caller's open from a new one. + */ + if (fd_table[guest_fd].type == FD_CLOSED || + fd_table[guest_fd].generation != generation) + return; + uint64_t ofd_id = fd_table[guest_fd].ofd_id; + if (!ofd_id) + return; + + /* Walk the allocation bitmap rather than the table: one word rules out 64 + * slots, and the flag sweeps run per fd at event-loop setup. + */ + for (int w = 0; w < FD_BITMAP_WORDS; w++) { + uint64_t used = ~fd_free_bitmap[w]; + while (used) { + int fd = w * 64 + bit_ctz64(used); + used &= used - 1; + if (fd_table[fd].type != FD_CLOSED && fd_table[fd].ofd_id == ofd_id) + fn(fd, ctx); + } + } +} + +typedef struct { + int mask, value; +} shadow_bits_ctx_t; + +static void set_shadow_bits_slot(int guest_fd, void *ctx) +{ + const shadow_bits_ctx_t *b = ctx; + fd_table[guest_fd].linux_flags = + (fd_table[guest_fd].linux_flags & ~b->mask) | (b->value & b->mask); +} + +void fd_set_shadow_flags(int guest_fd, uint64_t generation, int mask, int value) +{ + shadow_bits_ctx_t ctx = {.mask = mask, .value = value}; + pthread_mutex_lock(&fd_lock); + fd_for_each_alias_locked(guest_fd, generation, set_shadow_bits_slot, &ctx); + pthread_mutex_unlock(&fd_lock); +} + +bool fd_apply_guest_nonblock(int guest_fd, bool on) { if (!RANGE_CHECK(guest_fd, 0, FD_TABLE_SIZE)) return false; + + /* One fd_lock window for the question and the answer both. Reading the slot + * first and sweeping afterwards took the lock twice and revalidated the + * generation the first read had just produced, on a path every F_SETFL and + * every FIONBIO runs whatever the fd's type. + */ + shadow_bits_ctx_t ctx = {.mask = LINUX_O_NONBLOCK, + .value = on ? LINUX_O_NONBLOCK : 0}; pthread_mutex_lock(&fd_lock); - bool can_block = fd_table[guest_fd].can_block; + fd_entry_t *e = &fd_table[guest_fd]; + bool shadowed = e->type != FD_CLOSED && + fd_nonblock_shadowed(e->type, e->nonblock_owned); + if (shadowed) + fd_for_each_alias_locked(guest_fd, e->generation, set_shadow_bits_slot, + &ctx); pthread_mutex_unlock(&fd_lock); - return can_block; + return shadowed; } void fd_publish_linux_flags(int guest_fd, int linux_flags) @@ -681,7 +1096,8 @@ void fd_publish_linux_flags(int guest_fd, int linux_flags) if (!RANGE_CHECK(guest_fd, 0, FD_TABLE_SIZE)) return; pthread_mutex_lock(&fd_lock); - fd_table[guest_fd].linux_flags = linux_flags; + fd_table[guest_fd].linux_flags = + fd_flags_with_accmode(fd_table[guest_fd].type, linux_flags); pthread_mutex_unlock(&fd_lock); } @@ -707,6 +1123,31 @@ void (*fd_cleanup_for_type(int type))(int) return fd_type_cleanup[type]; } +/* Look up a guest FD, dup its host fd, and classify the slot, all in one + * fd_lock window. Caller owns the returned descriptor and must close it. + * + * Returns -1 on failure, with *st_out reporting FD_CLOSED. + */ +int fd_to_host_dup_state(int guest_fd, fd_block_state_t *st_out) +{ + /* fd_snapshot_and_dup already takes the whole entry and the dup in one + * fd_lock window, which is the atomicity a transfer needs: the pin keeps + * the description alive, and the classification has to describe that same + * description rather than whatever takes the fd number next. Composing with + * it rather than repeating it also inherits its host_fd < 0 guard, which a + * hand-rolled copy here did not have and would have dup(-1)'d for the FUSE + * types that carry no host descriptor. + */ + fd_entry_t snap; + int owned = fd_snapshot_and_dup(guest_fd, &snap); + if (owned < 0) { + *st_out = (fd_block_state_t) {.type = FD_CLOSED}; + return -1; + } + *st_out = fd_block_state_of(&snap); + return owned; +} + /* Look up a guest FD and return a dup'd host fd that the caller owns. The dup * is performed under fd_lock so that close() on another thread cannot * invalidate the host fd between lookup and dup. Caller must close the returned diff --git a/src/syscall/fs.c b/src/syscall/fs.c index f7303a3c..fcdd10e0 100644 --- a/src/syscall/fs.c +++ b/src/syscall/fs.c @@ -84,7 +84,14 @@ static int intercepted_fd_type(const char *path, int host_fd, int linux_flags) int type = opened_fd_type(host_fd, linux_flags); if (type < 0) return type; - if (type == FD_REGULAR && path && !strcmp(path, "/dev/urandom")) + + /* Both spellings, because procemu already serves them from one host device + * and Linux gives them one file_operations: /dev/random keeps O_ASYNC + * through random_fasync exactly as /dev/urandom does, and typing only one + * of them left the other reporting the flag cleared. + */ + if (type == FD_REGULAR && path && + (!strcmp(path, "/dev/urandom") || !strcmp(path, "/dev/random"))) return FD_URANDOM; return type; } @@ -410,12 +417,17 @@ void dir_stream_release(void *ds_ptr) } } +/* spec is the description this fd inherits from, or NULL for a fresh one. Only + * the magic-link open passes one: it implements the open as a dup, so the host + * flags are shared with the source and must not be probed or changed. + */ static int fd_alloc_opened_host(int host_fd, int type, int linux_flags, int min_guest_fd, void (*cleanup)(int), - const char *virtual_path) + const char *virtual_path, + const fd_alias_spec_t *spec) { dir_stream_t *ds = NULL; @@ -440,8 +452,9 @@ static int fd_alloc_opened_host(int host_fd, int guest_fd = min_guest_fd >= 0 - ? fd_alloc_from_relaxed(min_guest_fd, type, host_fd, cleanup, NULL) - : fd_alloc_from_relaxed(0, type, host_fd, cleanup, NULL); + ? fd_alloc_alias_relaxed(spec, -1, min_guest_fd, type, host_fd, + cleanup, NULL) + : fd_alloc_alias_relaxed(spec, -1, 0, type, host_fd, cleanup, NULL); if (guest_fd < 0) { int saved_errno = errno; if (ds) @@ -611,8 +624,8 @@ int64_t sys_openat_path(guest_t *g, close_keep_errno(host_fd); return linux_errno(); } - int guest_fd = - fd_alloc_opened_host(host_fd, type, linux_flags, -1, NULL, NULL); + int guest_fd = fd_alloc_opened_host(host_fd, type, linux_flags, -1, + NULL, NULL, NULL); if (guest_fd < 0) { close_keep_errno(host_fd); return linux_errno(); @@ -651,9 +664,34 @@ int64_t sys_openat_path(guest_t *g, } int min_guest_fd = (!strncmp(tx.intercept_path, "/dev/", 5)) ? -1 : 128; - int guest_fd = fd_alloc_opened_host( - intercepted, type, linux_flags, min_guest_fd, - fd_cleanup_for_type(type), tx.intercept_path); + + /* An fd magic link (/dev/stdin, /dev/fd/N, /proc/self/fd/N) is + * served by dup'ing a descriptor this process already holds, so the + * new slot aliases an open file description that already exists. + * Say so, or the allocator probes it: taking O_NONBLOCK ownership + * of the launcher's terminal leaves it nonblocking after elfuse + * exits, and re-probing a description elfuse already owns would + * answer the same thing twice. + */ + fd_entry_t alias_src; + fd_alias_spec_t spec = {0}; + int alias_fd = path_fd_magiclink_guest_fd(tx.intercept_path); + bool aliased = alias_fd >= 0 && fd_snapshot(alias_fd, &alias_src); + if (aliased) { + /* Ownership yes, identity no. Linux gives an opened magic link + * its own open file description with its own status flags, so + * this must not join the source's alias set: an F_SETFL on + * /proc/self/fd/0 would otherwise sweep onto fd 0. elfuse + * implements the open as a dup, so the host flags really are + * shared and the description really is foreign, which is + * exactly what fd_alias_host_shared claims and no more. + */ + spec = fd_alias_host_shared(&alias_src); + } + int guest_fd = + fd_alloc_opened_host(intercepted, type, linux_flags, + min_guest_fd, fd_cleanup_for_type(type), + tx.intercept_path, aliased ? &spec : NULL); if (guest_fd < 0) { proc_pty_forget_host_fd(intercepted); close_keep_errno(intercepted); @@ -679,8 +717,8 @@ int64_t sys_openat_path(guest_t *g, close_keep_errno(host_fd); return linux_errno(); } - int guest_fd = - fd_alloc_opened_host(host_fd, type, linux_flags, -1, NULL, NULL); + int guest_fd = fd_alloc_opened_host(host_fd, type, linux_flags, -1, + NULL, NULL, NULL); if (guest_fd < 0) { close_keep_errno(host_fd); return linux_errno(); @@ -704,7 +742,7 @@ int64_t sys_openat_path(guest_t *g, return linux_errno(); } int guest_fd = - fd_alloc_opened_host(host_fd, type, linux_flags, -1, NULL, NULL); + fd_alloc_opened_host(host_fd, type, linux_flags, -1, NULL, NULL, NULL); if (guest_fd < 0) { close_keep_errno(host_fd); return linux_errno(); @@ -941,17 +979,6 @@ static bool install_fd_alias_metadata_atomic(int dst_fd, dir_stream_t *ds, uint64_t expected_gen) { - /* LINUX_O_NONBLOCK is a file-status flag preserved by dup(2)/dup2(2). - * Required for FD_TIMERFD (and any other type that stores NONBLOCK in - * linux_flags rather than on the host fd) so a duplicated non-blocking - * timerfd does not silently turn blocking. - */ - int preserved_flags = - src_snap->linux_flags & - (LINUX_O_ACCMODE | LINUX_O_PATH | LINUX_O_DIRECTORY | LINUX_O_NOFOLLOW | - LINUX_O_DIRECT | LINUX_O_LARGEFILE | LINUX_O_NONBLOCK | LINUX_O_ASYNC); - int final_flags = preserved_flags | linux_flags; - bool installed = false; pthread_mutex_lock(&fd_lock); @@ -963,8 +990,9 @@ static bool install_fd_alias_metadata_atomic(int dst_fd, if (fd_table[dst_fd].type == expected_type && fd_table[dst_fd].host_fd == expected_host_fd && fd_table[dst_fd].generation == expected_gen) { - fd_table[dst_fd].linux_flags = final_flags; - fd_table[dst_fd].ofd_id = src_snap->ofd_id; + /* linux_flags and ofd_id are not written here: the allocator installed + * both from the alias spec, in the window that published the slot. + */ fd_table[dst_fd].fasync_owner_type = src_snap->fasync_owner_type; fd_table[dst_fd].fasync_owner = src_snap->fasync_owner; fd_table[dst_fd].seals = src_snap->seals; @@ -972,9 +1000,14 @@ static bool install_fd_alias_metadata_atomic(int dst_fd, sizeof(fd_table[dst_fd].proc_path)); if (ds) fd_table[dst_fd].dir = ds; + + /* Read the mode back from the slot the allocator published rather than + * from a local copy of it, so there is one answer to what this fd's + * access mode is. + */ bool readable_urandom = expected_type == FD_URANDOM && - (final_flags & LINUX_O_ACCMODE) != LINUX_O_WRONLY; + (fd_table[dst_fd].linux_flags & LINUX_O_ACCMODE) != LINUX_O_WRONLY; shim_globals_mark_urandom_fd(dst_fd, readable_urandom); installed = true; } @@ -1069,11 +1102,20 @@ static int duplicate_guest_fd(int src_fd, int new_type = (src_snap.type == FD_STDIO) ? FD_REGULAR : src_snap.type; void (*cleanup)(int) = fd_cleanup_for_type(new_type); uint64_t alloc_gen = 0; - int guest_fd = - fixed_slot ? fd_alloc_at_relaxed(fixed_guest_fd, new_type, new_host_fd, - cleanup, &alloc_gen) - : fd_alloc_from_relaxed(min_guest_fd, new_type, new_host_fd, - cleanup, &alloc_gen); + + /* The new slot aliases src_snap's description, whatever type it ends up + * with, so the allocator inherits its status-flag answers instead of + * probing a description it does not own. The dup's own bits ride along with + * the description's, so the allocator publishes the final value inside the + * window that creates the slot and install_fd_alias_metadata_atomic no + * longer writes flags or identity at all. Two writers of one field is how a + * dup'd epoll lost its ofd_id. + */ + fd_alias_spec_t spec = fd_alias_of(src_fd, &src_snap); + spec.linux_flags |= linux_flags; + int guest_fd = fd_alloc_alias_relaxed( + &spec, fixed_slot ? fixed_guest_fd : -1, min_guest_fd, new_type, + new_host_fd, cleanup, &alloc_gen); if (guest_fd < 0) { if (fixed_slot) errno = EBADF; @@ -1424,19 +1466,19 @@ int64_t sys_fcntl(guest_t *g, int fd, int cmd, uint64_t arg) pthread_mutex_unlock(&fd_lock); return 0; case 3: { /* F_GETFL */ - if (fuse_fd) - return fd_snap.linux_flags; - - /* Linux timerfd F_GETFL reports O_RDWR plus the writable status bits - * the kernel honors. Surface only those bits from the shadow rather - * than echoing arbitrary linux_flags bits so stray F_SETFL args cannot - * leak through here. O_ASYNC stays off because timerfd_fops lacks - * ->fasync, so generic_setfl drops it. + /* One rule: the host answers for the bits it is authoritative for, the + * shadow for the rest. fd_host_flag_mask names the first set from the + * type; O_NONBLOCK leaves it per fd, since ownership is not a property + * of the type alone. */ - if (fd_type == FD_TIMERFD) - return LINUX_O_RDWR | - (fd_snap.linux_flags & - (LINUX_O_APPEND | LINUX_O_NONBLOCK | LINUX_O_NOATIME)); + int host_mask = fd_host_flag_mask(fd_snap.type); + if (fd_nonblock_shadowed(fd_snap.type, fd_snap.nonblock_owned)) + host_mask &= ~LINUX_O_NONBLOCK; + + int shadow_fl = fd_snap.linux_flags & ~FD_GETFL_HIDDEN; + if (!host_mask) + return shadow_fl; + host_fd_ref_t host_ref; if (host_fd_ref_open(fd, &host_ref) < 0) return -LINUX_EBADF; @@ -1444,19 +1486,8 @@ int64_t sys_fcntl(guest_t *g, int fd, int cmd, uint64_t arg) host_fd_ref_close(&host_ref); if (mac_fl < 0) return linux_errno(); - int linux_fl = mac_to_linux_status_flags(mac_fl); - if (fd_snap.type == FD_REGULAR || fd_snap.type == FD_DIR || - fd_snap.type == FD_PATH || fd_snap.type == FD_URANDOM) - linux_fl = (linux_fl & ~O_ACCMODE) | (fd_snap.linux_flags & 3); - linux_fl |= fd_snap.linux_flags & - (LINUX_O_PATH | LINUX_O_DIRECTORY | LINUX_O_NOFOLLOW | - LINUX_O_DIRECT | LINUX_O_LARGEFILE); - - /* O_ASYNC is tracked in the shadow (never armed on the host fd), so - * surface it from there. See linux_to_mac_status_flags in translate.c. - */ - linux_fl |= fd_snap.linux_flags & LINUX_O_ASYNC; - return linux_fl; + return (shadow_fl & ~host_mask) | + (mac_to_linux_status_flags(mac_fl) & host_mask); } case 4: /* F_SETFL */ { @@ -1491,46 +1522,108 @@ int64_t sys_fcntl(guest_t *g, int fd, int cmd, uint64_t arg) return 0; } - /* Timerfd: kqueue host fd rejects fcntl(F_SETFL), so mirror Linux's - * file-status word in the linux_flags shadow. Of Linux's writable - * status flags (O_APPEND, O_ASYNC, O_DIRECT, O_NOATIME, O_NONBLOCK) the - * timerfd kernel object honors O_APPEND, O_NONBLOCK, and O_NOATIME. - * O_ASYNC is silently dropped (timerfd_fops lacks ->fasync). O_DIRECT - * returns -EINVAL because the inode lacks FMODE_CAN_ODIRECT. Bits - * outside the writable set (access mode, CLOEXEC, - * O_PATH/DIRECTORY/NOFOLLOW/etc.) are silently ignored, matching how - * Linux F_SETFL drops them. + /* An fd elfuse emulates whole has no host description to tell. The fd + * behind it is elfuse's own pipe or kqueue, held at the flags the + * emulation needs, and a kqueue rejects fcntl(F_SETFL) outright: this + * used to be a hand-written timerfd branch, while signalfd, inotify, + * eventfd, epoll, pidfd and netlink fell through to the host call below + * and set flags on elfuse's own descriptor. fd_host_flag_mask says + * which types those are, and answers the same question F_GETFL asks it. + * + * Of Linux's writable status flags (O_APPEND, O_ASYNC, O_DIRECT, + * O_NOATIME, O_NONBLOCK) these anon-inode objects honor O_APPEND, + * O_NONBLOCK and O_NOATIME. O_DIRECT is refused because the inode lacks + * FMODE_CAN_ODIRECT. Bits outside the writable set (access mode, + * CLOEXEC, O_PATH and friends) are silently dropped, as Linux drops + * them. */ - if (fd_type == FD_TIMERFD) { + if (fd_host_flag_mask(fd_snap.type) == 0) { const int setfl_mask = LINUX_O_APPEND | LINUX_O_NONBLOCK | LINUX_O_NOATIME; - pthread_mutex_lock(&fd_lock); - if (fd_table[fd].type != FD_TIMERFD || - fd_table[fd].generation != fd_snap.generation) { - pthread_mutex_unlock(&fd_lock); - return -LINUX_EBADF; - } - if ((int) arg & LINUX_O_DIRECT) { - pthread_mutex_unlock(&fd_lock); + if ((int) arg & LINUX_O_DIRECT) return -LINUX_EINVAL; - } - fd_table[fd].linux_flags = - (fd_table[fd].linux_flags & ~setfl_mask) | - ((int) arg & setfl_mask); + + /* The sweep below revalidates the generation and does nothing when + * it moved, which would report success for a write that never + * happened; a closed or reopened slot owes the guest EBADF. + */ + pthread_mutex_lock(&fd_lock); + bool live = fd_table[fd].type == fd_snap.type && + fd_table[fd].generation == fd_snap.generation; pthread_mutex_unlock(&fd_lock); + if (!live) + return -LINUX_EBADF; + + /* Every bit here is answered from the shadow, and each one is per + * open file description, so the sweep has to reach every alias + * rather than only the name the guest passed. O_NONBLOCK needs no + * second pass for that: it is in setfl_mask like the rest. + * + * The sweep revalidates the generation itself, so a close+reopen + * between the snapshot and here changes nothing and the separate + * lock-and-check this replaced is gone. + */ + fd_set_shadow_flags(fd, fd_snap.generation, setfl_mask, (int) arg); + asyncio_apply(fd, fd_snap.generation, ((int) arg & LINUX_O_ASYNC)); return 0; } + + /* A socket has no O_DIRECT to set: Linux answers EINVAL, since the + * inode has no FMODE_CAN_ODIRECT. A pipe does accept it -- that is + * packet mode, not a filesystem property -- so the reject is by type + * and not by "regular files only". Measured against qemu-aarch64: pipe + * rc=0 reported=1, socket rc=-1 errno=22, regular rc=0. + * + * Rejected before anything is applied, as Linux rejects it: setfl() + * checks O_DIRECT ahead of the flag store, so a call that fails here + * must not have landed the other bits it carried. + */ + if (((int) arg & LINUX_O_DIRECT) && fd_snap.type == FD_SOCKET) + return -LINUX_EINVAL; + host_fd_ref_t host_ref; if (host_fd_ref_open(fd, &host_ref) < 0) return -LINUX_EBADF; - int rc = - fcntl(host_ref.fd, F_SETFL, linux_to_mac_status_flags((int) arg)); + + /* An owned fd keeps O_NONBLOCK on the host whatever the guest asks; the + * request is recorded in the shadow below and the transfer paths read + * it from there. + */ + int mac_fl = linux_to_mac_status_flags((int) arg); + + /* The host flag is not the guest's to change on these: elfuse holds it + * set, either to keep the transfer non-parking or because the host fd + * is its own pipe behind a synthetic fd. The request is recorded in the + * shadow below instead. + */ + if (fd_nonblock_shadowed(fd_snap.type, fd_snap.nonblock_owned)) + mac_fl |= O_NONBLOCK; + int rc = fcntl(host_ref.fd, F_SETFL, mac_fl); if (rc < 0) { int64_t err = linux_errno(); host_fd_ref_close(&host_ref); return err; } + /* The settable bits macOS has no equivalent for are answered from the + * shadow (fd_host_flag_mask), so F_SETFL has to write them there or + * F_GETFL keeps reporting whatever open() recorded. O_DIRECT and + * O_NOATIME are those two; O_NONBLOCK and O_ASYNC have their own paths + * below. Measured before this: Linux takes O_NOATIME from 0 to 1 and + * back, elfuse stayed at 0 throughout. + */ + int shadow_setfl = (LINUX_O_DIRECT | LINUX_O_NOATIME) & + ~fd_host_flag_mask(fd_snap.type); + if (shadow_setfl) + fd_set_shadow_flags(fd, fd_snap.generation, shadow_setfl, + (int) arg); + + /* The guest's O_NONBLOCK for an owned fd lives in the shadow, which is + * what the transfer paths and F_GETFL read. It reaches every dup alias + * because Linux keeps O_NONBLOCK on the open file description. + */ + fd_apply_guest_nonblock(fd, ((int) arg & LINUX_O_NONBLOCK) != 0); + /* O_ASYNC is elfuse-managed: track the armed bit and (dis)arm the SIGIO * watcher. asyncio_apply rescans the slot under fd_lock and uses each * alias's real backing fd, not host_ref.fd (a per-syscall dup for @@ -2049,16 +2142,26 @@ int64_t sys_pipe2(guest_t *g, uint64_t fds_gva, int linux_flags) return linux_errno(); } - /* Apply O_NONBLOCK to host FDs if requested */ + /* The host fds are already nonblocking: fd_alloc owns O_NONBLOCK on a pipe + * so a transfer can report EAGAIN instead of parking a vCPU thread. Record + * what the guest asked for, which is what F_GETFL and the wait paths read. + */ + int shadow = linux_flags & (LINUX_O_CLOEXEC | LINUX_O_NONBLOCK); + fd_publish_linux_flags(guest_fds[0], shadow); + fd_publish_linux_flags(guest_fds[1], shadow); + + /* fd_alloc owns O_NONBLOCK on a pipe, so the guest's request is recorded + * above and the host fds are already nonblocking. If ownership was refused + * -- only a failing fcntl does that -- the guest's request still has to + * reach the host fd, since nothing else will answer for it. + */ if (linux_flags & LINUX_O_NONBLOCK) { - fcntl(host_fds[0], F_SETFL, O_NONBLOCK); - fcntl(host_fds[1], F_SETFL, O_NONBLOCK); + for (int i = 0; i < 2; i++) { + if (!fd_block_state(guest_fds[i]).nonblock_owned) + fd_set_nonblock(host_fds[i]); + } } - /* Propagate O_CLOEXEC if set in flags */ - fd_table[guest_fds[0]].linux_flags = linux_flags & LINUX_O_CLOEXEC; - fd_table[guest_fds[1]].linux_flags = linux_flags & LINUX_O_CLOEXEC; - int32_t fds[2] = {guest_fds[0], guest_fds[1]}; if (guest_write_small(g, fds_gva, fds, sizeof(fds)) < 0) { fd_mark_closed(guest_fds[0]); diff --git a/src/syscall/fuse.c b/src/syscall/fuse.c index 58f13ad4..e553976d 100644 --- a/src/syscall/fuse.c +++ b/src/syscall/fuse.c @@ -1816,6 +1816,9 @@ static int fuse_materialize_open_file_locked(fuse_session_t *session, if (sizeof(tmp_template) > outsz) return -LINUX_ENAMETOOLONG; + /* Not tmpfile_anon: execve needs a path to hand the loader, so this one + * keeps its name and is unlinked once the exec has taken it. + */ int tmp_fd = mkstemp(tmp_template); if (tmp_fd < 0) return linux_errno(); @@ -2403,8 +2406,14 @@ int64_t fuse_dev_read(int guest_fd, uint64_t buf_gva, uint64_t count) { + /* Take the blocking mode with the descriptor, not from a later lookup: a + * sibling reusing this fd number in between would otherwise decide how this + * read waits on a slot that is no longer the one being read. + */ host_fd_ref_t notify_ref; - if (host_fd_ref_open_io(guest_fd, ¬ify_ref) < 0) + fd_block_state_t dev_st; + uint64_t dev_gen = 0; + if (host_fd_ref_open_io_state(guest_fd, ¬ify_ref, &dev_gen, &dev_st) < 0) return -LINUX_EBADF; pthread_mutex_lock(&fuse_lock); @@ -2417,9 +2426,25 @@ int64_t fuse_dev_read(int guest_fd, fuse_session_get_locked(session); pthread_mutex_unlock(&fuse_lock); + /* The state came from the descriptor; the session came from the fd number. + * A sibling closing and reopening guest_fd between the two makes them + * describe different objects, and the read would then take its blocking + * mode from one and its queue from the other: a nonblocking replacement + * waits, a blocking one gets EAGAIN. The generation was captured with the + * descriptor, so comparing it here proves the pair belongs together. Taken + * after fuse_lock is dropped, since fd_lock orders above it. + */ + if (fd_block_state(guest_fd).generation != dev_gen) { + pthread_mutex_lock(&fuse_lock); + fuse_session_put_locked(session); + pthread_mutex_unlock(&fuse_lock); + host_fd_ref_close(¬ify_ref); + return -LINUX_EBADF; + } + pthread_mutex_lock(&session->lock); while (!session->closed && !session->queue_head) { - if (fd_table[guest_fd].linux_flags & LINUX_O_NONBLOCK) { + if (dev_st.guest_nonblock) { pthread_mutex_unlock(&session->lock); pthread_mutex_lock(&fuse_lock); fuse_session_put_locked(session); @@ -2827,12 +2852,18 @@ int fuse_dup_fd(int src_fd, int new_host_fd = snap.type == FD_FUSE_DEV ? dup(snap.host_fd) : -1; if (snap.type == FD_FUSE_DEV && new_host_fd < 0) return -1; + + /* Allocate as an alias, so the slot is published already carrying the + * description's identity and flags. Minting a fresh ofd_id and patching it + * afterwards leaves a window in which the slot is visible under an identity + * no other name shares, and an alias sweep running then skips it. + */ uint64_t alloc_gen = 0; - int guest_fd = - fixed_slot ? fd_alloc_at_relaxed(fixed_guest_fd, snap.type, new_host_fd, - fuse_fd_cleanup, &alloc_gen) - : fd_alloc_from_relaxed(min_guest_fd, snap.type, new_host_fd, - fuse_fd_cleanup, &alloc_gen); + fd_alias_spec_t spec = fd_alias_of(src_fd, &snap); + spec.linux_flags |= linux_flags; + int guest_fd = fd_alloc_alias_relaxed( + &spec, fixed_slot ? fixed_guest_fd : -1, min_guest_fd, snap.type, + new_host_fd, fuse_fd_cleanup, &alloc_gen); if (guest_fd < 0) { if (new_host_fd >= 0) close(new_host_fd); @@ -2895,13 +2926,9 @@ int fuse_dup_fd(int src_fd, if (fd_table[guest_fd].type == snap.type && fd_table[guest_fd].host_fd == new_host_fd && fd_table[guest_fd].generation == alloc_gen) { - int preserved_flags = - snap.linux_flags & - (LINUX_O_ACCMODE | LINUX_O_PATH | LINUX_O_DIRECTORY | - LINUX_O_NOFOLLOW | LINUX_O_DIRECT | LINUX_O_LARGEFILE | - LINUX_O_NONBLOCK | LINUX_O_ASYNC); - fd_table[guest_fd].linux_flags = preserved_flags | linux_flags; - fd_table[guest_fd].ofd_id = snap.ofd_id; + /* Flags and identity came with the allocation; only the fasync owner is + * left for this window to carry across. + */ fd_table[guest_fd].fasync_owner_type = snap.fasync_owner_type; fd_table[guest_fd].fasync_owner = snap.fasync_owner; if (fd_table[guest_fd].linux_flags & LINUX_O_ASYNC) diff --git a/src/syscall/inotify.c b/src/syscall/inotify.c index b7e753f2..32594015 100644 --- a/src/syscall/inotify.c +++ b/src/syscall/inotify.c @@ -31,6 +31,7 @@ #include #include #include +#include #include #include #include @@ -40,9 +41,9 @@ #include "syscall/linux-wire.h" #include "syscall/inotify.h" #include "syscall/internal.h" +#include "syscall/io.h" /* io_wait_fd_or_interrupted */ #include "syscall/path.h" -#include "runtime/thread.h" /* thread_stop_requested */ -#include "syscall/proc.h" /* proc_exit_group_requested */ +#include "syscall/proc.h" /* proc_exit_group_requested */ static void inotify_close(int guest_fd); @@ -100,7 +101,6 @@ typedef struct { int pipe_rd; /* Self-pipe read end (poll/epoll) */ int pipe_wr; /* Self-pipe write end */ int wd_counter; /* Next WD to allocate (1-based) */ - int nonblock; /* IN_NONBLOCK flag */ inotify_watch_t watches[INOTIFY_WATCHES]; /* Watch table */ uint8_t event_buf[INOTIFY_BUFSIZE]; /* Queued inotify events */ size_t event_used; /* Bytes used in event_buf */ @@ -642,12 +642,18 @@ int64_t sys_inotify_init1(int flags) inst->pipe_rd = pipefd[0]; inst->pipe_wr = pipefd[1]; inst->wd_counter = 1; /* WDs are 1-based */ - inst->nonblock = (flags & IN_NONBLOCK) ? 1 : 0; inst->event_used = 0; memset(inst->watches, 0, sizeof(inst->watches)); pthread_mutex_unlock(&inotify_lock); - fd_publish_linux_flags(gfd, (flags & IN_CLOEXEC) ? LINUX_O_CLOEXEC : 0); + /* Linux opens the inotify inode O_RDONLY (anon_inode_getfd in + * fs/notify/inotify/inotify_user.c), and O_NONBLOCK goes to the shadow + * rather than the internal pipe, which stays nonblocking so the emulation + * can do its own waiting. + */ + fd_publish_linux_flags(gfd, + ((flags & IN_CLOEXEC) ? LINUX_O_CLOEXEC : 0) | + ((flags & IN_NONBLOCK) ? LINUX_O_NONBLOCK : 0)); return gfd; } @@ -869,6 +875,12 @@ int64_t sys_inotify_rm_watch(int inotify_fd, int wd) int64_t inotify_read(int guest_fd, guest_t *g, uint64_t buf_gva, uint64_t count) { + /* Before inotify_lock: fd_guest_nonblock takes fd_lock, which orders ahead + * of this one. The guest's O_NONBLOCK lives in the fd_table shadow because + * the host fd behind an inotify fd is elfuse's own pipe. + */ + bool nonblock = fd_guest_nonblock(guest_fd); + pthread_mutex_lock(&inotify_lock); int slot = inotify_find(guest_fd); if (slot < 0) { @@ -880,7 +892,7 @@ int64_t inotify_read(int guest_fd, guest_t *g, uint64_t buf_gva, uint64_t count) /* If no buffered events, poll kqueue for new ones */ if (inst->event_used == 0) { - int n = collect_events(inst); + collect_events(inst); /* collect_events may release the lock for directory I/O; bail if the * instance was closed in that window. @@ -890,34 +902,39 @@ int64_t inotify_read(int guest_fd, guest_t *g, uint64_t buf_gva, uint64_t count) return -LINUX_EBADF; } - if (n == 0) { - if (inst->nonblock) { + /* Nothing buffered: wait for an event rather than reporting one of the + * two things a blocking inotify read cannot return. Linux blocks here + * until a watch fires; the loop this replaced polled the kqueue 300 + * times at a second each and then answered EAGAIN, and a kevent that + * woke with no event fell through to report EAGAIN as well. + * + * Looping on event_used rather than on the kevent result is what makes + * that hold. A vnode event the watch mask filters out leaves the buffer + * empty, and this thread has to go back to waiting exactly as it does + * when it lost the readiness to a sibling. + */ + while (inst->event_used == 0) { + if (nonblock) { pthread_mutex_unlock(&inotify_lock); return -LINUX_EAGAIN; } - /* Blocking read: release lock, wait on the kqueue for events. The - * self-pipe makes poll/select/epoll work, but for direct read() - * calls inotify emulation polls the kqueue with a moderate timeout - * and retry to avoid hanging indefinitely (allows signal delivery). + /* The kqueue is pollable on macOS, so the shared wait covers it: + * teardown and guest signals end this read on the same terms as + * every other synthetic reader, and no vCPU parks in kevent. */ int kq_fd = inst->kq_fd; pthread_mutex_unlock(&inotify_lock); + int64_t waited = io_wait_fd_or_interrupted(kq_fd, POLLIN); + if (waited < 0) + return waited; + struct kevent kev; - struct timespec ts = {1, 0}; /* 1 second per attempt */ - int nev = 0; - for (int attempt = 0; attempt < 300; attempt++) { - nev = kevent(kq_fd, NULL, 0, &kev, 1, &ts); - if (nev > 0) - break; - if (nev < 0 && errno != EINTR) - return linux_errno(); - if (thread_stop_requested()) - return -LINUX_EINTR; - } - if (nev <= 0) - return -LINUX_EAGAIN; + struct timespec collect = {0, 0}; + int nev = kevent(kq_fd, NULL, 0, &kev, 1, &collect); + if (nev < 0 && errno != EINTR) + return linux_errno(); /* Re-acquire lock and re-validate slot */ pthread_mutex_lock(&inotify_lock); @@ -927,6 +944,13 @@ int64_t inotify_read(int guest_fd, guest_t *g, uint64_t buf_gva, uint64_t count) } inst = &inotify_state[slot]; + /* No event to read after all: a sibling on this same inotify fd + * took the one the wait reported. kev holds nothing, so wait again + * rather than decoding it. + */ + if (nev <= 0) + continue; + /* Process the received event (same named-directory diff as the * non-blocking collect path). */ diff --git a/src/syscall/internal.h b/src/syscall/internal.h index a6587029..5e7ac6a0 100644 --- a/src/syscall/internal.h +++ b/src/syscall/internal.h @@ -154,6 +154,163 @@ void fdtable_init(void); */ int fd_alloc(int type, int host_fd, void (*cleanup)(int)); +/* The status bits that belong to the open file description rather than to the + * fd slot naming it, so a dup carries them to the alias and an alias sweep may + * write them to every slot sharing an ofd_id. + * + * One list because four hand-written copies of it had already drifted into two + * memberships, and neither carried O_APPEND or O_NOATIME -- which F_SETFL on a + * timerfd writes to the shadow, so a dup of one reported the flag on one name + * for it and not the other. The mode and the open-time bits are here for the + * same reason O_NONBLOCK is: dup(2) gives the alias the same description, so + * whatever the shadow answers for one name it has to answer for all of them. + */ +#define FD_DESCRIPTION_FLAGS \ + (LINUX_O_ACCMODE | LINUX_O_PATH | LINUX_O_DIRECTORY | LINUX_O_NOFOLLOW | \ + LINUX_O_DIRECT | LINUX_O_LARGEFILE | LINUX_O_NONBLOCK | LINUX_O_ASYNC | \ + LINUX_O_APPEND | LINUX_O_NOATIME) + +/* What a new fd slot inherits from an open file description that already + * exists. Every field is a separate claim, and a site has to make each one on + * purpose: build this with one of the constructors below rather than by + * initializer, so "not set" cannot quietly mean "not foreign, not owned, fresh + * identity" the way a partial fd_entry_t did. Four of the seven alias sites + * shipped a defect while it did. + * + * Both ownership fields matter. A dup of the launcher's stdin is typed + * FD_REGULAR, so a type test cannot see what it aliases, and taking O_NONBLOCK + * ownership there leaves the launching shell's terminal nonblocking after + * elfuse exits; carrying foreign_description keeps that fact through a dup of a + * dup, and through fork, which rebuilds the whole table from descriptions the + * parent already holds. Carrying nonblock_owned also saves the probe: an alias + * shares the description, so its answer cannot differ from its source's. + */ +typedef struct { + uint64_t ofd_id; /* 0: mint a fresh description identity */ + int linux_flags; /* guest-visible flags to publish with the slot */ + bool foreign_description; + bool nonblock_owned; + + /* The live source to re-read under fd_lock, or -1 for a site with no + * in-process source. The fields above are a snapshot the caller took before + * the allocation, and an F_SETFL landing in between sweeps the aliases that + * exist at that moment -- which does not include the one being built. + * Publishing from the snapshot then gives the new name a shadow the rest of + * the description has already moved past: a dup of a blocking pipe reports + * blocking through F_GETFL and waits in io_xfer while every other name for + * it is nonblocking. + * + * src_generation is what makes the re-read safe. A close+reopen in the same + * window puts a different description behind the same number, and + * re-reading then would copy identity and flags from a file the caller + * never saw. When the generation has moved the snapshot is used as-is, + * which is the behaviour this field replaces rather than a new risk. + */ + int src_guest_fd; + uint64_t src_generation; +} fd_alias_spec_t; + +/* A full alias: same description, same identity, same status flags. dup(2), + * dup2, dup3, F_DUPFD and the Rosetta socket upgrade. + * + * src_guest_fd is the number the snapshot came from, so the allocator can take + * the description state again under the lock that publishes the new slot. Pass + * -1 for a source that is no longer addressable by number -- the Rosetta socket + * upgrade rebuilds the very slot it snapshotted, so re-reading it would find + * the replacement it is in the middle of installing. + */ +static inline fd_alias_spec_t fd_alias_of(int src_guest_fd, + const fd_entry_t *src) +{ + return (fd_alias_spec_t) { + .ofd_id = src->ofd_id, + .linux_flags = src->linux_flags & FD_DESCRIPTION_FLAGS, + .foreign_description = src->foreign_description, + .nonblock_owned = src->nonblock_owned, + .src_guest_fd = src_guest_fd, + .src_generation = src->generation, + }; +} + +/* The same host description, but an identity of its own: opening a magic link + * (/proc/self/fd/N, /dev/stdin) dups the descriptor, so the host flags are + * shared and must not be touched, while Linux gives the result a new open file + * description. Sweeping the source's aliases from it would be wrong. + */ +static inline fd_alias_spec_t fd_alias_host_shared(const fd_entry_t *src) +{ + return (fd_alias_spec_t) { + .src_guest_fd = -1, + .foreign_description = src->foreign_description, + .nonblock_owned = src->nonblock_owned, + }; +} + +/* The same identity, with the flags the caller has already worked out: a dup of + * a synthetic fd, whose host fd is elfuse's own pipe or kqueue. Not foreign + * (elfuse opened it) and not owned (fd_nonblock_shadowed answers from the type + * for those), so the alias sweeps find it and nothing else changes. + * + * Passing the flags here rather than writing them after the allocation is the + * point: a slot published first and patched second is observable in between + * with the wrong flags, which for an EFD_NONBLOCK eventfd means a sibling + * reading it as blocking. + */ +static inline fd_alias_spec_t fd_alias_identity(uint64_t ofd_id, + int linux_flags) +{ + return (fd_alias_spec_t) { + .src_guest_fd = -1, .ofd_id = ofd_id, .linux_flags = linux_flags}; +} + +/* Ownership facts only, for the two sites with no source slot to point at: a + * descriptor arriving over SCM_RIGHTS, and the fork rebuild, which remaps + * identities itself once every slot exists. + */ +static inline fd_alias_spec_t fd_alias_carried(bool foreign, bool owned) +{ + return (fd_alias_spec_t) { + .src_guest_fd = -1, + .foreign_description = foreign, + .nonblock_owned = owned, + }; +} + +/* Allocate a slot that inherits from `spec`, applying the inheritance inside + * the same fd_lock window that publishes the slot: a close+reopen in a gap + * would otherwise take the alias's identity and be swept as though it shared a + * description it never saw. + * + * fixed_fd >= 0 asks for that exact slot; otherwise the lowest free slot at or + * above minfd. The _relaxed variant skips the lock for the generation read when + * this is the only active thread, matching fd_alloc_from_relaxed. + */ +int fd_alloc_alias(const fd_alias_spec_t *spec, + int type, + int host_fd, + void (*cleanup)(int)); +int fd_alloc_alias_at(const fd_alias_spec_t *spec, + int fd, + int type, + int host_fd, + void (*cleanup)(int), + uint64_t *out_gen); +int fd_alloc_alias_relaxed(const fd_alias_spec_t *spec, + int fixed_fd, + int minfd, + int type, + int host_fd, + void (*cleanup)(int), + uint64_t *out_gen); +int fd_alloc_alias_dir(const fd_alias_spec_t *spec, + int fixed_fd, + int minfd, + int type, + int host_fd, + void (*cleanup)(int), + void *dir, + int linux_flags); + /* Allocate the lowest available FD and publish type, host_fd, dir, and * linux_flags in one fd_lock critical section, so the slot never becomes * visible to a concurrent close/scan as type-set-but-dir-NULL. For fds (epoll) @@ -274,11 +431,84 @@ int fd_snapshot_and_dup(int guest_fd, fd_entry_t *out); */ int fd_get_type(int guest_fd); -/* True when a host read/write on this guest fd may block (pipe, socket, fifo, - * char/tty). Regular files and directories never block. Callers use this to - * decide whether to route a blocking I/O through the interruptible wait path. +/* The fields a transfer needs to decide how to run, read in one go. type is + * FD_CLOSED when the slot is closed or out of range. guest_nonblock is what the + * guest asked for, which on an owned fd is the only place it is recorded. seals + * rides along so a write path can reject a sealed memfd from the state it + * pinned, rather than from a second lookup that may describe another file. */ -bool fd_can_block(int guest_fd); +typedef struct { + int type; + uint64_t generation; + unsigned seals; + bool can_block; + bool nonblock_owned; + bool guest_nonblock; +} fd_block_state_t; + +fd_block_state_t fd_block_state(int guest_fd); + +/* Set or clear the guest's O_NONBLOCK shadow on every slot sharing guest_fd's + * open file description, anchored on guest_fd's generation so a close+reopen in + * the window changes nothing. O_NONBLOCK is a per-description flag on Linux, so + * a dup alias has to observe what the original asked for. On an fd whose + * O_NONBLOCK elfuse owns (see fd_init_entry) the host flag can no longer carry + * that, since elfuse holds it set, which is why the shadow has to be walked by + * hand. Call fn for every fd sharing guest_fd's open file description, + * including guest_fd itself, or not at all when the slot moved under the caller + * (a close+reopen that reused the number). The caller must hold fd_lock, and fn + * runs under it: the sweeps this serves mutate per-description state, and + * dropping the lock between finding an alias and touching it would let a + * sibling close retire the fd in between. + * + * Per-description state has no home of its own in this tree. O_NONBLOCK, + * O_ASYNC and the SIGIO owner all live per fd entry and are kept in step by + * sweeping the aliases, and this is the one place that knows how. + * + * The callers hold fd_lock; the sweep does not take it. Every writer of + * per-description state goes through here, so the lock covers the whole sweep + * rather than each slot in turn. + */ +void fd_for_each_alias_locked(int guest_fd, + uint64_t generation, + void (*fn)(int guest_fd, void *ctx), + void *ctx); + +/* Apply the bits of value selected by mask to the guest-visible status flags of + * every fd sharing guest_fd's open file description. + * + * Every bit answered from the shadow belongs to the description, not to the + * slot, so a change through one alias has to reach the rest: F_GETFL on a dup + * of a timerfd reported stale O_APPEND and O_NOATIME while only O_NONBLOCK was + * being swept. + */ +void fd_set_shadow_flags(int guest_fd, + uint64_t generation, + int mask, + int value); + +/* The guest's O_NONBLOCK for a fd whose host description is elfuse's own: a + * synthetic fd is backed by a pipe or a kqueue held nonblocking so the + * emulation can drive the waiting itself, so the host flag says nothing about + * what the guest asked for and the shadow is the only record. + * + * Every synthetic reader answers from here: eventfd, signalfd, timerfd, inotify + * and netlink. When one of them kept its own copy of the flag instead, a guest + * that set O_NONBLOCK with fcntl after creating the fd had it reported back + * correctly and then blocked forever on an empty read. + * + * Takes fd_lock, so callers must not already hold a lock that orders after it + * (sfd_lock=5a, inotify_lock); read the flag before taking those. + */ +bool fd_guest_nonblock(int guest_fd); + +/* Route a guest O_NONBLOCK request (F_SETFL, ioctl FIONBIO) to the shadow when + * elfuse owns the host flag on this fd. + * + * Returns false when it does not, and the caller should apply the request to + * the host fd itself. + */ +bool fd_apply_guest_nonblock(int guest_fd, bool on); /* Publish linux_flags for a guest fd under fd_lock. Use after fd_alloc when the * creating syscall needs to set linux_flags atomically with respect to a @@ -318,6 +548,76 @@ static inline bool fd_type_is_synthetic(int type) type == FD_EPOLL; } +/* The status bits the host description is authoritative for. Everything outside + * this mask is answered from the shadow, which is the inversion of how F_GETFL + * used to read: it asked the host and then overrode the answer bit by bit, + * eight special cases deep, taking the access mode from the shadow in two + * separate places for two disjoint type sets. + * + * Zero means the host has nothing to say, which is the honest answer for a type + * elfuse emulates whole: the fd behind it is elfuse's own pipe or kqueue, so + * F_GETFL cannot ask it what the guest opened and F_SETFL must not tell it. + * That second half matters -- a kqueue rejects fcntl(F_SETFL), which is why + * timerfd already had a hand-written branch to skip the host call while + * signalfd and inotify fell through to it. + * + * O_NONBLOCK is not decided here because it is not a property of the type + * alone: fd_nonblock_shadowed answers it per fd, and F_GETFL clears the bit + * from this mask when it does. + */ +static inline int fd_host_flag_mask(int type) +{ + /* An fd elfuse serves out of its own descriptor: nothing to ask. */ + if (fd_type_is_synthetic(type) || type == FD_FUSE_DEV || + type == FD_FUSE_FILE || type == FD_FUSE_DIR) + return 0; + + /* Bits the host description never carries for anyone: elfuse tracks O_ASYNC + * itself (it is never armed on the host fd), and the open-time bits are + * Linux spellings macOS has no equivalent for. + */ + int mask = + ~(LINUX_O_PATH | LINUX_O_DIRECTORY | LINUX_O_NOFOLLOW | LINUX_O_DIRECT | + LINUX_O_LARGEFILE | LINUX_O_ASYNC | LINUX_O_NOATIME); + + /* And the access mode, for the types elfuse opens on the host with a mode + * of its own choosing: an O_PATH or directory fd is opened read-only + * whatever the guest asked for. A pipe, socket or inherited stdio really + * was opened the way the guest sees it, so the host answers for those. + */ + switch (type) { + case FD_REGULAR: + case FD_DIR: + case FD_PATH: + case FD_URANDOM: + mask &= ~LINUX_O_ACCMODE; + break; + default: + break; + } + return mask; +} + +/* Bits the shadow holds that F_GETFL must never report: CLOEXEC is a descriptor + * flag, answered by F_GETFD, and Linux does not surface it here. + */ +#define FD_GETFL_HIDDEN (LINUX_O_CLOEXEC) + + +/* True when fd_entry_t.linux_flags, not the host description, is where this + * fd's O_NONBLOCK lives. Two ways to get there: elfuse owns the host flag so a + * transfer can report EAGAIN instead of parking a vCPU (fd_init_entry), or the + * host fd is elfuse's own pipe or kqueue and the guest is not talking to it at + * all, in which case the host flag has to stay as the emulation needs it. + * + * Everything that answers or records the flag agrees through this: F_GETFL, + * F_SETFL, ioctl FIONBIO, the transfer paths, and the synthetic readers. + */ +static inline bool fd_nonblock_shadowed(int type, bool nonblock_owned) +{ + return nonblock_owned || fd_type_is_synthetic(type); +} + /* Look up a guest FD and return a dup'd host fd owned by the caller. * Thread-safe: dup is performed under fd_lock. * @@ -325,6 +625,14 @@ static inline bool fd_type_is_synthetic(int type) */ int fd_to_host_dup(int guest_fd); +/* The same dup, with the slot's transfer classification taken in the same + * fd_lock window. A caller that pins a host fd and then asks what kind of fd it + * was has a gap: the pin keeps the description alive, but the guest fd number + * can be reused for another kind of object in between, and the answer then + * describes something the caller is not holding. + */ +int fd_to_host_dup_state(int guest_fd, fd_block_state_t *st_out); + /* Mark an FD slot as closed (set type = FD_CLOSED and update bitmap). Does NOT * close the host FD or free type-specific resources (DIR*, epoll instance); * caller must do that first. @@ -457,6 +765,35 @@ static inline int host_fd_ref_open(guest_fd_t guest_fd, host_fd_ref_t *ref) return 0; } +/* Pin the host fd and classify the slot together, so a transfer acts on the + * object it is holding rather than on whatever took that fd number afterwards. + * With one active thread there is no mutator and the two relaxed reads are + * already consistent; with siblings alive both come from one fd_lock window. + */ +static inline int host_fd_ref_open_state(guest_fd_t guest_fd, + host_fd_ref_t *ref, + fd_block_state_t *st_out) +{ + ref->fd = -1; + ref->owned = false; + + if (thread_is_single_active()) { + int host_fd = fd_to_host(guest_fd); + if (host_fd < 0) + return -1; + *st_out = fd_block_state(guest_fd); + ref->fd = host_fd; + return 0; + } + + int host_fd = fd_to_host_dup_state(guest_fd, st_out); + if (host_fd < 0) + return -1; + ref->fd = host_fd; + ref->owned = true; + return 0; +} + static inline void host_fd_ref_close(host_fd_ref_t *ref) { /* Preserve errno across close(2). Callers commonly invoke this on the @@ -482,28 +819,20 @@ static inline int host_dirfd_ref_open(guest_fd_t dirfd, host_fd_ref_t *ref) return host_fd_ref_open(dirfd, ref); } -/* Open a host fd reference, rejecting O_PATH (FD_PATH) entries with -EBADF. Use - * this for syscalls that operate on the underlying file -- read/write, lseek, - * ftruncate, fsync/fdatasync, flock, fsetxattr/fremovexattr, ioctl, etc. Linux - * returns EBADF on those calls when the fd was opened O_PATH; the host fd here - * is a plain O_RDONLY descriptor, so without this gate the host call would - * silently succeed and diverge from Linux semantics. - * - * Calls that are explicitly allowed on O_PATH (fstat, fstatfs, fchdir, close, - * dup, fcntl get/set CLOEXEC, *at() dirfd) keep using host_{fd,dirfd}_ref_open - * helpers above. +/* The transfer classification carried by an entry already in hand. Callers that + * snapshot and dup in one window get their state from here rather than looking + * the slot up again, which is what makes the two describe the same object. */ -static inline int64_t host_fd_ref_open_io(guest_fd_t guest_fd, - host_fd_ref_t *ref) +static inline fd_block_state_t fd_block_state_of(const fd_entry_t *e) { - fd_entry_t snap; - if (!fd_snapshot(guest_fd, &snap)) - return -LINUX_EBADF; - if (snap.type == FD_PATH) - return -LINUX_EBADF; - if (host_fd_ref_open(guest_fd, ref) < 0) - return -LINUX_EBADF; - return 0; + return (fd_block_state_t) { + .type = e->type, + .generation = e->generation, + .seals = e->seals, + .can_block = e->can_block, + .nonblock_owned = e->nonblock_owned, + .guest_nonblock = (e->linux_flags & LINUX_O_NONBLOCK) != 0, + }; } /* host_fd_ref_open_io() that also reports the fd generation the reference was @@ -520,15 +849,30 @@ static inline int64_t host_fd_ref_open_io(guest_fd_t guest_fd, * * *out_gen is 0 on failure. * + * st_out, when given, receives the transfer classification taken in that same + * window, which is what lets a transfer act on the object it is holding rather + * than on whatever took the fd number afterwards. Both out-params are optional. + * * Returns 0 on success, -LINUX_EBADF otherwise. */ -static inline int64_t host_fd_ref_open_io_gen(guest_fd_t guest_fd, - host_fd_ref_t *ref, - uint64_t *out_gen) +static inline int64_t host_fd_ref_open_io_state(guest_fd_t guest_fd, + host_fd_ref_t *ref, + uint64_t *out_gen, + fd_block_state_t *st_out) { ref->fd = -1; ref->owned = false; - *out_gen = 0; + if (st_out) + *st_out = (fd_block_state_t) {.type = FD_CLOSED}; + + /* Both out-params are optional. Writing through out_gen unconditionally is + * a null dereference for a caller that wants only the classification, and + * the compiler is entitled to assume that cannot happen: clang proved the + * UB and compiled the whole of fuse_dev_read to a single brk #1, so every + * FUSE read trapped before doing anything. + */ + if (out_gen) + *out_gen = 0; fd_entry_t snap; if (thread_is_single_active()) { @@ -538,8 +882,11 @@ static inline int64_t host_fd_ref_open_io_gen(guest_fd_t guest_fd, if (!fd_snapshot(guest_fd, &snap) || snap.type == FD_PATH || snap.host_fd < 0) return -LINUX_EBADF; + if (st_out) + *st_out = fd_block_state_of(&snap); ref->fd = snap.host_fd; - *out_gen = snap.generation; + if (out_gen) + *out_gen = snap.generation; return 0; } @@ -552,12 +899,46 @@ static inline int64_t host_fd_ref_open_io_gen(guest_fd_t guest_fd, errno = saved_errno; return -LINUX_EBADF; } + if (st_out) + *st_out = fd_block_state_of(&snap); ref->fd = host_fd; ref->owned = true; - *out_gen = snap.generation; + if (out_gen) + *out_gen = snap.generation; return 0; } +/* Open a host fd reference, rejecting O_PATH (FD_PATH) entries with -EBADF. Use + * this for syscalls that operate on the underlying file -- read/write, lseek, + * ftruncate, fsync/fdatasync, flock, fsetxattr/fremovexattr, ioctl, etc. Linux + * returns EBADF on those calls when the fd was opened O_PATH; the host fd here + * is a plain O_RDONLY descriptor, so without this gate the host call would + * silently succeed and diverge from Linux semantics. + * + * Calls that are explicitly allowed on O_PATH (fstat, fstatfs, fchdir, close, + * dup, fcntl get/set CLOEXEC, *at() dirfd) keep using host_{fd,dirfd}_ref_open + * helpers above. + */ +static inline int64_t host_fd_ref_open_io(guest_fd_t guest_fd, + host_fd_ref_t *ref) +{ + /* Hand-rolling the FD_PATH check here would take fd_lock twice and reject + * on a snapshot the pin does not have to agree with. One window, one + * decision; the caller just does not want what it classified. + */ + return host_fd_ref_open_io_state(guest_fd, ref, NULL, NULL); +} + +/* The generation-only spelling, for callers that do not run a transfer with the + * descriptor they pin. + */ +static inline int64_t host_fd_ref_open_io_gen(guest_fd_t guest_fd, + host_fd_ref_t *ref, + uint64_t *out_gen) +{ + return host_fd_ref_open_io_state(guest_fd, ref, out_gen, NULL); +} + /* A guest timeout at or above this many seconds means "wait indefinitely", and * the wait path spells indefinite as timeout_ms = -1. * diff --git a/src/syscall/io.c b/src/syscall/io.c index 825440e1..39d0cf4d 100644 --- a/src/syscall/io.c +++ b/src/syscall/io.c @@ -25,6 +25,7 @@ #include #include #include +#include #include #include #include @@ -36,8 +37,8 @@ #include "core/rosetta.h" #include "core/shim-globals.h" #include "hvutil.h" +#include "runtime/futex.h" /* futex_interrupt_consume, in the tty drain */ #include "runtime/procemu.h" -#include "runtime/futex.h" #include "runtime/thread.h" #include "syscall/linux-wire.h" @@ -59,6 +60,7 @@ /* Linux terminal struct types. */ /* Linux struct winsize (same layout as macOS) */ + typedef struct { uint16_t ws_row, ws_col, ws_xpixel, ws_ypixel; } linux_winsize_t; @@ -192,6 +194,32 @@ static int64_t linux_siocgifhwaddr(guest_t *g, uint64_t arg) return 0; } +/* One backoff step, without asking whether anything wants this thread out. + * + * Split from io_retry_backoff for the callers that cannot honor an interrupt: + * the two stream copiers hold a chunk already drained out of an unrewindable + * input, so abandoning the write drops the guest's bytes and the only thing + * left is to keep trying. Doing that at full speed costs a whole core -- a + * splice into a full pipe with a pending SIGALRM burned 6.9 s of CPU in 8 s of + * wall clock -- because io_xfer answers the pending signal immediately and + * every round is a bare retry. The sleep bounds the cost without changing who + * decides to stop. + */ +static void io_backoff_sleep(unsigned *backoff_us) +{ + if (*backoff_us == 0) { + sched_yield(); + *backoff_us = IO_RETRY_BACKOFF_START_US; + return; + } + + unsigned us = *backoff_us; + usleep(us); + + us *= 2; + *backoff_us = us > IO_RETRY_BACKOFF_MAX_US ? IO_RETRY_BACKOFF_MAX_US : us; +} + int64_t io_retry_backoff(unsigned *backoff_us) { /* Materialize an expired guest interval timer first. ITIMER_REAL is virtual @@ -216,17 +244,7 @@ int64_t io_retry_backoff(unsigned *backoff_us) * the current scheduling quantum is the common case, and a sleep would turn * it into a timer round trip that macOS rounds up well past the request. */ - if (*backoff_us == 0) { - sched_yield(); - *backoff_us = IO_RETRY_BACKOFF_START_US; - return 0; - } - - unsigned us = *backoff_us; - usleep(us); - - us *= 2; - *backoff_us = us > IO_RETRY_BACKOFF_MAX_US ? IO_RETRY_BACKOFF_MAX_US : us; + io_backoff_sleep(backoff_us); return 0; } @@ -270,17 +288,24 @@ int64_t io_wait_fd_or_interrupted(int host_fd, short events) /* Ignored/default-ignore signals do not interrupt; restartable handlers * still need to run promptly through the syscall epilogue. * - * The futex interrupt is left alone on the leader-work path, which the - * single ||-chain this replaced did by short-circuiting. It is a - * process-wide one-shot standing in for SIGCHLD when the last - * clone-thread exits, and consuming it here would hand its EINTR to a - * restart that swallows it, so no thread would ever observe the edge. - * Left set, it is still there for the next interruptible wait in any - * thread, which is the ordering this wants: a ready fd is reported - * first and the one-shot keeps until something waits again. + * The futex interrupt one-shot is deliberately not consulted here. It + * is process-wide, raised when the last clone-thread exits to unstick a + * futex wait that would otherwise miss the edge, and every caller of + * this wait is a data transfer rather than a futex waiter. Consuming it + * here truncates the transfer: a 1 MiB blocking write that has moved + * one pipe buffer reports 65536 and the guest, which asked for a + * blocking write and saw no signal, is handed a short count Linux would + * never produce. Measured, not theorized -- a sibling thread exiting + * before an unrelated big write was enough, and the reader waiting for + * the rest of the stream deadlocked. + * + * Nothing is lost by leaving it set. Teardown is answered above by + * thread_stop_requested, a real signal by the test below, and the + * wakeup pipe plus thread_interrupt_all -- which the one-shot's raiser + * sends alongside it -- already break this poll. The edge stays where + * it belongs, for the next futex or poll wait that is entitled to it. */ - if ((!leader_only && futex_interrupt_consume()) || - signal_pending_interruption(NULL)) + if (signal_pending_interruption(NULL)) return -LINUX_EINTR; /* Bounded wait even when the wakeup pipe exists: the pipe is a @@ -302,27 +327,307 @@ int64_t io_wait_fd_or_interrupted(int host_fd, short events) } } -/* Route a blocking read/write on a fd that can block (pipe, socket, fifo, - * char/tty) through the interruptible wait so the vCPU thread stays reachable - * by hv_vcpus_exit + the wakeup pipe. No-op for regular files, nonblocking fds, - * and direction mismatches (a POLLIN wait on an O_WRONLY fd would hang; the - * read then fails EBADF like Linux). - * - * Returns 0 to proceed or a negative Linux errno (EINTR) to abort. +/* True when a transfer in this direction can work on a description opened with + * these status flags. A POLLIN wait on an O_WRONLY description would never + * return, and the transfer that follows reports EBADF the way Linux does. */ -static int64_t io_block_wait(int fd, int host_fd, short events) +static bool io_fl_direction_ok(int fl, short events) +{ + int acc = fl & O_ACCMODE; + if ((events & POLLIN) && acc == O_WRONLY) + return false; + if ((events & POLLOUT) && acc == O_RDONLY) + return false; + return true; +} + +/* True when a transfer on a description elfuse does not own has to be gated on + * a wait: the guest has not asked for O_NONBLOCK and the direction matches the + * access mode. Only inherited stdio and its aliases reach here, and only they + * pay the fcntl: an owned fd answers from the shadow and a socket is asked + * later, once its transfer has actually reported EAGAIN. + */ +static bool io_foreign_should_block(int host_fd, short events) { - if (!fd_can_block(fd)) - return 0; int fl = fcntl(host_fd, F_GETFL); if (fl < 0 || (fl & O_NONBLOCK)) + return false; + return io_fl_direction_ok(fl, events); +} + +/* After a transfer reported EAGAIN on an fd whose transfer cannot block: does + * the guest's mode say to wait and retry, or to report it? + * + * Asking here rather than before the transfer is what keeps a host call off + * every successful read and write. An owned fd answers from the shadow with no + * host call at all, which now includes sockets; only a description elfuse did + * not open costs an fcntl, and only on the path that was going to wait anyway. + */ +static bool io_eagain_should_wait(const fd_block_state_t *st, int host_fd) +{ + if (st->nonblock_owned) + return !st->guest_nonblock; + return sock_op_should_block(st, host_fd, 0); +} + +/* One transfer attempt, in the form that reports EAGAIN rather than parking the + * caller. + * + * Sockets take a per-call MSG_DONTWAIT, which leaves the open file description + * alone. Everything else relies on elfuse owning O_NONBLOCK on the host fd + * (fd_init_entry); macOS has no per-call equivalent for a pipe and no private + * open file description to borrow either, since dup shares the status flags + * (measured) and a pipe has no path to reopen. The one kind of fd elfuse does + * not own is inherited stdio, where this blocks: the loop above it goes round + * again only if a host signal truncated the transfer, which is the residual + * parking window TODO.md records for those three descriptors. + */ +static ssize_t io_xfer_once(int host_fd, + bool is_socket, + bool is_read, + struct iovec *iov, + int iovcnt) +{ + if (is_socket) { + /* Every read/write caller has one buffer, and the flat form skips a + * msghdr the kernel would only have to walk back. + */ + if (iovcnt == 1) + return is_read ? recv(host_fd, iov->iov_base, iov->iov_len, + MSG_DONTWAIT) + : send(host_fd, iov->iov_base, iov->iov_len, + MSG_DONTWAIT); + struct msghdr msg = {.msg_iov = iov, .msg_iovlen = iovcnt}; + return is_read ? recvmsg(host_fd, &msg, MSG_DONTWAIT) + : sendmsg(host_fd, &msg, MSG_DONTWAIT); + } + if (iovcnt == 1) + return is_read ? read(host_fd, iov->iov_base, iov->iov_len) + : write(host_fd, iov->iov_base, iov->iov_len); + return is_read ? readv(host_fd, iov, iovcnt) : writev(host_fd, iov, iovcnt); +} + +/* Drop the entries a partial transfer already moved and trim the first + * survivor. + * + * Returns how many entries are now spent, so the caller resumes at iov + spent + * with iovcnt - spent entries. + */ +static int iov_advance(struct iovec *iov, int iovcnt, size_t moved) +{ + /* The index arithmetic is proved in proved/iov.h, which is where the two + * facts this trim depends on come from: the index is in range, and the + * remainder is strictly inside the entry it names, so neither subtracting + * it from iov_len nor adding it to iov_base can leave the entry. What is + * left here is the pointer bump, which no contract in this tree can carry: + * iov_base points into guest memory whose extent the analyzer cannot name. + */ + size_t rem = 0; + int spent = iov_advance_index(iov, iovcnt, moved, &rem); + if (spent < iovcnt) { + iov[spent].iov_base = (char *) iov[spent].iov_base + rem; + iov[spent].iov_len -= rem; + } + return spent; +} + + +int64_t io_xfer(int fd, + int host_fd, + short events, + struct iovec *iov, + int iovcnt, + ssize_t *out, + const fd_block_state_t *pinned) +{ + bool is_read = (events & POLLIN) != 0; + fd_block_state_t st = *pinned; + bool is_socket = st.type == FD_SOCKET; + + /* Nothing that can block: a regular file, a directory, or a closed slot on + * its way to EBADF. One host call, nothing asked before it. + */ + if (st.type == FD_CLOSED || !st.can_block) { + *out = io_xfer_once(host_fd, is_socket, is_read, iov, iovcnt); return 0; - int acc = fl & O_ACCMODE; - if ((events & POLLIN) && acc == O_WRONLY) + } + + /* An owned fd transfers without blocking, so the transfer runs first and + * answers the readiness question itself. A description elfuse does not own + * has a transfer that really blocks, so it keeps the gate in front of it. + * + * Ownership rather than type: a socket used to be counted as nonblocking on + * the strength of the MSG_DONTWAIT that io_xfer_once passes, and macOS does + * not honour that on AF_UNIX. A socket elfuse opened is now owned and lands + * on the left of this; one that arrived over SCM_RIGHTS is not owned, and + * belongs on the right with the other foreign descriptions. + */ + bool nb_transfer = st.nonblock_owned; + bool wait_first = false; + if (!nb_transfer) { + if (!io_foreign_should_block(host_fd, events)) { + *out = io_xfer_once(host_fd, is_socket, is_read, iov, iovcnt); + return 0; + } + wait_first = true; + } + + /* Sum through the proved add rather than by hand: the total feeds the + * completion test below, and iov_total_add is where this tree states that + * an iovec sum cannot carry past SSIZE_MAX. Every caller reaching here has + * already been clamped, so the reject is unreachable today -- which is the + * same "unreachable by provenance rather than by construction" the header + * was written about. + */ + uint64_t want = 0; + for (int i = 0; i < iovcnt; i++) { + if (!iov_total_add(want, iov[i].iov_len, &want)) + return -LINUX_EINVAL; + } + + /* Nothing to move: read(fd, buf, 0) and write(fd, buf, 0) report 0 on a + * blocking fd rather than waiting for one. Every caller guards this today, + * and the wait below would never end if one stopped. + */ + if (want == 0) { + *out = 0; return 0; - if ((events & POLLOUT) && acc == O_RDONLY) + } + + /* Every way a round can end badly leaves through the tail below, which + * states the reporting rule once: a partial count outranks the failure, the + * way an interrupted write(2) reports what it moved. `fail` carries a + * negative Linux errno for the failures elfuse names itself; a zero `fail` + * with a negative `xfer_ret` means the host transfer failed and its own + * return and errno are the answer. Both are set only on the round that + * breaks, so neither can go stale. + */ + ssize_t total = 0, xfer_ret = 0; + int xfer_errno = 0; + int64_t fail = 0; + unsigned backoff = 0, spins = 0; + for (;;) { + if (wait_first) { + int64_t waited = io_wait_fd_or_interrupted(host_fd, events); + if (waited < 0) { + /* Interrupted, by a guest signal or by elfuse's own teardown + * and execve-handoff wakes. + */ + fail = waited; + break; + } + } + + ssize_t ret = io_xfer_once(host_fd, is_socket, is_read, iov, iovcnt); + if (ret < 0) { + /* Everything below can call something that sets errno -- the pty + * lookup takes two locks, and the socket case of + * io_eagain_should_wait runs an fcntl -- so hold the transfer's own + * errno and put it back on the paths that report it. + */ + xfer_ret = ret; + xfer_errno = errno; + if (xfer_errno != EAGAIN) + break; + + /* Nothing there, and the pty this reads from has lost every slave: + * Linux answers EIO, and waiting cannot help because elfuse's own + * keepalive slave holds the pty open, so no hangup will ever arrive + * to end the wait. A pty master is a char device, so pipes and + * sockets -- every contended transfer -- skip the lookup, which + * costs two locks and a table scan to answer "no". + */ + if (is_read && st.type != FD_PIPE && !is_socket && + proc_pty_master_hung_up(fd, st.generation)) { + fail = -LINUX_EIO; + break; + } + + /* The guest asked for a nonblocking fd, so EAGAIN is the answer + * rather than something to wait out. + */ + if (nb_transfer && !io_eagain_should_wait(&st, host_fd)) + break; + + /* The bytes the transfer wanted went to somebody else, or never + * arrived. Wait for the next lot. + * + * The wait reports ready on any revents, POLLHUP and POLLERR + * included, so an fd whose readiness a nonblocking transfer keeps + * answering with EAGAIN would spin here at the cost of a whole + * core. Back off once that repeats, which also puts an interrupt + * check in the loop. + */ + wait_first = true; + if (++spins >= IO_XFER_SPIN_LIMIT) { + int64_t rc = io_retry_backoff(&backoff); + if (rc < 0) { + fail = rc; + break; + } + } + continue; + } + spins = 0; + backoff = 0; + xfer_ret = 0; + + total += ret; + + /* A read stops on the first return: Linux gives a reader whatever has + * arrived rather than waiting for the whole request. + */ + if (is_read || ret == 0 || (uint64_t) total == want) + break; + + /* A writer that asked for a nonblocking fd gets the partial count. + * + * A socket used to stop here unconditionally, one clause above, because + * nothing maintained its shadow: st.guest_nonblock read false for every + * socket including one the guest had set nonblocking, so the general + * test could not be trusted and a nonblocking socket write that filled + * the buffer would have waited for a reader instead of reporting what + * it moved (tests/test-socket-shortwrite.c). Now that sockets are owned + * the shadow is true for them and the general test covers both: a + * nonblocking socket write reports its short count here, and a blocking + * one goes round again for the remainder, which is what Linux does and + * what the old unconditional stop could not express. + */ + if (st.guest_nonblock) + break; + + /* Wait before the next round. Every round asks for the whole remainder, + * so a short return means the buffer filled, and retrying the transfer + * first would only add a failing syscall: the reader would have to + * drain in the microseconds between two calls for it to pay off. + * Measured both ways on a 1 MiB pipe write, no difference outside + * noise, so this keeps the cheaper shape. + */ + wait_first = true; + int spent = iov_advance(iov, iovcnt, (size_t) ret); + iov += spent; + iovcnt -= spent; + } + + /* A write that moved bytes and then found the reader gone reports the + * count, and Linux still raises SIGPIPE: pipe_write sends the signal even + * when it has something to return. io_write_result cannot do it, since the + * value it sees is a non-negative count with no errno attached, so the + * round that ended the loop says so here. Measured against qemu-aarch64: + * both sides return the same partial count, and only Linux signalled. + */ + if (!is_read && total > 0 && xfer_ret < 0 && xfer_errno == EPIPE) + signal_queue(LINUX_SIGPIPE); + + if (total == 0 && fail < 0) + return fail; + if (total == 0 && xfer_ret < 0) { + *out = xfer_ret; + errno = xfer_errno; return 0; - return io_wait_fd_or_interrupted(host_fd, events); + } + *out = total; + return 0; } static int64_t io_check_access(int host_fd, short events) @@ -330,12 +635,7 @@ static int64_t io_check_access(int host_fd, short events) int fl = fcntl(host_fd, F_GETFL); if (fl < 0) return linux_errno(); - int acc = fl & O_ACCMODE; - if ((events & POLLIN) && acc == O_WRONLY) - return -LINUX_EBADF; - if ((events & POLLOUT) && acc == O_RDONLY) - return -LINUX_EBADF; - return 0; + return io_fl_direction_ok(fl, events) ? 0 : -LINUX_EBADF; } /* Interruptible output drain, mirroring tty_wait_until_sent(): the Linux kernel @@ -1006,27 +1306,34 @@ static uint32_t mac_lflag_to_linux(tcflag_t mf) /* read/write and positional variants. */ -/* Open a host fd reference for regular I/O, checking type and seals under - * fd_lock for thread safety. +/* Open a host fd reference for regular I/O, rejecting path-only fds and + * write-sealed memfds. * * Returns -LINUX_EBADF for path-only or closed fds, -LINUX_EPERM for - * write-sealed fds (when check_write_seal is set), or 0 on success. + * write-sealed ones, 0 on success. st_out, when given, receives the + * classification taken with the descriptor rather than looked up afterwards; a + * transfer needs that pairing, since the guest fd number can be reused for + * another kind of object in between. The checks read that same state, so the + * seal that is enforced belongs to the description being written to and not to + * whatever occupied the slot a moment earlier. */ static int64_t host_fd_ref_open_checked(int guest_fd, host_fd_ref_t *ref, - bool check_write_seal) + fd_block_state_t *st_out) { - if (check_write_seal) { - fd_entry_t snap; - if (!fd_snapshot(guest_fd, &snap)) - return -LINUX_EBADF; - if (snap.type == FD_PATH) - return -LINUX_EBADF; - if (snap.seals & LINUX_F_SEAL_WRITE) - return -LINUX_EPERM; - return host_fd_ref_open(guest_fd, ref) < 0 ? -LINUX_EBADF : 0; + fd_block_state_t st; + if (host_fd_ref_open_state(guest_fd, ref, &st) < 0) + return -LINUX_EBADF; + + if (st.type == FD_PATH || (st.seals & LINUX_F_SEAL_WRITE)) { + int64_t err = (st.type == FD_PATH) ? -LINUX_EBADF : -LINUX_EPERM; + host_fd_ref_close(ref); + return err; } - return host_fd_ref_open_io(guest_fd, ref); + + if (st_out) + *st_out = st; + return 0; } /* True when a read on this pty master must fail with EIO. @@ -1049,20 +1356,7 @@ static bool pty_read_hangs_up(int guest_fd, uint64_t gen, int host_fd) return poll(&drain, 1, 0) <= 0 || !(drain.revents & POLLIN); } -static int64_t host_fd_ref_open_regular_io(int guest_fd, host_fd_ref_t *ref) -{ - return host_fd_ref_open_io(guest_fd, ref); -} -/* host_fd_ref_open_regular_io() that also pins the generation the reference was - * resolved against, in the same fd_lock window. See host_fd_ref_open_io_gen(). - */ -static int64_t host_fd_ref_open_regular_io_gen(int guest_fd, - host_fd_ref_t *ref, - uint64_t *out_gen) -{ - return host_fd_ref_open_io_gen(guest_fd, ref, out_gen); -} static int64_t proc_try_read_intercept(int fd, int host_fd, @@ -1205,7 +1499,8 @@ int64_t sys_write(guest_t *g, int fd, uint64_t buf_gva, uint64_t count) return netlink_send(fd, g, buf_gva, count); host_fd_ref_t host_ref; - int64_t err = host_fd_ref_open_checked(fd, &host_ref, true); + fd_block_state_t write_st; + int64_t err = host_fd_ref_open_checked(fd, &host_ref, &write_st); if (err < 0) return err; @@ -1242,20 +1537,18 @@ int64_t sys_write(guest_t *g, int fd, uint64_t buf_gva, uint64_t count) } } - /* A blocking write on a full pipe/socket buffer would park this vCPU thread - * in an uninterruptible host write() where the preempt thread's - * hv_vcpus_exit cannot reach it. Wait for POLLOUT (or a guest signal) - * first. Unlike the socket send paths there is no per-call nonblocking flag - * for write(), so the tiny window where the buffer refills between the wait - * and write() can still block; that matches sys_read and the receive paths. + /* Wait for POLLOUT (or a guest signal) and transfer without ever parking + * this vCPU thread in a host call. io_xfer completes short pipe writes and + * returns a short socket write. */ - int64_t wwait = io_block_wait(fd, host_ref.fd, POLLOUT); + struct iovec iov = {.iov_base = buf, .iov_len = count}; + ssize_t ret; + int64_t wwait = io_xfer(fd, host_ref.fd, POLLOUT, &iov, 1, &ret, &write_st); if (wwait < 0) { host_fd_ref_close(&host_ref); return wwait; } - ssize_t ret = write(host_ref.fd, buf, count); host_fd_ref_close(&host_ref); return io_write_result(ret); } @@ -1292,7 +1585,8 @@ int64_t sys_read(guest_t *g, int fd, uint64_t buf_gva, uint64_t count) */ host_fd_ref_t host_ref; uint64_t read_gen; - int64_t err = host_fd_ref_open_regular_io_gen(fd, &host_ref, &read_gen); + fd_block_state_t read_st; + int64_t err = host_fd_ref_open_io_state(fd, &host_ref, &read_gen, &read_st); if (err < 0) return err; @@ -1336,15 +1630,17 @@ int64_t sys_read(guest_t *g, int fd, uint64_t buf_gva, uint64_t count) } /* Wait interruptibly when the fd can block on a read (pipe, socket, fifo, - * char/tty). Regular files never block and skip this. + * char/tty), then take the bytes without parking. Regular files never block + * and transfer straight through. */ - int64_t rwait = io_block_wait(fd, host_ref.fd, POLLIN); + struct iovec iov = {.iov_base = buf, .iov_len = count}; + ssize_t ret; + int64_t rwait = io_xfer(fd, host_ref.fd, POLLIN, &iov, 1, &ret, &read_st); if (rwait < 0) { host_fd_ref_close(&host_ref); return rwait; } - ssize_t ret = read(host_ref.fd, buf, count); int64_t result = ret < 0 ? recv_eof_or_errno(host_ref.fd, fd) : ret; host_fd_ref_close(&host_ref); return result; @@ -1360,7 +1656,7 @@ int64_t sys_pread64(guest_t *g, return fuse_pread_fd(g, fd, buf_gva, count, offset); host_fd_ref_t host_ref; - int64_t err = host_fd_ref_open_regular_io(fd, &host_ref); + int64_t err = host_fd_ref_open_io(fd, &host_ref); if (err < 0) return err; @@ -1396,7 +1692,7 @@ int64_t sys_pwrite64(guest_t *g, int64_t offset) { host_fd_ref_t host_ref; - int64_t err = host_fd_ref_open_checked(fd, &host_ref, true); + int64_t err = host_fd_ref_open_checked(fd, &host_ref, NULL); if (err < 0) return err; @@ -1706,7 +2002,9 @@ int64_t sys_readv(guest_t *g, int fd, uint64_t iov_gva, int iovcnt) host_fd_ref_t host_ref; uint64_t readv_gen; - int64_t err = host_fd_ref_open_regular_io_gen(fd, &host_ref, &readv_gen); + fd_block_state_t readv_st; + int64_t err = + host_fd_ref_open_io_state(fd, &host_ref, &readv_gen, &readv_st); if (err < 0) return err; @@ -1747,14 +2045,15 @@ int64_t sys_readv(guest_t *g, int fd, uint64_t iov_gva, int iovcnt) } } - int64_t rwait = io_block_wait(fd, host_ref.fd, POLLIN); + ssize_t ret; + int64_t rwait = + io_xfer(fd, host_ref.fd, POLLIN, host_iov.iov, iovcnt, &ret, &readv_st); if (rwait < 0) { host_iov_free(&host_iov); host_fd_ref_close(&host_ref); return rwait; } - ssize_t ret = readv(host_ref.fd, host_iov.iov, iovcnt); int64_t result = ret < 0 ? recv_eof_or_errno(host_ref.fd, fd) : ret; host_iov_free(&host_iov); host_fd_ref_close(&host_ref); @@ -1795,7 +2094,8 @@ int64_t sys_writev(guest_t *g, int fd, uint64_t iov_gva, int iovcnt) } host_fd_ref_t host_ref; - int64_t err = host_fd_ref_open_checked(fd, &host_ref, true); + fd_block_state_t writev_st; + int64_t err = host_fd_ref_open_checked(fd, &host_ref, &writev_st); if (err < 0) return err; @@ -1823,14 +2123,15 @@ int64_t sys_writev(guest_t *g, int fd, uint64_t iov_gva, int iovcnt) } } - int64_t wwait = io_block_wait(fd, host_ref.fd, POLLOUT); + ssize_t ret; + int64_t wwait = io_xfer(fd, host_ref.fd, POLLOUT, host_iov.iov, iovcnt, + &ret, &writev_st); if (wwait < 0) { host_iov_free(&host_iov); host_fd_ref_close(&host_ref); return wwait; } - ssize_t ret = writev(host_ref.fd, host_iov.iov, iovcnt); int64_t result = io_write_result(ret); host_iov_free(&host_iov); host_fd_ref_close(&host_ref); @@ -1861,7 +2162,7 @@ int64_t sys_preadv(guest_t *g, } host_fd_ref_t host_ref; - int64_t err = host_fd_ref_open_regular_io(fd, &host_ref); + int64_t err = host_fd_ref_open_io(fd, &host_ref); if (err < 0) return err; @@ -1908,7 +2209,7 @@ int64_t sys_pwritev(guest_t *g, } host_fd_ref_t host_ref; - int64_t err = host_fd_ref_open_checked(fd, &host_ref, true); + int64_t err = host_fd_ref_open_checked(fd, &host_ref, NULL); if (err < 0) return err; @@ -1941,7 +2242,7 @@ static int64_t sys_pwritev_append(guest_t *g, bool update_file_offset) { host_fd_ref_t host_ref; - int64_t err = host_fd_ref_open_checked(fd, &host_ref, true); + int64_t err = host_fd_ref_open_checked(fd, &host_ref, NULL); if (err < 0) return err; @@ -2040,7 +2341,7 @@ int64_t sys_pwritev2(guest_t *g, /* RWF_SYNC/RWF_DSYNC: sync after successful write */ if (r > 0 && (flags & (RWF_SYNC | RWF_DSYNC))) { host_fd_ref_t host_ref; - if (host_fd_ref_open_regular_io(fd, &host_ref) == 0) { + if (host_fd_ref_open_io(fd, &host_ref) == 0) { fsync(host_ref.fd); host_fd_ref_close(&host_ref); } @@ -2229,12 +2530,12 @@ int64_t sys_ioctl(guest_t *g, int fd, uint64_t request, uint64_t arg) * host fd's FD_CLOEXEC, which is per-descriptor and would be lost on the * dup that host_fd_ref hands multi-threaded callers, so mirror the F_SETFD * path in sys_fcntl). They need no host fd, so dispatch them before - * host_fd_ref_open_regular_io(): that helper rejects O_PATH (FD_PATH) fds - * with EBADF, but Linux allows these ioctls -- like fcntl(F_SETFD) -- on - * O_PATH descriptors. Validate the slot and mutate the flag in a single - * fd_lock section so there is no validate-then-mutate window in which a - * concurrent close/reuse could flip CLOEXEC on a different file that took - * the slot. The arg is ignored. + * host_fd_ref_open_io(): that helper rejects O_PATH (FD_PATH) fds with + * EBADF, but Linux allows these ioctls -- like fcntl(F_SETFD) -- on O_PATH + * descriptors. Validate the slot and mutate the flag in a single fd_lock + * section so there is no validate-then-mutate window in which a concurrent + * close/reuse could flip CLOEXEC on a different file that took the slot. + * The arg is ignored. */ if (request == LINUX_FIOCLEX || request == LINUX_FIONCLEX) { if (!RANGE_CHECK(fd, 0, FD_TABLE_SIZE)) @@ -2265,7 +2566,7 @@ int64_t sys_ioctl(guest_t *g, int fd, uint64_t request, uint64_t arg) host_fd_ref_t host_ref; uint64_t ioctl_gen; - int64_t err = host_fd_ref_open_regular_io_gen(fd, &host_ref, &ioctl_gen); + int64_t err = host_fd_ref_open_io_gen(fd, &host_ref, &ioctl_gen); if (err < 0) return err; int host_fd = host_ref.fd; @@ -2895,6 +3196,17 @@ int64_t sys_ioctl(guest_t *g, int fd, uint64_t request, uint64_t arg) host_fd_ref_close(&host_ref); return -LINUX_EFAULT; } + + /* An owned fd stays nonblocking on the host whatever the guest asks for + * here, the same way F_SETFL treats it: the request goes to the shadow, + * which is where F_GETFL and the transfer paths read it, and reaches + * every dup alias of the same open file description. + */ + if (fd_apply_guest_nonblock(fd, on != 0)) { + host_fd_ref_close(&host_ref); + return 0; + } + int r = fd_update_status_flag(host_fd, O_NONBLOCK, on != 0); host_fd_ref_close(&host_ref); return r < 0 ? linux_errno() : 0; @@ -2911,7 +3223,7 @@ int64_t sys_ioctl(guest_t *g, int fd, uint64_t request, uint64_t arg) int64_t sys_fallocate(int fd, int mode, int64_t offset, int64_t len) { host_fd_ref_t host_ref; - int64_t err = host_fd_ref_open_regular_io(fd, &host_ref); + int64_t err = host_fd_ref_open_io(fd, &host_ref); if (err < 0) return err; @@ -3045,15 +3357,78 @@ int64_t sys_fallocate(int fd, int mode, int64_t offset, int64_t len) * * Returns the byte count moved, or a negative Linux errno only when the very * first read or write failed (partial transfers report the count so the caller - * can still write offsets back). + * can still write offsets back). Rewind a position-based input by the bytes a + * read consumed but the write never sent, so the fd's position matches what + * Linux advances it by. + * + * Returns false when the input cannot be put back: offset-based input never + * consumed anything, but a pipe cannot seek, and a pipe is the usual splice + * input. A caller reporting an interrupted transfer over an input it could not + * rewind must forbid the SVC restart, since re-running the original arguments + * would read past the bytes that went missing. */ -static int64_t copy_fd_range(int in_gfd, - int in_hfd, - int out_hfd, +static bool io_rewind_unsent(int64_t off_in, int in_hfd, ssize_t unsent) +{ + if (off_in >= 0) + return true; + + /* Nothing to put back, and the guard is not only for the caller's benefit: + * negating SSIZE_MIN is undefined, and this is the one expression in the + * transfer path that negates a signed count it did not compute itself. + * Every caller passes a chunk-bounded value today, so this is unreachable + * by provenance; the comparison makes it unreachable by construction. + */ + if (unsent <= 0) + return true; + + return lseek(in_hfd, (off_t) -unsent, SEEK_CUR) >= 0; +} + +/* An interrupted write of a chunk already drained out of the input: give up on + * it, or keep writing? + * + * If the unsent tail can be put back, the interruption is reportable and the + * guest loses nothing. If it cannot -- a pipe, the usual splice input -- + * abandoning it drops those bytes outright, so the write has to continue: an + * ordinary SIGCHLD arriving mid-splice must not eat 64 KiB of the guest's + * stream. Teardown is the one interruption that wins anyway, because the thread + * is going away and the image with it, so there is nobody left for the bytes to + * reach; without it the caller would spin, since io_xfer reports the stop + * request every time it is asked. + * + * Both stream copiers ask this, and they asked it in opposite spellings before + * it was one function. *rewound_out says which of the two answers a true return + * came from, because only the caller can spell syscall_restart_forbid() where + * scripts/check-eintr-contract.py can see it, and only the teardown answer owes + * it: an input the rewind put back is one the restart may safely re-read. + */ +static bool io_give_up_unsent(int64_t off_in, + int in_hfd, + ssize_t unsent, + bool *rewound_out) +{ + bool rewound = io_rewind_unsent(off_in, in_hfd, unsent); + *rewound_out = rewound; + return rewound || thread_stop_requested(); +} + +typedef struct { + int in_gfd, in_hfd; + int out_gfd, out_hfd; + fd_block_state_t in_st, out_st; +} copy_ends_t; + +static int64_t copy_fd_range(const copy_ends_t *ends, int64_t *off_in, int64_t *off_out, uint64_t len) { + int in_gfd = ends->in_gfd, in_hfd = ends->in_hfd; + int out_gfd = ends->out_gfd, out_hfd = ends->out_hfd; + + fd_block_state_t in_st = ends->in_st; + fd_block_state_t out_st = ends->out_st; + char *buf = malloc(IO_COPY_BUF_SIZE); if (!buf) return -LINUX_ENOMEM; @@ -3073,8 +3448,20 @@ static int64_t copy_fd_range(int in_gfd, } else { int64_t intercepted = proc_try_chunk_read_intercept(in_gfd, in_hfd, buf, chunk, 0, 0); - nr = (intercepted != INT64_MIN) ? intercepted - : read(in_hfd, buf, chunk); + if (intercepted != INT64_MIN) { + nr = intercepted; + } else { + /* An owned input fd is nonblocking on the host, so the read has + * to go through io_xfer to keep the guest's blocking semantics. + */ + struct iovec iov = {.iov_base = buf, .iov_len = chunk}; + int64_t waited = + io_xfer(in_gfd, in_hfd, POLLIN, &iov, 1, &nr, &in_st); + if (waited < 0) { + ret = total > 0 ? (int64_t) total : waited; + goto done; + } + } } if (nr < 0) { ret = total > 0 ? (int64_t) total : linux_errno(); @@ -3083,8 +3470,44 @@ static int64_t copy_fd_range(int in_gfd, if (nr == 0) break; /* EOF */ - ssize_t nw = (*off_out >= 0) ? pwrite(out_hfd, buf, nr, *off_out) - : write(out_hfd, buf, nr); + ssize_t nw; + if (*off_out >= 0) { + nw = pwrite(out_hfd, buf, nr, *off_out); + } else { + struct iovec iov = {.iov_base = buf, .iov_len = (size_t) nr}; + unsigned backoff = 0; + for (;;) { + int64_t waited = + io_xfer(out_gfd, out_hfd, POLLOUT, &iov, 1, &nw, &out_st); + if (waited >= 0) + break; + + /* The retry has to be this inner loop and not the outer one: + * continuing there would read a fresh chunk over the nr bytes + * still sitting in buf and drop them, which is exactly what + * keeping the write alive exists to prevent. io_xfer leaves iov + * untouched when it reports a negative, since it only rewrites + * the vector once something has moved and never fails after + * that, so the same iov is safe to hand back. sendfile does not + * reject a pipe in_fd the way Linux does, so an unrewindable + * input is reachable here too. + */ + bool rewound; + if (io_give_up_unsent(*off_in, in_hfd, nr, &rewound)) { + if (!rewound) + syscall_restart_forbid(); + ret = total > 0 ? (int64_t) total : waited; + goto done; + } + + /* Nothing to give up on and nothing tearing this thread down, + * so the write has to keep going. io_xfer answers the pending + * signal on entry, so retrying it bare is a spin; sleep between + * rounds instead. + */ + io_backoff_sleep(&backoff); + } + } if (nw < 0) { if (errno == EPIPE) signal_queue(LINUX_SIGPIPE); @@ -3108,8 +3531,7 @@ static int64_t copy_fd_range(int in_gfd, * (which sendfile/copy_file_range do not accept) simply keeps the * prior behavior. */ - if (*off_in < 0) - (void) lseek(in_hfd, (off_t) (nw - nr), SEEK_CUR); + (void) io_rewind_unsent(*off_in, in_hfd, nr - nw); break; } } @@ -3127,10 +3549,11 @@ int64_t sys_sendfile(guest_t *g, uint64_t count) { host_fd_ref_t out_ref, in_ref; - int64_t err = host_fd_ref_open_regular_io(out_fd, &out_ref); + fd_block_state_t out_st, in_st; + int64_t err = host_fd_ref_open_io_state(out_fd, &out_ref, NULL, &out_st); if (err < 0) return err; - err = host_fd_ref_open_regular_io(in_fd, &in_ref); + err = host_fd_ref_open_io_state(in_fd, &in_ref, NULL, &in_st); if (err < 0) { host_fd_ref_close(&out_ref); return err; @@ -3153,8 +3576,13 @@ int64_t sys_sendfile(guest_t *g, /* sendfile has no output offset, so out always uses write(). */ int64_t off_out = -1; - int64_t moved = - copy_fd_range(in_fd, in_ref.fd, out_ref.fd, &offset, &off_out, count); + int64_t moved = copy_fd_range(&(copy_ends_t) {.in_gfd = in_fd, + .in_hfd = in_ref.fd, + .in_st = in_st, + .out_gfd = out_fd, + .out_hfd = out_ref.fd, + .out_st = out_st}, + &offset, &off_out, count); if (moved < 0) { err = moved; goto out_sendfile; @@ -3191,10 +3619,11 @@ int64_t sys_copy_file_range(guest_t *g, return -LINUX_EINVAL; host_fd_ref_t in_ref, out_ref; - int64_t err = host_fd_ref_open_regular_io(fd_in, &in_ref); + fd_block_state_t in_st, out_st; + int64_t err = host_fd_ref_open_io_state(fd_in, &in_ref, NULL, &in_st); if (err < 0) return err; - err = host_fd_ref_open_regular_io(fd_out, &out_ref); + err = host_fd_ref_open_io_state(fd_out, &out_ref, NULL, &out_st); if (err < 0) { host_fd_ref_close(&in_ref); return err; @@ -3216,8 +3645,13 @@ int64_t sys_copy_file_range(guest_t *g, } /* Emulate with a pread/pwrite loop. */ - int64_t moved = - copy_fd_range(fd_in, in_ref.fd, out_ref.fd, &off_in, &off_out, len); + int64_t moved = copy_fd_range(&(copy_ends_t) {.in_gfd = fd_in, + .in_hfd = in_ref.fd, + .in_st = in_st, + .out_gfd = fd_out, + .out_hfd = out_ref.fd, + .out_st = out_st}, + &off_in, &off_out, len); if (moved < 0) { err = moved; goto out_copy_file_range; @@ -3246,7 +3680,105 @@ int64_t sys_copy_file_range(guest_t *g, /* splice/tee. */ -/* splice: emulate by reading from in_fd and writing to out_fd */ +/* splice: emulate by reading from in_fd and writing to out_fd One splice chunk + * in flight: the two ends, and the guest offsets that advance with it. Kept + * together so the drain below can be a function rather than a fourth level of + * nesting inside sys_splice. + */ +typedef struct { + int fd_in, in_hfd; + int fd_out, out_hfd; + int64_t off_in, off_out; /* -1 when the guest passed no offset */ + + /* Both ends classified once, from the descriptors sys_splice pinned. The + * per-chunk alternative lets a sibling reusing either fd number change the + * transfer form partway through a copy that is still holding the old + * descriptors. + */ + fd_block_state_t in_st, out_st; +} splice_state_t; + +/* Why the drain stopped, when it stopped early. */ +typedef struct { + bool stop; /* the outer loop must end */ + bool rw_error; /* a write failed; errno is in saved_errno */ + int saved_errno; /* preserved across the guest writes at done */ + int64_t wait_err; /* interrupted wait, reported only if nothing moved */ +} splice_fail_t; + +/* Drain one read chunk to the output the way splice does: a short write is + * continued rather than reported, so the chunk either lands whole or the caller + * stops. + * + * Returns the bytes written, which is the caller's whole accounting; the input + * rewind that a partial chunk needs happens here, since only this loop knows + * how much of the chunk never left. + */ +static size_t splice_drain_chunk(splice_state_t *st, + uint8_t *buf, + size_t n, + splice_fail_t *f) +{ + size_t written = 0; + unsigned backoff = 0; + while (written < n) { + ssize_t w; + if (st->off_out >= 0) { + w = pwrite(st->out_hfd, buf + written, n - written, st->off_out); + } else { + struct iovec iov = {.iov_base = buf + written, + .iov_len = n - written}; + int64_t waited = io_xfer(st->fd_out, st->out_hfd, POLLOUT, &iov, 1, + &w, &st->out_st); + if (waited < 0) { + bool rewound; + if (io_give_up_unsent(st->off_in, st->in_hfd, + (ssize_t) (n - written), &rewound)) { + if (!rewound) + syscall_restart_forbid(); + f->wait_err = waited; + f->stop = true; + return written; + } + + /* Nothing to give up on and nothing tearing this thread down, + * so the write has to keep going. io_xfer answers the pending + * signal on entry, so retrying it bare is a spin; sleep between + * rounds instead. + */ + io_backoff_sleep(&backoff); + continue; + } + backoff = 0; + } + if (w <= 0) { + if (w < 0) { + f->rw_error = true; + f->saved_errno = errno; + if (f->saved_errno == EPIPE) + signal_queue(LINUX_SIGPIPE); + } + + /* Position-based input: read() consumed all n bytes but only + * written were moved, so rewind the input by the difference to + * match Linux advancing only by bytes transferred. Best-effort; a + * pipe input, the common splice case, cannot seek and keeps the + * prior behavior. + */ + (void) io_rewind_unsent(st->off_in, st->in_hfd, + (ssize_t) (n - written)); + f->stop = true; + return written; + } + written += (size_t) w; + if (st->off_in >= 0) + st->off_in += w; + if (st->off_out >= 0) + st->off_out += w; + } + return written; +} + int64_t sys_splice(guest_t *g, int fd_in, uint64_t off_in_gva, @@ -3257,10 +3789,11 @@ int64_t sys_splice(guest_t *g, { (void) flags; host_fd_ref_t in_ref, out_ref; - int64_t err = host_fd_ref_open_regular_io(fd_in, &in_ref); + fd_block_state_t in_st, out_st; + int64_t err = host_fd_ref_open_io_state(fd_in, &in_ref, NULL, &in_st); if (err < 0) return err; - err = host_fd_ref_open_regular_io(fd_out, &out_ref); + err = host_fd_ref_open_io_state(fd_out, &out_ref, NULL, &out_st); if (err < 0) { host_fd_ref_close(&in_ref); return err; @@ -3284,8 +3817,8 @@ int64_t sys_splice(guest_t *g, } /* Emulate with a read/write loop over a heap buffer. splice fully drains - * each read chunk (inner write loop) rather than stopping on a short write, - * so it does not share copy_fd_range. + * each read chunk rather than stopping on a short write, which is why it + * does not share copy_fd_range; splice_drain_chunk is that drain. */ uint8_t *buf = malloc(IO_COPY_BUF_SIZE); if (!buf) { @@ -3295,54 +3828,45 @@ int64_t sys_splice(guest_t *g, } size_t chunk = len > IO_COPY_BUF_SIZE ? IO_COPY_BUF_SIZE : len; + splice_state_t st = {.fd_in = fd_in, + .in_hfd = in_ref.fd, + .fd_out = fd_out, + .out_hfd = out_ref.fd, + .off_in = off_in, + .off_out = off_out, + .in_st = in_st, + .out_st = out_st}; + splice_fail_t f = {0}; size_t total = 0; - int saved_errno = 0; /* Preserve errno across guest_write */ - bool rw_error = false; /* Track whether read or write failed */ int64_t ret; while (total < len) { size_t n = (len - total) > chunk ? chunk : (len - total); - ssize_t r = (off_in >= 0) ? pread(in_ref.fd, buf, n, off_in) - : read(in_ref.fd, buf, n); + ssize_t r; + if (st.off_in >= 0) { + r = pread(st.in_hfd, buf, n, st.off_in); + } else { + /* An owned fd is nonblocking on the host, so the transfer keeps the + * guest's blocking semantics only by going through io_xfer. + */ + struct iovec iov = {.iov_base = buf, .iov_len = n}; + int64_t waited = + io_xfer(st.fd_in, st.in_hfd, POLLIN, &iov, 1, &r, &st.in_st); + if (waited < 0) { + f.wait_err = waited; + goto done; + } + } if (r < 0) { - rw_error = true; - saved_errno = errno; + f.rw_error = true; + f.saved_errno = errno; break; } if (r == 0) break; /* EOF */ - if (off_in >= 0) - off_in += r; - - size_t written = 0; - while (written < (size_t) r) { - ssize_t w = - (off_out >= 0) - ? pwrite(out_ref.fd, buf + written, r - written, off_out) - : write(out_ref.fd, buf + written, r - written); - if (w <= 0) { - if (w < 0) { - rw_error = true; - saved_errno = errno; - } - if (w < 0 && saved_errno == EPIPE) - signal_queue(LINUX_SIGPIPE); - total += written; /* Account for partial bytes written */ - /* Position-based input: read() consumed all r bytes but only - * written were moved, so rewind the input fd by r - written to - * match Linux advancing only by bytes transferred. Best-effort; - * a pipe input (common for splice) cannot seek and keeps the - * prior behavior. saved_errno is restored at done. - */ - if (off_in < 0 && written < (size_t) r) - (void) lseek(in_ref.fd, (off_t) ((ssize_t) written - r), - SEEK_CUR); - goto done; - } - written += w; - if (off_out >= 0) - off_out += w; - } - total += r; + + total += splice_drain_chunk(&st, buf, (size_t) r, &f); + if (f.stop) + goto done; } done: @@ -3352,18 +3876,20 @@ int64_t sys_splice(guest_t *g, * sendfile/copy_file_range). A failed off_in writeback skips the off_out * writeback, matching the kernel. */ - if (off_in_gva && off_in >= 0 && - guest_write_small(g, off_in_gva, &off_in, sizeof(off_in)) < 0) { + if (off_in_gva && st.off_in >= 0 && + guest_write_small(g, off_in_gva, &st.off_in, sizeof(st.off_in)) < 0) { ret = total > 0 ? (int64_t) total : -LINUX_EFAULT; - } else if (off_out_gva && off_out >= 0 && - guest_write_small(g, off_out_gva, &off_out, sizeof(off_out)) < - 0) { + } else if (off_out_gva && st.off_out >= 0 && + guest_write_small(g, off_out_gva, &st.off_out, + sizeof(st.off_out)) < 0) { ret = total > 0 ? (int64_t) total : -LINUX_EFAULT; } else if (total > 0) { ret = (int64_t) total; - } else if (rw_error) { + } else if (f.wait_err) { + ret = f.wait_err; + } else if (f.rw_error) { /* Restore saved_errno; the guest writes above may have clobbered it. */ - errno = saved_errno; + errno = f.saved_errno; ret = linux_errno(); } else { ret = 0; @@ -3383,8 +3909,15 @@ int64_t sys_vmsplice(guest_t *g, unsigned int flags) { (void) flags; + + /* One pin and one classification for the whole call: the descriptor is + * pinned here and every segment below transfers on it, so re-resolving the + * slot per segment would let a sibling reusing this fd number change the + * transfer form partway through, up to 1024 times. + */ host_fd_ref_t host_ref; - int64_t err = host_fd_ref_open_regular_io(fd, &host_ref); + fd_block_state_t vm_st; + int64_t err = host_fd_ref_open_io_state(fd, &host_ref, NULL, &vm_st); if (err < 0) return err; if (nr_segs > 1024) { @@ -3413,7 +3946,13 @@ int64_t sys_vmsplice(guest_t *g, if (len > avail) len = avail; - ssize_t w = write(host_ref.fd, src, len); + struct iovec iov = {.iov_base = src, .iov_len = len}; + ssize_t w; + int64_t waited = io_xfer(fd, host_ref.fd, POLLOUT, &iov, 1, &w, &vm_st); + if (waited < 0) { + host_fd_ref_close(&host_ref); + return total > 0 ? (int64_t) total : waited; + } if (w < 0) { if (errno == EPIPE) signal_queue(LINUX_SIGPIPE); diff --git a/src/syscall/io.h b/src/syscall/io.h index 49bfdc1b..d1b6f848 100644 --- a/src/syscall/io.h +++ b/src/syscall/io.h @@ -16,7 +16,9 @@ #pragma once #include +#include #include "core/guest.h" +#include "syscall/internal.h" /* fd_block_state_t */ /* I/O syscall handlers. */ @@ -42,6 +44,75 @@ void io_init(void); */ int64_t io_wait_fd_or_interrupted(int host_fd, short events); +/* Consecutive EAGAINs from a transfer the wait called ready, before io_xfer + * stops retrying at full speed and starts backing off. + * + * Sized to tell two things apart that look identical from here. Losing a race + * to a sibling is normal and self-limiting: the wait blocks again until the fd + * genuinely has something, so a contended pipe can lose several times in one + * syscall and every loss still made progress somewhere. An fd whose readiness + * the transfer never honours -- poll reporting POLLHUP or POLLERR that a + * nonblocking transfer answers with EAGAIN -- never blocks in the wait at all + * and reaches this in microseconds. High enough that contention almost never + * pays for it, low enough that the pathological case cannot hold a core. + */ +#define IO_XFER_SPIN_LIMIT 16 + +/* Move iov through host_fd with the blocking semantics the guest asked for, + * without parking this vCPU thread in a host call. + * + * A readiness poll reserves nothing. Between the poll and the transfer a + * sibling thread, or a forked process sharing the open file description, can + * take the bytes the poll promised, and the transfer then blocks where neither + * hv_vcpus_exit nor the wakeup pipe reaches it; an execve teardown counts that + * thread as a sibling that would not leave. So the wait is interruptible and + * the transfer itself reports EAGAIN rather than blocking, and a steal only + * sends the caller back to the wait. A write keeps going until every byte has + * moved, which is what a blocking write(2) promises, and reports the partial + * count when a signal arrives with bytes already gone. + * + * Two kinds of fd fall outside the no-parking guarantee, and a caller relying + * on it during exec teardown has to know which. A socket transfers with + * MSG_DONTWAIT, which macOS ignores for AF_UNIX sends, so a send into a full + * buffer blocks in the kernel. Inherited stdio is a description elfuse does not + * own, so its transfer is a plain blocking read or write. Both are recorded in + * TODO.md; everything else elfuse owns O_NONBLOCK on and cannot park here. + * + * events picks the direction: POLLIN reads, POLLOUT writes. iov is scratch the + * caller owns, and a partial write rewrites it. Regular files, fds the guest + * set nonblocking, and direction mismatches transfer straight through. + * + * Returns 0 with *out set to the raw host result (errno live when it is -1), or + * a negative Linux errno, in which case nothing moved and iov is untouched: the + * wait was interrupted (EINTR), the fd is a pty master whose slaves are all + * gone (EIO), or the iovec lengths do not sum (EINVAL). A caller that cannot + * answer those itself has to hand the value back to the guest. The same + * transfer, classified from the state pinned with the host fd rather than + * looked up again. + * + * io_xfer resolves the slot itself, which is correct only while nothing can + * reuse the fd number underneath it. A caller that already holds a host fd took + * it from a slot that a sibling may since have closed and reopened as another + * kind of object; classifying that new object and transferring on the old + * descriptor is how a pinned pipe comes to be sent recv(MSG_DONTWAIT) and + * answers ENOTSOCK, and how a pinned socket comes to take the plain blocking + * read this file exists to avoid. host_fd_ref_open_state takes both together. + */ +int64_t io_xfer(int fd, + int host_fd, + short events, + struct iovec *iov, + int iovcnt, + ssize_t *out, + const fd_block_state_t *pinned); + +/* pinned is required, not optional: it is dereferenced unconditionally. Making + * it nullable would put this function one step from the failure that already + * happened once here -- an out-param written before its NULL check let clang + * prove the UB and compile a whole caller to a trap. Every caller pins and + * classifies in one window, so there is no caller that would want NULL. + */ + /* Backoff bounds for io_retry_backoff. These replace a blocking host call that * returned the instant the resource freed, so the ceiling is the added latency * a guest pays after the holder releases: 2 ms costs at most 500 wakeups/s on a diff --git a/src/syscall/linux-wire.h b/src/syscall/linux-wire.h index 422c79c2..7c3fb425 100644 --- a/src/syscall/linux-wire.h +++ b/src/syscall/linux-wire.h @@ -458,8 +458,11 @@ typedef struct { #define STATX_BASIC_STATS 0x07FFU #define STATX_BTIME 0x0800U -/* FD table. */ -#define FD_TABLE_SIZE 1024 +/* FD table. The bound itself lives in elfuse-limits.h, which is also where the + * host descriptor reserve is derived from it; two copies of the same 1024 would + * be a legal redefinition rather than a build error, so this includes it. + */ +#include "elfuse-limits.h" #define FD_CLOSED 0 #define FD_STDIO 1 @@ -569,6 +572,17 @@ typedef struct { * once at allocation via fstat so the interruptible wait * path can skip fds that never block. */ + bool foreign_description; /* The open file description behind this fd came + * from outside elfuse -- the launcher's stdio, + * or an alias of it -- so its status flags are + * not elfuse's to change. Inherited by every + * alias; see fd_init_entry. + */ + bool nonblock_owned; /* elfuse set O_NONBLOCK on the host fd and emulates + * the guest's blocking semantics on top of it, so + * linux_flags -- not the host flag -- is what the + * guest asked for. See fd_init_entry. + */ int32_t fasync_owner_type; /* FASYNC_OWNER_* recipient kind (0 = none) */ int32_t fasync_owner; /* pid/pgrp/tid for SIGIO/SIGURG delivery */ sock_opt_cache_t sock; /* Socket option cache (zeroed for non-sockets) */ diff --git a/src/syscall/mem.c b/src/syscall/mem.c index b79b28c4..0f1e4a6d 100644 --- a/src/syscall/mem.c +++ b/src/syscall/mem.c @@ -4985,14 +4985,12 @@ int mmap_fork_prepare_anon_shared(guest_t *g, uint64_t len = end - start; uint64_t aligned_len = ALIGN_UP(len, hps); - char tmpl[] = "/tmp/elfuse-anonsh-XXXXXX"; - int fd = mkstemp(tmpl); + int fd = tmpfile_anon("anonsh"); if (fd < 0) { - log_warn("fork-prep: mkstemp for anon-shared region: %s", + log_warn("fork-prep: temp file for anon-shared region: %s", strerror(errno)); continue; } - unlink(tmpl); if (ftruncate(fd, (off_t) aligned_len) < 0) { log_warn("fork-prep: ftruncate(%llu) failed: %s", (unsigned long long) aligned_len, strerror(errno)); diff --git a/src/syscall/net-msg.c b/src/syscall/net-msg.c index 5b9162d5..966327d1 100644 --- a/src/syscall/net-msg.c +++ b/src/syscall/net-msg.c @@ -17,6 +17,7 @@ #include #include +#include "proved/iov.h" #include "utils.h" #include "proved/cmsg.h" @@ -148,7 +149,8 @@ int64_t sys_sendmsg(guest_t *g, int fd, uint64_t msg_gva, int linux_flags) return netlink_sendmsg(fd, g, msg_gva, linux_flags); host_fd_ref_t host_ref; - if (host_fd_ref_open(fd, &host_ref) < 0) + fd_block_state_t sock_st; + if (host_fd_ref_open_state(fd, &host_ref, &sock_st) < 0) return -LINUX_EBADF; linux_msghdr_t lmsg; @@ -189,7 +191,7 @@ int64_t sys_sendmsg(guest_t *g, int fd, uint64_t msg_gva, int linux_flags) } bool blocking = - len > 0 && sock_op_should_block(host_ref.fd, linux_flags); + len > 0 && sock_op_should_block(&sock_st, host_ref.fd, linux_flags); int host_flags = mac_flags | (blocking ? MSG_DONTWAIT : 0); ssize_t ret; for (;;) { @@ -381,7 +383,7 @@ int64_t sys_sendmsg(guest_t *g, int fd, uint64_t msg_gva, int linux_flags) }; bool blocking = host_iov_has_payload(&host_iov, send_iovcnt) && - sock_op_should_block(host_ref.fd, linux_flags); + sock_op_should_block(&sock_st, host_ref.fd, linux_flags); int host_flags = mac_flags | (blocking ? MSG_DONTWAIT : 0); ssize_t ret; for (;;) { @@ -411,6 +413,54 @@ int64_t sys_sendmsg(guest_t *g, int fd, uint64_t msg_gva, int linux_flags) return ret; } +/* The gathering form of MSG_WAITALL for the msghdr paths. + * + * Same reason as recv_gathers_waitall, which this defers to: the host does not + * answer for the flag, and a guest recvmsg(MSG_WAITALL) that reaches it parks + * the vCPU for good. Stripping the flag alone would fix the hang and leave a + * blocking recvmsg reporting a short count where Linux waits, so the loop + * gathers across the iovec with iov_advance_index, the same proved bound the + * transfer path uses. + * + * A caller that asked for ancillary data is excluded rather than gathered. A + * second round would have to either overwrite the control buffer, losing the + * SCM_RIGHTS descriptors the first round installed, or refuse it and let the + * kernel discard them. Neither is worth doing to a message that has already + * been delivered, so those calls get the flag stripped and a single round -- + * the hang is gone, and MSG_WAITALL with ancillary data still short-counts. + */ +static bool msg_gathers_waitall(int host_fd, int linux_flags, size_t controllen) +{ + if (controllen != 0) + return false; + return recv_gathers_waitall(host_fd, linux_flags); +} + +/* Advance an iovec past bytes already received, so the next round writes after + * them. + * + * Returns the number of leading entries fully consumed. + */ +static int msg_iov_advance(struct iovec *iov, int iovcnt, size_t moved) +{ + size_t rem = 0; + int spent = iov_advance_index(iov, iovcnt, moved, &rem); + if (spent < iovcnt && rem > 0) { + iov[spent].iov_base = (char *) iov[spent].iov_base + rem; + iov[spent].iov_len -= rem; + } + return spent; +} + +/* Total bytes an iovec can still accept. */ +static size_t msg_iov_total(const struct iovec *iov, int iovcnt) +{ + size_t total = 0; + for (int i = 0; i < iovcnt; i++) + total += iov[i].iov_len; + return total; +} + /* NOLINTNEXTLINE(readability-function-size) */ int64_t sys_recvmsg(guest_t *g, int fd, uint64_t msg_gva, int flags) { @@ -418,7 +468,8 @@ int64_t sys_recvmsg(guest_t *g, int fd, uint64_t msg_gva, int flags) return netlink_recvmsg(fd, g, msg_gva, flags); host_fd_ref_t host_ref; - if (host_fd_ref_open(fd, &host_ref) < 0) + fd_block_state_t sock_st; + if (host_fd_ref_open_state(fd, &host_ref, &sock_st) < 0) return -LINUX_EBADF; linux_msghdr_t lmsg; @@ -457,8 +508,9 @@ int64_t sys_recvmsg(guest_t *g, int fd, uint64_t msg_gva, int flags) } int64_t waited = - len > 0 ? net_wait_or_interrupted(host_ref.fd, POLLIN, flags) - : net_recv_zero_payload_gate(host_ref.fd, flags); + len > 0 + ? net_wait_or_interrupted(&sock_st, host_ref.fd, POLLIN, flags) + : net_recv_zero_payload_gate(&sock_st, host_ref.fd, flags); if (waited < 0) { host_fd_ref_close(&host_ref); return waited; @@ -474,7 +526,51 @@ int64_t sys_recvmsg(guest_t *g, int fd, uint64_t msg_gva, int flags) .msg_flags = 0, }; - ssize_t ret = recvmsg(host_ref.fd, &msg, mac_flags); + /* Retry the EAGAIN a blocking guest socket must not see: elfuse owns + * O_NONBLOCK on the host descriptor, so it reports one for a readiness + * a sibling took. MSG_WAITALL is gathered here rather than handed to + * the host, which does not answer for it (msg_gathers_waitall). This + * branch carries no ancillary buffer, so the gather is unconditional + * once the predicate holds. + */ + bool gather = msg_gathers_waitall(host_ref.fd, flags, 0); + if (recv_strip_waitall(flags)) + mac_flags &= ~MSG_WAITALL; + + ssize_t ret; + uint64_t total = 0; + for (;;) { + msg.msg_flags = 0; + ret = recvmsg(host_ref.fd, &msg, mac_flags); + + if (ret > 0 && gather) { + total += (uint64_t) ret; + if (total >= len || + !sock_op_should_block(&sock_st, host_ref.fd, flags)) + break; + host_iov.iov_base = (char *) base + total; + host_iov.iov_len = len - total; + waited = net_wait_or_interrupted(&sock_st, host_ref.fd, POLLIN, + flags); + if (waited < 0) + break; /* interrupted after moving bytes: report the count + */ + continue; + } + + if (!net_recv_should_retry(&sock_st, host_ref.fd, flags, ret)) + break; + waited = + net_wait_or_interrupted(&sock_st, host_ref.fd, POLLIN, flags); + if (waited < 0) { + if (total > 0) + break; + host_fd_ref_close(&host_ref); + return waited; + } + } + if (total > 0) + ret = (ssize_t) total; if (ret < 0) { int64_t r = recv_eof_or_errno(host_ref.fd, fd); if (r != 0) { @@ -516,9 +612,10 @@ int64_t sys_recvmsg(guest_t *g, int fd, uint64_t msg_gva, int flags) host_fd_ref_close(&host_ref); return iov_err; } - int64_t waited = host_iov_has_payload(&host_iov, recv_iovcnt) - ? net_wait_or_interrupted(host_ref.fd, POLLIN, flags) - : net_recv_zero_payload_gate(host_ref.fd, flags); + int64_t waited = + host_iov_has_payload(&host_iov, recv_iovcnt) + ? net_wait_or_interrupted(&sock_st, host_ref.fd, POLLIN, flags) + : net_recv_zero_payload_gate(&sock_st, host_ref.fd, flags); if (waited < 0) { host_iov_free(&host_iov); host_fd_ref_close(&host_ref); @@ -566,7 +663,59 @@ int64_t sys_recvmsg(guest_t *g, int fd, uint64_t msg_gva, int flags) int scm_hfds[128]; int scm_nfds = 0; - ssize_t ret = recvmsg(host_ref.fd, &msg, mac_flags); + /* Retry the EAGAIN a blocking guest socket must not see; see the single-iov + * branch above. + */ + bool gather = msg_gathers_waitall(host_ref.fd, flags, ctrl_alloc); + if (recv_strip_waitall(flags)) + mac_flags &= ~MSG_WAITALL; + size_t want = gather ? msg_iov_total(msg.msg_iov, (int) msg.msg_iovlen) : 0; + + ssize_t ret; + uint64_t total = 0; + for (;;) { + msg.msg_namelen = lmsg.msg_name ? sa_len : 0; + msg.msg_controllen = ctrl_alloc; + msg.msg_flags = 0; + ret = recvmsg(host_ref.fd, &msg, mac_flags); + + if (ret > 0 && gather) { + total += (uint64_t) ret; + if (total >= want || + !sock_op_should_block(&sock_st, host_ref.fd, flags)) + break; + int spent = msg_iov_advance(msg.msg_iov, (int) msg.msg_iovlen, + (size_t) ret); + msg.msg_iov += spent; + msg.msg_iovlen -= spent; + if (msg.msg_iovlen == 0) + break; + + /* Only the first round may deliver a name; a later one would + * overwrite it with the same peer's address for no gain. + */ + msg.msg_name = NULL; + waited = + net_wait_or_interrupted(&sock_st, host_ref.fd, POLLIN, flags); + if (waited < 0) + break; /* interrupted after moving bytes: report the count */ + continue; + } + + if (!net_recv_should_retry(&sock_st, host_ref.fd, flags, ret)) + break; + waited = net_wait_or_interrupted(&sock_st, host_ref.fd, POLLIN, flags); + if (waited < 0) { + if (total > 0) + break; + free(mac_ctrl_heap); + host_iov_free(&host_iov); + host_fd_ref_close(&host_ref); + return waited; + } + } + if (total > 0) + ret = (ssize_t) total; if (ret < 0) { int64_t r = recv_eof_or_errno(host_ref.fd, fd); free(mac_ctrl_heap); @@ -723,7 +872,36 @@ int64_t sys_recvmsg(guest_t *g, int fd, uint64_t msg_gva, int flags) size_t nfds = data_len / sizeof(int); for (size_t i = 0; i < nfds; i++) { int host_recv_fd = fds[i]; - int gfd = fd_alloc(FD_REGULAR, fds[i], NULL); + + /* A received descriptor belongs to whoever sent it, so + * elfuse never sets O_NONBLOCK on one: that would mutate a + * description this process did not create, and a guest that + * passes fd 0 to itself would leave the launching shell's + * terminal nonblocking after exit. Marked foreign so a dup + * of it inherits the same restraint. + * + * Which view of the flag is the guest's still has to be + * decided, and the flag as found answers it. A description + * already carrying O_NONBLOCK is one elfuse itself owns: + * every pipe and fifo it opens carries the flag precisely + * because it emulates blocking on top, and an ordinary + * program's description does not. Adopting that emulation + * here sets nothing (the flag is already there) and gives a + * plain read the wait it asked for. Reading the host flag + * as the guest's view instead would hand a blocking read a + * spurious EAGAIN, which a program that never requested + * O_NONBLOCK has no handling for. + * + * What stays wrong is a sender that meant the description + * to be nonblocking: the receiver sees blocking. That is + * the cross-process divergence in TODO.md, and this is its + * survivable direction -- a wait the sender would not have + * waited, rather than an error on an fd that has none. + */ + int recv_fl = fcntl(host_recv_fd, F_GETFL); + fd_alias_spec_t spec = fd_alias_carried( + true, recv_fl >= 0 && (recv_fl & O_NONBLOCK)); + int gfd = fd_alloc_alias(&spec, FD_REGULAR, fds[i], NULL); if (gfd < 0) { close(fds[i]); fds[i] = -1; @@ -882,7 +1060,8 @@ int64_t sys_sendmmsg(guest_t *g, bool suppress_sigpipe = (flags & 0x4000) != 0; host_fd_ref_t host_ref; - if (host_fd_ref_open(fd, &host_ref) < 0) + fd_block_state_t sock_st; + if (host_fd_ref_open_state(fd, &host_ref, &sock_st) < 0) return -LINUX_EBADF; if (guest_read_small(g, msg_gva, &lmsg, sizeof(lmsg)) < 0) { host_fd_ref_close(&host_ref); @@ -916,7 +1095,8 @@ int64_t sys_sendmmsg(guest_t *g, len = (size_t) avail; } - bool blocking = len > 0 && sock_op_should_block(host_ref.fd, flags); + bool blocking = + len > 0 && sock_op_should_block(&sock_st, host_ref.fd, flags); int host_flags = mac_flags | (blocking ? MSG_DONTWAIT : 0); ssize_t ret; for (;;) { @@ -979,7 +1159,8 @@ int64_t sys_recvmmsg(guest_t *g, int mac_flags = translate_msg_flags(flags); host_fd_ref_t host_ref; - if (host_fd_ref_open(fd, &host_ref) < 0) + fd_block_state_t sock_st; + if (host_fd_ref_open_state(fd, &host_ref, &sock_st) < 0) return -LINUX_EBADF; if (guest_read_small(g, msg_gva, &lmsg, sizeof(lmsg)) < 0) { host_fd_ref_close(&host_ref); @@ -1018,14 +1199,55 @@ int64_t sys_recvmmsg(guest_t *g, .msg_iov = &host_iov, .msg_iovlen = 1, }; - int64_t waited = - len > 0 ? net_wait_or_interrupted(host_ref.fd, POLLIN, flags) - : net_recv_zero_payload_gate(host_ref.fd, flags); + int64_t waited = len > 0 ? net_wait_or_interrupted( + &sock_st, host_ref.fd, POLLIN, flags) + : net_recv_zero_payload_gate( + &sock_st, host_ref.fd, flags); if (waited < 0) { host_fd_ref_close(&host_ref); return waited; } - ssize_t ret = recvmsg(host_ref.fd, &host_msg, mac_flags); + + /* Retry the EAGAIN a blocking guest socket must not see, and gather + * MSG_WAITALL rather than hand it to the host; see sys_recvmsg. + */ + bool gather = msg_gathers_waitall(host_ref.fd, flags, 0); + if (recv_strip_waitall(flags)) + mac_flags &= ~MSG_WAITALL; + + ssize_t ret; + uint64_t total = 0; + for (;;) { + host_msg.msg_flags = 0; + ret = recvmsg(host_ref.fd, &host_msg, mac_flags); + + if (ret > 0 && gather) { + total += (uint64_t) ret; + if (total >= len || + !sock_op_should_block(&sock_st, host_ref.fd, flags)) + break; + host_iov.iov_base = (char *) base + total; + host_iov.iov_len = len - total; + waited = net_wait_or_interrupted(&sock_st, host_ref.fd, + POLLIN, flags); + if (waited < 0) + break; + continue; + } + + if (!net_recv_should_retry(&sock_st, host_ref.fd, flags, ret)) + break; + waited = net_wait_or_interrupted(&sock_st, host_ref.fd, POLLIN, + flags); + if (waited < 0) { + if (total > 0) + break; + host_fd_ref_close(&host_ref); + return waited; + } + } + if (total > 0) + ret = (ssize_t) total; if (ret < 0) { int64_t r = recv_eof_or_errno(host_ref.fd, fd); if (r != 0) { diff --git a/src/syscall/net.c b/src/syscall/net.c index 11c457f8..78ab41d7 100644 --- a/src/syscall/net.c +++ b/src/syscall/net.c @@ -47,6 +47,8 @@ #define LINUX_MSG_OOB 0x01 /* Linux MSG_DONTWAIT: recv/send skip the interruptible wait when set. */ #define LINUX_MSG_DONTWAIT 0x40 +#define LINUX_MSG_PEEK 0x02 +#define LINUX_MSG_WAITALL 0x100 /* Wait for a blocking socket op (recv/accept/connect/send) to become ready or * be interrupted by a guest signal, so a vCPU thread parked in the host call @@ -56,64 +58,150 @@ * * Returns 0 to proceed or a negative Linux errno (EINTR) to abort. */ -int64_t net_wait_or_interrupted(int host_fd, short events, int msg_flags) +int64_t net_wait_or_interrupted(const fd_block_state_t *st, + int host_fd, + short events, + int msg_flags) { - if (msg_flags & LINUX_MSG_DONTWAIT) - return 0; - int fl = fcntl(host_fd, F_GETFL); - if (fl < 0 || (fl & O_NONBLOCK)) + if (!sock_op_should_block(st, host_fd, msg_flags)) return 0; return io_wait_fd_or_interrupted(host_fd, events); } +/* Is there anything for a zero-length receive to return, asked without + * consuming it? + * + * poll answers a different question -- "the descriptor is readable" -- and its + * answer is already stale by the time the guest's call runs: a sibling on the + * same socket can take the byte in between, and the zero-length recv then + * returns 0 to a guest that asked to block until something arrived. A one-byte + * MSG_PEEK asks the socket buffer itself, atomically against it, and takes + * nothing away from whoever ends up reading it. + * + * A peek of 0 is end of file, which is also a case where Linux lets a + * zero-length recv through, so it counts as ready. Same for a zero-length + * datagram, which is indistinguishable here and wants the same answer. + * + * Returns 1 ready, 0 nothing there, -1 with errno set on a real failure. + */ +static int zero_len_ready(int host_fd) +{ + char probe; + ssize_t n = recv(host_fd, &probe, 1, MSG_PEEK | MSG_DONTWAIT); + if (n >= 0) + return 1; + if (errno == EAGAIN) + return 0; + return -1; +} + /* Linux clamps a socket receive's low-water target to one byte (sock_rcvlowat * returns v ?: 1), so a zero-payload recv/recvfrom/recvmsg on an empty socket * blocks -- or fails EAGAIN when nonblocking -- instead of returning 0 the way * the macOS host call does. (read() is the exception: sock_read_iter returns 0 * for a zero count, so sys_read stays untouched.) Gate the host call on - * readability: an interruptible wait for blocking callers, a zero-timeout - * readiness probe for nonblocking ones. EOF counts as readable in both, and the - * host call then returns 0 like Linux. + * readability: an interruptible wait for blocking callers, an immediate probe + * for nonblocking ones, both answered by zero_len_ready below. EOF counts as + * ready in both, and the host call then returns 0 like Linux. * * Returns 0 to proceed or a negative Linux errno (EINTR/EAGAIN). */ -int64_t net_recv_zero_payload_gate(int host_fd, int msg_flags) + +int64_t net_recv_zero_payload_gate(const fd_block_state_t *st, + int host_fd, + int msg_flags) { /* Linux's urgent-data receive path never waits for readiness: with no - * urgent data queued, recv(MSG_OOB) fails EINVAL immediately whether the - * socket blocks or not (tcp_recv_urg, unix_stream_recv_urg; verified on - * 6.12). Pass straight to the host call, which fails the same way. + * urgent data queued, recv(MSG_OOB) fails immediately whether the socket + * blocks or not (tcp_recv_urg, unix_stream_recv_urg; verified on 6.12). + * Pass straight to the host call, which also fails immediately. + * + * It does not fail with the same errno on AF_UNIX, and this used to claim + * it did. Linux supports out-of-band data there and answers EINVAL when + * none is queued; macOS does not support it on AF_UNIX at all and answers + * EOPNOTSUPP. Measured against the reference kernel, 22 against 95. The + * errno is left as the host gives it rather than rewritten, because a guest + * that reads EOPNOTSUPP as "no OOB here" is being told the truth about this + * host, while EINVAL would invite it to keep asking. */ if (msg_flags & LINUX_MSG_OOB) return 0; - if (sock_op_should_block(host_fd, msg_flags)) - return io_wait_fd_or_interrupted(host_fd, POLLIN); - struct pollfd pfd = {.fd = host_fd, .events = POLLIN}; - int ready = poll(&pfd, 1, 0); + + if (sock_op_should_block(st, host_fd, msg_flags)) { + /* Wait, then ask the buffer, and go back to waiting if a sibling got + * there first. Each round blocks until the socket says something, so + * this cannot spin. + */ + for (;;) { + int64_t waited = io_wait_fd_or_interrupted(host_fd, POLLIN); + if (waited < 0) + return waited; + int ready = zero_len_ready(host_fd); + if (ready < 0) + return linux_errno(); + if (ready > 0) + return 0; + } + } + + int ready = zero_len_ready(host_fd); if (ready < 0) return linux_errno(); - if (ready == 0) - return -LINUX_EAGAIN; - return 0; + return ready > 0 ? 0 : -LINUX_EAGAIN; } /* True when a socket send/recv should wait interruptibly and retry rather than * surface EAGAIN: the guest asked for blocking semantics (no MSG_DONTWAIT, fd - * not O_NONBLOCK). The send/recv paths probe with a per-call MSG_DONTWAIT and - * loop on EAGAIN when this holds, so a post-readiness buffer-full/steal race - * retries instead of parking the vCPU in an uninterruptible host call. Using - * MSG_DONTWAIT rather than toggling the fd's O_NONBLOCK keeps the flag off the - * shared open file description, so a sibling thread on the same fd is never hit - * with a spurious EAGAIN. + * not O_NONBLOCK). The send/recv paths loop on EAGAIN when this holds, so a + * post-readiness buffer-full/steal race retries instead of parking the vCPU in + * an uninterruptible host call. + * + * The answer comes from the pinned state and not from the host descriptor. + * elfuse owns O_NONBLOCK on the sockets it creates, so their host flag is + * always set and records nothing about what the guest asked for. Ownership is + * what makes the retry loop real: this used to lean on a per-call MSG_DONTWAIT + * instead, and macOS does not honour that flag on AF_UNIX -- a send on a full + * stream socket writes what fits and then blocks in the kernel for the rest, + * with the flag set (measured: one send moved 8192 bytes and sat for three + * seconds). + * + * A description elfuse did not create -- one that arrived over SCM_RIGHTS -- is + * not owned, and there the host flag is still the only record of it. */ -bool sock_op_should_block(int host_fd, int msg_flags) +bool sock_op_should_block(const fd_block_state_t *st, + int host_fd, + int msg_flags) { if (msg_flags & LINUX_MSG_DONTWAIT) return false; + if (fd_nonblock_shadowed(st->type, st->nonblock_owned)) + return !st->guest_nonblock; int fl = fcntl(host_fd, F_GETFL); return fl >= 0 && !(fl & O_NONBLOCK); } +int sock_creation_flags(int nonblock, int cloexec) +{ + return (nonblock ? LINUX_O_NONBLOCK : 0) | (cloexec ? LINUX_O_CLOEXEC : 0); +} + +bool net_recv_should_retry(const fd_block_state_t *st, + int host_fd, + int msg_flags, + ssize_t ret) +{ + if (ret >= 0 || errno != EAGAIN) + return false; + + /* sock_op_should_block may run an fcntl for a description elfuse does not + * own, and the caller reports the transfer's errno when this says no. + */ + int saved_errno = errno; + bool retry = sock_op_should_block(st, host_fd, msg_flags); + errno = saved_errno; + return retry; +} + /* Drive an already-nonblocking connect to completion or interruption: start it, * wait for POLLOUT (or a guest signal), then read SO_ERROR. * @@ -168,13 +256,34 @@ static int64_t connect_nonblock_wait(int host_fd, * or one that cannot be flipped, falls back to a plain connect (best-effort * interruptibility, never a lost connect); its EINPROGRESS surfaces unchanged. */ -static int64_t connect_or_interrupted(int host_fd, +static int64_t connect_or_interrupted(const fd_block_state_t *st, + int host_fd, const struct sockaddr *sa, socklen_t len) { + /* The guest asked for a nonblocking connect: hand it straight through, and + * EINPROGRESS with it. + */ + if (!sock_op_should_block(st, host_fd, 0)) + return connect(host_fd, sa, len) < 0 ? linux_errno() : 0; + + /* An owned socket is already nonblocking at the host and has to stay that + * way -- the flag is elfuse's, not the guest's, and putting it back would + * park the next transfer. Nothing to toggle, so just drive the connect. + * + * Reading the host flag here instead of the shadow is what broke when + * sockets became owned: every socket looked nonblocking, so every blocking + * connect returned EINPROGRESS to a guest that had asked to wait + * (tests/test-exec-handoff.c caught it). + */ + if (st->nonblock_owned) + return connect_nonblock_wait(host_fd, sa, len); + + /* A description elfuse does not own, whose flag really is the guest's: + * borrow O_NONBLOCK for the wait and give it back. + */ int fl = fcntl(host_fd, F_GETFL); - bool blocking = fl >= 0 && !(fl & O_NONBLOCK); - if (!blocking || fcntl(host_fd, F_SETFL, fl | O_NONBLOCK) < 0) + if (fl < 0 || fcntl(host_fd, F_SETFL, fl | O_NONBLOCK) < 0) return connect(host_fd, sa, len) < 0 ? linux_errno() : 0; int64_t result = connect_nonblock_wait(host_fd, sa, len); @@ -263,8 +372,7 @@ int64_t sys_socket(guest_t *g, int domain, int type, int protocol) close(fd); return -LINUX_EMFILE; } - if (cloexec) - fd_table[gfd].linux_flags |= LINUX_O_CLOEXEC; + fd_table[gfd].linux_flags |= sock_creation_flags(nonblock, cloexec); net_socket_cache_init_defaults(gfd, domain, original_type); return gfd; } @@ -290,10 +398,7 @@ int64_t sys_socket(guest_t *g, int domain, int type, int protocol) return -LINUX_EMFILE; } - int linux_flags = 0; - if (cloexec) - linux_flags |= LINUX_O_CLOEXEC; - fd_table[gfd].linux_flags = linux_flags; + fd_table[gfd].linux_flags = sock_creation_flags(nonblock, cloexec); net_socket_cache_init_defaults(gfd, domain, original_type); return gfd; @@ -345,7 +450,7 @@ int64_t sys_socketpair(guest_t *g, return -LINUX_EMFILE; } - int linux_flags = cloexec ? LINUX_O_CLOEXEC : 0; + int linux_flags = sock_creation_flags(nonblock, cloexec); fd_table[gfd0].linux_flags = linux_flags; fd_table[gfd1].linux_flags = linux_flags; net_socket_cache_init_defaults(gfd0, domain, original_type); @@ -453,9 +558,10 @@ static int64_t do_accept(guest_t *g, uint64_t listener_generation = 0; int listener_passcred_fallback = 0; int listener_type = FD_CLOSED; + fd_block_state_t sock_st = {.type = FD_CLOSED}; if (thread_is_single_active()) { - if (host_fd_ref_open(fd, &host_ref) < 0) + if (host_fd_ref_open_state(fd, &host_ref, &sock_st) < 0) return -LINUX_EBADF; listener_type = fd_table[fd].type; listener_generation = fd_table[fd].generation; @@ -469,6 +575,7 @@ static int64_t do_accept(guest_t *g, return -LINUX_EBADF; host_ref.fd = host_fd; host_ref.owned = true; + sock_st = fd_block_state_of(&listener_snap); listener_type = listener_snap.type; listener_generation = listener_snap.generation; if (listener_type == FD_SOCKET) @@ -481,19 +588,34 @@ static int64_t do_accept(guest_t *g, return -LINUX_ENOTSOCK; } - int64_t waited = net_wait_or_interrupted(host_ref.fd, POLLIN, 0); - if (waited < 0) { - host_fd_ref_close(&host_ref); - return waited; - } - struct sockaddr_storage mac_sa; - socklen_t mac_len = sizeof(mac_sa); + socklen_t mac_len; + int new_fd; + + /* Wait, accept, and retry the EAGAIN a blocking guest must not see. elfuse + * owns O_NONBLOCK on the listener, so a sibling that takes the connection + * this wait reported leaves EAGAIN here rather than leaving the accept to + * block, which is the same race the recv paths run and the same answer: the + * guest asked to wait, so wait again. + */ + for (;;) { + int64_t waited = + net_wait_or_interrupted(&sock_st, host_ref.fd, POLLIN, 0); + if (waited < 0) { + host_fd_ref_close(&host_ref); + return waited; + } - int new_fd = accept(host_ref.fd, (struct sockaddr *) &mac_sa, &mac_len); + mac_len = sizeof(mac_sa); + new_fd = accept(host_ref.fd, (struct sockaddr *) &mac_sa, &mac_len); + if (new_fd >= 0) + break; + if (!net_recv_should_retry(&sock_st, host_ref.fd, 0, -1)) { + host_fd_ref_close(&host_ref); + return linux_errno(); + } + } host_fd_ref_close(&host_ref); - if (new_fd < 0) - return linux_errno(); int listener_passcred = listener_passcred_fallback; (void) net_socket_cached_int_get_if_generation( @@ -514,7 +636,7 @@ static int64_t do_accept(guest_t *g, close(new_fd); return -LINUX_EMFILE; } - fd_table[gfd].linux_flags = cloexec ? LINUX_O_CLOEXEC : 0; + fd_table[gfd].linux_flags = sock_creation_flags(nonblock, cloexec); net_socket_cache_init_accept(gfd, listener_passcred); /* Write back peer address if requested. The accept has already succeeded @@ -576,7 +698,8 @@ int64_t sys_accept4(guest_t *g, int64_t sys_connect(guest_t *g, int fd, uint64_t addr_gva, uint32_t addrlen) { host_fd_ref_t host_ref; - if (host_fd_ref_open(fd, &host_ref) < 0) + fd_block_state_t sock_st; + if (host_fd_ref_open_state(fd, &host_ref, &sock_st) < 0) return -LINUX_EBADF; uint8_t linux_sa[128]; @@ -649,10 +772,17 @@ int64_t sys_connect(guest_t *g, int fd, uint64_t addr_gva, uint32_t addrlen) setsockopt(pair[0], SOL_SOCKET, SO_NOSIGPIPE, &one, sizeof(one)); setsockopt(pair[1], SOL_SOCKET, SO_NOSIGPIPE, &one, sizeof(one)); - int old_status = fcntl(host_ref.fd, F_GETFL, 0); + /* pair[0] is a fresh description wearing the old slot's identity, so + * the host flag has to be set here rather than inherited. The alias + * spec below claims nonblock_owned from the snapshot and the allocator + * takes that claim at its word -- an alias normally shares a + * description that already carries the flag, and this one does not. + * Testing the old host status instead would have been vacuous now that + * every socket elfuse opens carries O_NONBLOCK. + */ fd_entry_t snap; bool have_snap = fd_snapshot(fd, &snap); - if ((old_status >= 0 && (old_status & O_NONBLOCK) && + if ((have_snap && snap.nonblock_owned && fd_set_nonblock(pair[0]) < 0) || (have_snap && (snap.linux_flags & LINUX_O_CLOEXEC) && fd_set_cloexec(pair[0]) < 0)) { @@ -662,8 +792,25 @@ int64_t sys_connect(guest_t *g, int fd, uint64_t addr_gva, uint32_t addrlen) return linux_errno(); } - if (fd_alloc_at(fd, FD_SOCKET, pair[0], absock_unregister_fd, NULL) < - 0) { + /* Rebuilding the slot must not mint a fresh description identity: any + * dup alias of this fd keeps the old ofd_id, and the two would stop + * being swept together by the O_NONBLOCK, O_ASYNC and SIGIO-owner + * walks. The aliases still carry the pre-upgrade host fd, which is a + * deeper divergence recorded in TODO.md; keeping the identity is what + * stops this path from also breaking the sweeps. + */ + fd_alias_spec_t spec; + if (have_snap) + + /* -1, not fd: this path is rebuilding the very slot it snapshotted, + * so re-reading it under the publish lock would find the + * replacement being installed rather than the source. + */ + spec = fd_alias_of(-1, &snap); + int alloc_rc = + fd_alloc_alias_at(have_snap ? &spec : NULL, fd, FD_SOCKET, pair[0], + absock_unregister_fd, NULL); + if (alloc_rc < 0) { close(pair[0]); close(pair[1]); host_fd_ref_close(&host_ref); @@ -689,8 +836,9 @@ int64_t sys_connect(guest_t *g, int fd, uint64_t addr_gva, uint32_t addrlen) return -LINUX_EPROTOTYPE; } - int64_t crc = connect_or_interrupted( - host_ref.fd, (struct sockaddr *) &mac_sa, (socklen_t) mac_len); + int64_t crc = connect_or_interrupted(&sock_st, host_ref.fd, + (struct sockaddr *) &mac_sa, + (socklen_t) mac_len); host_fd_ref_close(&host_ref); return crc; } @@ -833,7 +981,8 @@ int64_t sys_sendto(guest_t *g, return netlink_send(fd, g, buf_gva, len); host_fd_ref_t host_ref; - if (host_fd_ref_open(fd, &host_ref) < 0) + fd_block_state_t send_st; + if (host_fd_ref_open_state(fd, &host_ref, &send_st) < 0) return -LINUX_EBADF; uint64_t avail = 0; @@ -876,7 +1025,8 @@ int64_t sys_sendto(guest_t *g, dest_len = (socklen_t) mac_len; } - bool blocking = len > 0 && sock_op_should_block(host_ref.fd, linux_flags); + bool blocking = + len > 0 && sock_op_should_block(&send_st, host_ref.fd, linux_flags); int host_flags = mac_flags | (blocking ? MSG_DONTWAIT : 0); ssize_t ret; for (;;) { @@ -900,6 +1050,58 @@ int64_t sys_sendto(guest_t *g, return ret; } +/* MSG_WAITALL never reaches the host, and this says whether elfuse then has to + * gather the request itself. + * + * Two separate things, because the answers differ. macOS does not answer for + * that flag the way Linux does under any of the shapes measured here, so it is + * stripped unconditionally by recv_strip_waitall below. Whether to loop + * afterwards is the narrower question, and only a stream socket without + * MSG_PEEK says yes. + * + * Every rule here is a measurement against the qemu reference kernel, not a + * reading of the manual page, and two of them contradict what the manual page + * suggests: + * + * plain stream, 2 of 16 queued: Linux blocks for the rest, macOS blocks + * forever even with MSG_DONTWAIT also set. Gathered here. + * MSG_PEEK on a stream, 2 of 16 queued: Linux returns 2 and does not wait, + * because a peek does not consume and it will not spin re-reading the same + * bytes. macOS returns 16, reporting fourteen bytes of whatever the guest + * buffer already held as received data. One host call, no gather. + * SOCK_SEQPACKET, two 4-byte messages queued, 16 requested: Linux returns 4. + * One recv is one message and MSG_WAITALL does not join them. Gathering + * would concatenate them and destroy the boundary. + * SOCK_DGRAM: Linux ignores MSG_WAITALL entirely. + */ +/* buf + total, with NULL preserved. + * + * A zero-length recv passes a NULL buffer, and NULL + 0 is undefined even + * though every compiler here folds it to NULL; UBSan says so out loud. + */ +static void *recv_at(void *buf, uint64_t total) +{ + return buf ? (char *) buf + total : NULL; +} + +bool recv_strip_waitall(int linux_flags) +{ + return (linux_flags & LINUX_MSG_WAITALL) != 0; +} + +bool recv_gathers_waitall(int host_fd, int linux_flags) +{ + if (!(linux_flags & LINUX_MSG_WAITALL) || (linux_flags & LINUX_MSG_PEEK)) + return false; + + int sotype = 0; + socklen_t sotype_len = sizeof(sotype); + if (getsockopt(host_fd, SOL_SOCKET, SO_TYPE, &sotype, &sotype_len) < 0) + return false; + return sotype == SOCK_STREAM; +} + + int64_t sys_recvfrom(guest_t *g, int fd, uint64_t buf_gva, @@ -912,7 +1114,8 @@ int64_t sys_recvfrom(guest_t *g, return netlink_recv(fd, g, buf_gva, len, flags, src_gva, addrlen_gva); host_fd_ref_t host_ref; - if (host_fd_ref_open(fd, &host_ref) < 0) + fd_block_state_t recv_st; + if (host_fd_ref_open_state(fd, &host_ref, &recv_st) < 0) return -LINUX_EBADF; uint64_t avail = 0; @@ -927,29 +1130,86 @@ int64_t sys_recvfrom(guest_t *g, int mac_flags = translate_msg_flags(flags); - /* A single interruptible wait (not a MSG_DONTWAIT probe loop) preserves - * MSG_WAITALL semantics; the tiny ready-then-stolen window can still block, - * matching sys_read. A zero-length recv takes the readiness gate instead: - * unlike read(), Linux blocks it on an empty socket. + /* Wait interruptibly, then retry the recv for as long as the guest asked + * for blocking semantics. The retry is what preserves them now that elfuse + * owns O_NONBLOCK on the host socket: the descriptor answers EAGAIN both + * when a sibling took the readiness this wait reported and when MSG_WAITALL + * has less than the full request queued. A zero-length recv takes the + * readiness gate instead: unlike read(), Linux blocks it on an empty + * socket. */ - int64_t waited = len > 0 - ? net_wait_or_interrupted(host_ref.fd, POLLIN, flags) - : net_recv_zero_payload_gate(host_ref.fd, flags); + int64_t waited = + len > 0 ? net_wait_or_interrupted(&recv_st, host_ref.fd, POLLIN, flags) + : net_recv_zero_payload_gate(&recv_st, host_ref.fd, flags); if (waited < 0) { host_fd_ref_close(&host_ref); return waited; } struct sockaddr_storage mac_sa; - socklen_t mac_len = sizeof(mac_sa); + socklen_t mac_len; + + bool gather = recv_gathers_waitall(host_ref.fd, flags); + if (recv_strip_waitall(flags)) + mac_flags &= ~MSG_WAITALL; ssize_t ret; - if (src_gva && addrlen_gva) { - ret = recvfrom(host_ref.fd, buf, len, mac_flags, - (struct sockaddr *) &mac_sa, &mac_len); - } else { - ret = recv(host_ref.fd, buf, len, mac_flags); + uint64_t total = 0; + for (;;) { + /* Reset per round: a recvfrom that failed may still have written it. */ + mac_len = sizeof(mac_sa); + if (src_gva && addrlen_gva) { + ret = recvfrom(host_ref.fd, recv_at(buf, total), len - total, + mac_flags, (struct sockaddr *) &mac_sa, &mac_len); + } else { + ret = + recv(host_ref.fd, recv_at(buf, total), len - total, mac_flags); + } + + if (ret > 0) { + total += (uint64_t) ret; + + /* Whole request in hand, or nothing asked us to gather more. */ + if (!gather || total >= len) + break; + + /* Linux stops a gathering recv at the first partial return when the + * guest asked not to wait, rather than reporting EAGAIN over bytes + * it has already moved. + */ + if (!sock_op_should_block(&recv_st, host_ref.fd, flags)) + break; + + waited = + net_wait_or_interrupted(&recv_st, host_ref.fd, POLLIN, flags); + if (waited < 0) + break; /* interrupted after moving bytes: report the count */ + continue; + } + + /* EOF, or an error. Either ends the gathering: what has been moved is + * the answer, and a stream that has closed will not deliver the rest. + */ + if (ret == 0) + break; + if (!net_recv_should_retry(&recv_st, host_ref.fd, flags, ret)) + break; + waited = net_wait_or_interrupted(&recv_st, host_ref.fd, POLLIN, flags); + if (waited < 0) { + if (total > 0) + break; + host_fd_ref_close(&host_ref); + return waited; + } } + + /* Bytes already delivered outrank whatever ended the loop, which is what + * Linux reports and what keeps a partial gather from being retried by a + * guest that would then read the same stream twice. + */ + if (total > 0) + ret = (ssize_t) total; + if (ret < 0) { int64_t result = recv_eof_or_errno(host_ref.fd, fd); if (result < 0) { diff --git a/src/syscall/net.h b/src/syscall/net.h index 946c300b..d17b0b0a 100644 --- a/src/syscall/net.h +++ b/src/syscall/net.h @@ -16,6 +16,7 @@ #include #include "core/guest.h" #include "syscall/linux-wire.h" /* linux_iovec_t */ +#include "syscall/internal.h" /* fd_block_state_t */ /* Linux address families. */ #define LINUX_AF_UNSPEC 0 @@ -164,7 +165,10 @@ int64_t sys_shutdown(int fd, int how); * * Returns 0 to proceed or a negative Linux errno (EINTR). */ -int64_t net_wait_or_interrupted(int host_fd, short events, int msg_flags); +int64_t net_wait_or_interrupted(const fd_block_state_t *st, + int host_fd, + short events, + int msg_flags); /* Readiness gate for a zero-payload recv/recvfrom/recvmsg: Linux clamps the * receive low-water target to one byte, so an empty socket blocks (EINTR on @@ -174,7 +178,9 @@ int64_t net_wait_or_interrupted(int host_fd, short events, int msg_flags); * * Returns 0 to proceed or a negative Linux errno (EINTR/EAGAIN). */ -int64_t net_recv_zero_payload_gate(int host_fd, int msg_flags); +int64_t net_recv_zero_payload_gate(const fd_block_state_t *st, + int host_fd, + int msg_flags); /* True when a socket send/recv should wait interruptibly and retry on EAGAIN * rather than surface it (guest wants blocking semantics: no MSG_DONTWAIT, fd @@ -182,7 +188,43 @@ int64_t net_recv_zero_payload_gate(int host_fd, int msg_flags); * so the interruptible wait cannot be defeated by a post-readiness buffer-full * or steal race, without toggling the fd's shared O_NONBLOCK flag. */ -bool sock_op_should_block(int host_fd, int msg_flags); +bool sock_op_should_block(const fd_block_state_t *st, + int host_fd, + int msg_flags); + +/* The guest-visible status flags a socket created with SOCK_NONBLOCK and/or + * SOCK_CLOEXEC carries. + * + * O_NONBLOCK has to be recorded here and not only on the host descriptor: + * elfuse owns the host flag on every socket it opens (fd_init_entry), so the + * host flag is set whatever the guest asked for, and the shadow is the only + * record left. Without it a SOCK_NONBLOCK socket read as blocking everywhere + * that matters -- F_GETFL reported no O_NONBLOCK, connect waited instead of + * reporting EINPROGRESS, and a read of an empty socket never returned. + */ +int sock_creation_flags(int nonblock, int cloexec); + +/* True when MSG_WAITALL is removed before this recv reaches the host. */ +bool recv_strip_waitall(int linux_flags); + +/* True when this recv must gather MSG_WAITALL itself. See the definition in + * net.c for the measurement. + */ +bool recv_gathers_waitall(int host_fd, int linux_flags); + +/* True when a socket receive that just reported EAGAIN has to be waited out and + * retried rather than reported to the guest. + * + * elfuse owns O_NONBLOCK on the sockets it opens, so a host recv answers EAGAIN + * in two cases a blocking guest socket must not see: the readiness the wait + * reported was taken by a sibling, and MSG_WAITALL with less than the full + * request queued, which macOS reports as EWOULDBLOCK while consuming nothing. + * Preserves errno, which the caller reports when this returns false. + */ +bool net_recv_should_retry(const fd_block_state_t *st, + int host_fd, + int msg_flags, + ssize_t ret); int64_t sys_sendmsg(guest_t *g, int fd, uint64_t msg_gva, int flags); int64_t sys_recvmsg(guest_t *g, int fd, uint64_t msg_gva, int flags); int64_t sys_sendmmsg(guest_t *g, diff --git a/src/syscall/netlink.c b/src/syscall/netlink.c index c1b7b5eb..f797f355 100644 --- a/src/syscall/netlink.c +++ b/src/syscall/netlink.c @@ -831,16 +831,13 @@ int64_t netlink_sendmsg(int guest_fd, guest_t *g, uint64_t msg_gva, int flags) * * On success returns 0 with nl_lock still held and ns valid. On EAGAIN, EINTR, * EIO, or if the socket was closed underneath the poll, releases nl_lock and - * returns the negative Linux errno. flags carries MSG_DONTWAIT; pass 0 for - * read(2), which only honors O_NONBLOCK. + * returns the negative Linux errno. */ static int64_t nl_wait_readable_locked(netlink_state_t *ns, int guest_fd, - int flags) + bool nonblock) { while (ns->buf_pos >= ns->buf_len) { - bool nonblock = (flags & LINUX_MSG_DONTWAIT) || - (fd_table[guest_fd].linux_flags & LINUX_O_NONBLOCK); if (nonblock) { pthread_mutex_unlock(&nl_lock); return -LINUX_EAGAIN; @@ -938,8 +935,8 @@ static void nl_write_kernel_src(guest_t *g, * of them hands out a message split across two calls. * * Returns the byte count, or a negative Linux errno. Takes nl_lock and releases - * it before returning. flags carries MSG_DONTWAIT; pass 0 for read(2), which - * only honors O_NONBLOCK. + * it before returning. nonblock is sampled before nl_lock so this leaf lock + * does not nest fd_lock. */ static int64_t netlink_recv_iov(int guest_fd, guest_t *g, @@ -947,6 +944,7 @@ static int64_t netlink_recv_iov(int guest_fd, int iovcnt, int flags) { + bool nonblock = (flags & LINUX_MSG_DONTWAIT) || fd_guest_nonblock(guest_fd); pthread_mutex_lock(&nl_lock); netlink_state_t *ns = nl_find(guest_fd); if (!ns) { @@ -968,7 +966,7 @@ static int64_t netlink_recv_iov(int guest_fd, return 0; } - int64_t werr = nl_wait_readable_locked(ns, guest_fd, flags); + int64_t werr = nl_wait_readable_locked(ns, guest_fd, nonblock); if (werr < 0) return werr; diff --git a/src/syscall/path.c b/src/syscall/path.c index 74eb88f9..36b27a6b 100644 --- a/src/syscall/path.c +++ b/src/syscall/path.c @@ -285,6 +285,11 @@ static int parse_fd_magiclink(const char *path) return path_parse_proc_name(rest); } +int path_fd_magiclink_guest_fd(const char *path) +{ + return parse_fd_magiclink(path); +} + int path_fd_magiclink_dup(const char *path) { int fd = parse_fd_magiclink(path); diff --git a/src/syscall/path.h b/src/syscall/path.h index b1a35902..12ade317 100644 --- a/src/syscall/path.h +++ b/src/syscall/path.h @@ -282,6 +282,15 @@ int path_openat2_check_fd_xdev(int guest_fd, int start_class); */ int path_parse_proc_name(const char *name); +/* The guest descriptor an absolute fd magic link names, or -1 when the path is + * not that shape. Unlike path_fd_magiclink_dup this hands back the guest fd + * number itself, for callers that need the table entry rather than the object: + * opening one of these paths produces a second name for a description the + * process already holds, so the new slot has to inherit that entry's answers + * instead of probing (see fd_alias_host_shared). + */ +int path_fd_magiclink_guest_fd(const char *path); + /* An owned dup of the descriptor an absolute fd magic link names * ("/proc/self/fd/", the own-pid spelling, "/dev/fd/", "/dev/std*"), or * -1 when the path is not that shape or its descriptor is not backed by a plain diff --git a/src/syscall/poll.c b/src/syscall/poll.c index 1aec46ca..7cdc5b23 100644 --- a/src/syscall/poll.c +++ b/src/syscall/poll.c @@ -1048,6 +1048,14 @@ int epoll_dup_fd(int src_fd, return -1; } inst->refcount++; + + /* The alias names the same open file description, so it carries the + * source's status flags and its ofd_id rather than a freshly built set; + * every alias sweep (O_NONBLOCK shadow, O_ASYNC, the SIGIO owner) matches + * on ofd_id, and a rebuilt one hides the alias from all of them. + */ + int src_flags = fd_table[src_fd].linux_flags & FD_DESCRIPTION_FLAGS; + uint64_t src_ofd_id = fd_table[src_fd].ofd_id; int new_host_fd = dup(src_host_fd); if (new_host_fd < 0) { epoll_instance_unref_locked(inst); @@ -1059,14 +1067,19 @@ int epoll_dup_fd(int src_fd, /* Publish type, host_fd, the shared dir, and flags in one fd_lock critical * section (fd_alloc_dir_*), so the slot is never observable as FD_EPOLL - * with a NULL dir -- matching sys_epoll_create1. + * with a NULL dir -- matching sys_epoll_create1. The access mode comes from + * fd_type_accmode on publish, the same as it does there. + */ + int lflags = src_flags | (linux_flags & LINUX_O_CLOEXEC); + + /* The new slot aliases the source's description: the allocator installs its + * ofd_id, foreign_description and nonblock_owned inside the window that + * publishes the slot, rather than this path patching them on afterwards. */ - int lflags = linux_flags & LINUX_O_CLOEXEC; - int new_guest_fd = fixed_slot - ? fd_alloc_dir_at(fixed_guest_fd, FD_EPOLL, - new_host_fd, NULL, inst, lflags) - : fd_alloc_dir_from(min_guest_fd, FD_EPOLL, - new_host_fd, NULL, inst, lflags); + fd_alias_spec_t spec = fd_alias_identity(src_ofd_id, 0); + int new_guest_fd = fd_alloc_alias_dir( + &spec, fixed_slot ? fixed_guest_fd : -1, min_guest_fd, FD_EPOLL, + new_host_fd, NULL, inst, lflags); if (new_guest_fd < 0) { /* fd_alloc_dir_at fails only when fixed_guest_fd is out of range or * over RLIMIT_NOFILE; dup2/dup3 report that as EBADF, not the EMFILE @@ -1081,6 +1094,7 @@ int epoll_dup_fd(int src_fd, errno = saved_errno; return -1; } + return new_guest_fd; } @@ -1188,6 +1202,10 @@ int64_t sys_epoll_create1(int flags) return -LINUX_ENOMEM; } + /* No access mode here: FD_EPOLL is in fd_type_accmode, and every publish + * forces the mode that table names back in (fd_flags_with_accmode), so a + * creator naming it again is a second place for one fact to live. + */ int lflags = 0; if (flags & LINUX_EPOLL_CLOEXEC) lflags |= LINUX_O_CLOEXEC; diff --git a/src/syscall/syscall.c b/src/syscall/syscall.c index fba03113..7db794f8 100644 --- a/src/syscall/syscall.c +++ b/src/syscall/syscall.c @@ -1864,18 +1864,23 @@ static int64_t sc_memfd_create(guest_t *g, if (guest_read_small(g, x0, &first, sizeof(first)) < 0) return -LINUX_EFAULT; - char template[] = "/tmp/elfuse-memfd-XXXXXX"; - int fd = mkstemp(template); + int fd = tmpfile_anon("memfd"); if (fd < 0) return linux_errno(); - unlink(template); int gfd = fd_alloc(FD_REGULAR, fd, NULL); if (gfd < 0) { close(fd); return linux_errno(); } - if (flags & LINUX_MFD_CLOEXEC) - fd_table[gfd].linux_flags |= LINUX_O_CLOEXEC; + + /* Linux opens a memfd O_RDWR (shmem_file_setup then get_unused_fd_flags), + * and F_GETFL answers a regular file's access mode from the shadow because + * O_PATH and directory fds have no macOS equivalent. The type table cannot + * speak for this one: a memfd is FD_REGULAR like any other file. + */ + fd_publish_linux_flags( + gfd, + LINUX_O_RDWR | ((flags & LINUX_MFD_CLOEXEC) ? LINUX_O_CLOEXEC : 0)); fd_table[gfd].seals = (flags & LINUX_MFD_ALLOW_SEALING) ? 0 : LINUX_F_SEAL_SEAL; return gfd; @@ -2602,17 +2607,6 @@ int syscall_dispatch(hv_vcpu_t vcpu, guest_t *g, int *exit_code, bool verbose) tp != FD_SOCKET) goto slow_path; - /* Same racy-but-benign read as tp above, and no worse than the - * shipped tp-based divert: a concurrent close+reopen (only possible - * with a live sibling thread; a single active thread has no - * mutator) that flips this slot to a blocking fd could skip the - * divert for one call. The guest is already reading an fd it is - * concurrently reopening, so the pinned fd it gets is undefined - * regardless; the slow path carries the identical window. Not worth - * a lock on the hot regular-file read. - */ - bool can_block = fd_table[fd].can_block; - /* Proc-backed fds may need synthetic read/write handling (for * example, oom_* rereads recompute content on each read and proc * dirfds steer relative *at() resolution). Keep them on the slow @@ -2621,12 +2615,25 @@ int syscall_dispatch(hv_vcpu_t vcpu, guest_t *g, int *exit_code, bool verbose) if (fd_table[fd].proc_path[0] != '\0') goto slow_path; + /* Pin the descriptor and classify it together: io_xfer would + * otherwise look the slot up again, and a sibling reusing this fd + * number in between makes the two describe different objects. + */ host_fd_ref_t host_ref; - if (host_fd_ref_open(fd, &host_ref) != 0) + fd_block_state_t fast_st; + if (host_fd_ref_open_state(fd, &host_ref, &fast_st) != 0) goto slow_path; - /* Check seals after dup; the fd is still valid */ - if (nr == SYS_write && (fd_table[fd].seals & LINUX_F_SEAL_WRITE)) { + /* Both of the decisions below come from fast_st, the state taken + * with the descriptor, and not from a second look at the table. The + * pre-filter above can afford its racy read because a wrong answer + * only costs a diversion to the slow path; these two cannot. A + * can_block read that disagrees with the pinned fd picks the wrong + * transfer form for it, which is the parked-vCPU failure this path + * is built to avoid, and a seals read that disagrees enforces the + * seal of whatever occupied the slot a moment ago. + */ + if (nr == SYS_write && (fast_st.seals & LINUX_F_SEAL_WRITE)) { host_fd_ref_close(&host_ref); goto slow_path; } @@ -2647,30 +2654,30 @@ int syscall_dispatch(hv_vcpu_t vcpu, guest_t *g, int *exit_code, bool verbose) goto slow_path; } - /* A blocking read/write on a pipe, socket, fifo, or char device - * would park this vCPU thread in an uninterruptible host call where - * the preempt thread's hv_vcpus_exit cannot reach it. Probe - * non-blocking: read waits for POLLIN, write for POLLOUT; if the fd - * would block, divert to the slow path where sys_read/sys_write - * wait interruptibly (poll + wakeup pipe). Regular files never - * block (can_block is false) and stay on the fast path. + /* Readiness is not a reservation: a sibling thread or a forked + * process on the same open file description can take the bytes + * before this thread gets to them, and a transfer that then blocks + * parks this vCPU where hv_vcpus_exit and the wakeup pipe cannot + * reach it. io_xfer runs the transfer without that risk, waiting + * interruptibly when it has to, and reports a negative only for the + * cases the slow path has to answer: an interrupted wait, and a + * read on a pty master whose slaves are gone. + * + * Regular files never block (can_block is false) and go straight to + * the host call. */ - if (can_block) { + ssize_t ret; + if (fast_st.can_block) { short ev = (nr == SYS_read) ? POLLIN : POLLOUT; - struct pollfd pfd = {.fd = host_ref.fd, .events = ev}; - - /* Divert on not-ready (0) or probe error (< 0, e.g. EINTR): a - * blocking call here cannot be preempted, so let the - * interruptible slow path handle both. - */ - if (poll(&pfd, 1, 0) <= 0) { + struct iovec iov = {.iov_base = buf, .iov_len = count}; + if (io_xfer(fd, host_ref.fd, ev, &iov, 1, &ret, &fast_st) < 0) { host_fd_ref_close(&host_ref); goto slow_path; } + } else { + ret = (nr == SYS_read) ? read(host_ref.fd, buf, count) + : write(host_ref.fd, buf, count); } - - ssize_t ret = (nr == SYS_read) ? read(host_ref.fd, buf, count) - : write(host_ref.fd, buf, count); if (ret >= 0) { host_fd_ref_close(&host_ref); result = ret; diff --git a/src/utils.h b/src/utils.h index 29a602de..5f39c844 100644 --- a/src/utils.h +++ b/src/utils.h @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -296,6 +297,82 @@ static inline int fd_set_nonblock(int fd) return fd_update_status_flag(fd, O_NONBLOCK, true); } +/* Create a temp file that exists only as long as the returned fd: mkstemp under + * /tmp with an elfuse-- prefix, unlinked before it is handed back. + * + * The unlink is the point. Four places wanted this shape and each spelled out + * the create-then-unlink pair, which is a file left on disk the first time + * somebody adds an early return between the two. Callers that keep the name + * (the Rosetta AOT cache staging its output for a rename, and the FUSE exec + * materializer, which unlinks after the exec) genuinely differ and stay as they + * are; a flag to suppress the unlink here would just move their decision + * somewhere it reads as an afterthought. + * + * Returns the fd, or -1 with errno set. The path is never reported because + * nothing can reach it: that is what makes it anonymous. + */ +static inline int tmpfile_anon(const char *what) +{ + char path[64]; + int n = snprintf(path, sizeof(path), "/tmp/elfuse-%s-XXXXXX", what); + if (n < 0 || (size_t) n >= sizeof(path)) { + errno = ENAMETOOLONG; + return -1; + } + + int fd = mkstemp(path); + if (fd < 0) + return -1; + + /* The name has to go, or the fd is not anonymous and the caller has no way + * to remove a path it never sees. Retry EINTR, and fail the call rather + * than hand back a descriptor with a name still attached. + */ + int rc; + do { + rc = unlink(path); + } while (rc < 0 && errno == EINTR); + + /* Any failure other than ENOENT leaves a named file behind that the caller + * cannot see to remove, so the call fails rather than hand back a + * descriptor that is not anonymous. Retrying would not help: the error is a + * property of the path or the directory, not a transient of this call, and + * EINTR is already handled above. + */ + if (rc < 0 && errno != ENOENT) { + int unlink_errno = errno; + close(fd); + errno = unlink_errno; + return -1; + } + + /* Neither answer the unlink can give is proof the file is anonymous, so ask + * the descriptor. A success is not proof: another process with this uid can + * hard-link the entry between mkstemp and here, and then the unlink removes + * the name it was given while the descriptor stays linked under the other + * one. An ENOENT is not proof either: the same race with rename leaves the + * unlink missing entirely. The link count is the fact that matters in both, + * so it is checked in both. + */ + struct stat anon; + if (fstat(fd, &anon) != 0) { + /* Its own errno, not EEXIST. Folding the two together reported a + * spurious EEXIST out of memfd_create for anything fstat could fail + * with. + */ + int fstat_errno = errno; + close(fd); + errno = fstat_errno; + return -1; + } + if (anon.st_nlink != 0) { + close(fd); + errno = EEXIST; + return -1; + } + return fd; +} + /* Carry overflow/underflow between tv_nsec and tv_sec so the result is a * canonical timespec with 0 <= tv_nsec < 1e9. Uses div/mod (which truncate * toward zero in C99) plus a single borrow so the LONG_MIN case never negates diff --git a/tests/bench-hot-guard.c b/tests/bench-hot-guard.c index b9f669c1..ae4f3576 100644 --- a/tests/bench-hot-guard.c +++ b/tests/bench-hot-guard.c @@ -4,12 +4,23 @@ * Copyright 2026 elfuse contributors * SPDX-License-Identifier: Apache-2.0 * - * Minimal bench that measures the four labels the guardrail script checks: + * Minimal bench that measures the eleven labels the guardrail script checks, + * all of which it extracts by name and holds to a ceiling: * * getpid (raw SVC; shim identity fast path) * clock_gettime (vDSO trampoline; see -DGUARD_USE_LIBC_CG below) * read-urandom1 (raw read; shim urandom ring fast path) * stat-path (full SVC round trip through guest_read_path) + * pipe-roundtrip (write + read on a pipe; the read/write transfer path) + * pipe-eagain (read of an empty nonblocking pipe; per-transfer cost) + * fd-create (open + close; fd_init_entry including its path work) + * pipe-create (pipe + close; the same allocation without a path) + * pipe-bulk (a megabyte into a pipe a sibling drains) + * getpid-mt (the identity path with a sibling thread alive) + * pipe-eagain-mt (the transfer path with a sibling thread alive) + * + * The last three run after the single-threaded cases, so nothing above pays for + * a second thread existing. * * Built twice from this single source: * build/bench-hot-guard -- static glibc. Compiled without @@ -40,7 +51,9 @@ */ #include +#include #include +#include #include #include #include @@ -146,6 +159,23 @@ static uint64_t monotonic_ns(clock_gettime_fn cg) return (uint64_t) ts.tv_sec * 1000000000ULL + (uint64_t) ts.tv_nsec; } +/* A lane that cannot be set up must not simply vanish from the output. The + * guardrail treats an absent label as a deterministic MISS and does not retry + * it, so a transient malloc or thread failure would read exactly like a + * regression. Say which lane and why, and exit non-zero so the run is a setup + * failure rather than a measurement. + * + * The error comes in as an argument because pthread_create reports through its + * return value and is not required to touch errno; musl does not, so reading + * errno here would print whatever the last unrelated call left behind. + */ +static void bench_setup_failed(const char *lane, const char *what, int err) +{ + fprintf(stderr, "bench-hot-guard: %s unavailable: %s failed: %s\n", lane, + what, strerror(err)); + exit(2); +} + static long bench_getpid(void *ctx) { (void) ctx; @@ -178,6 +208,132 @@ static long bench_read_urandom1(void *ctx) return read(fd, &byte, 1); } +/* The transfer lane: one byte out and the same byte back through a pipe, so + * each iteration is two guest read/write syscalls over a fd that can block, + * with the data always ready. + * + * This is the path every guest doing real work spends its time on, and until + * this lane existed nothing here measured it: the other four are served by the + * shim, the vDSO, or the path helpers, and a 30-50% regression in the + * read/write transfer path passed the guardrail clean. It regresses as a slope + * (a host call or a lock acquisition added per transfer), which is why it is + * checked as a ratio to getpid like stat-path rather than absolutely. + */ +typedef struct { + int rd, wr; +} pipe_ctx_t; + +/* The same transfer path with the data movement and the wakeup taken out: a + * read of an empty pipe the guest itself set nonblocking. It reaches the fd + * lookup, the block-state decision and the transfer attempt, and returns EAGAIN + * without touching the pipe buffer or waking anybody, so it measures + * per-transfer overhead with far less scheduling noise than the round trip. + */ +static long bench_pipe_eagain(void *ctx) +{ + pipe_ctx_t *p = ctx; + char c; + return read(p->rd, &c, 1); +} + +static long bench_pipe_roundtrip(void *ctx) +{ + pipe_ctx_t *p = ctx; + char c = 'x'; + if (write(p->wr, &c, 1) != 1) + return -1; + return read(p->rd, &c, 1); +} + +/* The bulk lane: a large write into a pipe a sibling thread is draining, so the + * write fills the buffer, waits, and resumes. That wait-and-resume loop is what + * a single-byte transfer never reaches, and it is where the ready-poll rewrite + * left a regression the other lanes could not see: measured 21-31% slower than + * the parent commit, filed as a P2 with no lane to hold it. This is that lane. + * + * Reported as ns/op over a fixed transfer size, so a slope in the retry loop + * shows up directly rather than through a ratio. + */ +#define BULK_BYTES (1u << 20) + +typedef struct { + int rd, wr; + unsigned char *buf; +} bulk_ctx_t; + +static void *bulk_drain(void *arg) +{ + bulk_ctx_t *b = arg; + unsigned char sink[65536]; + for (;;) { + ssize_t n = read(b->rd, sink, sizeof(sink)); + if (n <= 0) + break; + } + return NULL; +} + +static long bench_pipe_bulk(void *ctx) +{ + bulk_ctx_t *b = ctx; + size_t sent = 0; + while (sent < BULK_BYTES) { + ssize_t n = write(b->wr, b->buf + sent, BULK_BYTES - sent); + if (n <= 0) + return -1; + sent += (size_t) n; + } + return (long) sent; +} + +/* The same per-transfer op as pipe-eagain, with a sibling thread alive. + * + * elfuse takes fd_lock on the fd-table reads it can skip when only one thread + * is running (thread_is_single_active), so every lane above measures the + * lock-free path exclusively. A guest doing real work has more than one thread, + * and a lock added to the transfer path would be invisible here without this. + */ +static void *idle_sibling(void *arg) +{ + volatile int *stop = arg; + while (!*stop) { + struct timespec ts = {.tv_sec = 0, .tv_nsec = 20 * 1000 * 1000}; + nanosleep(&ts, NULL); + } + return NULL; +} + +/* fd creation with the path work taken out: pipe() makes two descriptors from + * nothing, so what is left is the table allocation itself. Both slots run + * fd_init_entry, which decides O_NONBLOCK ownership with two fcntls inside the + * fd-table lock; the open-based lane below cannot see that cost under the path + * resolution it also pays. + */ +static long bench_pipe_create(void *ctx) +{ + (void) ctx; + int p[2]; + if (pipe(p) != 0) + return -1; + close(p[0]); + close(p[1]); + return 0; +} + +/* The fd-creation lane: open and close the same path, so each iteration runs a + * full fd_init_entry, which stats the host fd and may set O_NONBLOCK on it, + * both inside the fd-table lock. Nothing else here allocates a descriptor. + */ +static long bench_fd_create(void *ctx) +{ + (void) ctx; + int fd = open("/dev/null", O_RDWR); + if (fd < 0) + return -1; + close(fd); + return 0; +} + /* The path-resolving lane. Every syscall that takes a path pays guest_read_path * (guest_read_str_small, then guest_read_str), and stat pays a * guest_write_small for the result struct on top, so this is the densest @@ -248,18 +404,95 @@ int main(int argc, char **argv) return 1; } + int pipefd[2]; + if (pipe(pipefd) != 0) { + perror("pipe"); + close(urandomfd); + return 1; + } + + int eagain_fd[2]; + if (pipe(eagain_fd) != 0) { + perror("pipe"); + close(pipefd[0]); + close(pipefd[1]); + close(urandomfd); + return 1; + } + fcntl(eagain_fd[0], F_SETFL, fcntl(eagain_fd[0], F_GETFL) | O_NONBLOCK); + pipe_ctx_t eagain_ctx = {.rd = eagain_fd[0], .wr = eagain_fd[1]}; + cg_ctx_t cg_ctx = {.fn = vdso_cg}; struct stat stat_buf; + pipe_ctx_t pipe_ctx = {.rd = pipefd[0], .wr = pipefd[1]}; const bench_case_t cases[] = { {"getpid", bench_getpid, NULL}, {"clock_gettime", bench_clock_gettime, &cg_ctx}, {"read-urandom1", bench_read_urandom1, &urandomfd}, {"stat-path", bench_stat_path, &stat_buf}, + {"pipe-roundtrip", bench_pipe_roundtrip, &pipe_ctx}, + {"pipe-eagain", bench_pipe_eagain, &eagain_ctx}, + {"fd-create", bench_fd_create, NULL}, + {"pipe-create", bench_pipe_create, NULL}, }; for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) run_case(vdso_cg, &cases[i], iters); + /* The bulk lane needs a reader on the other end, and the sibling lane needs + * a thread that is merely alive. Both run after the single-threaded cases + * so nothing above pays for a second thread existing. + */ + int bulk_fd[2]; + unsigned char *bulk_buf = malloc(BULK_BYTES); + if (bulk_buf && pipe(bulk_fd) == 0) { + memset(bulk_buf, 0x5a, BULK_BYTES); + bulk_ctx_t bulk_ctx = { + .rd = bulk_fd[0], .wr = bulk_fd[1], .buf = bulk_buf}; + pthread_t drain; + int rc_drain = pthread_create(&drain, NULL, bulk_drain, &bulk_ctx); + if (rc_drain == 0) { + /* A megabyte per op, so far fewer iterations than the syscall + * lanes; the guardrail divides by its own count. + */ + unsigned long bulk_iters = iters / 200 ? iters / 200 : 1; + bench_case_t bulk = {"pipe-bulk", bench_pipe_bulk, &bulk_ctx}; + run_case(vdso_cg, &bulk, bulk_iters); + close(bulk_fd[1]); + pthread_join(drain, NULL); + close(bulk_fd[0]); + } else { + bench_setup_failed("pipe-bulk", "pthread_create", rc_drain); + close(bulk_fd[0]); + close(bulk_fd[1]); + } + } else { + bench_setup_failed("pipe-bulk", bulk_buf ? "pipe" : "malloc", errno); + } + free(bulk_buf); + + volatile int stop = 0; + pthread_t sibling; + int rc_sibling = + pthread_create(&sibling, NULL, idle_sibling, (void *) &stop); + if (rc_sibling == 0) { + bench_case_t mt[] = { + {"getpid-mt", bench_getpid, NULL}, + {"pipe-eagain-mt", bench_pipe_eagain, &eagain_ctx}, + }; + for (size_t i = 0; i < sizeof(mt) / sizeof(mt[0]); i++) + run_case(vdso_cg, &mt[i], iters); + stop = 1; + pthread_join(sibling, NULL); + } else { + bench_setup_failed("getpid-mt/pipe-eagain-mt", "pthread_create", + rc_sibling); + } + + close(pipefd[0]); + close(pipefd[1]); + close(eagain_fd[0]); + close(eagain_fd[1]); close(urandomfd); return 0; } diff --git a/tests/lib/report.sh b/tests/lib/report.sh index 26876ce0..baed096d 100644 --- a/tests/lib/report.sh +++ b/tests/lib/report.sh @@ -6,9 +6,17 @@ # Sources tests/lib/test-runner.sh and exposes report_pass / report_fail / # report_skip on top of test_report so per-binary output matches the matrix # runner's aarch64 format ([ OK ] / [ FAIL ] / [ SKIP ] aligned to -# TEST_LABEL_WIDTH). Each script still owns its pass/fail /skip/total counters; -# this lib only centralizes the report sites and the trailing Results: summary -# line that tests/test-matrix.sh scrapes. +# TEST_LABEL_WIDTH) and the trailing Results: summary line that +# tests/test-matrix.sh scrapes. +# +# The pass/fail/skip counters the report functions increment are initialized by +# tests/lib/test-runner.sh, which this sources below and which eight scripts +# source without going through here. That is where they have to live for those +# eight, so setting them here as well would be dead code: the source that +# follows re-runs the same three assignments. Twelve scripts used to declare +# them a third time at their own top level, which shellcheck reads, correctly, +# as assignments nobody in that file goes on to use. A script that wants its own +# total still keeps it: report_results takes one as an argument. # Align the LABEL column with tests/test-matrix.sh so the aggregated matrix # output looks uniform across aarch64 and x86_64 modes. diff --git a/tests/manifest.txt b/tests/manifest.txt index 99f2c95d..46206b3c 100644 --- a/tests/manifest.txt +++ b/tests/manifest.txt @@ -63,6 +63,7 @@ test-shim-urandom-wrap [section] I/O subsystem tests test-eventfd test-eventfd-dup +test-eventfd-semaphore-contended test-signalfd test-signalfd-hardening test-epoll @@ -79,6 +80,13 @@ test-pty test-ioctl-fioasync test-getdents-refcount test-dev-shm-paths +test-fcntl-flags +test-socket-shortwrite +test-socket-blockwrite-signal +test-socket-accept-contended +test-socket-waitall +test-synthetic-wait-signal +test-sigpipe [section] Threading tests test-thread # diff=skip @@ -144,6 +152,8 @@ test-robust-futex [section] FD table race tests test-fd-race +test-dup-setfl-race +test-pipe-steal [section] Multithreaded fork tests test-mt-fork diff --git a/tests/test-bench-guardrail.sh b/tests/test-bench-guardrail.sh index 0624c5eb..5941103f 100755 --- a/tests/test-bench-guardrail.sh +++ b/tests/test-bench-guardrail.sh @@ -10,9 +10,12 @@ # clock_gettime(libc) <= 50 ns/op (vDSO CNTVCT fast path) # read(/dev/urandom, 1) <= 400 ns/op (shim urandom ring fast path) # stat("/dev/null") <= Nx getpid (guest_read_path, no fast path) +# read(empty nonblocking pipe) <= Nx getpid (per-transfer overhead) +# pipe write+read <= Nx getpid (transfer plus wait/wakeup) # -# The first three are absolute; stat-path is a ratio. See the threshold block -# below for why the two kinds of limit are not interchangeable. +# The first three are absolute; every other lane is a ratio to a getpid from the +# same run. See the threshold block below for why the two kinds of limit are not +# interchangeable. # # The static (musl) bench is the baseline; the dynamic-glibc bench verifies that # glibc 2.41's vDSO probe (NT_GNU_ABI_TAG PT_NOTE) keeps clock_gettime on the @@ -69,6 +72,82 @@ THRESH_URANDOM=400 THRESH_STAT_RATIO_STATIC=105 THRESH_STAT_RATIO_GLIBC=500 +# The two transfer lanes, both ratios to getpid for the same reason stat-path +# is: they are host calls whose cost is a slope, and dividing by a lane measured +# on the same machine moments earlier cancels the machine out. +# +# pipe-eagain is the detector. A read of an empty pipe the guest set nonblocking +# reaches the fd lookup, the block-state decision and the transfer attempt, then +# returns without touching the pipe buffer or waking anything, so it measures +# per-transfer overhead and nothing else. Observed 52-66 across runs on a loaded +# machine, a 25% spread, which is tight enough to gate at 85. +# +# The number this exists to catch: read/write on a fd that can block used to +# probe with poll(), divert to the slow path when the probe said "not ready", +# and re-resolve the fd there. Measured on this lane at 259-857 against 52-66 +# after that divert was removed. Anything that puts a host call or a lock +# acquisition back into the per-transfer path lands in the same range, and the +# guardrail had no lane that could see it: a 30-50% regression in the read/write +# path passed this script clean while it was being written. +# +# pipe-roundtrip moves a byte out and back, so it covers the wait and wakeup +# machinery pipe-eagain skips. It also inherits the scheduler noise that comes +# with them: observed 77-176 on the same runs where pipe-eagain held a 25% +# spread. It is a gross-regression arm, kept for the coverage and ceilinged so +# it does not flake. Do not tighten it toward the observed median chasing small +# regressions, and do not read a pass here as evidence the transfer path is +# clean; that is pipe-eagain's job. +THRESH_PIPE_EAGAIN_RATIO=85 +THRESH_PIPE_RT_RATIO=220 + +# The same transfer with a sibling thread alive, divided by getpid from that +# same state, so the pair isolates what a second thread costs and nothing else. +# It is not a duplicate of pipe-eagain: with one active thread elfuse borrows +# the host fd, and with a sibling it dups and closes it around every fd syscall +# to keep a racing close from retiring it. That is two host syscalls a guest +# doing real work pays on every read and write, measured at 47.8x against 58.3x +# here, and no single-threaded lane can see it. Ceiled at 110 so the dup pair +# has room but a third host call does not. +THRESH_PIPE_EAGAIN_MT_RATIO=110 + +# Bulk transfer: a megabyte written into a pipe a sibling drains, so the write +# fills the buffer, waits, and resumes. Every other lane moves one byte, which +# is why the ready-poll rewrite could leave bulk writes 21-31% slower with +# nothing here to notice. +# +# This lane is a gross-regression arm, and the ceiling says so. It cannot catch +# the regression that motivated it: idle it measures ~6900x, a busy host alone +# put it at 9900x, and a 25% regression would read ~8600x -- inside the band +# load produces on its own. Bulk throughput is dominated by pipe capacity and +# scheduling rather than by syscall entry cost, so dividing by getpid does not +# cancel the load the way it does for stat-path. A tighter number here would +# fail on clean trees and be ignored, which costs more than it catches. +# +# What catches a 25% slope is the A/B in the TODO entry: two builds, alternating +# passes, medians, on an idle machine. The lane's job is to make that comparison +# possible at all by existing. +# +# The ceiling is 15000 against an idle figure near 6900, which is 2.17x: it +# trips just short of a 2.2x collapse, and lets a 2x one through. It is written +# that way because a busy host alone reached 1.4x, and the gap between the two +# is the whole margin this lane has: a number tight enough to catch 2x would +# fail on clean trees. +THRESH_PIPE_BULK_RATIO=15000 + +# Descriptor creation, with and without path resolution. fd_init_entry stats the +# host fd and may set O_NONBLOCK on it inside the fd-table lock; pipe-create is +# the same allocation with the path work removed, so the pair says how much of +# the cost is resolution. +# +# fd-create splits by variant for the same reason stat-path does: under dynamic +# glibc the open resolves through the sysroot, which is most of the lane. It +# measured 999x against a first-cut 1000x ceiling, which would have flaked on +# the next busy run rather than caught anything. pipe-create allocates without a +# path, so one number covers both. +THRESH_FD_CREATE_RATIO_STATIC=1000 +THRESH_FD_CREATE_RATIO_GLIBC=1700 +THRESH_PIPE_CREATE_RATIO=1000 + C_RED='\033[0;31m' C_GREEN='\033[0;32m' C_YELLOW='\033[0;33m' @@ -176,8 +255,8 @@ check_ratio() run_one_pass() { - local variant="$1" bench="$2" stat_ratio="$3" - shift 3 + local variant="$1" bench="$2" stat_ratio="$3" fd_ratio="$4" + shift 4 local out out="$(mktemp)" if ! "$ELFUSE" "$@" "$bench" "$ITERS" > "$out" 2>&1; then @@ -199,6 +278,28 @@ run_one_pass() "$(extract_ns "$out" read-urandom1)" "$THRESH_URANDOM" check_ratio "$variant" "stat-path" \ "$(extract_ns "$out" stat-path)" "$getpid_ns" "$stat_ratio" + check_ratio "$variant" "pipe-eagain" \ + "$(extract_ns "$out" pipe-eagain)" "$getpid_ns" \ + "$THRESH_PIPE_EAGAIN_RATIO" + check_ratio "$variant" "pipe-roundtrip" \ + "$(extract_ns "$out" pipe-roundtrip)" "$getpid_ns" \ + "$THRESH_PIPE_RT_RATIO" + check_ratio "$variant" "fd-create" \ + "$(extract_ns "$out" fd-create)" "$getpid_ns" "$fd_ratio" + check_ratio "$variant" "pipe-create" \ + "$(extract_ns "$out" pipe-create)" "$getpid_ns" \ + "$THRESH_PIPE_CREATE_RATIO" + check_ratio "$variant" "pipe-bulk" \ + "$(extract_ns "$out" pipe-bulk)" "$getpid_ns" \ + "$THRESH_PIPE_BULK_RATIO" + + # The sibling-alive lanes divide by their own getpid, measured with that + # sibling running, so the ratio carries only the per-transfer difference. + local getpid_mt_ns + getpid_mt_ns="$(extract_ns "$out" getpid-mt)" + check_ratio "$variant" "pipe-eagain-mt" \ + "$(extract_ns "$out" pipe-eagain-mt)" "$getpid_mt_ns" \ + "$THRESH_PIPE_EAGAIN_MT_RATIO" rm -f "$out" } @@ -254,13 +355,14 @@ echo "=== bench-guardrail (iters=$ITERS) ===" if [ "$run_static" = 1 ]; then echo "[static (musl)]" - run_and_check static "$STATIC_BENCH" "$THRESH_STAT_RATIO_STATIC" + run_and_check static "$STATIC_BENCH" "$THRESH_STAT_RATIO_STATIC" \ + "$THRESH_FD_CREATE_RATIO_STATIC" fi if [ -x "$GLIBC_BENCH" ] && [ -d "$GLIBC_SYSROOT" ]; then echo "[dynamic-glibc]" run_and_check dyn-glibc "$GLIBC_BENCH" "$THRESH_STAT_RATIO_GLIBC" \ - --sysroot "$GLIBC_SYSROOT" + "$THRESH_FD_CREATE_RATIO_GLIBC" --sysroot "$GLIBC_SYSROOT" else /usr/bin/printf " ${C_YELLOW}SKIP${C_RESET} dyn-glibc cross-toolchain absent: %s\n" \ "$GLIBC_TOOLCHAIN" diff --git a/tests/test-config.sh b/tests/test-config.sh index a11e2dde..1f936623 100755 --- a/tests/test-config.sh +++ b/tests/test-config.sh @@ -65,7 +65,8 @@ elfuse_resolve_host_nofile() elfuse_test_host_nofile() { local manifest="$1" - local name="$(basename "$2")" + local name + name="$(basename "$2")" local spec spec=$(awk -v wanted="$name" ' $1 == wanted { diff --git a/tests/test-dup-setfl-race.c b/tests/test-dup-setfl-race.c new file mode 100644 index 00000000..683b19e0 --- /dev/null +++ b/tests/test-dup-setfl-race.c @@ -0,0 +1,156 @@ +/* + * A dup taken while a sibling flips O_NONBLOCK agrees with the description + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + * + * O_NONBLOCK belongs to the open file description, so every name for one must + * report the same value. elfuse keeps that in a per-fd shadow and sweeps the + * aliases on F_SETFL, which leaves one window: an alias built from a snapshot + * taken before it is published is not in the table when the sweep runs, so the + * sweep cannot reach it and the publish restores the stale value. Nothing later + * notices, because F_SETFL stamps no new generation. + * + * Concretely: T1 snapshots a blocking pipe on its way into dup(). T2 sets + * O_NONBLOCK and sweeps the aliases that exist. T1 publishes. The new name + * reports blocking and waits in io_xfer where the guest asked for EAGAIN, on a + * description whose other names are nonblocking. + * + * The comparison happens only at quiescence, and that is the whole design of + * this test. Reading two names while a third thread is still flipping the flag + * cannot tell a stale alias from a value that changed between the two reads -- + * an earlier version of this test did exactly that and reported disagreements + * on a tree that had none. So each round dups a batch, parks the flipper, and + * waits for it to acknowledge; only then must every name agree, because no + * writer remains and the last sweep covered every slot that existed. + * + * Passes on real Linux, where one description simply has one flag. + * + * Syscalls exercised: pipe2(59), dup(23), fcntl(25), close(57), clone(220), + * futex(98), sched_yield(124) + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "test-harness.h" + +int passes = 0, fails = 0; + +/* 700 x 24 dups, about two seconds. The window is a few instructions wide, so + * one round hits it rarely: at 60 rounds a deliberately broken tree was caught + * in one run out of three, which is not a gate. Sized from that measurement so + * a regression is caught essentially every run, and the loop exits as soon as + * it finds one. + */ +#define ROUNDS 700 +#define BATCH 24 + +static int pipe_rd, pipe_wr; +static atomic_int stop, pause_req, paused; +static int stale_round = -1, stale_alias = -1, stale_src = -1; + +static void *flipper(void *arg) +{ + (void) arg; + while (!atomic_load(&stop)) { + if (atomic_load(&pause_req)) { + atomic_store(&paused, 1); + while (atomic_load(&pause_req) && !atomic_load(&stop)) + sched_yield(); + atomic_store(&paused, 0); + continue; + } + int fl = fcntl(pipe_wr, F_GETFL); + if (fl >= 0) + (void) fcntl(pipe_wr, F_SETFL, fl ^ O_NONBLOCK); + } + atomic_store(&paused, 1); + return NULL; +} + +int main(void) +{ + int fds[2]; + if (pipe(fds) != 0) { + FAIL("pipe failed"); + SUMMARY("test-dup-setfl-race"); + return 1; + } + pipe_rd = fds[0]; + pipe_wr = fds[1]; + + pthread_t t; + if (pthread_create(&t, NULL, flipper, NULL) != 0) { + FAIL("pthread_create failed"); + SUMMARY("test-dup-setfl-race"); + return 1; + } + + int dups[BATCH]; + int dups_taken = 0; + + for (int r = 0; r < ROUNDS && stale_round < 0; r++) { + int n = 0; + for (int i = 0; i < BATCH; i++) { + int d = dup(pipe_wr); + if (d >= 0) + dups[n++] = d; + } + + /* Park the flipper and wait for it to say so. From here nothing can + * write the flag, so any name that still disagrees kept a value the + * description has moved past. + */ + atomic_store(&pause_req, 1); + while (!atomic_load(&paused)) + sched_yield(); + + int src = fcntl(pipe_wr, F_GETFL); + for (int i = 0; i < n; i++) { + int alias = fcntl(dups[i], F_GETFL); + if (src >= 0 && alias >= 0 && ((src ^ alias) & O_NONBLOCK) && + stale_round < 0) { + stale_round = r; + stale_alias = alias & O_NONBLOCK; + stale_src = src & O_NONBLOCK; + } + } + + atomic_store(&pause_req, 0); + for (int i = 0; i < n; i++) + close(dups[i]); + dups_taken += n; + } + + atomic_store(&stop, 1); + atomic_store(&pause_req, 0); + pthread_join(t, NULL); + + if (stale_round >= 0) + printf(" round %d: alias O_NONBLOCK=%d, description=%d\n", + stale_round, stale_alias ? 1 : 0, stale_src ? 1 : 0); + + TEST("every dup taken during an F_SETFL sweep sees the description's flag"); + EXPECT_TRUE(stale_round < 0, + "a dup kept an O_NONBLOCK the description had moved past"); + + /* Count what the rounds actually produced, not that they ran. Asserting on + * a counter the loop bumps unconditionally says only that the loop body + * executed, which the loop condition already guarantees; a dup() failing + * every time would still have passed it. + */ + TEST("the race window was actually exercised"); + EXPECT_TRUE(dups_taken >= BATCH, + "no dup was taken, so nothing was compared"); + + close(pipe_rd); + close(pipe_wr); + SUMMARY("test-dup-setfl-race"); + return fails ? 1 : 0; +} diff --git a/tests/test-eventfd-semaphore-contended.c b/tests/test-eventfd-semaphore-contended.c new file mode 100644 index 00000000..6b441d3d --- /dev/null +++ b/tests/test-eventfd-semaphore-contended.c @@ -0,0 +1,148 @@ +/* + * Two threads blocking-reading one EFD_SEMAPHORE eventfd + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + * + * elfuse signals an eventfd's readability through an internal pipe, and posts + * to it on the counter's 0-to-nonzero edge. That edge is the whole story for a + * plain eventfd, whose read takes the counter to zero every time. It is half + * the story for EFD_SEMAPHORE, whose read decrements by one: a counter of 2 + * read once stays readable while the pipe goes empty, because the reader + * consumed the single byte and the edge will not come again until the counter + * returns to zero. A second reader blocked on that pipe then sleeps through a + * count it was entitled to, and its vCPU thread is parked where neither + * hv_vcpus_exit nor the wakeup pipe reaches it. + * + * Two readers and one writer make the window easy to hit: measured three hangs + * in five runs before the read path learned to re-arm the pipe, and five clean + * runs out of five against the qemu reference kernel, which is what says the + * expectation below is Linux's and not elfuse's. + * + * The second assertion is the older bug the same path had: a sibling taking the + * byte first must not turn a blocking read into EAGAIN. + * + * Syscalls exercised: eventfd2(19), read(63), write(64), clone(220), + * futex(98), nanosleep(101) + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "test-harness.h" + +int passes = 0, fails = 0; + +/* Linux spelling; the guest headers may not expose eventfd(). SYS_eventfd2 + * comes from , which already knows it. + */ +#define L_EFD_SEMAPHORE 1 + +#define POSTS 400 + +static int efd; +static atomic_int eagains, got, stop; + +static void *reader(void *arg) +{ + (void) arg; + while (!atomic_load(&stop)) { + uint64_t v; + ssize_t n = read(efd, &v, sizeof(v)); + if (n < 0) { + if (errno == EAGAIN) + atomic_fetch_add(&eagains, 1); + return NULL; + } + atomic_fetch_add(&got, (int) v); + } + return NULL; +} + +int main(void) +{ + efd = (int) syscall(SYS_eventfd2, 0, L_EFD_SEMAPHORE); + if (efd < 0) { + FAIL("eventfd2 failed"); + SUMMARY("test-eventfd-semaphore-contended"); + return 1; + } + + pthread_t a, b; + if (pthread_create(&a, NULL, reader, NULL) != 0 || + pthread_create(&b, NULL, reader, NULL) != 0) { + FAIL("pthread_create failed"); + SUMMARY("test-eventfd-semaphore-contended"); + return 1; + } + + /* Aim at the window rather than hoping to stumble into it. Let both readers + * reach the wait, then post two units in one write: the first reader takes + * one and consumes the single pipe byte, and the counter it leaves behind + * is exactly the state the 0-to-nonzero edge cannot signal again. Without + * the re-arm the second reader sleeps here. + * + * Detection of the lost wakeup is not certain, and the reason is a window + * no guest can reach: a reader only waits when it has just seen a zero + * counter, so a second reader has to be between that check and its poll at + * the moment the first consumes the byte. One reader cannot reproduce it at + * all, because after a take that leaves a remainder it simply loops and + * takes again rather than parking. + * + * Measured against a tree with the re-arm removed: this shape catches it in + * roughly two runs in five (4/5 and 3/8 across two sittings), and the fixed + * tree passed 5/5. Repeating the burst and draining between rounds was + * tried and measured worse, 2/6, because a full drain destroys the leftover + * count the bug needs. The EAGAIN assertion below is the reliable half: + * origin/main fails it in four runs out of five. + */ + usleep(50 * 1000); + uint64_t burst = 2; + int expect = POSTS; + if (write(efd, &burst, sizeof(burst)) == (ssize_t) sizeof(burst)) { + expect += 2; + } else { + /* Do not go on to wait for units this write never posted: the loop + * below would hang for a count that cannot arrive, and the harness + * would report a timeout rather than the write that failed. + */ + FAIL("burst write failed"); + } + + for (int i = 0; i < POSTS; i++) { + uint64_t one = 1; + if (write(efd, &one, sizeof(one)) != (ssize_t) sizeof(one)) { + FAIL("write failed"); + break; + } + } + + /* Every posted unit has to come back out. A missed wakeup shows up here as + * a test that never finishes, which the harness timeout reports. + */ + while (atomic_load(&got) < expect) + usleep(1000); + + atomic_store(&stop, 1); + uint64_t two = 2; + write(efd, &two, sizeof(two)); /* release both readers */ + pthread_join(a, NULL); + pthread_join(b, NULL); + + TEST("every posted unit reaches a blocked reader"); + EXPECT_TRUE(atomic_load(&got) >= expect, + "a semaphore eventfd lost a wakeup"); + + TEST("a blocking read never reports EAGAIN to the guest"); + EXPECT_TRUE(atomic_load(&eagains) == 0, + "a sibling taking the byte turned a blocking read into EAGAIN"); + + close(efd); + SUMMARY("test-eventfd-semaphore-contended"); + return fails ? 1 : 0; +} diff --git a/tests/test-fcntl-flags.c b/tests/test-fcntl-flags.c new file mode 100644 index 00000000..1c27ae02 --- /dev/null +++ b/tests/test-fcntl-flags.c @@ -0,0 +1,463 @@ +/* + * F_GETFL / F_SETFL across every fd type elfuse answers for + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + * + * elfuse answers the file status flags from three different places depending on + * the fd: the host description, a per-fd shadow, or a fixed reply. FUSE fds are + * pure shadow, a timerfd is a kqueue the host will not take F_SETFL on, O_ASYNC + * is never armed on the host fd, the access mode of a regular file is kept in + * the shadow because O_PATH and O_DIRECTORY have no macOS equivalent, and + * O_NONBLOCK is elfuse's own on the fds whose transfers it owns. + * + * Each of those is defensible alone, and together they are a matrix nothing + * pinned. This walks it: what F_GETFL reports for a freshly opened fd of each + * kind, which bits survive a round trip through F_SETFL, and which the kernel + * is supposed to ignore. It is written to hold across a refactor of how the + * flags are stored, so it asserts what Linux answers, not how elfuse gets + * there. + * + * Syscalls exercised: fcntl(25), openat(56), pipe2(59), socket(198), + * timerfd_create(85), eventfd2(19), ioctl(29) + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "test-harness.h" + +int passes = 0, fails = 0; + +#ifndef O_PATH +#define O_PATH 010000000 +#endif + +/* Written where the guest can create it under either sysroot. */ +#define NOATIME_FILE "/tmp/fcntl-noatime.tmp" + +static void check_accmode(const char *what, int fd, int want) +{ + TEST(what); + int fl = fcntl(fd, F_GETFL); + if (fl < 0) + FAIL("F_GETFL failed"); + else + EXPECT_EQ(fl & O_ACCMODE, want, "wrong access mode"); +} + +/* A bit the guest sets through F_SETFL must read back through F_GETFL, and + * clearing it must stick. Both directions, because a shadow that is written but + * not read (or read but not written) passes a one-way check. + */ +static void check_roundtrip(const char *what, int fd, int bit) +{ + TEST(what); + int base = fcntl(fd, F_GETFL); + if (base < 0) { + FAIL("F_GETFL failed"); + return; + } + if (fcntl(fd, F_SETFL, base | bit) < 0) { + FAIL("F_SETFL failed"); + return; + } + int set = fcntl(fd, F_GETFL); + if (fcntl(fd, F_SETFL, base & ~bit) < 0) { + FAIL("F_SETFL clear failed"); + return; + } + int cleared = fcntl(fd, F_GETFL); + EXPECT_TRUE((set & bit) && !(cleared & bit), "bit did not round trip"); +} + +/* F_SETFL cannot change the access mode: Linux masks the argument down to the + * settable set and leaves O_ACCMODE alone. + */ +static void check_accmode_immutable(const char *what, int fd, int want) +{ + TEST(what); + int fl = fcntl(fd, F_GETFL); + if (fl < 0) { + FAIL("F_GETFL failed"); + return; + } + /* Ask for the opposite access mode plus a settable bit. */ + int other = (want == O_RDONLY) ? O_WRONLY : O_RDONLY; + fcntl(fd, F_SETFL, (fl & ~O_ACCMODE) | other); + int after = fcntl(fd, F_GETFL); + EXPECT_EQ(after & O_ACCMODE, want, "F_SETFL changed the access mode"); +} + +int main(void) +{ + printf("test-fcntl-flags: status flags across fd types\n"); + + /* Regular files, one per access mode. */ + int rd = open("/etc/hostname", O_RDONLY); + if (rd < 0) + rd = open("/proc/self/cmdline", O_RDONLY); + int wr = + open("/tmp/elfuse-fcntl-flags", O_WRONLY | O_CREAT | O_TRUNC, 0600); + int rw = open("/tmp/elfuse-fcntl-flags", O_RDWR); + + check_accmode("regular O_RDONLY", rd, O_RDONLY); + check_accmode("regular O_WRONLY", wr, O_WRONLY); + check_accmode("regular O_RDWR", rw, O_RDWR); + check_accmode_immutable("regular access mode is immutable", wr, O_WRONLY); + check_roundtrip("regular O_APPEND round trip", rw, O_APPEND); + check_roundtrip("regular O_NONBLOCK round trip", rw, O_NONBLOCK); + + /* A pipe: elfuse owns O_NONBLOCK on both ends. */ + int p[2]; + if (pipe(p) == 0) { + check_accmode("pipe read end is O_RDONLY", p[0], O_RDONLY); + check_accmode("pipe write end is O_WRONLY", p[1], O_WRONLY); + check_roundtrip("pipe O_NONBLOCK round trip", p[0], O_NONBLOCK); + check_roundtrip("pipe O_ASYNC round trip", p[0], O_ASYNC); + check_accmode_immutable("pipe access mode is immutable", p[0], + O_RDONLY); + + TEST("pipe2(O_NONBLOCK) reports it at once"); + int q[2]; + if (pipe2(q, O_NONBLOCK) == 0) { + EXPECT_TRUE(fcntl(q[0], F_GETFL) & O_NONBLOCK, "not reported"); + close(q[0]); + close(q[1]); + } else { + FAIL("pipe2 failed"); + } + close(p[0]); + close(p[1]); + } + + /* A nonblocking write bigger than the pipe buffer reports what it moved. + * Waiting for the remainder is exactly what O_NONBLOCK said not to do, and + * a transfer path that loops until the whole request is out hangs here + * instead of returning. + */ + int nbw[2]; + if (pipe(nbw) == 0) { + fcntl(nbw[1], F_SETFL, fcntl(nbw[1], F_GETFL) | O_NONBLOCK); + static char big[1 << 20]; + ssize_t n = write(nbw[1], big, sizeof(big)); + TEST("nonblocking write past the pipe buffer reports a partial count"); + EXPECT_TRUE(n > 0 && (size_t) n < sizeof(big), "not a partial count"); + + TEST("and the next one reports EAGAIN"); + EXPECT_ERRNO(write(nbw[1], big, sizeof(big)), EAGAIN, + "write did not report EAGAIN"); + close(nbw[0]); + close(nbw[1]); + } + + /* A socket: never owned, so the host flag is the guest's. */ + int sock = socket(AF_UNIX, SOCK_STREAM, 0); + if (sock >= 0) { + check_accmode("socket is O_RDWR", sock, O_RDWR); + check_roundtrip("socket O_NONBLOCK round trip", sock, O_NONBLOCK); + close(sock); + } + + /* A timerfd is a kqueue on the host, which refuses F_SETFL, so its flags + * live entirely in the shadow. + */ + int tfd = timerfd_create(CLOCK_MONOTONIC, 0); + if (tfd >= 0) { + check_accmode("timerfd is O_RDWR", tfd, O_RDWR); + check_roundtrip("timerfd O_NONBLOCK round trip", tfd, O_NONBLOCK); + close(tfd); + } + + int efd = eventfd(0, 0); + if (efd >= 0) { + check_accmode("eventfd is O_RDWR", efd, O_RDWR); + check_roundtrip("eventfd O_NONBLOCK round trip", efd, O_NONBLOCK); + + /* Reporting the flag is not the same as honouring it. A synthetic fd is + * backed by elfuse's own pipe, so the emulation has to read the guest's + * O_NONBLOCK from the same place F_GETFL answers from; when it did not, + * this read blocked forever instead of reporting EAGAIN. + */ + uint64_t v; + fcntl(efd, F_SETFL, fcntl(efd, F_GETFL) | O_NONBLOCK); + TEST("eventfd honours O_NONBLOCK set by fcntl"); + EXPECT_ERRNO(read(efd, &v, sizeof(v)), EAGAIN, "read did not EAGAIN"); + + int on = 1; + fcntl(efd, F_SETFL, fcntl(efd, F_GETFL) & ~O_NONBLOCK); + ioctl(efd, FIONBIO, &on); + TEST("eventfd honours O_NONBLOCK set by FIONBIO"); + EXPECT_ERRNO(read(efd, &v, sizeof(v)), EAGAIN, "read did not EAGAIN"); + close(efd); + } + + int efd_nb = eventfd(0, EFD_NONBLOCK); + if (efd_nb >= 0) { + uint64_t v; + TEST("eventfd honours EFD_NONBLOCK from creation"); + EXPECT_ERRNO(read(efd_nb, &v, sizeof(v)), EAGAIN, + "read did not EAGAIN"); + close(efd_nb); + } + + /* Same shape for signalfd, whose flag lived in the same private field. */ + sigset_t mask; + sigemptyset(&mask); + sigaddset(&mask, SIGUSR1); + int sfd = signalfd(-1, &mask, 0); + if (sfd >= 0) { + check_accmode("signalfd is O_RDWR", sfd, O_RDWR); + char sbuf[128]; + fcntl(sfd, F_SETFL, fcntl(sfd, F_GETFL) | O_NONBLOCK); + TEST("signalfd honours O_NONBLOCK set by fcntl"); + EXPECT_ERRNO(read(sfd, sbuf, sizeof(sbuf)), EAGAIN, + "read did not EAGAIN"); + close(sfd); + } + + /* inotify keeps the same shape: O_RDONLY on Linux, and a private copy of + * the nonblock flag would strand a guest that set it after creation. An + * alias of a synthetic fd shares its open file description, so it inherits + * the mode and the flag. The dup paths for these types rebuild linux_flags + * by hand, which is how they came to drop both. + */ + int edup_src = eventfd(0, EFD_NONBLOCK); + if (edup_src >= 0) { + int alias = dup(edup_src); + if (alias >= 0) { + uint64_t v; + check_accmode("dup(eventfd) keeps O_RDWR", alias, O_RDWR); + TEST("dup(eventfd) keeps O_NONBLOCK"); + EXPECT_TRUE(fcntl(alias, F_GETFL) & O_NONBLOCK, "flag lost"); + TEST("dup(eventfd) honours it"); + EXPECT_ERRNO(read(alias, &v, sizeof(v)), EAGAIN, + "read did not EAGAIN"); + close(alias); + } + close(edup_src); + } + + int tdup_src = timerfd_create(CLOCK_MONOTONIC, 0); + if (tdup_src >= 0) { + int alias = dup(tdup_src); + if (alias >= 0) { + /* F_SETFL on one alias is a change to the description, so the other + * one sees it too. + */ + fcntl(tdup_src, F_SETFL, fcntl(tdup_src, F_GETFL) | O_NONBLOCK); + TEST("F_SETFL on a timerfd reaches its alias"); + EXPECT_TRUE(fcntl(alias, F_GETFL) & O_NONBLOCK, "alias missed it"); + + /* Every bit the shadow answers belongs to the description, not just + * O_NONBLOCK: O_APPEND is one Linux keeps for a timerfd. + */ + fcntl(tdup_src, F_SETFL, fcntl(tdup_src, F_GETFL) | O_APPEND); + TEST("every shadowed timerfd bit reaches the alias"); + EXPECT_TRUE(fcntl(alias, F_GETFL) & O_APPEND, "alias missed it"); + close(alias); + } + close(tdup_src); + } + + int ifd = inotify_init(); + if (ifd >= 0) { + check_accmode("inotify is O_RDONLY", ifd, O_RDONLY); + char ibuf[512]; + fcntl(ifd, F_SETFL, fcntl(ifd, F_GETFL) | O_NONBLOCK); + TEST("inotify honours O_NONBLOCK set by fcntl"); + EXPECT_ERRNO(read(ifd, ibuf, sizeof(ibuf)), EAGAIN, + "read did not EAGAIN"); + close(ifd); + } + + int ifd_nb = inotify_init1(IN_NONBLOCK); + if (ifd_nb >= 0) { + char ibuf[512]; + TEST("inotify honours IN_NONBLOCK from creation"); + EXPECT_ERRNO(read(ifd_nb, ibuf, sizeof(ibuf)), EAGAIN, + "read did not EAGAIN"); + close(ifd_nb); + } + + /* O_DIRECT is settable on a pipe (that is packet mode) and refused on a + * socket, where Linux has no FMODE_CAN_ODIRECT to offer. Both spellings + * measured against qemu-aarch64; recording O_DIRECT for a socket would + * report a mode the guest cannot have. + */ + int odp[2]; + if (pipe(odp) == 0) { + TEST("a pipe accepts O_DIRECT"); + fcntl(odp[1], F_SETFL, fcntl(odp[1], F_GETFL) | O_DIRECT); + EXPECT_TRUE(fcntl(odp[1], F_GETFL) & O_DIRECT, "bit did not stick"); + close(odp[0]); + close(odp[1]); + } + int odsock = socket(AF_UNIX, SOCK_STREAM, 0); + if (odsock >= 0) { + TEST("a socket refuses O_DIRECT"); + EXPECT_ERRNO(fcntl(odsock, F_SETFL, fcntl(odsock, F_GETFL) | O_DIRECT), + EINVAL, "F_SETFL did not report EINVAL"); + TEST("and does not report it afterwards"); + EXPECT_EQ(fcntl(odsock, F_GETFL) & O_DIRECT, 0, "bit leaked in"); + close(odsock); + } + + /* O_NOATIME is settable, and macOS has no equivalent, so the shadow owns + * it: F_SETFL has to write it there or F_GETFL keeps answering with what + * open() recorded. Checked in both directions, since a shadow that is only + * ever set looks correct until something clears it. + */ + int nafd = open(NOATIME_FILE, O_RDWR | O_CREAT, 0600); + if (nafd >= 0) { + TEST("O_NOATIME is not reported before it is set"); + EXPECT_EQ(fcntl(nafd, F_GETFL) & O_NOATIME, 0, "reported unset flag"); + + fcntl(nafd, F_SETFL, fcntl(nafd, F_GETFL) | O_NOATIME); + TEST("F_SETFL records O_NOATIME"); + EXPECT_TRUE(fcntl(nafd, F_GETFL) & O_NOATIME, "bit did not stick"); + + fcntl(nafd, F_SETFL, fcntl(nafd, F_GETFL) & ~O_NOATIME); + TEST("F_SETFL clears O_NOATIME"); + EXPECT_EQ(fcntl(nafd, F_GETFL) & O_NOATIME, 0, "bit did not clear"); + close(nafd); + unlink(NOATIME_FILE); + } + + /* O_PATH and O_DIRECTORY have no macOS equivalent and are carried in the + * shadow; both must survive F_GETFL. + */ + int dirfd = open("/tmp", O_RDONLY | O_DIRECTORY); + if (dirfd >= 0) { + TEST("O_DIRECTORY survives F_GETFL"); + EXPECT_TRUE(fcntl(dirfd, F_GETFL) & O_DIRECTORY, "bit lost"); + close(dirfd); + } + + int pathfd = open("/tmp", O_PATH); + if (pathfd >= 0) { + TEST("O_PATH survives F_GETFL"); + EXPECT_TRUE(fcntl(pathfd, F_GETFL) & O_PATH, "bit lost"); + close(pathfd); + } + + /* O_ASYNC sticks only where the object supports it. Linux does not carry + * FASYNC in SETFL_MASK: setfl() lands the bit by calling + * file_operations->fasync, so an object whose fops lack one keeps O_ASYNC + * clear however often the guest sets it. The expectations here were + * measured under qemu-aarch64, and this table is why they cannot drift: + * elfuse used to answer 1 for all eleven. + */ + struct { + const char *name; + int fd, want; + } fasync[] = { + {"timerfd drops O_ASYNC", timerfd_create(CLOCK_MONOTONIC, 0), 0}, + {"eventfd drops O_ASYNC", eventfd(0, 0), 0}, + {"signalfd drops O_ASYNC", signalfd(-1, &mask, 0), 0}, + {"epoll drops O_ASYNC", epoll_create1(0), 0}, + {"pidfd drops O_ASYNC", (int) syscall(434, getpid(), 0), 0}, + {"inotify keeps O_ASYNC", inotify_init(), 1}, + {"netlink keeps O_ASYNC", socket(AF_NETLINK, SOCK_RAW, 0), 1}, + {"socket keeps O_ASYNC", socket(AF_UNIX, SOCK_STREAM, 0), 1}, + }; + for (size_t i = 0; i < sizeof(fasync) / sizeof(fasync[0]); i++) { + if (fasync[i].fd < 0) + continue; + TEST(fasync[i].name); + fcntl(fasync[i].fd, F_SETFL, fcntl(fasync[i].fd, F_GETFL) | O_ASYNC); + int got = (fcntl(fasync[i].fd, F_GETFL) & O_ASYNC) ? 1 : 0; + EXPECT_EQ(got, fasync[i].want, "O_ASYNC does not match Linux"); + close(fasync[i].fd); + } + + /* Character devices are the case a type test cannot answer: elfuse types + * them all FD_REGULAR, and Linux lets the flag stick on /dev/urandom + * (random_fasync) but not on /dev/null or /dev/zero. Deciding from "can + * this block" got all three wrong. + */ + struct { + const char *name, *path; + int want; + } chardev[] = { + {"/dev/null drops O_ASYNC", "/dev/null", 0}, + {"/dev/zero drops O_ASYNC", "/dev/zero", 0}, + {"/dev/urandom keeps O_ASYNC", "/dev/urandom", 1}, + {"/dev/random keeps O_ASYNC", "/dev/random", 1}, + }; + for (size_t i = 0; i < sizeof(chardev) / sizeof(chardev[0]); i++) { + int cfd = open(chardev[i].path, O_RDONLY); + if (cfd < 0) + continue; + TEST(chardev[i].name); + fcntl(cfd, F_SETFL, fcntl(cfd, F_GETFL) | O_ASYNC); + int got = (fcntl(cfd, F_GETFL) & O_ASYNC) ? 1 : 0; + EXPECT_EQ(got, chardev[i].want, "O_ASYNC does not match Linux"); + close(cfd); + } + + /* A pipe keeps it too, and both ends of one are worth checking: the write + * end is the alias path that lost flags before. + */ + int afds[2]; + if (pipe(afds) == 0) { + for (int i = 0; i < 2; i++) { + TEST(i == 0 ? "pipe read end keeps O_ASYNC" + : "pipe write end keeps O_ASYNC"); + fcntl(afds[i], F_SETFL, fcntl(afds[i], F_GETFL) | O_ASYNC); + EXPECT_TRUE(fcntl(afds[i], F_GETFL) & O_ASYNC, "bit did not stick"); + close(afds[i]); + } + } + + /* Every synthetic fd answers its access mode from elfuse's shadow, because + * the host fd behind it is a pipe or a kqueue elfuse opened for its own + * purposes. Each one has to carry the mode Linux gives its anon inode. + */ + struct { + const char *name; + int fd, want; + } synth[] = { + {"epoll is O_RDWR", epoll_create1(0), O_RDWR}, + {"netlink is O_RDWR", socket(AF_NETLINK, SOCK_RAW, 0), O_RDWR}, + {"pidfd is O_RDWR", (int) syscall(434, getpid(), 0), O_RDWR}, + }; + for (size_t i = 0; i < sizeof(synth) / sizeof(synth[0]); i++) { + if (synth[i].fd < 0) + continue; + check_accmode(synth[i].name, synth[i].fd, synth[i].want); + close(synth[i].fd); + } + + int urand = open("/dev/urandom", O_RDONLY); + if (urand >= 0) { + check_accmode("urandom is O_RDONLY", urand, O_RDONLY); + close(urand); + } + + if (rd >= 0) + close(rd); + if (wr >= 0) + close(wr); + if (rw >= 0) + close(rw); + unlink("/tmp/elfuse-fcntl-flags"); + + SUMMARY("test-fcntl-flags"); + return fails > 0 ? 1 : 0; +} diff --git a/tests/test-fork-ipc-protocol-host.c b/tests/test-fork-ipc-protocol-host.c index 0b2c2c6b..ef2f151b 100644 --- a/tests/test-fork-ipc-protocol-host.c +++ b/tests/test-fork-ipc-protocol-host.c @@ -20,9 +20,10 @@ #define PREVIOUS_ELFL_MAGIC 0x454C464CU #define PREVIOUS_ELFM_MAGIC 0x454C464DU #define PREVIOUS_ELFN_MAGIC 0x454C464EU +#define PREVIOUS_ELFO_MAGIC 0x454C464FU -_Static_assert(FORK_IPC_PROTOCOL_MAGIC == 0x454C464FU, - "fork IPC protocol magic must remain ELFO until the next " +_Static_assert(FORK_IPC_PROTOCOL_MAGIC == 0x454C4650U, + "fork IPC protocol magic must remain ELFP until the next " "incompatible wire-format change"); _Static_assert(IPC_MAGIC_HEADER == FORK_IPC_PROTOCOL_MAGIC, "header magic must be the protocol identity"); @@ -34,6 +35,8 @@ _Static_assert(FORK_IPC_PROTOCOL_MAGIC != PREVIOUS_ELFM_MAGIC, "start_stack header field requires rejecting ELFM peers"); _Static_assert(FORK_IPC_PROTOCOL_MAGIC != PREVIOUS_ELFN_MAGIC, "region fork metadata requires rejecting ELFN peers"); +_Static_assert(FORK_IPC_PROTOCOL_MAGIC != PREVIOUS_ELFO_MAGIC, + "per-fd description ownership requires rejecting ELFO peers"); _Static_assert(IPC_MAGIC_SENTINEL != FORK_IPC_PROTOCOL_MAGIC, "process-state sentinel must not alias the header protocol"); diff --git a/tests/test-fuse-alpine.sh b/tests/test-fuse-alpine.sh index 882cfb1b..f78b6861 100755 --- a/tests/test-fuse-alpine.sh +++ b/tests/test-fuse-alpine.sh @@ -12,10 +12,19 @@ SYSROOT="${2:?Usage: $0 }" TEST_BIN="${3:?Usage: $0 }" SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" -TEST_LABEL_WIDTH=14 TEST_TIMEOUT=20 source "$SCRIPT_DIR/lib/test-runner.sh" +# This suite reports its own tallies rather than sourcing lib/report.sh, so it +# has to declare them. All four were only ever incremented, never set: a run +# with no failures reached "[ "$fail" -gt 0 ]" with fail unset and bash answered +# "integer expression expected" on stderr, and the summary line printed empty +# fields for skip and xfail, which nothing here ever touches. +pass=0 +fail=0 +skip=0 +expected_fail=0 + TEST_RUNNER=("$ELFUSE" --sysroot "$SYSROOT") if [ ! -d "$SYSROOT" ]; then diff --git a/tests/test-launch-flags.sh b/tests/test-launch-flags.sh index 70762e50..69d4d1ed 100755 --- a/tests/test-launch-flags.sh +++ b/tests/test-launch-flags.sh @@ -30,9 +30,6 @@ ENV_CAT="${4:-}" . "$(dirname "$0")/lib/report.sh" # Counters are per-script; see tests/lib/report.sh. -pass=0 -fail=0 -skip=0 # check check() diff --git a/tests/test-matrix.sh b/tests/test-matrix.sh index 54b8bfc4..97c922a5 100755 --- a/tests/test-matrix.sh +++ b/tests/test-matrix.sh @@ -496,6 +496,10 @@ run_summary_suite() fields="$(suite_summary_fields "$output")" if [ -n "$fields" ]; then local suite_pass=0 suite_fail=0 suite_skip=0 suite_total=0 + + # suite_total lands read's fourth field so the third does not absorb it. + # The matrix keeps its own running total, so nothing reads it back. + # shellcheck disable=SC2034 read -r suite_pass suite_fail suite_skip suite_total <<< "$fields" # Force decimal: a sub-suite that ever emits a zero-padded count ('08', @@ -786,7 +790,30 @@ run_unit_tests() test_rc "$runner" "test-threaded-exec-worker" 0 \ "$bindir/test-threaded-exec" worker test_rc "$runner" "test-exec-handoff" 0 "$bindir/test-exec-handoff" + test_rc "$runner" "test-pipe-steal" 0 "$bindir/test-pipe-steal" + test_check "$runner" "test-fcntl-flags" "0 failed" "$bindir/test-fcntl-flags" test_rc "$runner" "test-mprotect-mt" 0 "$bindir/test-mprotect-mt" + test_check "$runner" "test-dup-setfl-race" "0 failed" \ + "$bindir/test-dup-setfl-race" + + # The blocking-semantics surface elfuse emulates on top of an O_NONBLOCK it + # owns. Every assertion in these is Linux's own answer, which is the whole + # point of running them against the reference kernel as well: a test that + # only passes under elfuse documents a bug as a feature. + printf "\nBlocking semantics\n" + test_check "$runner" "test-eventfd-semaphore-contended" "0 failed" \ + "$bindir/test-eventfd-semaphore-contended" + test_check "$runner" "test-socket-shortwrite" "0 failed" \ + "$bindir/test-socket-shortwrite" + test_check "$runner" "test-socket-blockwrite-signal" "0 failed" \ + "$bindir/test-socket-blockwrite-signal" + test_check "$runner" "test-socket-accept-contended" "0 failed" \ + "$bindir/test-socket-accept-contended" + test_check "$runner" "test-socket-waitall" "0 failed" \ + "$bindir/test-socket-waitall" + test_check "$runner" "test-synthetic-wait-signal" "0 failed" \ + "$bindir/test-synthetic-wait-signal" + test_check "$runner" "test-sigpipe" "0 failed" "$bindir/test-sigpipe" printf "\nNegative tests\n" test_check "$runner" "test-negative" "0 failed" "$bindir/test-negative" @@ -1482,12 +1509,14 @@ detect_x86_64_host_class() # qemu-aarch64: none. test-poll used to diverge inside the qemu reference VM but # now passes there (observed on the self-hosted runner and in local captures); # the qemu row in EXPECTED_BASELINES therefore pins exactly zero failures. +# shellcheck disable=SC2034 # reference data for review and CI triage, not code KNOWN_FAILURES_QEMU_AARCH64="" # elfuse-x86_64: rosetta limitations documented in the upstream hyper-linux # audit. test-signal-thread fails because rosetta shadows signal state # internally (SA_RESETHAND not reset); test-thread / test-stress hang on # rosetta's TLS=0 corner case. +# shellcheck disable=SC2034 # reference data for review and CI triage, not code KNOWN_FAILURES_ELFUSE_X86_64="test-signal-thread test-tgkill-directed test-thread test-stress" verify_expected_counts() diff --git a/tests/test-pipe-steal.c b/tests/test-pipe-steal.c new file mode 100644 index 00000000..39176afb --- /dev/null +++ b/tests/test-pipe-steal.c @@ -0,0 +1,606 @@ +/* + * Readiness-poll steal: a reader that loses the race must not park a vCPU + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + * + * elfuse waits for readiness before every blocking read or write, and the wait + * reserves nothing. With several threads on one pipe, one byte per wakeup, all + * of them come back from the wait ready and one takes the byte; the losers then + * transfer. If that transfer is a plain blocking host call it parks the vCPU + * thread where neither hv_vcpus_exit nor the wakeup pipe reaches it, and the + * execve teardown counts it as a sibling that would not leave and kills the + * process (exit 128) instead of running the new image. + * + * The exec chain is what tests it: READERS threads share one blocking pipe, a + * writer feeds single bytes, and once the writer stops the losers are the ones + * left holding a transfer that will never complete. Then the process execs + * itself and checks the facts Linux guarantees in the new image. + * + * Iteration 0 also checks the other half of the contract: making the transfer + * non-blocking must not leak short writes to the guest. A blocking write(2) + * moves every byte, so a write and a writev larger than the pipe buffer must + * still report the full count, in order, with nothing dropped or repeated. + * + * Syscalls exercised: execve(221), clone(220), read(63), write(64), + * writev(66), pipe2(59), fcntl(25), ioctl(29), + * gettid(178), getpid(172), socketpair(199), + * sendmsg(211), recvmsg(212), ppoll(73) + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "test-harness.h" + +int passes = 0, fails = 0; + +/* drain_worker runs while main is also reporting, so its failures go through an + * atomic of its own and are folded in after the join. + */ +static atomic_int worker_fails; + +#define READERS 4 +#define TOTAL_EXECS 40 +#define BIG_WRITE (1u << 20) /* well past any pipe buffer */ + +static atomic_int ready; +static atomic_int exec_now; +static int pipe_rd = -1, pipe_wr = -1; +static const char *self_path; +static int g_iter; + +/* Byte i of the stream the full-write checks send. */ +static uint8_t stream_byte(size_t i) +{ + return (uint8_t) (i * 7u + 3u); +} + +/* Reader half of the full-write check: drains the pipe in small bites and + * verifies every byte arrives once, in order. A partial write reassembled with + * a bad offset shows up here as a mismatch, not as a lost byte count. + */ +static void *drain_worker(void *arg) +{ + size_t want = (size_t) (uintptr_t) arg; + size_t seen = 0; + uint8_t buf[4096]; + + while (seen < want) { + /* Wait with a deadline rather than blocking outright. A writer that + * reports the full count but moves less leaves this thread waiting for + * bytes nobody will send, and main is joining it: without the deadline + * the whole test hangs and says nothing, which is how a truncated + * blocking write hid here once already. + */ + struct pollfd pfd = {.fd = pipe_rd, .events = POLLIN}; + int pr = poll(&pfd, 1, 5000); + if (pr == 0) { + printf("\nstalled after %zu of %zu bytes\n", seen, want); + atomic_fetch_add(&worker_fails, 1); + return NULL; + } + if (pr < 0) { + if (errno == EINTR) + continue; + break; + } + + ssize_t n = read(pipe_rd, buf, sizeof(buf)); + if (n <= 0) { + if (n < 0 && errno == EINTR) + continue; + break; + } + for (ssize_t i = 0; i < n; i++) { + /* Both writes send the same BIG_WRITE pattern, so the expected byte + * wraps at that boundary. + */ + if (buf[i] != stream_byte((seen + (size_t) i) % BIG_WRITE)) { + printf("\nstream mismatch at byte %zu\n", seen + (size_t) i); + atomic_fetch_add(&worker_fails, 1); + return NULL; + } + } + seen += (size_t) n; + + /* Keep the writer against a full pipe rather than a drained one. */ + usleep(200); + } + + if (seen != want) { + printf("\ndrained %zu bytes, want %zu\n", seen, want); + atomic_fetch_add(&worker_fails, 1); + } + return NULL; +} + +/* A blocking write moves every byte it was given. Send more than the pipe + * buffer holds, once through write() and once through writev(), and require the + * full count both times. + */ +static void check_full_write(void) +{ + uint8_t *buf = malloc(BIG_WRITE); + if (!buf) { + FAIL("malloc"); + return; + } + for (size_t i = 0; i < BIG_WRITE; i++) + buf[i] = stream_byte(i); + + int fds[2]; + if (pipe(fds) != 0) { + FAIL("pipe"); + free(buf); + return; + } + pipe_rd = fds[0]; + pipe_wr = fds[1]; + + size_t split = BIG_WRITE / 3; + pthread_t drain; + if (pthread_create(&drain, NULL, drain_worker, + (void *) (uintptr_t) (BIG_WRITE + BIG_WRITE)) != 0) { + FAIL("pthread_create"); + goto out; + } + + TEST("blocking write moves every byte"); + ssize_t w = write(pipe_wr, buf, BIG_WRITE); + EXPECT_EQ(w, (ssize_t) BIG_WRITE, "short write on a blocking pipe"); + + /* The writev sends the same pattern a second time, split across three + * segments, so a partial write reassembled at the wrong offset shows up as + * a mismatch in the reader rather than only as a short count. + */ + struct iovec iov[3] = { + {.iov_base = buf, .iov_len = split}, + {.iov_base = buf + split, .iov_len = split}, + {.iov_base = buf + 2 * split, .iov_len = BIG_WRITE - 2 * split}, + }; + TEST("blocking writev moves every byte"); + ssize_t wv = writev(pipe_wr, iov, 3); + EXPECT_EQ(wv, (ssize_t) BIG_WRITE, "short writev on a blocking pipe"); + + pthread_join(drain, NULL); + fails += atomic_load(&worker_fails); + +out: + close(pipe_wr); + close(pipe_rd); + pipe_rd = pipe_wr = -1; + free(buf); +} + +/* Reading an empty pipe is how the guest observes its own O_NONBLOCK. */ +static void expect_eagain_read(int fd, const char *what) +{ + char c; + TEST(what); + EXPECT_ERRNO(read(fd, &c, 1), EAGAIN, "read did not report EAGAIN"); +} + +/* elfuse keeps O_NONBLOCK set on the host pipe so a transfer can report EAGAIN + * instead of parking, and answers the guest from its own shadow of the flag. + * The guest must see exactly what it asked for, through either spelling. + */ +static void check_nonblock_view(void) +{ + int fds[2]; + if (pipe2(fds, O_NONBLOCK) != 0) { + FAIL("pipe2(O_NONBLOCK)"); + return; + } + + TEST("pipe2(O_NONBLOCK) is visible"); + EXPECT_TRUE(fcntl(fds[0], F_GETFL) & O_NONBLOCK, "F_GETFL lost O_NONBLOCK"); + + expect_eagain_read(fds[0], "nonblocking read of an empty pipe"); + close(fds[0]); + close(fds[1]); + + if (pipe(fds) != 0) { + FAIL("pipe"); + return; + } + + TEST("a plain pipe reads as blocking"); + EXPECT_TRUE((fcntl(fds[0], F_GETFL) & O_NONBLOCK) == 0, + "F_GETFL invented O_NONBLOCK"); + + fcntl(fds[0], F_SETFL, fcntl(fds[0], F_GETFL) | O_NONBLOCK); + expect_eagain_read(fds[0], "F_SETFL O_NONBLOCK takes effect"); + + /* ioctl is the other spelling libuv uses on pipes. */ + int off = 0, on = 1; + ioctl(fds[0], FIONBIO, &off); + TEST("FIONBIO clears the guest's O_NONBLOCK"); + EXPECT_TRUE((fcntl(fds[0], F_GETFL) & O_NONBLOCK) == 0, + "F_GETFL still reports O_NONBLOCK"); + + ioctl(fds[0], FIONBIO, &on); + expect_eagain_read(fds[0], "FIONBIO sets it back"); + + close(fds[0]); + close(fds[1]); +} + +/* O_NONBLOCK lives on the open file description, so every dup alias observes a + * change made through any of them. elfuse owns the host flag on a pipe and + * answers from its own shadow, which is per-fd, so this is the case that shadow + * has to keep in step. + */ +static void check_dup_alias_flags(void) +{ + int fds[2]; + if (pipe(fds) != 0) { + FAIL("pipe"); + return; + } + int alias = dup(fds[0]); + if (alias < 0) { + FAIL("dup"); + close(fds[0]); + close(fds[1]); + return; + } + + fcntl(fds[0], F_SETFL, fcntl(fds[0], F_GETFL) | O_NONBLOCK); + TEST("F_SETFL reaches a dup alias"); + EXPECT_TRUE(fcntl(alias, F_GETFL) & O_NONBLOCK, + "the alias still reads as blocking"); + + expect_eagain_read(alias, "the alias transfers nonblocking too"); + + /* And back the other way, through the other spelling. */ + int off = 0; + ioctl(alias, FIONBIO, &off); + TEST("FIONBIO on the alias reaches the original"); + EXPECT_TRUE((fcntl(fds[0], F_GETFL) & O_NONBLOCK) == 0, + "the original still reads as nonblocking"); + + close(alias); + close(fds[0]); + close(fds[1]); +} + +/* Send one fd over a socketpair to ourselves and return what came back, or -1. + * The caller owns the result. + */ +static int scm_roundtrip(int fd) +{ + int sv[2]; + if (socketpair(AF_UNIX, SOCK_STREAM, 0, sv) != 0) + return -1; + + char cbuf[CMSG_SPACE(sizeof(int))] = {0}, data = 'x'; + struct iovec iov = {.iov_base = &data, .iov_len = 1}; + struct msghdr m = {.msg_iov = &iov, + .msg_iovlen = 1, + .msg_control = cbuf, + .msg_controllen = sizeof(cbuf)}; + struct cmsghdr *c = CMSG_FIRSTHDR(&m); + c->cmsg_level = SOL_SOCKET; + c->cmsg_type = SCM_RIGHTS; + c->cmsg_len = CMSG_LEN(sizeof(int)); + memcpy(CMSG_DATA(c), &fd, sizeof(fd)); + if (sendmsg(sv[0], &m, 0) < 0) { + close(sv[0]); + close(sv[1]); + return -1; + } + + char rbuf[8], rc[CMSG_SPACE(sizeof(int))] = {0}; + struct iovec riov = {.iov_base = rbuf, .iov_len = sizeof(rbuf)}; + struct msghdr rm = {.msg_iov = &riov, + .msg_iovlen = 1, + .msg_control = rc, + .msg_controllen = sizeof(rc)}; + int got = -1; + if (recvmsg(sv[1], &rm, 0) > 0) { + struct cmsghdr *rcm = CMSG_FIRSTHDR(&rm); + if (rcm && rcm->cmsg_type == SCM_RIGHTS) + memcpy(&got, CMSG_DATA(rcm), sizeof(got)); + } + close(sv[0]); + close(sv[1]); + return got; +} + +static void *late_writer(void *arg) +{ + usleep(50000); + ssize_t n = write(*(int *) arg, "y", 1); + (void) n; + return NULL; +} + +/* The received end of the same pipe must still read as blocking. + * + * elfuse keeps O_NONBLOCK on the host end of every pipe it opens and emulates + * the wait on top, so a descriptor arriving over SCM_RIGHTS from another elfuse + * process is found already nonblocking. Reporting that flag as the guest's view + * would give a plain read EAGAIN on a pipe the guest never set nonblocking, + * which is fatal to a program that has no handling for it. The receive path + * adopts the emulation instead, which sets nothing on a description it did not + * create. + */ +static void check_scm_recv_blocking(void) +{ + int fds[2]; + if (pipe(fds) != 0) { + FAIL("pipe failed"); + return; + } + + int rfd = scm_roundtrip(fds[0]); + if (rfd < 0) { + FAIL("SCM_RIGHTS roundtrip failed"); + close(fds[0]); + close(fds[1]); + return; + } + + TEST("a received pipe end reads as blocking"); + EXPECT_TRUE((fcntl(rfd, F_GETFL) & O_NONBLOCK) == 0, + "the received fd reports O_NONBLOCK the guest never set"); + + /* And behaves that way: with the data 50ms out, a blocking read waits for + * it and a leaked O_NONBLOCK returns EAGAIN at once. + */ + pthread_t t; + if (pthread_create(&t, NULL, late_writer, &fds[1]) != 0) { + FAIL("pthread_create failed"); + } else { + char c = 0; + ssize_t n = read(rfd, &c, 1); + TEST("a read on a received pipe end waits for the data"); + EXPECT_TRUE(n == 1 && c == 'y', "read did not return the late byte"); + pthread_join(t, NULL); + } + + close(rfd); + close(fds[0]); + close(fds[1]); +} + +/* Ask elfuse to duplicate the stdin it was handed, so the host side can check + * that nothing about elfuse's own use of O_NONBLOCK reached the description its + * launcher owns. See tests/test-stdio-nonblock-host.c. Pass stdin to ourselves + * over a socketpair. The description that comes back is the launcher's, reached + * through the one fd-creating path that has no source slot to inherit from, so + * it is the path most likely to take ownership of a description elfuse did not + * create. + */ +static int scm_pass_stdin_and_exit(void) +{ + int got = scm_roundtrip(0); + if (got < 0) + return 1; + close(got); + return 0; +} + +static int dup_stdin_and_exit(bool via_fork) +{ + /* A magic link is served by dup()ing the descriptor the process already + * holds, so it aliases the launcher's description exactly as dup(0) does, + * through a different path. + */ + int ml = open("/dev/stdin", O_RDONLY); + if (ml >= 0) + close(ml); + + int a = dup(0); + int b = fcntl(0, F_DUPFD, 20); + if (a < 0 || b < 0) + return 1; + + /* An alias of an alias still names the launcher's description. */ + int c = dup(a); + if (c < 0) + return 1; + if (dup2(0, 9) < 0) + return 1; + + /* The fork path rebuilds the child's fd table from scratch, so the alias + * has to survive that too: the child re-allocates a slot for the same + * description the launcher owns. + */ + if (via_fork) { + pid_t pid = fork(); + if (pid < 0) + return 1; + if (pid == 0) + _exit(0); + int status = 0; + if (waitpid(pid, &status, 0) < 0) + return 1; + return WIFEXITED(status) && WEXITSTATUS(status) == 0 ? 0 : 1; + } + return 0; +} + +static void exec_next(void) +{ + fflush(stdout); + fflush(stderr); + + char iterbuf[16], pidbuf[16]; + snprintf(iterbuf, sizeof(iterbuf), "%d", g_iter + 1); + snprintf(pidbuf, sizeof(pidbuf), "%d", (int) getpid()); + + extern char **environ; + char *argv[] = {(char *) self_path, iterbuf, pidbuf, NULL}; + execve(self_path, argv, environ); + + fprintf(stderr, "\ntest-pipe-steal: execve(%s) failed at iter %d (%s)\n", + self_path, g_iter, strerror(errno)); + _exit(1); +} + +/* Every reader wants the same single byte. Whoever loses is left holding a + * transfer that only the next write can satisfy, and the writer stops before + * the exec. + */ +static void *steal_reader(void *arg) +{ + (void) arg; + atomic_fetch_add(&ready, 1); + for (;;) { + char c; + ssize_t n = read(pipe_rd, &c, 1); + if (n == 0 || (n < 0 && errno != EINTR)) + return NULL; + } +} + +static void *feeder(void *arg) +{ + (void) arg; + atomic_fetch_add(&ready, 1); + while (!atomic_load(&exec_now)) { + char c = 'x'; + if (write(pipe_wr, &c, 1) != 1) + return NULL; + usleep(100); + } + return NULL; +} + +static int read_thread_count(void) +{ + FILE *f = fopen("/proc/self/status", "r"); + if (!f) + return -1; + + char line[256]; + int n = -1; + while (fgets(line, sizeof(line), f)) { + if (sscanf(line, "Threads: %d", &n) == 1) + break; + } + fclose(f); + return n; +} + +/* Fail fast: a per-iteration PASS line would print TOTAL_EXECS times. */ +static void check_post_exec(int iter, int want_pid) +{ + int nthreads = read_thread_count(); + if (nthreads != 1) { + printf("\niter %d: /proc/self/status Threads: %d, want 1\n", iter, + nthreads); + fails++; + } + + int pid = (int) getpid(); + if (pid != want_pid) { + printf("\niter %d: pid %d after exec, want %d\n", iter, pid, want_pid); + fails++; + } + + int tid = (int) syscall(SYS_gettid); + if (tid != pid) { + printf("\niter %d: gettid %d != getpid %d after exec\n", iter, tid, + pid); + fails++; + } +} + +static void spawn_contenders(void) +{ + int fds[2]; + if (pipe2(fds, O_CLOEXEC) != 0) { + FAIL("pipe2 failed"); + exit(1); + } + pipe_rd = fds[0]; + pipe_wr = fds[1]; + + atomic_store(&ready, 0); + atomic_store(&exec_now, 0); + + for (int i = 0; i < READERS; i++) { + pthread_t t; + if (pthread_create(&t, NULL, steal_reader, NULL) != 0) { + FAIL("pthread_create failed"); + exit(1); + } + pthread_detach(t); + } + + pthread_t w; + if (pthread_create(&w, NULL, feeder, NULL) != 0) { + FAIL("pthread_create failed"); + exit(1); + } + pthread_detach(w); + + while (atomic_load(&ready) < READERS + 1) + sched_yield(); +} + +int main(int argc, char **argv) +{ + self_path = argv[0]; + if (argc > 1 && strcmp(argv[1], "dupstdin") == 0) + return dup_stdin_and_exit(false); + if (argc > 1 && strcmp(argv[1], "dupstdin-fork") == 0) + return dup_stdin_and_exit(true); + if (argc > 1 && strcmp(argv[1], "scmpass") == 0) + return scm_pass_stdin_and_exit(); + + int iter = argc > 1 ? atoi(argv[1]) : 0; + + if (iter == 0) { + printf( + "test-pipe-steal: %d readers contending for one byte, %d execs\n", + READERS, TOTAL_EXECS); + check_nonblock_view(); + check_dup_alias_flags(); + check_scm_recv_blocking(); + check_full_write(); + if (fails > 0) + return 1; + TEST("exec chain under a contended pipe"); + } else { + check_post_exec(iter, argc > 2 ? atoi(argv[2]) : 0); + if (fails > 0) + return 1; + } + + if (iter >= TOTAL_EXECS) { + PASS(); + SUMMARY("test-pipe-steal"); + return fails > 0 ? 1 : 0; + } + + g_iter = iter; + spawn_contenders(); + + /* Let the readers cycle through the contended wakeup, then cut the feed so + * the losers stay where they are, and exec on top of them. + */ + usleep(20000); + atomic_store(&exec_now, 1); + usleep(2000); + exec_next(); + return 1; +} diff --git a/tests/test-rosetta-alpine.sh b/tests/test-rosetta-alpine.sh index 0ec1c664..289456d5 100755 --- a/tests/test-rosetta-alpine.sh +++ b/tests/test-rosetta-alpine.sh @@ -33,7 +33,6 @@ esac FIXTURES="${FIXTURES_DIR:-externals/test-fixtures}" STATICBIN_LONG="${FIXTURES}/x86_64-musl/staticbin/bin" -ROOTFS="${FIXTURES}/x86_64-musl/rootfs" ROSETTA_PATH="${MATRIX_ROSETTA_TRANSLATOR:-/Library/Apple/usr/libexec/oah/RosettaLinux/rosetta}" SHORTDIR=/tmp/elfuse-ra @@ -46,9 +45,6 @@ DATA="${SHORTDIR}/data" # shellcheck source=tests/lib/report.sh . "$(dirname "$0")/lib/report.sh" -pass=0 -fail=0 -skip=0 total=0 # Pre-flight. diff --git a/tests/test-rosetta-audit.sh b/tests/test-rosetta-audit.sh index 112ce53b..f89c1f10 100644 --- a/tests/test-rosetta-audit.sh +++ b/tests/test-rosetta-audit.sh @@ -13,7 +13,6 @@ case "$ELFUSE_INPUT" in *) ELFUSE="$(pwd)/$ELFUSE_INPUT" ;; esac -FIXTURES="${FIXTURES_DIR:-externals/test-fixtures}" ROSETTA_PATH="${MATRIX_ROSETTA_TRANSLATOR:-/Library/Apple/usr/libexec/oah/RosettaLinux/rosetta}" AUDIT_BIN="$(pwd)/tests/fixtures/rosetta/x86_64-rosetta-audit" TLS0_BIN="$(pwd)/tests/fixtures/rosetta/x86_64-rosetta-tls0" @@ -21,9 +20,6 @@ TLS0_BIN="$(pwd)/tests/fixtures/rosetta/x86_64-rosetta-tls0" # shellcheck source=tests/lib/report.sh . "$(dirname "$0")/lib/report.sh" -pass=0 -fail=0 -skip=0 total=0 if [ ! -x "$ROSETTA_PATH" ]; then diff --git a/tests/test-rosetta-cli.sh b/tests/test-rosetta-cli.sh index 6309601a..76d3d937 100755 --- a/tests/test-rosetta-cli.sh +++ b/tests/test-rosetta-cli.sh @@ -16,9 +16,6 @@ ELFUSE="${1:-build/elfuse}" # shellcheck source=tests/lib/report.sh . "$(dirname "$0")/lib/report.sh" -pass=0 -fail=0 -skip=0 total=0 tmpdir="$(mktemp -d "${TMPDIR:-/tmp}/elfuse-rosetta-cli.XXXXXX")" diff --git a/tests/test-rosetta-execfd.sh b/tests/test-rosetta-execfd.sh index 4b455337..7123bc76 100755 --- a/tests/test-rosetta-execfd.sh +++ b/tests/test-rosetta-execfd.sh @@ -39,9 +39,6 @@ ROSETTA_PATH="${MATRIX_ROSETTA_TRANSLATOR:-/Library/Apple/usr/libexec/oah/Rosett # shellcheck source=tests/lib/report.sh . "$(dirname "$0")/lib/report.sh" -pass=0 -fail=0 -skip=0 total=0 if [ ! -x "$ROSETTA_PATH" ]; then diff --git a/tests/test-rosetta-failure-modes.sh b/tests/test-rosetta-failure-modes.sh index 35377d28..bdfeeb53 100755 --- a/tests/test-rosetta-failure-modes.sh +++ b/tests/test-rosetta-failure-modes.sh @@ -40,9 +40,6 @@ SHORTDIR=/tmp/elfuse-rfm # shellcheck source=tests/lib/report.sh . "$(dirname "$0")/lib/report.sh" -pass=0 -fail=0 -skip=0 total=0 # Expect a non-zero exit AND a stderr fragment match. Args: