0.14.1 --- a copy of the space can start a thread

.github/workflows/ci.yml

@@ -329,6 +329,15 @@ jobs:
           #     first is deliberate: `pthread_self' IS a name musl defines, and
           #     this system's library would be handed a thread record of
           #     another library's layout.
+          #   _dyld_atfork_*, _dyld_fork_child,
+          #   _malloc_fork_*, _libc_fork_*,
+          #   _pthread_atfork_*, _mach_fork_child   the preparations this
+          #     system's `fork' performs around the duplication primitive, part
+          #     by part, which kal_space_start calls so that a copy can start a
+          #     thread (src/space.cpp, mcpplibs/openkal-musl#55). The routine
+          #     `fork' calls them from is not exported; each part is, under a
+          #     name no C library in this ecosystem defines. None of them calls
+          #     back into a C library above.
           #   __libc_start_main, main, _main    the hand-over, undefined here by
           #     construction.
           #   kal_*                             the interface itself.
@@ -348,7 +357,7 @@ jobs:
           #   ___dso_handle                     likewise the format's: emitted
           #     beside the weak references above, defined by the image's own
           #     start files.
-          permitted='^_?(memcpy|memmove|memset|memcmp|bzero|clock_gettime_nsec_np|pthread_create_from_mach_thread|pthread_create|pthread_join|_pthread_self|_pthread_get_stackaddr_np|_pthread_get_stacksize_np|__libc_start_main|main|kal_[a-z_]+|__stack_chk_guard|__stack_chk_fail|GCC_except_table.*|_ZN3okm.*|__Unwind_Resume|__dso_handle|section\$(start|end)\$__TEXT\$__(eh_frame|unwind_info))$'
+          permitted='^_?(memcpy|memmove|memset|memcmp|bzero|clock_gettime_nsec_np|pthread_create_from_mach_thread|pthread_create|pthread_join|_pthread_self|_pthread_get_stackaddr_np|_pthread_get_stacksize_np|_dyld_atfork_(prepare|parent)|_dyld_fork_child|_(malloc|libc)_fork_(prepare|parent|child)|_pthread_atfork_(prepare|parent|child)|_mach_fork_child|__libc_start_main|main|kal_[a-z_]+|__stack_chk_guard|__stack_chk_fail|GCC_except_table.*|_ZN3okm.*|__Unwind_Resume|__dso_handle|section\$(start|end)\$__TEXT\$__(eh_frame|unwind_info))$'
           # Reported with the object that references it. A symbol without the
           # object it came from names a fault and not a place, and the first
           # time this check fired the answer was in the object rather than in

README.md

@@ -8,7 +8,7 @@ written on the kernel's own calls.
 openkal = "0.16.0"
 
 [target.'cfg(os = "macos")'.dependencies]
-openkal-macos = "0.14.0"
+openkal-macos = "0.14.1"
 ```
 
 Its purpose is as much to test the specification as to be used. A specification

mcpp.toml

@@ -1,7 +1,7 @@
 [package]
 namespace   = "mcpplibs"
 name        = "openkal-macos"
-version     = "0.14.0"
+version     = "0.14.1"
 description = "An implementation of openkal for macOS, written on the kernel's own calls. Its purpose is as much to test the specification as to be used."
 license     = "Apache-2.0"
 
@@ -70,6 +70,31 @@ flags = [
                                  "-fno-stack-protector", "-fno-builtin"] },
 ]
 
+# THE PREPARATIONS FOR A DUPLICATE ARE LEFT FOR THE LOADER TO FIND.
+#
+# kal_space_start calls the parts of the system library that this system's
+# `fork' calls around the duplication primitive (src/space.cpp,
+# mcpplibs/openkal-musl#55). They are exported, and they are not part of the
+# interface the system documents, so a development stub need not list them.
+# `-U' states that a name may be left for the loader to find among the images
+# it has loaded, which is where it is; a name the stub does list is bound as
+# usual. A link made elsewhere reads port/libSystem.tbd, which lists them. The
+# flags reach every program above this package: mcpp forwards a dependency's
+# `ldflags' to its consumer's link.
+ldflags = ["-Wl,-U,__dyld_atfork_parent",
+           "-Wl,-U,__dyld_atfork_prepare",
+           "-Wl,-U,__dyld_fork_child",
+           "-Wl,-U,__libc_fork_child",
+           "-Wl,-U,__libc_fork_parent",
+           "-Wl,-U,__libc_fork_prepare",
+           "-Wl,-U,__mach_fork_child",
+           "-Wl,-U,__malloc_fork_child",
+           "-Wl,-U,__malloc_fork_parent",
+           "-Wl,-U,__malloc_fork_prepare",
+           "-Wl,-U,__pthread_atfork_child",
+           "-Wl,-U,__pthread_atfork_parent",
+           "-Wl,-U,__pthread_atfork_prepare"]
+
 [features]
 default = []
 

port/libSystem.tbd

@@ -28,10 +28,13 @@ targets:         [ arm64-macos, x86_64-macos ]
 install-name:    '/usr/lib/libSystem.B.dylib'
 current-version: 1351.0.0
 #
-# FIVE NAMES ARE CALLED AND ONE IS NOT.
+# EIGHTEEN NAMES ARE CALLED AND ONE IS NOT.
 #
-# The first four are names this implementation CALLS, and the enumeration above
-# found them. The three that 0.13.0 added answer the stack of the calling
+# The first eighteen are names this implementation CALLS. The thirteen that
+# 0.14.1 added are the preparations this system's `fork' performs around the
+# primitive, part by part, which a duplicate made by kal_space_start needs
+# before it can start a thread (src/space.cpp, mcpplibs/openkal-musl#55).
+# The three that 0.13.0 added answer the stack of the calling
 # thread (openkal 0.15, clause 11 entry 21), and they follow the same rule as
 # the two before them: this system's library defines them and no C library in
 # this ecosystem does, which is what makes them reachable from an
@@ -54,7 +57,12 @@ current-version: 1351.0.0
 # evidence about a format.
 exports:
   - targets:     [ arm64-macos, x86_64-macos ]
-    symbols:     [ _clock_gettime_nsec_np, _pthread_create_from_mach_thread,
+    symbols:     [ __dyld_atfork_parent, __dyld_atfork_prepare, __dyld_fork_child,
+                   __libc_fork_child, __libc_fork_parent, __libc_fork_prepare,
+                   __mach_fork_child, __malloc_fork_child, __malloc_fork_parent,
+                   __malloc_fork_prepare, __pthread_atfork_child, __pthread_atfork_parent,
+                   __pthread_atfork_prepare,
+                   _clock_gettime_nsec_np, _pthread_create_from_mach_thread,
                    _pthread_get_stackaddr_np, _pthread_get_stacksize_np,
                    __pthread_self, dyld_stub_binder ]
 ...

src/space.cpp

@@ -19,6 +19,63 @@
 
 extern "C" {
 
+// THE SYSTEM LIBRARY'S OWN PREPARATION FOR A DUPLICATE, WITHOUT WHICH THE
+// DUPLICATE CANNOT START A THREAD.
+//
+// This system's `fork' is not only the primitive. Around it each part of the
+// system library takes its locks, and in the copy each makes its state true of
+// the new process --- among the rest the name of the task, through which the
+// thread library allocates every stack, and the thread library's own record of
+// which threads exist. The primitive alone left the copy holding the original's
+// state, and the first thread it asked for failed (mcpplibs/openkal-musl#55).
+//
+// MEASURED on macOS 14.8.9, in a program that duplicated itself with the
+// primitive and asked the copy for a thread:
+//
+//     nothing                                  EAGAIN
+//     the two child preparations alone         the copy was killed
+//     and the thread library's locks around    0
+//     the whole set below, five times twenty   0
+//
+// The library's own `fork' calls these from a routine it does not export, so
+// the parts are called directly, in its order: locks before the primitive from
+// the outermost part inward, released in the original in the reverse order,
+// and the state renewed in the copy. Each is exported under a name no C library
+// in this ecosystem defines, so it is reachable beneath one as above one. The
+// handlers a program registers with `pthread_atfork' are not run: a copy made
+// here is not a `fork' the program asked for.
+void _dyld_atfork_prepare(void);
+void _dyld_atfork_parent(void);
+void _dyld_fork_child(void);
+void _malloc_fork_prepare(void);
+void _malloc_fork_parent(void);
+void _malloc_fork_child(void);
+void _libc_fork_prepare(void);
+void _libc_fork_parent(void);
+void _libc_fork_child(void);
+void _pthread_atfork_prepare(void);
+void _pthread_atfork_parent(void);
+void _pthread_atfork_child(void);
+void _mach_fork_child(void);
+
+}  // extern "C"
+
+namespace {
+
+void prepare() {
+    _dyld_atfork_prepare(); _malloc_fork_prepare(); _libc_fork_prepare(); _pthread_atfork_prepare();
+}
+void in_the_original() {
+    _pthread_atfork_parent(); _libc_fork_parent(); _malloc_fork_parent(); _dyld_atfork_parent();
+}
+void in_the_copy() {
+    _mach_fork_child(); _dyld_fork_child(); _pthread_atfork_child(); _malloc_fork_child(); _libc_fork_child();
+}
+
+}  // namespace
+
+extern "C" {
+
 int kal_space_start(void (*entry)(void*), void* arg, void* stack_top,
                     kal_process* out) {
     if (entry == nullptr || out == nullptr) return kal_err_invalid;
@@ -31,10 +88,12 @@ int kal_space_start(void (*entry)(void*), void* arg, void* stack_top,
     (void)stack_top;
 
     bool is_duplicate = false;
+    prepare();
     const okm_long child = okm::duplicate(is_duplicate);
-    if (okm::failed(child)) return okm::translate(child);
+    if (okm::failed(child)) { in_the_original(); return okm::translate(child); }
 
     if (is_duplicate) {
+        in_the_copy();
         entry(arg);
         // THE ENTRY IS NOT REQUIRED TO RETURN, AND IF IT DOES THE CONTEXT ENDS.
         //
@@ -46,6 +105,7 @@ int kal_space_start(void (*entry)(void*), void* arg, void* stack_top,
         for (;;) okm::sys(okm::nr_exit, 0);
     }
 
+    in_the_original();
     *out = kal_process{ static_cast<kal_uintptr>(child) };
     return kal_ok;
 }

tests/conformance_space_task.cpp

@@ -0,0 +1,64 @@
+// Conformance: a context started in a copy of the space can start a context.
+//
+// mcpplibs/openkal-musl#55: in a copy made by kal_space_start, the first
+// thread the copy asked for failed with EAGAIN, while the original started one.
+// The duplication primitive alone leaves the system library's state the
+// original's, and this system's `fork' prepares it around the primitive; the
+// implementation now does what `fork' does. Observed through the status, the
+// only channel a separate space has: 3 is a context started, joined and given
+// its least stack size in the copy, and any other value is the step that
+// failed.
+import openkal.types;
+import openkal.task;
+import openkal.space;
+import openkal.process;
+import openkal.stream;
+import openkal.abort;
+
+namespace {
+int failures = 0;
+void check(bool ok, const char* what) {
+    if (ok) return;
+    ++failures;
+    const char pre[] = "FAIL: ";
+    kal::write(kal::err(), pre, sizeof(pre) - 1);
+    kal_uintptr n = 0; while (what[n]) ++n;
+    kal::write(kal::err(), what, n);
+    kal::write(kal::err(), "\n", 1);
+}
+
+volatile int g_ran = 0;
+void ran(void*) { g_ran = 1; }
+
+void in_the_copy(void*) {
+    kal_task t{};
+    if (kal_task_start(ran, nullptr, &t) != kal_ok) kal_exit(4);
+    if (kal_task_join(t) != kal_ok || g_ran != 1) kal_exit(5);
+    kal_task s{};
+    if (kal_task_start_sized(ran, nullptr, 16u << 20, &s) != kal_ok) kal_exit(6);
+    kal_task_join(s);
+    kal_exit(3);
+}
+
+alignas(16) unsigned char g_stack[64 * 1024];
+}
+
+int main() {
+    for (int round = 0; round < 3; ++round) {
+        kal_process p{};
+        const int rc = kal_space_start(in_the_copy, nullptr, g_stack + sizeof g_stack, &p);
+        check(rc == kal_ok, "a copy of the space starts");
+        if (rc != kal_ok) break;
+        int status = 0, terminated = 0;
+        check(kal_process_wait(p, &status, &terminated) == kal_ok, "the copy is waited for");
+        kal_process_close(p);
+        check(terminated == 0 && status == 3, "the copy started a context, joined it, and started one with a least stack size");
+        if (status != 3) {
+            char line[] = "copy status: X\n";
+            line[13] = static_cast<char>('0' + (status % 10));
+            kal::write(kal::err(), line, sizeof line - 1);
+        }
+    }
+    if (failures == 0) kal::write(kal::out(), "openkal-macos: a copy of the space starts contexts\n", 51);
+    return failures == 0 ? 0 : 1;
+}