Streams

.clang-tidy

@@ -1,19 +1,23 @@
-# clang-tidy configuration for libioma. CLion picks this file up (it prefers a project .clang-tidy
-# over its own settings); from the shell, with the compile flags the Makefile uses:
+# clang-tidy configuration for libioxd. CLion picks this file up (it prefers a project .clang-tidy
+# over its own settings); from the shell, `make tidy` runs it with the flags the objects are built
+# with, which is
 #
-#   clang-tidy src/io/*.c src/http/*.c tests/server.c -- -std=gnu23 -D_GNU_SOURCE -Iinclude -Isrc -Ithird_party/picohttpparser
+#   clang-tidy lib/io/*.c lib/http/*.c tests/server.c -- -std=gnu23 -D_GNU_SOURCE -Iinclude -Ilib -Ithird_party/picohttpparser
 #
 # (CLion's bundled clang-tidy ships without clang's builtin headers; give it gcc's with
-#  --extra-arg=-isystem/usr/lib/gcc/x86_64-linux-gnu/13/include, or it cascades phantom warnings.)
+#  --extra-arg=-isystem$(gcc -print-file-name=include), or it cascades phantom warnings - which is
+#  what `make tidy TIDY=<clion>/bin/clang/linux/x64/bin/clang-tidy` does for you.)
 #
-# The checks that matter for a C library, minus the ones that only add noise here: the Annex K
+# The checks that matter for a C library - the threading ones included, since the workers share a
+# certificate store and a stop flag - minus the ones that only add noise here: the Annex K
 # "memcpy is insecure" nag (glibc has no *_s functions), include-what-you-use over deliberately
 # transitive private headers, the pragma-once portability nag, int-to-pointer casts that ARE the
 # tagged user_data design, and the readability checks that argue with idiomatic C.
 Checks: >
   -*,
   bugprone-*,
   clang-analyzer-*,
+  concurrency-*,
   misc-*,
   performance-*,
   portability-*,
@@ -36,4 +40,4 @@ Checks: >
   -misc-no-recursion
 CheckOptions:
   bugprone-signed-bitwise.IgnorePositiveIntegerLiterals: true
-HeaderFilterRegex: '(include|src)/.*'
+HeaderFilterRegex: '(include|lib)/.*'

.gitignore

@@ -1,17 +1,19 @@
 obj/
+obj-tiny/
 *.o
 
 # libraries and examples
-libioma.a
-libioma.so
-libioma.so.*
-ioma-hello
-ioma.pc
+libioxd.a
+libioxd.so
+libioxd.so.*
+libioxd-tiny.a
+ioxd-hello
+ioxd.pc
 
 # old binaries
-ioma
-ioma-tiny
-ioma-trace
+/ioxd
+ioxd-tiny
+ioxd-trace
 stackful
 stackful-tiny
 stackful-trace
@@ -25,5 +27,9 @@ compile_commands.json
 
 # IDE
 .idea/
-tests/ioma-test-server
-tests/ioma-unit
+tests/ioxd-test-server
+tests/ioxd-test-server-tiny
+tests/ioxd-unit
+tests/ioxd-pipe-server
+tests/ioxd-router-test
+tests/certs/

CMakeLists.txt

@@ -1,44 +1,44 @@
-# CMake build for the ioma library plus the playground examples.
+# CMake build for the ioxd library plus the playground examples.
 #
 # Consume it from another CMake project either as a subdirectory:
-#     add_subdirectory(ioma)
-#     target_link_libraries(myapp PRIVATE ioma::ioma)
+#     add_subdirectory(ioxd)
+#     target_link_libraries(myapp PRIVATE ioxd::ioxd)
 # or, after `cmake --install`, via find_package:
-#     find_package(ioma REQUIRED)
-#     target_link_libraries(myapp PRIVATE ioma::ioma)
+#     find_package(ioxd REQUIRED)
+#     target_link_libraries(myapp PRIVATE ioxd::ioxd)
 cmake_minimum_required(VERSION 3.21)   # PROJECT_IS_TOP_LEVEL, find_program(NO_CACHE)
 
 # The compiler: unless one is given (-DCMAKE_C_COMPILER, CC in the environment, a toolchain file),
 # the newest gcc on the PATH - a distro's `gcc` is often older than a `gcc-NN` installed beside it,
 # and the code is C23, which needs gcc 14. Decided before project(), where CMake fixes the
 # compiler; the make build does the same.
 if(NOT CMAKE_C_COMPILER AND NOT DEFINED ENV{CC} AND NOT CMAKE_TOOLCHAIN_FILE)
-    set(ioma_newest_gcc "")
-    set(ioma_newest_gcc_version 0)
+    set(ioxd_newest_gcc "")
+    set(ioxd_newest_gcc_version 0)
     foreach(candidate gcc gcc-14 gcc-15 gcc-16)
         unset(candidate_path)
         find_program(candidate_path ${candidate} NO_CACHE)
         if(candidate_path)
             execute_process(COMMAND ${candidate_path} -dumpfullversion
                             OUTPUT_VARIABLE candidate_version OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_QUIET)
-            if(candidate_version VERSION_GREATER ioma_newest_gcc_version)
-                set(ioma_newest_gcc ${candidate_path})
-                set(ioma_newest_gcc_version ${candidate_version})
+            if(candidate_version VERSION_GREATER ioxd_newest_gcc_version)
+                set(ioxd_newest_gcc ${candidate_path})
+                set(ioxd_newest_gcc_version ${candidate_version})
             endif()
         endif()
     endforeach()
-    if(ioma_newest_gcc)
-        set(CMAKE_C_COMPILER ${ioma_newest_gcc})
+    if(ioxd_newest_gcc)
+        set(CMAKE_C_COMPILER ${ioxd_newest_gcc})
     endif()
 endif()
 
-project(ioma VERSION 0.1.0 LANGUAGES C ASM)
+project(ioxd VERSION 0.1.0 LANGUAGES C ASM)
 
 set(CMAKE_C_STANDARD 23)
 set(CMAKE_C_STANDARD_REQUIRED ON)
 set(CMAKE_C_EXTENSIONS ON)                 # gnu23: _GNU_SOURCE, statement exprs
 if(CMAKE_C_COMPILER_ID STREQUAL "GNU" AND CMAKE_C_COMPILER_VERSION VERSION_LESS 14)
-    message(FATAL_ERROR "libioma is C23 and needs gcc 14 or newer; this is gcc ${CMAKE_C_COMPILER_VERSION}. "
+    message(FATAL_ERROR "libioxd is C23 and needs gcc 14 or newer; this is gcc ${CMAKE_C_COMPILER_VERSION}. "
                         "Configure with -DCMAKE_C_COMPILER=gcc-14 (in CLion: Settings > Build > Toolchains).")
 endif()
 set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
@@ -52,84 +52,140 @@ find_package(Threads REQUIRED)
 include(GNUInstallDirs)
 
 # --- the library (static by default; -DBUILD_SHARED_LIBS=ON for a .so) ---
-add_library(ioma
-        src/io/uring.c
-        src/io/coro.c
-        src/io/switch_x86_64.S
-        src/io/bufring.c
-        src/io/conn.c
-        src/io/proactor.c
-        src/http/engine.c
-        src/http/api.c
-        src/http/router.c
-        src/http/run.c
+add_library(ioxd
+        lib/io/uring.c
+        lib/io/coro.c
+        lib/io/switch_x86_64.S
+        lib/io/bufring.c
+        lib/io/conn.c
+        lib/io/proactor.c
+        lib/io/pipe.c
+        lib/http/engine.c
+        lib/http/api.c
+        lib/http/router.c
+        lib/http/run.c
+        lib/json/json.c
+        lib/tls/store.c
+        lib/tls/handshake.c
         third_party/picohttpparser/picohttpparser.c)
-add_library(ioma::ioma ALIAS ioma)
+add_library(ioxd::ioxd ALIAS ioxd)
 
-target_include_directories(ioma
+target_include_directories(ioxd
         PUBLIC
             $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
-            $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}/ioma>
+            $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>
         PRIVATE
-            src
+            lib
             third_party/picohttpparser)
-target_compile_definitions(ioma PRIVATE _GNU_SOURCE)
-target_compile_options(ioma PRIVATE -Wall -Wextra)
+target_compile_definitions(ioxd PRIVATE _GNU_SOURCE)
+# TLS: OpenSSL for the handshake only; the kernel does the records (TLS.md).
+option(IOXD_TLS "TLS listeners (needs OpenSSL 3)" ON)
+if(IOXD_TLS)
+    find_package(OpenSSL 3 REQUIRED)
+    target_compile_definitions(ioxd PRIVATE IOXD_TLS=1)
+    target_link_libraries(ioxd PUBLIC OpenSSL::SSL OpenSSL::Crypto)
+else()
+    target_compile_definitions(ioxd PRIVATE IOXD_TLS=0)
+endif()
+target_compile_options(ioxd PRIVATE -Wall -Wextra)
 # Fat LTO objects when supported: the archive stays plain-linkable, and consumers that link with
 # -flto get cross-file inlining (the demo below does).
 include(CheckCCompilerFlag)
-check_c_compiler_flag("-flto -ffat-lto-objects" IOMA_HAVE_FAT_LTO)
-if(IOMA_HAVE_FAT_LTO)
-    target_compile_options(ioma PRIVATE -flto -ffat-lto-objects)
-    target_link_options(ioma PRIVATE -flto)
+check_c_compiler_flag("-flto -ffat-lto-objects" IOXD_HAVE_FAT_LTO)
+if(IOXD_HAVE_FAT_LTO)
+    target_compile_options(ioxd PRIVATE -flto -ffat-lto-objects)
+    target_link_options(ioxd PRIVATE -flto)
+endif()
+# A coroutine's stack ends at a guard page, which only stops a frame that touches every page as it
+# grows: stack clash protection makes the compiler emit those probes.
+check_c_compiler_flag(-fstack-clash-protection IOXD_HAVE_STACK_CLASH)
+if(IOXD_HAVE_STACK_CLASH)
+    target_compile_options(ioxd PRIVATE -fstack-clash-protection)
+endif()
+# A shared build exports ioxd_* and nothing else, through the version script the make build uses.
+set(IOXD_VERSION_SCRIPT ${CMAKE_CURRENT_SOURCE_DIR}/cmake/ioxd.map)
+if(BUILD_SHARED_LIBS)
+    target_link_options(ioxd PRIVATE -Wl,--version-script,${IOXD_VERSION_SCRIPT})
+    set_target_properties(ioxd PROPERTIES LINK_DEPENDS ${IOXD_VERSION_SCRIPT})
 endif()
-target_link_libraries(ioma PUBLIC Threads::Threads)
-set_target_properties(ioma PROPERTIES VERSION ${PROJECT_VERSION} SOVERSION 0)
+target_link_libraries(ioxd PUBLIC Threads::Threads)
+set_target_properties(ioxd PROPERTIES VERSION ${PROJECT_VERSION} SOVERSION 0)
 
 # Vendored parser: silence its warnings, they are not ours to fix.
 set_source_files_properties(third_party/picohttpparser/picohttpparser.c
         PROPERTIES COMPILE_OPTIONS "-w")
 
-# --- playground examples (off when ioma is a subproject) ---
-option(IOMA_EXAMPLES "Build the playground examples" ${PROJECT_IS_TOP_LEVEL})
-if(IOMA_EXAMPLES)
-    add_executable(ioma-hello playground/hello/main.c)
-    target_link_libraries(ioma-hello PRIVATE ioma::ioma)
-    add_executable(ioma-test-server tests/server.c)
-    target_link_libraries(ioma-test-server PRIVATE ioma::ioma)
-    add_executable(ioma-unit tests/unit.c)
-    target_link_libraries(ioma-unit PRIVATE ioma::ioma)
+# --- playground examples (off when ioxd is a subproject) ---
+option(IOXD_EXAMPLES "Build the playground examples" ${PROJECT_IS_TOP_LEVEL})
+if(IOXD_EXAMPLES)
+    add_executable(ioxd-hello playground/hello/main.c)
+    target_link_libraries(ioxd-hello PRIVATE ioxd::ioxd)
+    add_executable(ioxd-test-server tests/server.c)
+    target_link_libraries(ioxd-test-server PRIVATE ioxd::ioxd)
+    add_executable(ioxd-pipe-server tests/pipe-server.c)
+    target_link_libraries(ioxd-pipe-server PRIVATE ioxd::ioxd)
+    add_executable(ioxd-unit tests/unit.c)
+    target_link_libraries(ioxd-unit PRIVATE ioxd::ioxd)
+    add_executable(ioxd-router-test tests/router_test.c)
+    target_link_libraries(ioxd-router-test PRIVATE ioxd::ioxd)
+    # The suites, through the one script `make check` runs, so the sequence lives in one place.
+    # The three that talk to the HTTP fixture share one entry, and so one fixture on one port, as
+    # they do under make: a fixture bound to a port the suite before it filled with TIME_WAIT
+    # connections has some of its new ones reset. Both entries bind fixed ports, so neither may
+    # run beside anything else.
     enable_testing()
-    add_test(NAME unit COMMAND ioma-unit)
-    if(IOMA_HAVE_FAT_LTO)
-        target_compile_options(ioma-hello PRIVATE -flto)
-        target_link_options(ioma-hello PRIVATE -flto)
-        target_compile_options(ioma-test-server PRIVATE -flto)
-        target_link_options(ioma-test-server PRIVATE -flto)
+    set(IOXD_CHECK_PORT 8099    CACHE STRING "first port the fixture listens on; it takes the two after it")
+    set(IOXD_PIPE_PORT  8102    CACHE STRING "port the pipe fixture listens on")
+    set(IOXD_TLS_PYTHON python3 CACHE STRING "a python with tlslite-ng, for tests/tls_early.py")
+    add_test(NAME unit COMMAND ioxd-unit)
+    add_test(NAME router COMMAND ioxd-router-test)
+    add_test(NAME suites COMMAND sh ${CMAKE_CURRENT_SOURCE_DIR}/tests/run-suites.sh
+             --suite smoke --suite conformance --suite stress --suite tls
+             --port ${IOXD_CHECK_PORT}
+             --server $<TARGET_FILE:ioxd-test-server>
+             --tls-python ${IOXD_TLS_PYTHON}
+             --work ${CMAKE_CURRENT_BINARY_DIR}/check/suites)
+    add_test(NAME pipes COMMAND sh ${CMAKE_CURRENT_SOURCE_DIR}/tests/run-suites.sh
+             --suite pipes
+             --pipe-port ${IOXD_PIPE_PORT}
+             --pipe-server $<TARGET_FILE:ioxd-pipe-server>
+             --work ${CMAKE_CURRENT_BINARY_DIR}/check/pipes)
+    set_tests_properties(suites pipes PROPERTIES RUN_SERIAL TRUE)
+    add_custom_target(check
+            COMMAND ${CMAKE_CTEST_COMMAND} --output-on-failure
+            DEPENDS ioxd-unit ioxd-router-test ioxd-test-server ioxd-pipe-server
+            USES_TERMINAL)
+    if(IOXD_HAVE_FAT_LTO)
+        target_compile_options(ioxd-hello PRIVATE -flto)
+        target_link_options(ioxd-hello PRIVATE -flto)
+        target_compile_options(ioxd-test-server PRIVATE -flto)
+        target_link_options(ioxd-test-server PRIVATE -flto)
     endif()
 endif()
 
-# --- install + export so find_package(ioma) works ---
-install(TARGETS ioma EXPORT iomaTargets
+# --- install + export so find_package(ioxd) works ---
+install(TARGETS ioxd EXPORT ioxdTargets
         ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
         LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR})
-install(FILES include/ioma.h DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/ioma)
-install(EXPORT iomaTargets
-        NAMESPACE ioma::
-        DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/ioma
-        FILE iomaTargets.cmake)
+install(FILES include/ioxd.h DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
+install(DIRECTORY include/ioxd DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
+install(EXPORT ioxdTargets
+        NAMESPACE ioxd::
+        DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/ioxd
+        FILE ioxdTargets.cmake)
 
-# A config file that re-finds the public Threads dependency, then loads the exported targets.
+# A config file that re-finds the public dependencies - Threads, and OpenSSL when this build has
+# TLS, which it is told through IOXD_TLS - then loads the exported targets.
 include(CMakePackageConfigHelpers)
 configure_package_config_file(
-        ${CMAKE_CURRENT_SOURCE_DIR}/cmake/iomaConfig.cmake.in
-        ${CMAKE_CURRENT_BINARY_DIR}/iomaConfig.cmake
-        INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/ioma)
+        ${CMAKE_CURRENT_SOURCE_DIR}/cmake/ioxdConfig.cmake.in
+        ${CMAKE_CURRENT_BINARY_DIR}/ioxdConfig.cmake
+        INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/ioxd)
 write_basic_package_version_file(
-        ${CMAKE_CURRENT_BINARY_DIR}/iomaConfigVersion.cmake
+        ${CMAKE_CURRENT_BINARY_DIR}/ioxdConfigVersion.cmake
         VERSION ${PROJECT_VERSION}
         COMPATIBILITY SameMajorVersion)
 install(FILES
-            ${CMAKE_CURRENT_BINARY_DIR}/iomaConfig.cmake
-            ${CMAKE_CURRENT_BINARY_DIR}/iomaConfigVersion.cmake
-        DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/ioma)
+            ${CMAKE_CURRENT_BINARY_DIR}/ioxdConfig.cmake
+            ${CMAKE_CURRENT_BINARY_DIR}/ioxdConfigVersion.cmake
+        DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/ioxd)

DESIGN.md

@@ -1,6 +1,14 @@
 # ioxide's TCP core, read for a minimal stackful C runtime
 
-> Note: this is the design record for **ioma** (github.com/MDA2AV/ioma). Paths like `ioxide/...` and `ringzero/...` refer to the author's sibling repos (github.com/MDA2AV/ioxide, github.com/MDA2AV/ringzero); ioma itself is the `stackful/`→`ioma/` runtime described in section 3 and shipped in this repository.
+> **Historical.** This is the v1 design record for **ioxd** (github.com/MDA2AV/ioxd): the reading
+> of ioxide's TCP core and the ~500-line runtime planned from it, written before any of it was
+> built. It does **not** describe what shipped. The v1 sketched here uses liburing, one-shot
+> accept/recv/send, no provided buffers and files named `worker.c`/`listener.c`; the library has
+> raw io_uring with no liburing, multishot accept and recv over a provided buffer ring, and the
+> layout in [`ARCHITECTURE.md`](ARCHITECTURE.md), which supersedes this document. Kept for the
+> reasoning - why a stackful coroutine removes half of ioxide's machinery, and the pitfalls in
+> section 4, which still hold. Paths like `ioxide/...` and `ringzero/...` are the author's sibling
+> repos (github.com/MDA2AV/ioxide, github.com/MDA2AV/ringzero).
 
 
 Scope: `ioxide/src/ioxide` — `io_uring/Ring.cs`, `Native/*`, `Reactor/*`, `Reactor/Transport/Tcp/*`,

FILES.md

@@ -0,0 +1,45 @@
+# Static files over the ring (design, branch `streams`)
+
+**Status.** None of this is built: there is no `ioxd_files_*` in the library, no tail capture in
+the router and no `lib/io/watch.c`. This is the design, written down before the work.
+
+The model is ioxide.file's: small, hot files served from a baked, immutable snapshot; large files
+as positional ring reads or, better, spliced from the descriptor straight into the socket - no
+thread pool either way, and with kernel TLS the splice is encrypted on its way through.
+
+## The snapshot
+
+`ioxd_files_new("/srv/www")` walks the root once and builds a table keyed by URL path. Files up to
+a threshold (256 KB) get their whole response baked: status line, `content-type` from the
+extension, `content-length`, `etag` (inode, size, mtime), `last-modified`, then the body, in one
+block, so serving one is a lookup and one send with nothing formatted. Larger files keep an open
+descriptor and their `statx`; serving one is the head from the snapshot and the body by
+`IORING_OP_SPLICE` from the descriptor through a pipe pair into the socket, in slices the writer
+paces, or by positional `IORING_OP_READ` into the reply slab where splice is not wanted. Conditional
+requests (`if-none-match`, `if-modified-since`) answer 304 from the snapshot alone. Ranges and
+precompressed `.gz`/`.br` siblings are second-round work.
+
+The router grows a tail capture, `/static/*path`, so a handler is one line:
+
+    IOXD_GET("/static/*path", assets);   /* ioxd_files_serve(ctx, files, ctx->req.route_params[0].value) */
+
+## Knowing a file changed
+
+Watching the file itself misses the case that matters: an atomic replace is a `rename` over the
+old name, a new inode, and the old watch dies with the old one. So the watch is on directories,
+one per directory in the tree (inotify watches are cheap; an asset tree of a few hundred
+directories is nothing), and the events that mean "the content under this name is different" are
+`IN_CLOSE_WRITE` (a write in place finished), `IN_MOVED_TO` (the atomic replace landed),
+`IN_CREATE` and `IN_DELETE`. The descriptor would be read as an ordinary ring read by the same
+control coroutine that watches certificates - one facility, `lib/io/watch.c`, with two clients.
+Neither the watcher nor that file exists yet; the certificate store's own rotation watcher is
+planned in the same terms (TLS.md), and today it reloads only when the application says so.
+
+Events are debounced, then the changed entries are confirmed with `statx` against the snapshot
+(inode, size, mtime), re-baked, and a new snapshot is published: an atomic pointer with a
+reference count, so a request that is mid-send from a baked block keeps the old snapshot alive
+until its send completes, and the old snapshot is freed when the last lease drops. That is
+ioxide.file's lease-safe reload, driven by the watch instead of only by `Reload()`, which stays
+for deployments that swap the tree whole. `fanotify` could watch a whole mount with one
+descriptor but needs privileges; inotify per directory does not. The fallback for filesystems
+without events is the same `statx` sweep on a timer.