[k2] implement shared-memory confdata

CMakeLists.txt

@@ -83,7 +83,7 @@ endif()
 add_custom_target(kphp ALL DEPENDS ${OBJS_DIR}/php_lib_version.sha256)
 
 if (COMPILE_RUNTIME_LIGHT)
-    add_dependencies(kphp kphp2cpp kphp-light-runtime-pic)
+    add_dependencies(kphp kphp2cpp kphp-light-runtime-pic kphp-confdata-pic)
 else ()
     add_dependencies(kphp kphp2cpp kphp-full-runtime-no-pic kphp-full-runtime-pic)
 endif ()
@@ -95,6 +95,13 @@ install_symlink(${VK_INSTALL_DIR}/bin/kphp2cpp ${CMAKE_INSTALL_PREFIX}/bin/kphp
 
 # k2 specific
 if(COMPILE_RUNTIME_LIGHT)
+    install(TARGETS kphp-confdata-pic
+            LIBRARY DESTINATION ${VK_INSTALL_DIR}/k2-components
+            COMPONENT KPHP-CONFDATA)
+
+    set(CPACK_DEBIAN_KPHP-CONFDATA_PACKAGE_NAME "k2kphp-confdata")
+    set(CPACK_DEBIAN_KPHP-CONFDATA_DESCRIPTION "K2 kPHP confdata component")
+
     install(FILES ${OBJS_DIR}/libk2kphp-rt.a
             COMPONENT KPHP
             DESTINATION ${INSTALL_KPHP_SOURCE}/objs)

builtin-functions/kphp-light/stdlib/confdata-functions.txt

@@ -2,11 +2,8 @@
 
 function is_confdata_loaded(): bool;
 
-/** @kphp-extern-func-info interruptible */
 function confdata_get_value($key ::: string): mixed;
 
-/** @kphp-extern-func-info interruptible */
 function confdata_get_values_by_any_wildcard($wildcard ::: string): mixed[];
 
-/** @kphp-extern-func-info interruptible */
 function confdata_get_values_by_predefined_wildcard($wildcard ::: string): mixed[];

runtime-common/core/memory-resource/resource_allocator.h

@@ -4,6 +4,7 @@
 
 #pragma once
 
+#include <forward_list>
 #include <functional>
 #include <list>
 #include <map>
@@ -88,6 +89,9 @@ using vector = std::vector<T, resource_allocator<T, Resource>>;
 template<class T, class Resource>
 using list = std::list<T, resource_allocator<T, Resource>>;
 
+template<class T, class Resource>
+using forward_list = std::forward_list<T, resource_allocator<T, Resource>>;
+
 template<class Resource>
 using string = std::basic_string<char, std::char_traits<char>, resource_allocator<char, Resource>>;
 } // namespace stl

runtime-common/core/std/containers.h

@@ -5,6 +5,7 @@
 #pragma once
 
 #include <deque>
+#include <forward_list>
 #include <functional>
 #include <list>
 #include <map>
@@ -41,6 +42,9 @@ using queue = std::queue<T, std::deque<T, Allocator<T>>>;
 template<class T, template<class> class Allocator>
 using list = std::list<T, Allocator<T>>;
 
+template<class T, template<class> class Allocator>
+using forward_list = std::forward_list<T, Allocator<T>>;
+
 template<class T, template<class> class Allocator>
 using vector = std::vector<T, Allocator<T>>;
 

runtime-light/components/confdata/bindings/bindings.cpp

@@ -15,6 +15,7 @@
 #include "runtime-light/stdlib/diagnostics/contextual-tags.h"
 #include "runtime-light/stdlib/diagnostics/error-handling-state.h"
 #include "runtime-light/stdlib/diagnostics/logs.h"
+#include "runtime-light/stdlib/string/string-state.h"
 
 namespace kphp::coro {
 
@@ -52,3 +53,7 @@ auto ErrorHandlingState::try_get() noexcept -> std::optional<std::reference_wrap
 auto RuntimeContext::get() noexcept -> RuntimeContext& {
   kphp::log::error("unexpected access to RuntimeContext"); // confdata doesn't have a runtime context
 }
+
+auto StringInstanceState::get() noexcept -> StringInstanceState& {
+  kphp::log::error("unexpected access to StringInstanceState"); // confdata doesn't actually use StringInstanceState
+}

runtime-light/components/confdata/confdata-component.cpp

@@ -85,12 +85,12 @@ VISIBILITY_DEFAULT k2::PollStatus k2_poll() {
 }
 
 VISIBILITY_DEFAULT const ImageInfo* k2_describe() {
-  static constexpr std::array extra_info{ImageInfo::KeyValuePair{.key = "compiler_version", .value = K2_CONFDATA_COMPILER_VERSION}};
-  static constexpr ImageInfo image_info{.image_name = kphp::confdata::COMPONENT_NAME.data(),
+  static constexpr std::array extra_info{ImageInfo::KeyValuePair{.key = "compiler_version", .value = KPHP_CONFDATA_COMPILER_VERSION}};
+  static constexpr ImageInfo image_info{.image_name = kphp::confdata::IMAGE_NAME.data(),
                                         .is_oneshot = 0,
-                                        .build_timestamp = K2_CONFDATA_BUILD_TIMESTAMP,
+                                        .build_timestamp = KPHP_CONFDATA_BUILD_TIMESTAMP,
                                         .header_h_version = K2_PLATFORM_HEADER_H_VERSION,
-                                        .version = "0.0.1",
+                                        .version = "1.0.0",
                                         .extra_info_size = extra_info.size(),
                                         .extra_info = extra_info.data()};
   return std::addressof(image_info);

runtime-light/components/confdata/confdata-proxy/sync-functions.h

@@ -9,6 +9,8 @@
 #include <cstddef>
 #include <cstdint>
 #include <expected>
+#include <functional>
+#include <optional>
 #include <span>
 #include <string_view>
 #include <utility>
@@ -34,17 +36,25 @@ struct pagination {
   bool m_has_synced{};
 };
 
-enum class subscribe_error : uint8_t { transport, old_offset, malformed_response, not_synced };
+using encoded_snapshot_page = kphp::stl::vector<std::byte, kphp::memory::script_allocator>;
+using encoded_snapshot_pages = kphp::stl::vector<encoded_snapshot_page, kphp::memory::script_allocator>;
 
-namespace details {
+struct snapshot final {
+  pagination m_pagination;
+  encoded_snapshot_pages m_pages;
+};
+
+enum class subscribe_error : uint8_t { transport, old_offset, malformed_response, not_synced, batch_rejected };
 
 // Performs a single confdata.subscribe round-trip.
 // On success, invokes `event_handler(events)` once with the batch of received events and updates `to` pagination.
+// If the handler returns false, the batch is rejected with `batch_rejected` and `to` is left unchanged so it can be requested again.
 // The batch is a view into the response buffer and is only valid for the duration of the call; empty batches are not delivered.
 // An empty event value means that the key has been deleted.
-template<std::invocable<std::span<const tl::confdata::KeyValuePair>> event_handler_type>
-auto subscribe(std::string_view confdata_proxy_actor, kphp::confdata::pagination& to,
-               const event_handler_type& event_handler) noexcept -> kphp::coro::task<std::expected<void, kphp::confdata::subscribe_error>> {
+template<std::predicate<std::span<const tl::confdata::KeyValuePair>> event_handler_type>
+auto subscribe(std::string_view confdata_proxy_actor, kphp::confdata::pagination& to, const event_handler_type& event_handler,
+               std::optional<std::reference_wrapper<encoded_snapshot_page>> retained_response = {}) noexcept
+    -> kphp::coro::task<std::expected<void, kphp::confdata::subscribe_error>> {
   // subscribe is a longpoll method, so the timeout must cover the time confdata-proxy may hold the request open
   static constexpr auto SUBSCRIBE_TIMEOUT{std::chrono::milliseconds{45'000}};
 
@@ -66,63 +76,94 @@ auto subscribe(std::string_view confdata_proxy_actor, kphp::confdata::pagination
   // client-side timeout must outlive the server-side longpoll (SUBSCRIBE_TIMEOUT); 10x is a safe margin
   auto expected_query{kphp::rpc::query::send(confdata_proxy_actor, SUBSCRIBE_TIMEOUT * 10, tls.view(), k2::RpcKind::TL_RPC)};
   if (!expected_query) [[unlikely]] {
-    kphp::log::warning("confdata: failed to send subscribe request: {}", expected_query.error());
+    kphp::log::warning("failed to send subscribe request: {}", expected_query.error());
     co_return std::unexpected{kphp::confdata::subscribe_error::transport};
   }
 
-  kphp::stl::vector<std::byte, kphp::memory::script_allocator> response_buffer{};
+  encoded_snapshot_page response_buffer{};
   auto expected_response{co_await kphp::rpc::query::response(std::move(*expected_query), [&response_buffer](size_t size) noexcept -> std::span<std::byte> {
     response_buffer.resize(size);
-    return {response_buffer.data(), response_buffer.size()};
+    return response_buffer;
   })};
   if (!expected_response) [[unlikely]] {
-    kphp::log::warning("confdata: failed to fetch subscribe response: {}", expected_response.error());
+    kphp::log::warning("failed to fetch subscribe response: {}", expected_response.error());
     co_return std::unexpected{kphp::confdata::subscribe_error::transport};
   }
 
   tl::fetcher tlf{*expected_response};
   tl::confdata::SubscribeResponse response{};
   if (!response.fetch(tlf)) [[unlikely]] {
-    kphp::log::warning("confdata: failed to parse subscribe response");
+    kphp::log::warning("failed to parse subscribe response");
     co_return std::unexpected{kphp::confdata::subscribe_error::malformed_response};
   }
 
-  co_return std::visit(
-      overloaded{
-          [&event_handler, &to](const tl::confdata::subscribeResponseOk& response) noexcept -> std::expected<void, kphp::confdata::subscribe_error> {
-            if (const auto& events{response.events}; events.size() != 0) {
-              std::invoke(event_handler, std::span<const tl::confdata::KeyValuePair>{events.value});
-            }
-
-            to.m_page = response.new_page.value;
-            to.m_offset = response.new_offset.value;
-            to.m_has_synced = response.new_has_synced.value;
-            return {};
-          },
-          [](const tl::confdata::subscribeResponseOldOffsetError& /* unused */) noexcept -> std::expected<void, kphp::confdata::subscribe_error> {
-            return std::unexpected{kphp::confdata::subscribe_error::old_offset};
-          },
-      },
-      response.value);
+  auto handled{
+      std::visit(overloaded{
+                     [&event_handler, &to](const tl::confdata::subscribeResponseOk& response) noexcept -> std::expected<void, kphp::confdata::subscribe_error> {
+                       if (const auto& events{response.events}; events.size() != 0) {
+                         if (!std::invoke(event_handler, std::span<const tl::confdata::KeyValuePair>{events.value})) {
+                           return std::unexpected{kphp::confdata::subscribe_error::batch_rejected};
+                         }
+                       }
+
+                       to.m_page = response.new_page.value;
+                       to.m_offset = response.new_offset.value;
+                       to.m_has_synced = response.new_has_synced.value;
+                       return {};
+                     },
+                     [](const tl::confdata::subscribeResponseOldOffsetError& /* unused */) noexcept -> std::expected<void, kphp::confdata::subscribe_error> {
+                       return std::unexpected{kphp::confdata::subscribe_error::old_offset};
+                     },
+                 },
+                 response.value)};
+  if (handled && retained_response.has_value()) {
+    retained_response->get() = std::move(response_buffer);
+  }
+  co_return std::move(handled);
 }
 
-} // namespace details
-
-// Paginates through a consistent snapshot of all subscribed keys until it has been fully synced.
-// Returns the final pagination that should be passed to `update`.
-//
-// `event_handler` is invoked once per round-trip with a batch of events; the batch is only valid
-// for the duration of the call and must be copied if it needs to be retained.
-template<std::invocable<std::span<const tl::confdata::KeyValuePair>> event_handler_type>
+// Fetches one consistent snapshot while retaining the encoded response pages.
+// The handler can collect metadata from the initial parse; replay()
+// reparses the same bytes later without another network synchronization.
+template<std::predicate<std::span<const tl::confdata::KeyValuePair>> event_handler_type>
 auto sync(std::string_view confdata_proxy_actor,
-          event_handler_type event_handler) noexcept -> kphp::coro::task<std::expected<kphp::confdata::pagination, kphp::confdata::subscribe_error>> {
-  kphp::confdata::pagination p{};
-  for (; !p.m_has_synced;) {
-    if (auto expected{co_await details::subscribe(confdata_proxy_actor, p, event_handler)}; !expected) [[unlikely]] {
+          event_handler_type event_handler) noexcept -> kphp::coro::task<std::expected<snapshot, kphp::confdata::subscribe_error>> {
+  snapshot snapshot{};
+  for (; !snapshot.m_pagination.m_has_synced;) {
+    snapshot.m_pages.emplace_back();
+    if (auto expected{co_await subscribe(confdata_proxy_actor, snapshot.m_pagination, event_handler, snapshot.m_pages.back())}; !expected) [[unlikely]] {
       co_return std::unexpected{expected.error()};
     }
   }
-  co_return std::move(p);
+  co_return std::move(snapshot);
+}
+
+// Takes ownership of previously fetched snapshot pages and releases each page after
+// its parsed event descriptors have been consumed. On failure, remaining pages are released too.
+template<std::predicate<std::span<const tl::confdata::KeyValuePair>> event_handler_type>
+auto replay(encoded_snapshot_pages pages, const event_handler_type& event_handler) noexcept -> std::expected<void, kphp::confdata::subscribe_error> {
+  for (auto& encoded_page : pages) {
+    {
+      tl::fetcher tlf{std::span<const std::byte>{encoded_page.data(), encoded_page.size()}};
+      tl::confdata::SubscribeResponse response{};
+      if (!response.fetch(tlf)) [[unlikely]] {
+        return std::unexpected{kphp::confdata::subscribe_error::malformed_response};
+      }
+
+      if (std::holds_alternative<tl::confdata::subscribeResponseOldOffsetError>(response.value)) {
+        return std::unexpected{kphp::confdata::subscribe_error::old_offset};
+      }
+
+      const auto& response_ok{std::get<tl::confdata::subscribeResponseOk>(response.value)};
+      if (const auto& events{response_ok.events};
+          events.size() != 0 && !std::invoke(event_handler, std::span<const tl::confdata::KeyValuePair>{events.value})) {
+        return std::unexpected{kphp::confdata::subscribe_error::batch_rejected};
+      }
+    }
+    // Make the processed page's memory reusable before parsing the next page.
+    encoded_snapshot_page{}.swap(encoded_page);
+  }
+  return {};
 }
 
 // Longpoll loop: invokes `event_handler` for each event as it arrives, throttled to at most one batch per second:
@@ -132,7 +173,7 @@ auto sync(std::string_view confdata_proxy_actor,
 //
 // `event_handler` is invoked once per round-trip with a batch of events; the batch is only valid
 // for the duration of the call and must be copied if it needs to be retained.
-template<std::invocable<std::span<const tl::confdata::KeyValuePair>> event_handler_type>
+template<std::predicate<std::span<const tl::confdata::KeyValuePair>> event_handler_type>
 auto update(std::string_view confdata_proxy_actor, kphp::confdata::pagination& from,
             event_handler_type event_handler) noexcept -> kphp::coro::task<std::expected<void, kphp::confdata::subscribe_error>> {
   // limits the update rate to at most one batch per interval
@@ -143,7 +184,7 @@ auto update(std::string_view confdata_proxy_actor, kphp::confdata::pagination& f
   }
 
   for (;;) {
-    if (auto expected{co_await details::subscribe(confdata_proxy_actor, from, event_handler)}; !expected) [[unlikely]] {
+    if (auto expected{co_await subscribe(confdata_proxy_actor, from, event_handler)}; !expected) [[unlikely]] {
       co_return std::unexpected{expected.error()};
     }
     co_await kphp::coro::io_scheduler::get().schedule(UPDATE_INTERVAL);

runtime-light/components/confdata/confdata.cmake

@@ -1,50 +1,61 @@
-# k2-confdata image: a standalone C++ component that shares the common
+# kphp-confdata image: a standalone C++ component that shares the common
 # runtime-light machinery but provides its own entry points and bindings
-set(K2_CONFDATA_COMPONENT_SRC
+set(KPHP_CONFDATA_COMPONENT_SRC
     ${RUNTIME_LIGHT_DIR}/components/confdata/confdata-component.cpp
     ${RUNTIME_LIGHT_DIR}/components/confdata/bindings/bindings.cpp
     ${RUNTIME_LIGHT_DIR}/components/confdata/state/component-state.cpp
-    ${RUNTIME_LIGHT_DIR}/components/confdata/state/instance-state.cpp)
+    ${RUNTIME_LIGHT_DIR}/components/confdata/state/instance-state.cpp
+    ${RUNTIME_LIGHT_DIR}/stdlib/confdata/confdata-storage.cpp
+    ${RUNTIME_LIGHT_DIR}/stdlib/confdata/confdata-keys.cpp
+    ${RUNTIME_LIGHT_DIR}/stdlib/confdata/predefined-wildcards.cpp)
 
-set(K2_CONFDATA_TL_SRC
+set(KPHP_CONFDATA_TL_SRC
     ${RUNTIME_LIGHT_DIR}/tl/tl-types.cpp
     ${RUNTIME_LIGHT_DIR}/tl/tl-functions.cpp)
 
-set(K2_CONFDATA_ALLOCATOR_SRC
-    ${RUNTIME_LIGHT_DIR}/allocator/runtime-light-allocator.cpp
+set(KPHP_CONFDATA_ALLOCATOR_SRC ${RUNTIME_LIGHT_ALLOCATOR_SRC})
+list(TRANSFORM KPHP_CONFDATA_ALLOCATOR_SRC PREPEND "${RUNTIME_LIGHT_DIR}/")
+list(APPEND KPHP_CONFDATA_ALLOCATOR_SRC
     ${RUNTIME_LIGHT_DIR}/memory-resource-impl/monotonic-light-buffer-resource.cpp)
 
-set(K2_CONFDATA_DIAGNOSTICS_SRC
+set(KPHP_CONFDATA_DIAGNOSTICS_SRC
     ${RUNTIME_LIGHT_DIR}/stdlib/diagnostics/backtrace.cpp
     ${RUNTIME_LIGHT_DIR}/stdlib/diagnostics/php-assert.cpp)
 
-set(K2_CONFDATA_MEMORY_RESOURCE_SRC
-    ${RUNTIME_COMMON_DIR}/core/memory-resource/unsynchronized_pool_resource.cpp
-    ${RUNTIME_COMMON_DIR}/core/memory-resource/monotonic_buffer_resource.cpp
-    ${RUNTIME_COMMON_DIR}/core/memory-resource/details/memory_chunk_tree.cpp
-    ${RUNTIME_COMMON_DIR}/core/memory-resource/details/memory_ordered_chunk_list.cpp)
-
-set(K2_CONFDATA_SRC
-    ${K2_CONFDATA_COMPONENT_SRC}
-    ${K2_CONFDATA_TL_SRC}
-    ${K2_CONFDATA_ALLOCATOR_SRC}
-    ${K2_CONFDATA_DIAGNOSTICS_SRC}
-    ${K2_CONFDATA_MEMORY_RESOURCE_SRC}
+set(KPHP_CONFDATA_SERIALIZATION_SRC
+    ${RUNTIME_COMMON_DIR}/stdlib/serialization/json-functions.cpp
+    ${RUNTIME_COMMON_DIR}/stdlib/serialization/serialize-functions.cpp
+    ${RUNTIME_COMMON_DIR}/stdlib/string/string-functions.cpp)
+
+set(KPHP_CONFDATA_RUNTIME_CORE_SRC ${CORE_SRC})
+list(TRANSFORM KPHP_CONFDATA_RUNTIME_CORE_SRC PREPEND "${RUNTIME_COMMON_DIR}/")
+
+set(KPHP_CONFDATA_SRC
+    ${KPHP_CONFDATA_COMPONENT_SRC}
+    ${KPHP_CONFDATA_TL_SRC}
+    ${KPHP_CONFDATA_ALLOCATOR_SRC}
+    ${KPHP_CONFDATA_DIAGNOSTICS_SRC}
+    ${KPHP_CONFDATA_SERIALIZATION_SRC}
+    ${KPHP_CONFDATA_RUNTIME_CORE_SRC}
     # link the alloc-wrapper objects directly (not as an archive) so that
     # __wrap_* definitions are always present regardless of link order
     $<TARGET_OBJECTS:libc-alloc-wrapper-pic>)
 
-vk_add_library_pic(k2-confdata-pic SHARED ${K2_CONFDATA_SRC})
-set_target_properties(k2-confdata-pic PROPERTIES PREFIX "" OUTPUT_NAME "k2-confdata" LIBRARY_OUTPUT_DIRECTORY ${OBJS_DIR})
-target_compile_options(k2-confdata-pic PUBLIC ${RUNTIME_LIGHT_COMPILE_FLAGS})
+vk_add_library_pic(kphp-confdata-pic SHARED ${KPHP_CONFDATA_SRC})
+set_target_properties(kphp-confdata-pic PROPERTIES PREFIX "" OUTPUT_NAME "kphp-confdata" LIBRARY_OUTPUT_DIRECTORY ${OBJS_DIR})
+# Hide runtime internals so unused functions (such as the JSON encoder) can be
+# discarded with their dependencies. K2 entry points are explicitly exported.
+# Keep this after the runtime flags to override their configurable visibility.
+target_compile_options(kphp-confdata-pic PRIVATE ${RUNTIME_LIGHT_COMPILE_FLAGS} -fvisibility=hidden)
+target_link_libraries(kphp-confdata-pic PRIVATE vk::pic::light-common)
 # reuse the common link flags; cmake drives the link through the compiler,
 # so bare ld options need the -Wl, prefix
-set(K2_CONFDATA_LINK_FLAGS ${RUNTIME_LIGHT_LINK_FLAGS})
+set(KPHP_CONFDATA_LINK_FLAGS ${RUNTIME_LIGHT_LINK_FLAGS})
 if(NOT APPLE)
-  list(TRANSFORM K2_CONFDATA_LINK_FLAGS REPLACE "^--" "-Wl,--")
+  list(TRANSFORM KPHP_CONFDATA_LINK_FLAGS REPLACE "^--" "-Wl,--")
 endif()
-target_link_options(k2-confdata-pic PUBLIC ${K2_CONFDATA_LINK_FLAGS})
+target_link_options(kphp-confdata-pic PUBLIC ${KPHP_CONFDATA_LINK_FLAGS})
 
-string(TIMESTAMP K2_CONFDATA_BUILD_TIMESTAMP "%s" UTC)
-target_compile_definitions(k2-confdata-pic PRIVATE K2_CONFDATA_BUILD_TIMESTAMP=${K2_CONFDATA_BUILD_TIMESTAMP}ULL
-                                                   K2_CONFDATA_COMPILER_VERSION="${CMAKE_CXX_COMPILER_ID}-${CMAKE_CXX_COMPILER_VERSION}")
+string(TIMESTAMP KPHP_CONFDATA_BUILD_TIMESTAMP "%s" UTC)
+target_compile_definitions(kphp-confdata-pic PRIVATE KPHP_CONFDATA_BUILD_TIMESTAMP=${KPHP_CONFDATA_BUILD_TIMESTAMP}ULL
+                                                   KPHP_CONFDATA_COMPILER_VERSION="${CMAKE_CXX_COMPILER_ID}-${CMAKE_CXX_COMPILER_VERSION}")