From 92222594d6f8224791bff3f81eafe2946ac4e03a Mon Sep 17 00:00:00 2001 From: Tyler Cloutier Date: Tue, 8 Sep 2026 15:01:58 -0400 Subject: [PATCH 01/34] Add database environment storage, SQL, and module bindings --- Cargo.lock | 7 + Cargo.toml | 1 + .../include/spacetimedb/abi/FFI.h | 1 + .../include/spacetimedb/abi/abi.h | 7 + .../include/spacetimedb/bsatn/reader.h | 6 +- .../include/spacetimedb/environment.h | 34 ++++ .../include/spacetimedb/procedure_context.h | 3 + .../include/spacetimedb/reducer_context.h | 3 + .../include/spacetimedb/tx_context.h | 2 + .../include/spacetimedb/view_context.h | 4 + crates/bindings-cpp/tests/unit/CMakeLists.txt | 1 + .../tests/unit/environment_unit_tests.cpp | 50 +++++ .../diag/snapshots/Module#FFI.verified.cs | 3 + .../snapshots/Module#FFI.verified.cs | 3 + .../server/snapshots/Module#FFI.verified.cs | 3 + crates/bindings-csharp/Codegen/Module.cs | 3 + .../Runtime/DatabaseEnvironment.cs | 26 +++ .../bindings-csharp/Runtime/HandlerContext.cs | 2 + .../bindings-csharp/Runtime/Internal/FFI.cs | 15 ++ .../Runtime/ProcedureContext.cs | 2 + crates/bindings-csharp/Runtime/bindings.c | 4 + crates/bindings-sys/src/lib.rs | 20 ++ .../src/lib/environment.ts | 5 + .../bindings-typescript/src/lib/reducers.ts | 2 + .../src/server/environment.ts | 6 + .../src/server/http_handlers.ts | 2 + .../bindings-typescript/src/server/index.ts | 2 + .../src/server/procedures.ts | 3 + .../bindings-typescript/src/server/runtime.ts | 5 + .../bindings-typescript/src/server/sys.d.ts | 7 + .../bindings-typescript/src/server/views.ts | 3 + crates/bindings/src/http.rs | 4 + crates/bindings/src/lib.rs | 29 +++ crates/bindings/src/rt.rs | 7 + crates/client-api/src/lib.rs | 7 +- crates/core/src/db/environment.rs | 131 +++++++++++++ crates/core/src/db/mod.rs | 2 + crates/core/src/error.rs | 4 + crates/core/src/host/instance_env.rs | 142 ++++++++++++++ crates/core/src/host/mod.rs | 1 + crates/core/src/host/v8/syscall/mod.rs | 2 + crates/core/src/host/v8/syscall/v2.rs | 18 ++ crates/core/src/host/wasm_common.rs | 4 + .../src/host/wasmtime/wasm_instance_env.rs | 41 +++- .../core/src/host/wasmtime/wasmtime_module.rs | 2 +- crates/core/src/sql/execute.rs | 109 ++++++++++- .../locking_tx_datastore/committed_state.rs | 2 + .../src/locking_tx_datastore/datastore.rs | 7 + crates/datastore/src/system_tables.rs | 12 +- .../src/system_tables/environment.rs | 45 +++++ crates/expr/src/errors.rs | 4 + crates/expr/src/statement.rs | 21 +++ crates/lib/src/environment.rs | 62 ++++++ crates/lib/src/lib.rs | 1 + crates/query/src/lib.rs | 6 +- crates/sql-parser/src/ast/sql.rs | 9 + crates/sql-parser/src/parser/sql.rs | 35 +++- crates/testing/tests/environment.rs | 177 ++++++++++++++++++ modules/environment-test/Cargo.toml | 13 ++ modules/environment-test/src/lib.rs | 69 +++++++ modules/module-test-cpp/src/lib.cpp | 12 ++ modules/module-test-cs/EnvironmentTests.cs | 25 +++ modules/module-test-ts/src/index.ts | 23 +++ 63 files changed, 1239 insertions(+), 22 deletions(-) create mode 100644 crates/bindings-cpp/include/spacetimedb/environment.h create mode 100644 crates/bindings-cpp/tests/unit/environment_unit_tests.cpp create mode 100644 crates/bindings-csharp/Runtime/DatabaseEnvironment.cs create mode 100644 crates/bindings-typescript/src/lib/environment.ts create mode 100644 crates/bindings-typescript/src/server/environment.ts create mode 100644 crates/core/src/db/environment.rs create mode 100644 crates/datastore/src/system_tables/environment.rs create mode 100644 crates/lib/src/environment.rs create mode 100644 crates/testing/tests/environment.rs create mode 100644 modules/environment-test/Cargo.toml create mode 100644 modules/environment-test/src/lib.rs create mode 100644 modules/module-test-cs/EnvironmentTests.cs diff --git a/Cargo.lock b/Cargo.lock index 6f9455400ec..9b030bc73ff 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2298,6 +2298,13 @@ dependencies = [ "log", ] +[[package]] +name = "environment-test" +version = "0.0.0" +dependencies = [ + "spacetimedb 2.10.0", +] + [[package]] name = "equivalent" version = "1.0.2" diff --git a/Cargo.toml b/Cargo.toml index 9f7852863c8..afdde4e3253 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -46,6 +46,7 @@ members = [ "modules/keynote-benchmarks", "modules/perf-test", "modules/module-test", + "modules/environment-test", "templates/basic-rs/spacetimedb", "templates/chat-console-rs/spacetimedb", "modules/sdk-test", diff --git a/crates/bindings-cpp/include/spacetimedb/abi/FFI.h b/crates/bindings-cpp/include/spacetimedb/abi/FFI.h index 32990f156d4..f3ad213452f 100644 --- a/crates/bindings-cpp/include/spacetimedb/abi/FFI.h +++ b/crates/bindings-cpp/include/spacetimedb/abi/FFI.h @@ -73,6 +73,7 @@ using ::identity; // ===== JWT ===== using ::get_jwt; +using ::env_get; // ===== Procedure Transactions ===== using ::procedure_start_mut_tx; diff --git a/crates/bindings-cpp/include/spacetimedb/abi/abi.h b/crates/bindings-cpp/include/spacetimedb/abi/abi.h index 99dc067c147..285974cff3c 100644 --- a/crates/bindings-cpp/include/spacetimedb/abi/abi.h +++ b/crates/bindings-cpp/include/spacetimedb/abi/abi.h @@ -39,6 +39,10 @@ #define STDB_IMPORT_10_5(name) \ __attribute__((import_module("spacetime_10.5"), import_name(#name))) extern +// ABI10.6 is reserved for the separate invocation-authority extension. +#define STDB_IMPORT_10_7(name) \ + __attribute__((import_module("spacetime_10.7"), import_name(#name))) extern + // Import opaque types into global namespace for C compatibility using SpacetimeDB::Status; using SpacetimeDB::TableId; @@ -59,6 +63,9 @@ using SpacetimeDB::ConsoleTimerId; extern "C" { +STDB_IMPORT_10_7(env_get) +Status env_get(const uint8_t* key, uint32_t key_len, BytesSource* out); + // ===== Table and Index Management ===== STDB_IMPORT(table_id_from_name) Status table_id_from_name(const uint8_t* name_ptr, size_t name_len, TableId* out); diff --git a/crates/bindings-cpp/include/spacetimedb/bsatn/reader.h b/crates/bindings-cpp/include/spacetimedb/bsatn/reader.h index 5fcba095c76..a61fc3bcc59 100644 --- a/crates/bindings-cpp/include/spacetimedb/bsatn/reader.h +++ b/crates/bindings-cpp/include/spacetimedb/bsatn/reader.h @@ -124,10 +124,10 @@ namespace SpacetimeDB::bsatn { template std::optional read_optional() { uint8_t tag = read_u8(); - if (tag == 0) { - return std::nullopt; - } else if (tag == 1) { + if (tag == 0) { // Some, matching the canonical BSATN option type. return SpacetimeDB::bsatn::deserialize(*this); + } else if (tag == 1) { // None. + return std::nullopt; } else { std::abort(); // Invalid optional tag in BSATN deserialization } diff --git a/crates/bindings-cpp/include/spacetimedb/environment.h b/crates/bindings-cpp/include/spacetimedb/environment.h new file mode 100644 index 00000000000..f2df08234d0 --- /dev/null +++ b/crates/bindings-cpp/include/spacetimedb/environment.h @@ -0,0 +1,34 @@ +#ifndef SPACETIMEDB_ENVIRONMENT_H +#define SPACETIMEDB_ENVIRONMENT_H +#include +#include +#include +#include +#include +#include + +namespace SpacetimeDB { +/// Read-only database environment. Reads use the current transaction, or a +/// short snapshot in a procedure outside a transaction. Values are not cached. +class Environment { +public: + std::optional get(std::string_view key) const { + if (key.empty() || key.size() > 256) LOG_PANIC("invalid environment variable name"); + BytesSource source{0}; + if (FFI::env_get(reinterpret_cast(key.data()), static_cast(key.size()), &source) != Status(0)) + LOG_PANIC("environment read failed"); + if (source == BytesSource{0}) return std::nullopt; + std::array buffer; + std::string value; + for (;;) { + size_t len = buffer.size(); + const auto status = FFI::bytes_source_read(source, buffer.data(), &len); + if ((status != 0 && status != -1) || len > buffer.size()) LOG_PANIC("environment source read failed"); + value.append(reinterpret_cast(buffer.data()), len); + if (status == -1) return value; + if (len == 0) LOG_PANIC("environment source made no progress"); + } + } +}; +} +#endif diff --git a/crates/bindings-cpp/include/spacetimedb/procedure_context.h b/crates/bindings-cpp/include/spacetimedb/procedure_context.h index ebfd958af7c..35c93c31475 100644 --- a/crates/bindings-cpp/include/spacetimedb/procedure_context.h +++ b/crates/bindings-cpp/include/spacetimedb/procedure_context.h @@ -15,6 +15,8 @@ #include #include +#include + namespace SpacetimeDB { /** @@ -57,6 +59,7 @@ struct ProcedureContext { Identity sender_; public: + Environment env; // Timestamp when the procedure was invoked Timestamp timestamp; diff --git a/crates/bindings-cpp/include/spacetimedb/reducer_context.h b/crates/bindings-cpp/include/spacetimedb/reducer_context.h index 41865b14f3a..14a6028c606 100644 --- a/crates/bindings-cpp/include/spacetimedb/reducer_context.h +++ b/crates/bindings-cpp/include/spacetimedb/reducer_context.h @@ -13,6 +13,8 @@ // Include database for DatabaseContext #include +#include + namespace SpacetimeDB { // Enhanced ReducerContext with database access - matches Rust pattern @@ -21,6 +23,7 @@ struct ReducerContext { Identity sender_; public: + Environment env; // Core fields - sender is exposed via sender() like Rust, other fields remain directly accessible std::optional connection_id; Timestamp timestamp; diff --git a/crates/bindings-cpp/include/spacetimedb/tx_context.h b/crates/bindings-cpp/include/spacetimedb/tx_context.h index 1a04ef027e1..c874a22b4ff 100644 --- a/crates/bindings-cpp/include/spacetimedb/tx_context.h +++ b/crates/bindings-cpp/include/spacetimedb/tx_context.h @@ -56,6 +56,7 @@ struct TxContext { // In C++, we explicitly expose references where possible and provide // accessors for fields exposed as methods on ReducerContext. DatabaseContext& db; + const Environment& env; const Timestamp& timestamp; const std::optional& connection_id; @@ -63,6 +64,7 @@ struct TxContext { explicit TxContext(ReducerContext& ctx) : ctx_(ctx), db(ctx.db), + env(ctx.env), timestamp(ctx.timestamp), connection_id(ctx.connection_id) {} diff --git a/crates/bindings-cpp/include/spacetimedb/view_context.h b/crates/bindings-cpp/include/spacetimedb/view_context.h index 63102699e42..6985c2e55b0 100644 --- a/crates/bindings-cpp/include/spacetimedb/view_context.h +++ b/crates/bindings-cpp/include/spacetimedb/view_context.h @@ -7,6 +7,8 @@ #include // For ReadOnlyDatabaseContext #include +#include + namespace SpacetimeDB { /** @@ -41,6 +43,7 @@ struct ViewContext { public: // Read-only database access - no mutations allowed ReadOnlyDatabaseContext db; + Environment env; QueryBuilder from; // Constructors @@ -76,6 +79,7 @@ struct ViewContext { struct AnonymousViewContext { // Read-only database access - no mutations allowed ReadOnlyDatabaseContext db; + Environment env; QueryBuilder from; // Constructors diff --git a/crates/bindings-cpp/tests/unit/CMakeLists.txt b/crates/bindings-cpp/tests/unit/CMakeLists.txt index 0ced4e0194c..da7b8705e1c 100644 --- a/crates/bindings-cpp/tests/unit/CMakeLists.txt +++ b/crates/bindings-cpp/tests/unit/CMakeLists.txt @@ -11,6 +11,7 @@ endif() add_executable(bindings_cpp_unit_tests main.cpp http_unit_tests.cpp + environment_unit_tests.cpp ) target_include_directories(bindings_cpp_unit_tests PRIVATE diff --git a/crates/bindings-cpp/tests/unit/environment_unit_tests.cpp b/crates/bindings-cpp/tests/unit/environment_unit_tests.cpp new file mode 100644 index 00000000000..c6bd0f58cd8 --- /dev/null +++ b/crates/bindings-cpp/tests/unit/environment_unit_tests.cpp @@ -0,0 +1,50 @@ +#include "test_harness.h" +#include "spacetimedb/environment.h" +#include "spacetimedb/bsatn/reader.h" +#include +#include + +using namespace SpacetimeDB; + +namespace { +size_t payload_offset; +std::string payload; +} + +extern "C" Status env_get(const uint8_t* key, uint32_t key_len, BytesSource* out) { + payload_offset = 0; + const std::string name(reinterpret_cast(key), key_len); + *out = BytesSource{name == "MISSING" ? 0u : 1u}; + return Status{0}; +} + +extern "C" int16_t bytes_source_read(BytesSource, uint8_t* out, size_t* len) { + *len = std::min(*len, payload.size() - payload_offset); + std::memcpy(out, payload.data() + payload_offset, *len); + payload_offset += *len; + return payload_offset == payload.size() ? -1 : 0; +} + +extern "C" void console_log(LogLevel, const uint8_t*, size_t, const uint8_t*, size_t, + uint32_t, const uint8_t*, size_t) {} + +TEST_CASE(environment_preserves_missing_empty_and_all_chunks_without_caching) { + Environment env; + ASSERT_TRUE(!env.get("MISSING").has_value()); + ASSERT_EQ(std::string{}, env.get("EMPTY").value()); + payload = std::string(8192, 'x'); + ASSERT_EQ(payload, env.get("LARGE").value()); + payload = std::string("a\0b", 3); + ASSERT_EQ(payload, env.get("NUL").value()); + payload = "updated"; + ASSERT_EQ(payload, env.get("NUL").value()); +} + +TEST_CASE(optional_reader_matches_canonical_bsatn_tags_and_preserves_following_bytes) { + const std::vector bytes{1, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 'a', 0, 'b', 42}; + bsatn::Reader reader(bytes.data(), bytes.size()); + ASSERT_TRUE(!bsatn::deserialize>(reader).has_value()); + ASSERT_EQ(std::string{}, bsatn::deserialize>(reader).value()); + ASSERT_EQ(std::string("a\0b", 3), bsatn::deserialize>(reader).value()); + ASSERT_EQ(uint8_t{42}, reader.read_u8()); +} diff --git a/crates/bindings-csharp/Codegen.Tests/fixtures/diag/snapshots/Module#FFI.verified.cs b/crates/bindings-csharp/Codegen.Tests/fixtures/diag/snapshots/Module#FFI.verified.cs index 6638bb12fc8..2eb70c4a352 100644 --- a/crates/bindings-csharp/Codegen.Tests/fixtures/diag/snapshots/Module#FFI.verified.cs +++ b/crates/bindings-csharp/Codegen.Tests/fixtures/diag/snapshots/Module#FFI.verified.cs @@ -649,6 +649,7 @@ public static class Handlers { } public sealed record ReducerContext : DbContext, Internal.IReducerContext { + public global::SpacetimeDB.DatabaseEnvironment Env => default; public readonly Identity Sender; public readonly ConnectionId? ConnectionId; public readonly Random Rng; @@ -892,6 +893,7 @@ public sealed record ViewContext : DbContext, Internal.I { public Identity Sender { get; } + public global::SpacetimeDB.DatabaseEnvironment Env => default; public QueryBuilder From => default; internal ViewContext(Identity sender, Internal.LocalReadOnly db) @@ -905,6 +907,7 @@ public sealed record AnonymousViewContext : DbContext, Internal.IAnonymousViewContext { + public global::SpacetimeDB.DatabaseEnvironment Env => default; public QueryBuilder From => default; internal AnonymousViewContext(Internal.LocalReadOnly db) diff --git a/crates/bindings-csharp/Codegen.Tests/fixtures/explicitnames/snapshots/Module#FFI.verified.cs b/crates/bindings-csharp/Codegen.Tests/fixtures/explicitnames/snapshots/Module#FFI.verified.cs index 015ec80b3ad..d05060e0bd3 100644 --- a/crates/bindings-csharp/Codegen.Tests/fixtures/explicitnames/snapshots/Module#FFI.verified.cs +++ b/crates/bindings-csharp/Codegen.Tests/fixtures/explicitnames/snapshots/Module#FFI.verified.cs @@ -51,6 +51,7 @@ public static class Handlers { } public sealed record ReducerContext : DbContext, Internal.IReducerContext { + public global::SpacetimeDB.DatabaseEnvironment Env => default; public readonly Identity Sender; public readonly ConnectionId? ConnectionId; public readonly Random Rng; @@ -275,6 +276,7 @@ public sealed record ViewContext : DbContext, Internal.I { public Identity Sender { get; } + public global::SpacetimeDB.DatabaseEnvironment Env => default; public QueryBuilder From => default; internal ViewContext(Identity sender, Internal.LocalReadOnly db) @@ -288,6 +290,7 @@ public sealed record AnonymousViewContext : DbContext, Internal.IAnonymousViewContext { + public global::SpacetimeDB.DatabaseEnvironment Env => default; public QueryBuilder From => default; internal AnonymousViewContext(Internal.LocalReadOnly db) diff --git a/crates/bindings-csharp/Codegen.Tests/fixtures/server/snapshots/Module#FFI.verified.cs b/crates/bindings-csharp/Codegen.Tests/fixtures/server/snapshots/Module#FFI.verified.cs index 82e3d7bdecf..7f196c87a30 100644 --- a/crates/bindings-csharp/Codegen.Tests/fixtures/server/snapshots/Module#FFI.verified.cs +++ b/crates/bindings-csharp/Codegen.Tests/fixtures/server/snapshots/Module#FFI.verified.cs @@ -493,6 +493,7 @@ public static class Handlers { } public sealed record ReducerContext : DbContext, Internal.IReducerContext { + public global::SpacetimeDB.DatabaseEnvironment Env => default; public readonly Identity Sender; public readonly ConnectionId? ConnectionId; public readonly Random Rng; @@ -726,6 +727,7 @@ public sealed record ViewContext : DbContext, Internal.I { public Identity Sender { get; } + public global::SpacetimeDB.DatabaseEnvironment Env => default; public QueryBuilder From => default; internal ViewContext(Identity sender, Internal.LocalReadOnly db) @@ -739,6 +741,7 @@ public sealed record AnonymousViewContext : DbContext, Internal.IAnonymousViewContext { + public global::SpacetimeDB.DatabaseEnvironment Env => default; public QueryBuilder From => default; internal AnonymousViewContext(Internal.LocalReadOnly db) diff --git a/crates/bindings-csharp/Codegen/Module.cs b/crates/bindings-csharp/Codegen/Module.cs index aad1bf54c02..e3f146ad3f2 100644 --- a/crates/bindings-csharp/Codegen/Module.cs +++ b/crates/bindings-csharp/Codegen/Module.cs @@ -2557,6 +2557,7 @@ public static class Handlers { ))}} } public sealed record ReducerContext : DbContext, Internal.IReducerContext { + public global::SpacetimeDB.DatabaseEnvironment Env => default; public readonly Identity Sender; public readonly ConnectionId? ConnectionId; public readonly Random Rng; @@ -2755,6 +2756,7 @@ public sealed record ViewContext : DbContext, Internal.I { public Identity Sender { get; } + public global::SpacetimeDB.DatabaseEnvironment Env => default; public QueryBuilder From => default; internal ViewContext(Identity sender, Internal.LocalReadOnly db) @@ -2766,6 +2768,7 @@ internal ViewContext(Identity sender, Internal.LocalReadOnly db) public sealed record AnonymousViewContext : DbContext, Internal.IAnonymousViewContext { + public global::SpacetimeDB.DatabaseEnvironment Env => default; public QueryBuilder From => default; internal AnonymousViewContext(Internal.LocalReadOnly db) diff --git a/crates/bindings-csharp/Runtime/DatabaseEnvironment.cs b/crates/bindings-csharp/Runtime/DatabaseEnvironment.cs new file mode 100644 index 00000000000..59ca2c26ec2 --- /dev/null +++ b/crates/bindings-csharp/Runtime/DatabaseEnvironment.cs @@ -0,0 +1,26 @@ +namespace SpacetimeDB; + +/// +/// Read-only database environment. Values are plaintext and accessible to database +/// collaborators. Procedure reads outside a transaction use a short snapshot. +/// +public readonly struct DatabaseEnvironment +{ + internal static readonly DatabaseEnvironment Instance = new(); + + /// Return null for a missing key, or an empty string for a present empty value. + public unsafe string? Get(string key) + { + ArgumentNullException.ThrowIfNull(key); + var bytes = System.Text.Encoding.UTF8.GetBytes(key); + fixed (byte* ptr = bytes) + { + Internal.FFI.env_get(ptr, checked((uint)bytes.Length), out var source); + if (source == Internal.BytesSource.INVALID) + { + return null; + } + return System.Text.Encoding.UTF8.GetString(Internal.Module.Consume(source)); + } + } +} diff --git a/crates/bindings-csharp/Runtime/HandlerContext.cs b/crates/bindings-csharp/Runtime/HandlerContext.cs index 76b13230426..9fc02d8b858 100644 --- a/crates/bindings-csharp/Runtime/HandlerContext.cs +++ b/crates/bindings-csharp/Runtime/HandlerContext.cs @@ -7,6 +7,7 @@ namespace SpacetimeDB; public abstract class HandlerContextBase { public Random Rng => txState.Rng; + public DatabaseEnvironment Env { get; } = DatabaseEnvironment.Instance; public Timestamp Timestamp => txState.Timestamp; // NOTE: The host rejects procedure HTTP requests while a mut transaction is open @@ -90,6 +91,7 @@ public abstract class HandlerTxContextBase(Internal.TxContext inner) : IRefresha void IRefreshableTxContext.Refresh(Internal.TxContext inner) => Refresh(inner); public LocalBase Db => (LocalBase)Inner.Db; + public DatabaseEnvironment Env { get; } = DatabaseEnvironment.Instance; public Timestamp Timestamp => Inner.Timestamp; public Random Rng => Inner.Rng; } diff --git a/crates/bindings-csharp/Runtime/Internal/FFI.cs b/crates/bindings-csharp/Runtime/Internal/FFI.cs index afc56abbc8f..f498e6ed8ea 100644 --- a/crates/bindings-csharp/Runtime/Internal/FFI.cs +++ b/crates/bindings-csharp/Runtime/Internal/FFI.cs @@ -109,6 +109,21 @@ internal static partial class FFI #endif ; + const string StdbNamespace10_7 = +#if EXPERIMENTAL_WASM_AOT + "spacetime_10.7" +#else + "bindings" +#endif + ; + + [LibraryImport(StdbNamespace10_7)] + public static unsafe partial CheckedStatus env_get( + byte* key, + uint keyLen, + out BytesSource source + ); + [NativeMarshalling(typeof(Marshaller))] public struct CheckedStatus { diff --git a/crates/bindings-csharp/Runtime/ProcedureContext.cs b/crates/bindings-csharp/Runtime/ProcedureContext.cs index a86711f2814..63ec0238ca0 100644 --- a/crates/bindings-csharp/Runtime/ProcedureContext.cs +++ b/crates/bindings-csharp/Runtime/ProcedureContext.cs @@ -3,6 +3,7 @@ namespace SpacetimeDB; #pragma warning disable STDB_UNSTABLE public abstract class ProcedureContextBase : Internal.IInternalProcedureContext { + public DatabaseEnvironment Env { get; } = DatabaseEnvironment.Instance; public static Identity Identity => Internal.IProcedureContext.GetIdentity(); public Identity Sender { get; } public ConnectionId? ConnectionId { get; } @@ -100,6 +101,7 @@ public abstract class ProcedureTxContextBase(Internal.TxContext inner) : IRefres void IRefreshableTxContext.Refresh(Internal.TxContext inner) => Refresh(inner); public LocalBase Db => (LocalBase)Inner.Db; + public DatabaseEnvironment Env { get; } = DatabaseEnvironment.Instance; public Identity Sender => Inner.Sender; public ConnectionId? ConnectionId => Inner.ConnectionId; public Timestamp Timestamp => Inner.Timestamp; diff --git a/crates/bindings-csharp/Runtime/bindings.c b/crates/bindings-csharp/Runtime/bindings.c index 6118ba6be11..f4d3635a5f2 100644 --- a/crates/bindings-csharp/Runtime/bindings.c +++ b/crates/bindings-csharp/Runtime/bindings.c @@ -135,6 +135,10 @@ IMPORT(Status, datastore_clear, (table_id, count)); #undef SPACETIME_MODULE_VERSION +#define SPACETIME_MODULE_VERSION "spacetime_10.7" +IMPORT(Status, env_get, (const uint8_t* key, uint32_t key_len, BytesSource* source), (key, key_len, source)); +#undef SPACETIME_MODULE_VERSION + #ifndef EXPERIMENTAL_WASM_AOT static MonoClass* ffi_class; diff --git a/crates/bindings-sys/src/lib.rs b/crates/bindings-sys/src/lib.rs index 0295b4616e5..2a3bd77b454 100644 --- a/crates/bindings-sys/src/lib.rs +++ b/crates/bindings-sys/src/lib.rs @@ -883,6 +883,18 @@ pub mod raw { pub fn datastore_clear(table_id: TableId, out: *mut u64) -> u16; } + // ABI10.6 is reserved for the separate invocation-authority extension. + #[link(wasm_import_module = "spacetime_10.7")] + unsafe extern "C" { + /// Read a UTF-8 environment value. Writes INVALID for a missing key; + /// present empty strings have a valid BytesSource. Returns ordinary errno. + /// Invalid keys return HOST_CALL_FAILURE. NO_SPACE means 256 byte + /// sources remain unconsumed; consume a source before retrying. + /// Calls outside a reducer/view + /// transaction or procedure return NOT_IN_TRANSACTION. + pub fn env_get(key: *const u8, key_len: usize, out: *mut BytesSource) -> u16; + } + /// What strategy does the database index use? /// /// See also: @@ -1493,6 +1505,14 @@ pub fn get_jwt(connection_id: [u8; 16]) -> Option { } } +/// Read a database environment value without exposing the system table. +#[inline] +pub fn env_get(key: &str) -> Option { + let source = unsafe { call(|out| raw::env_get(key.as_ptr(), key.len(), out)) } + .unwrap_or_else(|errno: Errno| panic!("Error reading environment: {errno}")); + (source != raw::BytesSource::INVALID).then_some(source) +} + pub struct RowIter { raw: raw::RowIter, } diff --git a/crates/bindings-typescript/src/lib/environment.ts b/crates/bindings-typescript/src/lib/environment.ts new file mode 100644 index 00000000000..3798cfb43d4 --- /dev/null +++ b/crates/bindings-typescript/src/lib/environment.ts @@ -0,0 +1,5 @@ +/** Read-only database environment access. Missing keys return null; empty values return "". */ +export interface Environment { + /** Reads the current transaction, or a short snapshot outside a procedure transaction. */ + get(key: string): string | null; +} diff --git a/crates/bindings-typescript/src/lib/reducers.ts b/crates/bindings-typescript/src/lib/reducers.ts index 27c57f0721b..eaa8de94b55 100644 --- a/crates/bindings-typescript/src/lib/reducers.ts +++ b/crates/bindings-typescript/src/lib/reducers.ts @@ -1,3 +1,4 @@ +import type { Environment } from './environment'; import type { DbView } from '../server/db_view'; import type { Random } from '../server/rng'; import type { ConnectionId } from './connection_id'; @@ -115,6 +116,7 @@ export type ReducerCtx = Readonly<{ timestamp: Timestamp; connectionId: ConnectionId | null; db: DbView; + env: Environment; senderAuth: AuthCtx; newUuidV4(): Uuid; newUuidV7(): Uuid; diff --git a/crates/bindings-typescript/src/server/environment.ts b/crates/bindings-typescript/src/server/environment.ts new file mode 100644 index 00000000000..349e8ffed8c --- /dev/null +++ b/crates/bindings-typescript/src/server/environment.ts @@ -0,0 +1,6 @@ +import { env_get } from 'spacetime:sys@2.3'; +import type { Environment } from '../lib/environment'; + +/** Values are not cached: transaction and procedure reads retain host semantics. */ +export const environment: Environment = Object.freeze({ get: env_get }); +export type { Environment } from '../lib/environment'; diff --git a/crates/bindings-typescript/src/server/http_handlers.ts b/crates/bindings-typescript/src/server/http_handlers.ts index 68d42a267f8..0c4d9469eee 100644 --- a/crates/bindings-typescript/src/server/http_handlers.ts +++ b/crates/bindings-typescript/src/server/http_handlers.ts @@ -1,3 +1,4 @@ +import type { Environment } from '../lib/environment'; import type { Identity } from '../lib/identity'; import type { HttpMethod, @@ -219,6 +220,7 @@ export type HandlerAliasViews = : {}; export interface HandlerContext { + readonly env: Environment; readonly timestamp: Timestamp; readonly http: HttpClient; readonly identity: Identity; diff --git a/crates/bindings-typescript/src/server/index.ts b/crates/bindings-typescript/src/server/index.ts index ae084f672d4..3ac3e8f0fbb 100644 --- a/crates/bindings-typescript/src/server/index.ts +++ b/crates/bindings-typescript/src/server/index.ts @@ -35,4 +35,6 @@ export { export type { HandlerContext, HttpHandlerExport } from './http'; export { ScheduleAt } from '../lib/schedule_at'; +export type { Environment } from './environment'; + import './polyfills'; // Ensure polyfills are loaded diff --git a/crates/bindings-typescript/src/server/procedures.ts b/crates/bindings-typescript/src/server/procedures.ts index 2dec68467c0..f4d57416e69 100644 --- a/crates/bindings-typescript/src/server/procedures.ts +++ b/crates/bindings-typescript/src/server/procedures.ts @@ -1,3 +1,4 @@ +import { environment, type Environment } from './environment'; import { AlgebraicType, ProductType, @@ -108,6 +109,7 @@ export type ProcedureAliasViews = : {}; export interface ProcedureCtx { + readonly env: Environment; readonly sender: Identity; readonly databaseIdentity: Identity; /** @deprecated Use `databaseIdentity` instead. */ @@ -226,6 +228,7 @@ const ProcedureCtxImpl = class ProcedureCtx #uuidCounter: { value: 0 } | undefined; #random: Random | undefined; #dbView: () => DbView; + readonly env = environment; #dispatches: SubmoduleDispatchInfo[]; #parentPrefix: string; #asViews: object | undefined; diff --git a/crates/bindings-typescript/src/server/runtime.ts b/crates/bindings-typescript/src/server/runtime.ts index cf579e87631..1fac18ac95b 100644 --- a/crates/bindings-typescript/src/server/runtime.ts +++ b/crates/bindings-typescript/src/server/runtime.ts @@ -1,3 +1,4 @@ +import { environment } from './environment'; import * as _syscalls2_0 from 'spacetime:sys@2.0'; import * as _syscalls2_1 from 'spacetime:sys@2.1'; @@ -246,6 +247,7 @@ export const ReducerCtxImpl = class ReducerCtx< timestamp: Timestamp; connectionId: ConnectionId | null; db: DbView; + readonly env = environment; as: AliasViews; constructor( @@ -627,6 +629,7 @@ class ModuleHooksImpl implements ModuleHooks { const { fn, deserializeParams, serializeReturn, returnTypeBaseSize } = viewFns![localId!]; const ctx: ViewCtx = freeze({ + env: environment, sender: new Identity(sender), db: dbView!, from: from!, @@ -677,6 +680,7 @@ class ModuleHooksImpl implements ModuleHooks { const { fn, deserializeParams, serializeReturn, returnTypeBaseSize } = anonViewFns![localId!]; const ctx: AnonymousViewCtx = freeze({ + env: environment, db: dbView!, from: from!, }); @@ -773,6 +777,7 @@ const BINARY_READER = new BinaryReader(new Uint8Array()); class HandlerContextImpl implements HandlerContext { + readonly env = environment; #identity: Identity | undefined; #uuidCounter: { value: number } | undefined; #random: Random | undefined; diff --git a/crates/bindings-typescript/src/server/sys.d.ts b/crates/bindings-typescript/src/server/sys.d.ts index f0315867cb3..1f74debd2fc 100644 --- a/crates/bindings-typescript/src/server/sys.d.ts +++ b/crates/bindings-typescript/src/server/sys.d.ts @@ -123,3 +123,10 @@ declare module 'spacetime:sys@2.0' { declare module 'spacetime:sys@2.1' { export function datastore_clear(table_id: u32): u64; } + +// sys2.2 is reserved for the separate invocation-authority extension. + +declare module 'spacetime:sys@2.3' { + /** Null means missing; an empty string is a present value. */ + export function env_get(key: string): string | null; +} diff --git a/crates/bindings-typescript/src/server/views.ts b/crates/bindings-typescript/src/server/views.ts index c7cb9ca0b4d..528c8319063 100644 --- a/crates/bindings-typescript/src/server/views.ts +++ b/crates/bindings-typescript/src/server/views.ts @@ -1,3 +1,4 @@ +import type { Environment } from '../lib/environment'; import { AlgebraicType, ProductType, @@ -81,11 +82,13 @@ export function makeAnonViewExport< export type ViewCtx = Readonly<{ sender: Identity; db: ReadonlyDbView; + env: Environment; from: QueryBuilder; }>; export type AnonymousViewCtx = Readonly<{ db: ReadonlyDbView; + env: Environment; from: QueryBuilder; }>; diff --git a/crates/bindings/src/http.rs b/crates/bindings/src/http.rs index 4c8ea487c47..3638d35f9e2 100644 --- a/crates/bindings/src/http.rs +++ b/crates/bindings/src/http.rs @@ -98,6 +98,9 @@ pub use spacetimedb_bindings_macro::http_router as router; #[cfg(feature = "unstable")] #[non_exhaustive] pub struct HandlerContext { + /// Read-only access to this database's environment store. + pub env: crate::Environment, + /// The time at which the handler was started. pub timestamp: Timestamp, @@ -117,6 +120,7 @@ pub struct HandlerContext { impl HandlerContext { pub(crate) fn new(timestamp: Timestamp) -> Self { Self { + env: crate::Environment::default(), timestamp, http: HttpClient {}, #[cfg(feature = "rand08")] diff --git a/crates/bindings/src/lib.rs b/crates/bindings/src/lib.rs index 2c7faa78c6b..d1548bca052 100644 --- a/crates/bindings/src/lib.rs +++ b/crates/bindings/src/lib.rs @@ -919,10 +919,29 @@ pub use spacetimedb_bindings_macro::view; pub struct QueryBuilder {} pub use query_builder::{Query, RawQuery}; +/// Read-only access to this database's environment store. +/// +/// Reads use the current transaction. In a procedure outside a transaction, +/// each read uses a short snapshot; use `with_tx` to read related keys together. +/// Values are stored in plaintext and may be read by database collaborators. +#[derive(Clone, Copy, Debug, Default)] +pub struct Environment { + _private: (), +} + +impl Environment { + /// Return None for a missing key and Some("") for a present empty value. + /// Keys must be POSIX environment names of at most 256 bytes. + pub fn get(&self, key: &str) -> Option { + rt::env_get(key) + } +} + /// One of two possible types that can be passed as the first argument to a `#[view]`. /// The other is [`ViewContext`]. /// Use this type if the view does not depend on the caller's identity. pub struct AnonymousViewContext { + pub env: Environment, pub db: LocalReadOnly, pub from: QueryBuilder, } @@ -930,6 +949,7 @@ pub struct AnonymousViewContext { impl Default for AnonymousViewContext { fn default() -> Self { Self { + env: Environment::default(), db: LocalReadOnly {}, from: QueryBuilder {}, } @@ -939,6 +959,7 @@ impl Default for AnonymousViewContext { /// The other is [`AnonymousViewContext`]. /// Use this type if the view depends on the caller's identity. pub struct ViewContext { + pub env: Environment, sender: Identity, pub db: LocalReadOnly, pub from: QueryBuilder, @@ -948,6 +969,7 @@ impl ViewContext { pub fn new(sender: Identity) -> Self { Self { sender, + env: Environment::default(), db: LocalReadOnly {}, from: QueryBuilder {}, } @@ -978,6 +1000,8 @@ impl ViewContext { /// Implements the `DbContext` trait for accessing views into a database. #[non_exhaustive] pub struct ReducerContext { + /// Read-only access to the database environment in this transaction. + pub env: Environment, /// The `Identity` of the client that invoked the reducer. sender: Identity, @@ -1042,6 +1066,7 @@ impl ReducerContext { #[doc(hidden)] pub fn __dummy() -> Self { Self { + env: Environment::default(), db: Local {}, sender: Identity::__dummy(), timestamp: Timestamp::UNIX_EPOCH, @@ -1057,6 +1082,7 @@ impl ReducerContext { #[doc(hidden)] fn new(db: Local, sender: Identity, connection_id: Option, timestamp: Timestamp) -> Self { Self { + env: Environment::default(), db, sender, timestamp, @@ -1254,6 +1280,8 @@ fn with_tx(body: impl Fn(&TxContext) -> T, identity: Identity, connection_id: /// and exposes methods for running transactions and performing side-effecting operations. #[non_exhaustive] pub struct ProcedureContext { + /// Read-only access to the database environment. + pub env: Environment, /// The `Identity` of the client that invoked the procedure. sender: Identity, @@ -1285,6 +1313,7 @@ impl ProcedureContext { sender, timestamp, connection_id, + env: Environment::default(), http: http::HttpClient {}, #[cfg(feature = "rand08")] rng: std::cell::OnceCell::new(), diff --git a/crates/bindings/src/rt.rs b/crates/bindings/src/rt.rs index 7d456d73fde..b09c998f5c3 100644 --- a/crates/bindings/src/rt.rs +++ b/crates/bindings/src/rt.rs @@ -1319,6 +1319,13 @@ pub fn get_jwt(connection_id: ConnectionId) -> Option { Some(std::str::from_utf8(&buf).unwrap().to_string()) } +pub(crate) fn env_get(key: &str) -> Option { + let source = sys::env_get(key)?; + let mut buf = IterBuf::take(); + read_bytes_source_into(source, &mut buf); + Some(String::from_utf8(buf.to_vec()).expect("host environment values are UTF-8")) +} + /// Read `source` from the host fully into `buf`. pub(crate) fn read_bytes_source_into(source: BytesSource, buf: &mut Vec) { const INVALID: i16 = NO_SUCH_BYTES as i16; diff --git a/crates/client-api/src/lib.rs b/crates/client-api/src/lib.rs index e8999e1e617..68fd0a61bb5 100644 --- a/crates/client-api/src/lib.rs +++ b/crates/client-api/src/lib.rs @@ -146,7 +146,8 @@ impl Host { .await .map_err(|_| (StatusCode::NOT_FOUND, "module not found".to_string()))?; - tracing::debug!(sql = body); + // Environment SQL contains values; routine request logs must omit them. + tracing::debug!(sql_bytes = body.len(), "executing SQL"); let mut header = vec![]; let sql_start = std::time::Instant::now(); let sql_span = tracing::trace_span!("execute_sql", total_duration = tracing::field::Empty,); @@ -164,8 +165,8 @@ impl Host { ) .await .map_err(|e| { - // TODO: Review log level after user SQL errors can be distinguished from internal database failures. - log::warn!("{e}"); + // Parser diagnostics can quote values. Return them only to the caller. + log::warn!("SQL request rejected"); (StatusCode::BAD_REQUEST, e.to_string()) })?; diff --git a/crates/core/src/db/environment.rs b/crates/core/src/db/environment.rs new file mode 100644 index 00000000000..87b7a7231f7 --- /dev/null +++ b/crates/core/src/db/environment.rs @@ -0,0 +1,131 @@ +//! Dedicated access to the private environment store. +//! +//! Mutation callers must authorize owner/admin access before calling these +//! helpers and commit through the normal module transaction machinery so +//! dependent views refresh. Helpers never acquire a second transaction. + +use super::relational_db::{MutTx, RelationalDB}; +use crate::error::DBError; +use spacetimedb_datastore::error::DatastoreError; +use spacetimedb_datastore::locking_tx_datastore::state_view::StateView; +use spacetimedb_datastore::system_tables::{StEnvFields, StEnvRow, ST_ENV_ID}; +use spacetimedb_lib::environment::{validate_key, validate_value, EnvironmentValidationError, MAX_ENV_VARS}; +use spacetimedb_sats::AlgebraicValue; +use std::collections::BTreeMap; + +#[derive(Debug, thiserror::Error)] +pub enum EnvironmentError { + #[error(transparent)] + Validation(#[from] EnvironmentValidationError), + #[error(transparent)] + Datastore(#[from] DatastoreError), + #[error(transparent)] + Database(#[from] DBError), +} + +/// Read from exactly the caller's snapshot, preserving missing versus empty. +pub fn get(state: &impl StateView, key: &str) -> Result, EnvironmentError> { + validate_key(key)?; + state + .iter_by_col_eq(ST_ENV_ID, StEnvFields::Key, &AlgebraicValue::String(key.into()))? + .next() + .map(|row| Ok(StEnvRow::try_from(row)?.value)) + .transpose() +} + +pub fn snapshot(state: &impl StateView) -> Result, EnvironmentError> { + state + .iter(ST_ENV_ID)? + .map(|row| { + let row = StEnvRow::try_from(row)?; + Ok((row.key, row.value)) + }) + .collect() +} + +/// Insert or replace one key. Validation occurs before any mutation. +pub fn set(db: &RelationalDB, tx: &mut MutTx, key: &str, value: &str) -> Result<(), EnvironmentError> { + validate_key(key)?; + validate_value(value)?; + let previous = get(tx, key)?; + if previous.is_none() && tx.table_row_count(ST_ENV_ID).unwrap_or(0) >= MAX_ENV_VARS as u64 { + return Err(EnvironmentValidationError::TooManyVariables.into()); + } + if previous.as_deref() == Some(value) { + return Ok(()); + } + delete(db, tx, key)?; + tx.insert_via_serialize_bsatn( + ST_ENV_ID, + &StEnvRow { + key: key.into(), + value: value.into(), + }, + )?; + Ok(()) +} + +pub fn delete(db: &RelationalDB, tx: &mut MutTx, key: &str) -> Result { + validate_key(key)?; + let pointer = tx + .iter_by_col_eq(ST_ENV_ID, StEnvFields::Key, &AlgebraicValue::String(key.into()))? + .next() + .map(|row| row.pointer()); + if let Some(pointer) = pointer { + db.delete(tx, ST_ENV_ID, [pointer]); + return Ok(true); + } + Ok(false) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::db::relational_db::tests_utils::TestDB; + use spacetimedb_datastore::execution_context::Workload; + + #[test] + fn missing_empty_nul_update_and_rollback() { + let db = TestDB::in_memory().unwrap(); + db.with_auto_commit(Workload::ForTests, |tx| -> Result<(), EnvironmentError> { + assert_eq!(get(tx, "EMPTY")?, None); + set(&db, tx, "EMPTY", "")?; + set(&db, tx, "NUL", "a\0b")?; + assert_eq!(get(tx, "EMPTY")?, Some(String::new())); + assert_eq!(get(tx, "NUL")?, Some("a\0b".into())); + Ok(()) + }) + .unwrap(); + let result = db.with_auto_commit(Workload::ForTests, |tx| -> Result<(), EnvironmentError> { + set(&db, tx, "EMPTY", "changed")?; + delete(&db, tx, "NUL")?; + Err(EnvironmentValidationError::InvalidKey.into()) + }); + assert!(result.is_err()); + db.with_read_only(Workload::ForTests, |tx| { + assert_eq!( + snapshot(tx).unwrap(), + BTreeMap::from([("EMPTY".into(), "".into()), ("NUL".into(), "a\0b".into())]) + ); + }); + } + + #[test] + fn capacity_and_value_limits_precede_mutation() { + let db = TestDB::in_memory().unwrap(); + db.with_auto_commit(Workload::ForTests, |tx| -> Result<(), EnvironmentError> { + for i in 0..MAX_ENV_VARS { + set(&db, tx, &format!("K{i}"), "")?; + } + assert!(set(&db, tx, "EXTRA", "").is_err()); + set(&db, tx, "K0", "updated")?; + assert!(set(&db, tx, "K0", &"x".repeat(8193)).is_err()); + assert_eq!(get(tx, "K0")?.as_deref(), Some("updated")); + assert!(delete(&db, tx, "K1")?); + assert!(!delete(&db, tx, "MISSING")?); + set(&db, tx, "EXTRA", "")?; + Ok(()) + }) + .unwrap(); + } +} diff --git a/crates/core/src/db/mod.rs b/crates/core/src/db/mod.rs index 6b1d2f6700b..a7117db5b71 100644 --- a/crates/core/src/db/mod.rs +++ b/crates/core/src/db/mod.rs @@ -1,3 +1,5 @@ +pub mod environment; + pub mod persistence { pub use spacetimedb_engine::persistence::*; } diff --git a/crates/core/src/error.rs b/crates/core/src/error.rs index efee537dbbe..0f8d5621b3f 100644 --- a/crates/core/src/error.rs +++ b/crates/core/src/error.rs @@ -30,6 +30,10 @@ impl From for DBError { #[derive(Error, Debug)] pub enum NodesError { + #[error("invalid environment variable name")] + InvalidEnvironmentKey, + #[error("too many outstanding byte sources for environment read")] + EnvironmentSourceLimit, #[error("Failed to decode row: {0}")] DecodeRow(#[source] DecodeError), #[error("Failed to decode value: {0}")] diff --git a/crates/core/src/host/instance_env.rs b/crates/core/src/host/instance_env.rs index 9bf807ebf7a..1bfb4f7f7c7 100644 --- a/crates/core/src/host/instance_env.rs +++ b/crates/core/src/host/instance_env.rs @@ -17,6 +17,7 @@ use spacetimedb_datastore::db_metrics::DB_METRICS; use spacetimedb_datastore::execution_context::Workload; use spacetimedb_datastore::locking_tx_datastore::state_view::StateView; use spacetimedb_datastore::locking_tx_datastore::{FuncCallType, IndexScanPointOrRange, MutTxId}; +use spacetimedb_datastore::system_tables::{is_module_restricted_index, is_module_restricted_table}; use spacetimedb_datastore::traits::IsolationLevel; use spacetimedb_lib::{http as st_http, ConnectionId, Identity, Timestamp}; use spacetimedb_metrics::utils::IntGaugeExt; @@ -281,6 +282,25 @@ impl InstanceEnv { self.replica_ctx.relational_db() } + /// Dedicated read-only environment access. Missing reads also register a + /// dependency so a later insert refreshes a view that observed absence. + pub(crate) fn env_get(&self, key: &str) -> Result, NodesError> { + use crate::db::environment; + use spacetimedb_datastore::system_tables::ST_ENV_ID; + spacetimedb_lib::environment::validate_key(key).map_err(|_| NodesError::InvalidEnvironmentKey)?; + let read = |state: &_| environment::get(state, key).map_err(|err| NodesError::from(DBError::Other(err.into()))); + if let Ok(mut tx) = self.get_tx() { + tx.record_table_scan(&self.func_type, ST_ENV_ID); + return read(&*tx); + } + if !matches!(self.func_type, FuncCallType::Procedure) { + return Err(NodesError::NotInTransaction); + } + self.relational_db().with_read_only(Workload::Internal, |tx| { + environment::get(tx, key).map_err(|err| NodesError::from(DBError::Other(err.into()))) + }) + } + pub(crate) fn get_jwt_payload(&self, connection_id: ConnectionId) -> Result, NodesError> { let tx = &mut *self.get_tx()?; Ok(tx.get_jwt_payload(connection_id).map_err(DBError::from)?) @@ -355,9 +375,19 @@ impl InstanceEnv { count } + /// Environment values are reachable only through their dedicated host interface. + fn require_module_table(table_id: TableId) -> Result<(), NodesError> { + if is_module_restricted_table(table_id) { + Err(NodesError::TableNotFound) + } else { + Ok(()) + } + } + pub fn insert(&self, table_id: TableId, buffer: &mut [u8]) -> Result { let stdb = self.relational_db(); let tx = &mut *self.get_tx()?; + Self::require_module_table(table_id)?; let (row_len, row_ptr, insert_flags) = stdb .insert(tx, table_id, buffer) @@ -436,6 +466,7 @@ impl InstanceEnv { pub fn update(&self, table_id: TableId, index_id: IndexId, buffer: &mut [u8]) -> Result { let stdb = self.relational_db(); let tx = &mut *self.get_tx()?; + Self::require_module_table(table_id)?; let (row_len, row_ptr, update_flags) = stdb .update(tx, table_id, index_id, buffer) @@ -479,6 +510,7 @@ impl InstanceEnv { // Find all rows in the table to delete. let (table_id, _, iter) = stdb.index_scan_point(tx, index_id, point)?; + Self::require_module_table(table_id)?; // Re. `SmallVec`, `delete_by_field` only cares about 1 element, so optimize for that. let rows_to_delete = iter.map(|row_ref| row_ref.pointer()).collect::>(); @@ -499,6 +531,7 @@ impl InstanceEnv { // Find all rows in the table to delete. let (table_id, iter) = stdb.index_scan_range(tx, index_id, prefix, prefix_elems, rstart, rend)?; + Self::require_module_table(table_id)?; // Re. `SmallVec`, `delete_by_field` only cares about 1 element, so optimize for that. let rows_to_delete = match iter { IndexScanPointOrRange::Point(_, iter) => iter.map(|row_ref| row_ref.pointer()).collect(), @@ -540,6 +573,7 @@ impl InstanceEnv { pub fn datastore_delete_all_by_eq_bsatn(&self, table_id: TableId, relation: &[u8]) -> Result { let stdb = self.relational_db(); let tx = &mut *self.get_tx()?; + Self::require_module_table(table_id)?; // Track the number of bytes coming from the caller tx.metrics.bytes_scanned += relation.len(); @@ -563,6 +597,7 @@ impl InstanceEnv { pub fn clear(&self, table_id: TableId) -> Result { let stdb = self.relational_db(); let tx = &mut *self.get_tx()?; + Self::require_module_table(table_id)?; let rows_deleted = stdb.clear_table(tx, table_id).map_err(NodesError::from)?; @@ -584,6 +619,7 @@ impl InstanceEnv { // Query the table id from the name. stdb.table_id_from_name_mut(tx, table_name)? + .filter(|id| !is_module_restricted_table(*id)) .ok_or(NodesError::TableNotFound) } @@ -598,6 +634,7 @@ impl InstanceEnv { // Query the index id from the name. stdb.index_id_from_name_mut(tx, index_name)? + .filter(|id| !is_module_restricted_index(*id)) .ok_or(NodesError::IndexNotFound) } @@ -609,6 +646,7 @@ impl InstanceEnv { pub fn datastore_table_row_count(&self, table_id: TableId) -> Result { let stdb = self.relational_db(); let tx = &mut *self.get_tx()?; + Self::require_module_table(table_id)?; // Query the row count for id. stdb.table_row_count_mut(tx, table_id) @@ -625,6 +663,7 @@ impl InstanceEnv { table_id: TableId, ) -> Result>, NodesError> { let tx = &mut *self.get_tx()?; + Self::require_module_table(table_id)?; // Open the iterator. let iter = self.relational_db().iter_mut(tx, table_id)?; @@ -652,6 +691,7 @@ impl InstanceEnv { // Open index iterator let (table_id, point, iter) = self.relational_db().index_scan_point(tx, index_id, point)?; + Self::require_module_table(table_id)?; // Scan the index and serialize rows to BSATN. let (chunks, rows_scanned, bytes_scanned) = ChunkedWriter::collect_iter(pool, iter); @@ -682,6 +722,7 @@ impl InstanceEnv { let (table_id, iter) = self.relational_db() .index_scan_range(tx, index_id, prefix, prefix_elems, rstart, rend)?; + Self::require_module_table(table_id)?; // Scan the index and serialize rows to BSATN. let (point, (chunks, rows_scanned, bytes_scanned)) = match iter { @@ -1446,6 +1487,107 @@ mod test { Ok(db) } + #[test] + fn environment_reads_use_active_transaction_and_track_missing_view_dependency() -> Result<()> { + use crate::db::environment; + use spacetimedb_datastore::locking_tx_datastore::ViewCallInfo; + use spacetimedb_primitives::ViewId; + let db = relational_db()?; + let (mut env, _runtime) = instance_env(db.clone())?; + assert!(matches!(env.env_get("A"), Err(NodesError::NotInTransaction))); + env.func_type = FuncCallType::Procedure; + assert_eq!(env.env_get("A")?, None); + let mut tx = begin_mut_tx(&db); + environment::set(&db, &mut tx, "A", "uncommitted")?; + env.tx.set_raw(tx); + assert_eq!(env.env_get("A")?.as_deref(), Some("uncommitted")); + let view = ViewCallInfo::anonymous(ViewId(88)); + env.func_type = FuncCallType::View(view.clone()); + assert_eq!(env.env_get("MISSING")?, None); + let tx = env.tx.take()?; + db.commit_tx(tx)?; + let mut tx = begin_mut_tx(&db); + environment::set(&db, &mut tx, "MISSING", "")?; + assert!(tx.views_for_refresh().any(|dependency| dependency == &view)); + let (_, metrics, reducer) = db.rollback_mut_tx(tx); + db.report_mut_tx_metrics(reducer, metrics, None); + env.func_type = FuncCallType::Procedure; + assert_eq!(env.env_get("MISSING")?, None); + assert!(matches!(env.env_get("A=B"), Err(NodesError::InvalidEnvironmentKey))); + Ok(()) + } + + #[test] + fn module_cannot_access_environment_by_guessed_table_and_index_ids() -> Result<()> { + use spacetimedb_datastore::system_tables::ST_ENV_ID; + let db = relational_db()?; + let (env, _runtime) = instance_env(db.clone())?; + let mut slot = env.tx.clone(); + let protected = [(ST_ENV_ID, "st_env", to_vec("TOKEN")?)]; + let tx = begin_mut_tx(&db); + let (tx, result) = slot.set(tx, || -> Result<()> { + for (table, name, point) in &protected { + // Host lookup remains available, independently of module lookup. + let (index, index_name) = { + let tx = env.get_tx()?; + let schema = db.schema_for_table_mut(&tx, *table)?; + let index = &schema.indexes[0]; + (index.index_id, index.index_name.to_string()) + }; + assert!(matches!(env.table_id_from_name(name), Err(NodesError::TableNotFound))); + assert!(matches!( + env.index_id_from_name(&index_name), + Err(NodesError::IndexNotFound) + )); + assert!(matches!(env.insert(*table, &mut []), Err(NodesError::TableNotFound))); + assert!(matches!( + env.update(*table, index, &mut []), + Err(NodesError::TableNotFound) + )); + assert!(matches!(env.clear(*table), Err(NodesError::TableNotFound))); + assert!(matches!( + env.datastore_table_row_count(*table), + Err(NodesError::TableNotFound) + )); + assert!(matches!( + env.datastore_table_scan_bsatn_chunks(&mut ChunkPool::default(), *table), + Err(NodesError::TableNotFound) + )); + assert!(matches!( + env.datastore_delete_all_by_eq_bsatn(*table, &[]), + Err(NodesError::TableNotFound) + )); + assert!(matches!( + env.datastore_index_scan_point_bsatn_chunks(&mut ChunkPool::default(), index, point), + Err(NodesError::TableNotFound) + )); + assert!(matches!( + env.datastore_delete_by_index_scan_point_bsatn(index, point), + Err(NodesError::TableNotFound) + )); + let bound = to_vec(&Bound::::Unbounded)?; + assert!(matches!( + env.datastore_index_scan_range_bsatn_chunks( + &mut ChunkPool::default(), + index, + &[], + 0.into(), + &bound, + &bound + ), + Err(NodesError::TableNotFound) + )); + assert!(matches!( + env.datastore_delete_by_index_scan_range_bsatn(index, &[], 0.into(), &bound, &bound), + Err(NodesError::TableNotFound) + )); + } + Ok(()) + }); + let _ = db.rollback_mut_tx(tx); + result + } + /// Generate a `ProductValue` for use in [create_table_with_index] fn product_row(i: usize) -> ProductValue { let str = i.to_string(); diff --git a/crates/core/src/host/mod.rs b/crates/core/src/host/mod.rs index 31aa62ec6bc..f28a515c910 100644 --- a/crates/core/src/host/mod.rs +++ b/crates/core/src/host/mod.rs @@ -191,6 +191,7 @@ pub enum AbiCall { Identity, JwtLength, GetJwt, + EnvGet, VolatileNonatomicScheduleImmediate, diff --git a/crates/core/src/host/v8/syscall/mod.rs b/crates/core/src/host/v8/syscall/mod.rs index a09e7cbba0c..029d5836282 100644 --- a/crates/core/src/host/v8/syscall/mod.rs +++ b/crates/core/src/host/v8/syscall/mod.rs @@ -62,6 +62,8 @@ fn resolve_sys_module_inner<'scope>( (1, 3) => Ok(v1::sys_v1_3(scope)), (2, 0) => Ok(v2::sys_v2_0(scope)), (2, 1) => Ok(v2::sys_v2_1(scope)), + // sys2.2 is reserved for invocation authority. + (2, 3) => Ok(v2::sys_v2_3(scope)), _ => Err(TypeError(format!( "Could not import {spec:?}, likely because this module was built for a newer version of SpacetimeDB.\n\ It requires sys module v{major}.{minor}, but that version is not supported by the database." diff --git a/crates/core/src/host/v8/syscall/v2.rs b/crates/core/src/host/v8/syscall/v2.rs index f49d2260549..8fb376ec7af 100644 --- a/crates/core/src/host/v8/syscall/v2.rs +++ b/crates/core/src/host/v8/syscall/v2.rs @@ -169,6 +169,24 @@ pub(super) fn sys_v2_1<'scope>(scope: &mut PinScope<'scope, '_>) -> Local<'scope ) } +pub(super) fn sys_v2_3<'scope>(scope: &mut PinScope<'scope, '_>) -> Local<'scope, Module> { + create_synthetic_module!(scope, "spacetime:sys@2.3", (with_sys_result, AbiCall::EnvGet, env_get),) +} + +fn env_get<'s>( + scope: &mut PinScope<'s, '_>, + args: FunctionCallbackArguments<'s>, +) -> SysCallResult> { + let key: String = deserialize_js(scope, args.get(0))?; + match get_env(scope)?.instance_env.env_get(&key)? { + Some(value) => Ok(value + .into_string(scope) + .map_err(|_| RangeError("environment value could not be represented").throw(scope))? + .into()), + None => Ok(v8::null(scope).into()), + } +} + /// Registers a function in `module` /// where the function has `name` and does `body`. fn register_module_fun( diff --git a/crates/core/src/host/wasm_common.rs b/crates/core/src/host/wasm_common.rs index 1e7fd18b5e1..dc8baa44227 100644 --- a/crates/core/src/host/wasm_common.rs +++ b/crates/core/src/host/wasm_common.rs @@ -362,6 +362,8 @@ pub fn err_to_errno(err: NodesError) -> Result<(NonZeroU16, Option), Nod NodesError::DecodeRow(_) => errno::BSATN_DECODE_ERROR, NodesError::DecodeValue(_) => errno::BSATN_DECODE_ERROR, NodesError::TableNotFound => errno::NO_SUCH_TABLE, + NodesError::InvalidEnvironmentKey => errno::HOST_CALL_FAILURE, + NodesError::EnvironmentSourceLimit => errno::NO_SPACE, NodesError::IndexNotFound => errno::NO_SUCH_INDEX, NodesError::IndexNotUnique => errno::INDEX_NOT_UNIQUE, NodesError::IndexRowNotFound => errno::NO_SUCH_ROW, @@ -442,6 +444,8 @@ macro_rules! abi_funcs { "spacetime_10.4"::datastore_delete_by_index_scan_point_bsatn, "spacetime_10.5"::datastore_clear, + // ABI10.6 is reserved for invocation authority. + "spacetime_10.7"::env_get, } $link_async! { diff --git a/crates/core/src/host/wasmtime/wasm_instance_env.rs b/crates/core/src/host/wasmtime/wasm_instance_env.rs index da4a2f3987f..23d33983440 100644 --- a/crates/core/src/host/wasmtime/wasm_instance_env.rs +++ b/crates/core/src/host/wasmtime/wasm_instance_env.rs @@ -31,6 +31,10 @@ use std::sync::Arc; use std::time::Instant; use wasmtime::{AsContext, Caller, StoreContextMut}; +/// Env reads may retain at most 2 MiB of value bytes outside the Wasm heap. +/// Other outstanding byte sources count against this interface's handle limit. +const MAX_OUTSTANDING_ENV_SOURCES: usize = 256; + /// A stream of bytes which the WASM module can read from /// using [`WasmInstanceEnv::bytes_source_read`]. /// @@ -253,7 +257,14 @@ impl WasmInstanceEnv { // This allows the module to avoid allocating and make a system call in those cases. if bytes.is_empty() { Ok(BytesSourceId::INVALID) - } else if bytes.len() > u32::MAX as usize { + } else { + self.create_present_bytes_source(bytes) + } + } + + /// Allocate a valid source even for an empty value when zero means absence. + fn create_present_bytes_source(&mut self, bytes: bytes::Bytes) -> RtResult { + if bytes.len() > u32::MAX as usize { // There's no inherent reason we need to error here, // other than that it makes it impossible to report the length in `bytes_source_remaining_length` // and that all of our usage of `BytesSource`s as of writing (pgoldman 2025-09-26) @@ -1572,6 +1583,34 @@ impl WasmInstanceEnv { }) } + /// Read an environment value as a nullable BytesSource. Zero means missing; + /// a present empty string always receives a nonzero, consumable source. + pub fn env_get( + caller: Caller<'_, Self>, + key: WasmPtr, + key_len: u32, + target_ptr: WasmPtr, + ) -> RtResult { + Self::cvt_ret(caller, AbiCall::EnvGet, target_ptr, |caller| { + if key_len == 0 || key_len > spacetimedb_lib::environment::MAX_ENV_KEY_BYTES as u32 { + return Err(crate::error::NodesError::InvalidEnvironmentKey.into()); + } + let (mem, env) = Self::mem_env(caller); + let key = mem.deref_str(key, key_len)?; + match env.instance_env.env_get(key)? { + None => Ok(0), + Some(value) => { + // These buffers live on the host heap until consumed or the + // invocation ends. Bound retained reads from hand-written Wasm. + if env.bytes_sources.len() >= MAX_OUTSTANDING_ENV_SOURCES { + return Err(crate::error::NodesError::EnvironmentSourceLimit.into()); + } + Ok(env.create_present_bytes_source(bytes::Bytes::from(value))?.0) + } + } + }) + } + /// Finds the JWT payload associated with `connection_id`. /// A `[ByteSourceId]` for the payload will be written to `target_ptr`. /// If nothing is found for the connection, `[ByteSourceId::INVALID]` (zero) is written to `target_ptr`. diff --git a/crates/core/src/host/wasmtime/wasmtime_module.rs b/crates/core/src/host/wasmtime/wasmtime_module.rs index e34367ee0de..a5316ceb95c 100644 --- a/crates/core/src/host/wasmtime/wasmtime_module.rs +++ b/crates/core/src/host/wasmtime/wasmtime_module.rs @@ -55,7 +55,7 @@ impl WasmtimeModule { WasmtimeModule { module } } - pub const IMPLEMENTED_ABI: abi::VersionTuple = abi::VersionTuple::new(10, 5); + pub const IMPLEMENTED_ABI: abi::VersionTuple = abi::VersionTuple::new(10, 7); pub(super) fn link_imports(linker: &mut Linker) -> anyhow::Result<()> { link_imports(linker, AsyncImportMode::SyncStub) diff --git a/crates/core/src/sql/execute.rs b/crates/core/src/sql/execute.rs index ddd57d780c5..e927921c886 100644 --- a/crates/core/src/sql/execute.rs +++ b/crates/core/src/sql/execute.rs @@ -88,7 +88,23 @@ fn run_inner( // We parse the sql statement in a mutable transaction. // If it turns out to be a query, we downgrade the tx. let (tx, stmt) = db.with_auto_rollback(db.begin_mut_tx(IsolationLevel::Serializable, Workload::Sql), |tx| { - compile_sql_stmt(&sql_text, &SchemaViewer::new(tx, &auth), &auth) + let stmt = compile_sql_stmt(&sql_text, &SchemaViewer::new(tx, &auth), &auth)?; + // Check mutation authority while the automatic rollback guard owns + // the transaction, including rejected administrative statements. + if matches!(&stmt, Statement::DML(_) | Statement::Environment(_)) && !auth.has_write_access() { + return Err(anyhow!( + "Caller {} is not authorized to run SQL mutations", + auth.caller() + )); + } + if let Statement::DML(dml) = &stmt + && dml.table_id() == spacetimedb_datastore::system_tables::ST_ENV_ID + { + return Err(anyhow!( + "Use SET env.KEY or DELETE env.KEY to modify database environment variables" + )); + } + Ok(stmt) })?; let mut metrics = ExecutionMetrics::default(); @@ -141,14 +157,21 @@ fn run_inner( trapped, )) } - Statement::DML(stmt) => { - // An extra layer of auth is required for DML - if !auth.has_write_access() { - return Err(anyhow!("Caller {} is not authorized to run SQL DML statements", auth.caller()).into()); - } - + stmt @ (Statement::DML(_) | Statement::Environment(_)) => { // Evaluate the mutation - let (mut tx, _) = db.with_auto_rollback(tx, |tx| execute_dml_stmt(&auth, stmt, tx, &mut metrics))?; + let (mut tx, _) = db.with_auto_rollback(tx, |tx| -> anyhow::Result<()> { + match stmt { + Statement::DML(stmt) => execute_dml_stmt(&auth, stmt, tx, &mut metrics)?, + Statement::Environment(environment) => match environment.value { + Some(value) => crate::db::environment::set(&db, tx, &environment.key, &value)?, + None => { + crate::db::environment::delete(&db, tx, &environment.key)?; + } + }, + Statement::Select(_) => unreachable!(), + } + Ok(()) + })?; // Update transaction metrics tx.metrics.merge(metrics); @@ -243,6 +266,76 @@ pub(crate) mod tests { use spacetimedb_schema::schema::{ColumnSchema, TableSchema}; use spacetimedb_schema::table_name::TableName; + #[test] + fn environment_sql_enforces_permissions_escaping_limits_and_rollback() { + use spacetimedb_lib::identity::SqlPermission; + let db = TestDB::in_memory().unwrap(); + let runtime = tokio::runtime::Runtime::new().unwrap(); + let owner = AuthCtx::for_current(Identity::ZERO); + let viewer = AuthCtx::with_permissions( + Identity::ONE, + Arc::new(|permission| matches!(permission, SqlPermission::Read(_))), + ); + let outsider = AuthCtx::new(Identity::ZERO, Identity::ONE); + let execute = |statement: &str, auth: AuthCtx| { + runtime.block_on(run(db.clone(), statement.to_string(), auth, None, None, &mut vec![])) + }; + let value = "it's \\quoted;\nUTF-8 é\0tail"; + execute( + &format!( + "/* prefix */ SET env.Mixed_Key = '{}'; -- suffix", + value.replace('\'', "''") + ), + owner.clone(), + ) + .unwrap(); + execute("SET env.EMPTY TO ''", owner.clone()).unwrap(); + let rows = execute("SELECT value FROM st_env WHERE key = 'Mixed_Key'", viewer.clone()) + .unwrap() + .rows; + assert_eq!(rows, vec![product![value]]); + assert!(execute("SELECT * FROM st_env", outsider.clone()).is_err()); + for auth in [viewer, outsider] { + assert!(execute("SET env.Mixed_Key = 'forbidden'", auth.clone()).is_err()); + assert!(execute("DELETE env.Mixed_Key", auth).is_err()); + } + for statement in [ + "SET env.EMPTY = 5".to_string(), + "SET env.\"BAD-KEY\" = 'value'".to_string(), + format!("SET env.EMPTY = '{}'", "x".repeat(8193)), + "SET env.EMPTY = 'changed'; DELETE env.Mixed_Key".to_string(), + "DELETE env.Mixed_Key WHERE true".to_string(), + "INSERT INTO st_env (key, value) VALUES ('BYPASS', 'value')".to_string(), + "UPDATE st_env SET value = 'bypass'".to_string(), + "DELETE FROM st_env".to_string(), + ] { + assert!( + execute(&statement, owner.clone()).is_err(), + "unexpectedly accepted {statement}" + ); + } + // Every rejected path released its transaction and preserved old data. + assert_eq!( + execute("SELECT value FROM st_env WHERE key = 'EMPTY'", owner.clone()) + .unwrap() + .rows, + vec![product![""]] + ); + assert_eq!( + execute("SELECT value FROM st_env WHERE key = 'Mixed_Key'", owner.clone()) + .unwrap() + .rows, + vec![product![value]] + ); + execute("SET env.EMPTY = 'updated'", owner.clone()).unwrap(); + execute("DELETE env.EMPTY", owner.clone()).unwrap(); + execute("DELETE env.EMPTY", owner.clone()).unwrap(); + assert!(execute("SELECT value FROM st_env WHERE key = 'EMPTY'", owner) + .unwrap() + .rows + .is_empty()); + } + /// Short-cut for simplify test execution pub(crate) fn run_for_testing(db: &Arc, sql_text: &str) -> Result, DBError> { let (subs, runtime) = ModuleSubscriptions::for_test_new_runtime(db.clone()); diff --git a/crates/datastore/src/locking_tx_datastore/committed_state.rs b/crates/datastore/src/locking_tx_datastore/committed_state.rs index 0ce629524c4..9169a96b45c 100644 --- a/crates/datastore/src/locking_tx_datastore/committed_state.rs +++ b/crates/datastore/src/locking_tx_datastore/committed_state.rs @@ -357,6 +357,8 @@ impl CommittedState { self.create_table(ST_TABLE_ACCESSOR_ID, schemas[ST_TABLE_ACCESSOR_IDX].clone()); self.create_table(ST_INDEX_ACCESSOR_ID, schemas[ST_INDEX_ACCESSOR_IDX].clone()); self.create_table(ST_COLUMN_ACCESSOR_ID, schemas[ST_COLUMN_ACCESSOR_IDX].clone()); + let env = crate::system_tables::st_env_schema(); + self.create_table(env.table_id, env.into()); // Insert the sequences into `st_sequences` let (st_sequences, blob_store, pool) = diff --git a/crates/datastore/src/locking_tx_datastore/datastore.rs b/crates/datastore/src/locking_tx_datastore/datastore.rs index 7293a81fba0..cf5fd662c63 100644 --- a/crates/datastore/src/locking_tx_datastore/datastore.rs +++ b/crates/datastore/src/locking_tx_datastore/datastore.rs @@ -1079,6 +1079,7 @@ pub(crate) mod tests { use super::*; use crate::error::IndexError; use crate::locking_tx_datastore::tx_state::PendingSchemaChange; + use crate::system_tables::ST_ENV_ID; use crate::system_tables::{ system_tables, StColumnRow, StConnectionCredentialsFields, StConstraintData, StConstraintFields, StConstraintRow, StEventTableFields, StIndexAlgorithm, StIndexFields, StIndexRow, StRowLevelSecurityFields, @@ -1558,6 +1559,7 @@ pub(crate) mod tests { TableRow { id: ST_TABLE_ACCESSOR_ID.into(), name: ST_TABLE_ACCESSOR_NAME, ty: StTableType::System, access: StAccess::Public, primary_key: None }, TableRow { id: ST_INDEX_ACCESSOR_ID.into(), name: ST_INDEX_ACCESSOR_NAME, ty: StTableType::System, access: StAccess::Public, primary_key: None }, TableRow { id: ST_COLUMN_ACCESSOR_ID.into(), name: ST_COLUMN_ACCESSOR_NAME, ty: StTableType::System, access: StAccess::Public, primary_key: None }, + TableRow { id: ST_ENV_ID.into(), name: "st_env", ty: StTableType::System, access: StAccess::Private, primary_key: Some(ColId(0)) }, ])); #[rustfmt::skip] @@ -1655,6 +1657,8 @@ pub(crate) mod tests { ColRow { table: ST_COLUMN_ACCESSOR_ID.into(), pos: 0, name: "table_name", ty: AlgebraicType::String }, ColRow { table: ST_COLUMN_ACCESSOR_ID.into(), pos: 1, name: "col_name", ty: AlgebraicType::String }, ColRow { table: ST_COLUMN_ACCESSOR_ID.into(), pos: 2, name: "accessor_name", ty: AlgebraicType::String }, + ColRow { table: ST_ENV_ID.into(), pos: 0, name: "key", ty: AlgebraicType::String }, + ColRow { table: ST_ENV_ID.into(), pos: 1, name: "value", ty: AlgebraicType::String }, ])); #[rustfmt::skip] assert_eq!(query.scan_st_indexes()?, map_array([ @@ -1687,6 +1691,7 @@ pub(crate) mod tests { IndexRow { id: 27, table: ST_INDEX_ACCESSOR_ID.into(), col: col(1), name: "st_index_accessor_accessor_name_idx_btree", }, IndexRow { id: 28, table: ST_COLUMN_ACCESSOR_ID.into(), col: col_list![0, 1], name: "st_column_accessor_table_name_col_name_idx_btree", }, IndexRow { id: 29, table: ST_COLUMN_ACCESSOR_ID.into(), col: col_list![0, 2], name: "st_column_accessor_table_name_accessor_name_idx_btree", }, + IndexRow { id: 30, table: ST_ENV_ID.into(), col: col_list![0], name: "st_env_key_idx_btree", }, ])); let start = ST_RESERVED_SEQUENCE_RANGE as i128 + 1; #[rustfmt::skip] @@ -1732,6 +1737,7 @@ pub(crate) mod tests { ConstraintRow { constraint_id: 23, table_id: ST_INDEX_ACCESSOR_ID.into(), unique_columns: col(1), constraint_name: "st_index_accessor_accessor_name_key", }, ConstraintRow { constraint_id: 24, table_id: ST_COLUMN_ACCESSOR_ID.into(), unique_columns: col_list![0, 1], constraint_name: "st_column_accessor_table_name_col_name_key", }, ConstraintRow { constraint_id: 25, table_id: ST_COLUMN_ACCESSOR_ID.into(), unique_columns: col_list![0, 2], constraint_name: "st_column_accessor_table_name_accessor_name_key", }, + ConstraintRow { constraint_id: 26, table_id: ST_ENV_ID.into(), unique_columns: col_list![0], constraint_name: "st_env_key_key", }, ])); // Verify we get back the tables correctly with the proper ids... @@ -2165,6 +2171,7 @@ pub(crate) mod tests { IndexRow { id: 27, table: ST_INDEX_ACCESSOR_ID.into(), col: col(1), name: "st_index_accessor_accessor_name_idx_btree", }, IndexRow { id: 28, table: ST_COLUMN_ACCESSOR_ID.into(), col: col_list![0, 1], name: "st_column_accessor_table_name_col_name_idx_btree", }, IndexRow { id: 29, table: ST_COLUMN_ACCESSOR_ID.into(), col: col_list![0, 2], name: "st_column_accessor_table_name_accessor_name_idx_btree", }, + IndexRow { id: 30, table: ST_ENV_ID.into(), col: col_list![0], name: "st_env_key_idx_btree", }, IndexRow { id: seq_start, table: FIRST_NON_SYSTEM_ID, col: col(0), name: "Foo_id_idx_btree", }, IndexRow { id: seq_start + 1, table: FIRST_NON_SYSTEM_ID, col: col(1), name: "Foo_name_idx_btree", }, IndexRow { id: seq_start + 2, table: FIRST_NON_SYSTEM_ID, col: col(2), name: "Foo_age_idx_btree", }, diff --git a/crates/datastore/src/system_tables.rs b/crates/datastore/src/system_tables.rs index 2635093cc2d..d39cd14923c 100644 --- a/crates/datastore/src/system_tables.rs +++ b/crates/datastore/src/system_tables.rs @@ -207,7 +207,7 @@ pub enum SystemTable { st_event_table = ST_EVENT_TABLE_ID.0 as _, } -pub fn system_tables() -> [TableSchema; 20] { +pub fn system_tables() -> [TableSchema; 21] { [ // The order should match the `id` of the system table, that start with [ST_TABLE_IDX]. st_table_schema(), @@ -230,6 +230,7 @@ pub fn system_tables() -> [TableSchema; 20] { st_table_accessor_schema(), st_index_accessor_schema(), st_column_accessor_schema(), + st_env_schema(), ] } @@ -313,6 +314,9 @@ macro_rules! st_fields_enum { } } +mod environment; +pub use environment::*; + // WARNING: For a stable schema, don't change the field names and discriminants. st_fields_enum!(enum StTableFields { "table_id", TableId = 0, @@ -670,6 +674,8 @@ fn system_module_def() -> ModuleDef { .with_unique_constraint(st_column_accessor_table_alias_cols) .with_index_no_accessor_name(btree(st_column_accessor_table_alias_cols)); + environment::register_table(&mut builder); + let result = builder .finish() .try_into() @@ -694,6 +700,7 @@ fn system_module_def() -> ModuleDef { validate_system_table::(&result, ST_EVENT_TABLE_NAME); validate_system_table::(&result, ST_TABLE_ACCESSOR_NAME); validate_system_table::(&result, ST_INDEX_ACCESSOR_NAME); + environment::validate_table(&result); validate_system_table::(&result, ST_COLUMN_ACCESSOR_NAME); result @@ -743,6 +750,7 @@ lazy_static::lazy_static! { m.insert("st_index_accessor_accessor_name_key", ConstraintId(23)); m.insert("st_column_accessor_table_name_col_name_key", ConstraintId(24)); m.insert("st_column_accessor_table_name_accessor_name_key", ConstraintId(25)); + m.insert("st_env_key_key", ConstraintId(26)); m }; } @@ -781,6 +789,7 @@ lazy_static::lazy_static! { m.insert("st_index_accessor_accessor_name_idx_btree", IndexId(27)); m.insert("st_column_accessor_table_name_col_name_idx_btree", IndexId(28)); m.insert("st_column_accessor_table_name_accessor_name_idx_btree", IndexId(29)); + m.insert("st_env_key_idx_btree", IndexId(30)); m }; } @@ -970,6 +979,7 @@ pub(crate) fn system_table_schema(table_id: TableId) -> Option { ST_TABLE_ACCESSOR_ID => Some(st_table_accessor_schema()), ST_INDEX_ACCESSOR_ID => Some(st_index_accessor_schema()), ST_COLUMN_ACCESSOR_ID => Some(st_column_accessor_schema()), + ST_ENV_ID => Some(st_env_schema()), _ => None, } } diff --git a/crates/datastore/src/system_tables/environment.rs b/crates/datastore/src/system_tables/environment.rs new file mode 100644 index 00000000000..82395f4bd77 --- /dev/null +++ b/crates/datastore/src/system_tables/environment.rs @@ -0,0 +1,45 @@ +//! Private database environment state. Values follow ordinary table durability. +use super::*; +pub const ST_ENV_ID: TableId = TableId(21); +pub const ST_ENV_NAME: &str = "st_env"; +st_fields_enum!(enum StEnvFields { "key", Key = 0, "value", Value = 1, }); +#[derive(Debug, Clone, PartialEq, Eq, SpacetimeType)] +#[sats(crate = spacetimedb_lib)] +pub struct StEnvRow { + pub key: String, + pub value: String, +} +impl TryFrom> for StEnvRow { + type Error = DatastoreError; + fn try_from(row: RowRef<'_>) -> Result { + read_via_bsatn(row) + } +} +impl From for ProductValue { + fn from(row: StEnvRow) -> Self { + to_product_value(&row) + } +} +pub(super) fn register_table(builder: &mut RawModuleDefV9Builder) { + let ty = builder.add_type::(); + builder + .build_table(ST_ENV_NAME, *ty.as_ref().expect("system row must be a product")) + .with_type(TableType::System) + .with_access(v9::TableAccess::Private) + .with_primary_key(ColId(0)) + .with_unique_constraint(ColId(0)) + .with_index_no_accessor_name(btree(ColId(0))); +} +pub(super) fn validate_table(def: &ModuleDef) { + validate_system_table::(def, ST_ENV_NAME); +} +pub(crate) fn st_env_schema() -> TableSchema { + st_schema(ST_ENV_NAME, ST_ENV_ID) +} +/// Module code must use env_get even when it guesses numeric identifiers. +pub fn is_module_restricted_table(table: TableId) -> bool { + table == ST_ENV_ID +} +pub fn is_module_restricted_index(index: IndexId) -> bool { + index == IndexId(30) +} diff --git a/crates/expr/src/errors.rs b/crates/expr/src/errors.rs index 1e7315cb4eb..d61ade64f54 100644 --- a/crates/expr/src/errors.rs +++ b/crates/expr/src/errors.rs @@ -132,6 +132,10 @@ pub struct DmlOnView { #[derive(Error, Debug)] pub enum TypingError { + #[error(transparent)] + Environment(#[from] spacetimedb_lib::environment::EnvironmentValidationError), + #[error("environment values must be SQL string literals")] + EnvironmentValueType, #[error(transparent)] Unsupported(#[from] Unsupported), #[error(transparent)] diff --git a/crates/expr/src/statement.rs b/crates/expr/src/statement.rs index 9fbdb869a20..3293553f1b6 100644 --- a/crates/expr/src/statement.rs +++ b/crates/expr/src/statement.rs @@ -31,6 +31,12 @@ use super::{ pub enum Statement { Select(ProjectList), DML(DML), + Environment(EnvironmentWrite), +} + +pub struct EnvironmentWrite { + pub key: Box, + pub value: Option>, } pub enum DML { @@ -454,6 +460,21 @@ pub fn parse_and_type_sql(sql: &str, tx: &impl SchemaView, _auth: &AuthCtx) -> T SqlAst::Update(update) => Ok(Statement::DML(DML::Update(type_update(update, tx)?))), SqlAst::Set(set) => Ok(Statement::DML(DML::Insert(type_and_rewrite_set(set, tx)?))), SqlAst::Show(show) => Ok(Statement::Select(type_and_rewrite_show(show, tx)?)), + SqlAst::Environment(environment) => { + // Resolve through the normal private-table visibility check. + tx.schema("st_env").ok_or_else(|| Unresolved::table("st_env"))?; + let key = &*environment.key.0; + spacetimedb_lib::environment::validate_key(key)?; + let value = match environment.value { + None => None, + Some(SqlLiteral::Str(value)) => { + spacetimedb_lib::environment::validate_value(&value)?; + Some(value) + } + Some(_) => return Err(TypingError::EnvironmentValueType), + }; + Ok(Statement::Environment(EnvironmentWrite { key: key.into(), value })) + } } } diff --git a/crates/lib/src/environment.rs b/crates/lib/src/environment.rs new file mode 100644 index 00000000000..17b5c3f3293 --- /dev/null +++ b/crates/lib/src/environment.rs @@ -0,0 +1,62 @@ +//! Limits shared by the database environment store and its clients. + +pub const MAX_ENV_KEY_BYTES: usize = 256; +pub const MAX_ENV_VALUE_BYTES: usize = 8 * 1024; +pub const MAX_ENV_VARS: usize = 256; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum EnvironmentValidationError { + InvalidKey, + ValueTooLarge, + TooManyVariables, +} + +impl std::fmt::Display for EnvironmentValidationError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(match self { + Self::InvalidKey => "invalid POSIX environment variable name (maximum 256 bytes)", + Self::ValueTooLarge => "environment value exceeds 8192 UTF-8 bytes", + Self::TooManyVariables => "environment store exceeds 256 variables", + }) + } +} + +impl std::error::Error for EnvironmentValidationError {} + +pub fn validate_key(key: &str) -> Result<(), EnvironmentValidationError> { + let bytes = key.as_bytes(); + if bytes.is_empty() + || bytes.len() > MAX_ENV_KEY_BYTES + || !(bytes[0].is_ascii_alphabetic() || bytes[0] == b'_') + || !bytes.iter().all(|b| b.is_ascii_alphanumeric() || *b == b'_') + { + return Err(EnvironmentValidationError::InvalidKey); + } + Ok(()) +} + +/// NUL is representable in the database. Container launch separately rejects it. +pub fn validate_value(value: &str) -> Result<(), EnvironmentValidationError> { + if value.len() > MAX_ENV_VALUE_BYTES { + return Err(EnvironmentValidationError::ValueTooLarge); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn validates_utf8_byte_limits_and_posix_keys_without_container_policy() { + for key in ["", "1FIRST", "A=B", "A\0B", "é", "A-B"] { + assert_eq!(validate_key(key), Err(EnvironmentValidationError::InvalidKey)); + } + assert!(validate_key(&"A".repeat(256)).is_ok()); + assert!(validate_key(&"A".repeat(257)).is_err()); + assert!(validate_key("SPACETIMEDB_USER_DATA").is_ok()); + assert!(validate_value("\0").is_ok()); + assert!(validate_value(&"é".repeat(4096)).is_ok()); + assert!(validate_value(&"é".repeat(4097)).is_err()); + } +} diff --git a/crates/lib/src/lib.rs b/crates/lib/src/lib.rs index aedded78ad1..55d5621a16d 100644 --- a/crates/lib/src/lib.rs +++ b/crates/lib/src/lib.rs @@ -13,6 +13,7 @@ use std::collections::{btree_map, BTreeMap}; pub mod connection_id; pub mod db; mod direct_index_key; +pub mod environment; pub mod error; mod filterable_value; pub mod http; diff --git a/crates/query/src/lib.rs b/crates/query/src/lib.rs index 9225393eb34..69168032a0a 100644 --- a/crates/query/src/lib.rs +++ b/crates/query/src/lib.rs @@ -29,7 +29,7 @@ pub fn compile_subscription( auth: &AuthCtx, ) -> Result<(Vec, TableId, TableName, bool)> { if sql.len() > MAX_SQL_LENGTH { - bail!("SQL query exceeds maximum allowed length: \"{sql:.120}...\"") + bail!("SQL query exceeds maximum allowed length") } let (plan, mut has_param) = parse_and_type_sub(sql, tx, auth)?; @@ -59,11 +59,11 @@ pub fn compile_subscription( /// A utility for parsing and type checking a sql statement pub fn compile_sql_stmt(sql: &str, tx: &impl SchemaView, auth: &AuthCtx) -> Result { if sql.len() > MAX_SQL_LENGTH { - bail!("SQL query exceeds maximum allowed length: \"{sql:.120}...\"") + bail!("SQL query exceeds maximum allowed length") } match parse_and_type_sql(sql, tx, auth)? { - stmt @ Statement::DML(_) => Ok(stmt), + stmt @ (Statement::DML(_) | Statement::Environment(_)) => Ok(stmt), Statement::Select(expr) => Ok(Statement::Select(resolve_views_for_sql(tx, expr, auth)?)), } } diff --git a/crates/sql-parser/src/ast/sql.rs b/crates/sql-parser/src/ast/sql.rs index be7b753f395..a7593cc4582 100644 --- a/crates/sql-parser/src/ast/sql.rs +++ b/crates/sql-parser/src/ast/sql.rs @@ -19,6 +19,15 @@ pub enum SqlAst { Set(SqlSet), /// SHOW var Show(SqlShow), + /// Administrative environment mutation, distinct from generic table DML. + Environment(SqlEnvironment), +} + +#[derive(Debug)] +pub struct SqlEnvironment { + pub key: SqlIdent, + /// None is DELETE; a string literal, including empty, is SET. + pub value: Option, } impl SqlAst { diff --git a/crates/sql-parser/src/parser/sql.rs b/crates/sql-parser/src/parser/sql.rs index 0508baae5b2..a70ea4d5b39 100644 --- a/crates/sql-parser/src/parser/sql.rs +++ b/crates/sql-parser/src/parser/sql.rs @@ -133,11 +133,13 @@ use sqlparser::{ Value, Values, }, dialect::PostgreSqlDialect, + keywords::Keyword, parser::Parser, + tokenizer::Token, }; use crate::ast::{ - sql::{SqlAst, SqlDelete, SqlInsert, SqlSelect, SqlSet, SqlShow, SqlUpdate, SqlValues}, + sql::{SqlAst, SqlDelete, SqlEnvironment, SqlInsert, SqlSelect, SqlSet, SqlShow, SqlUpdate, SqlValues}, SqlIdent, }; @@ -148,7 +150,36 @@ use super::{ /// Parse a SQL string pub fn parse_sql(sql: &str) -> SqlParseResult { - let mut stmts = Parser::parse_sql(&PostgreSqlDialect {}, sql)?; + // DELETE env.KEY is a SpacetimeDB administrative statement, not the + // PostgreSQL DELETE FROM grammar. Use the same tokenizer and expression + // parser, including comments and SQL string escaping, for this extension. + let mut parser = Parser::new(&PostgreSqlDialect {}).try_with_sql(sql)?; + let verb = parser.peek_token().token; + let environment_prefix = matches!(parser.peek_nth_token(1).token, + Token::Word(word) if word.quote_style.is_none() && word.value.eq_ignore_ascii_case("env")) + && parser.peek_nth_token(2).token == Token::Period; + if environment_prefix + && matches!(&verb, Token::Word(word) if matches!(word.keyword, Keyword::SET | Keyword::DELETE)) + { + parser.next_token(); + parser.next_token(); + parser.next_token(); + let key = SqlIdent(parser.parse_identifier()?.value.into()); + let value = if matches!(verb, Token::Word(word) if word.keyword == Keyword::SET) { + if !parser.parse_keyword(Keyword::TO) { + parser.expect_token(&Token::Eq)?; + } + Some(parse_literal_expr(parser.parse_expr()?, SqlUnsupported::Assignment)?) + } else { + None + }; + let _ = parser.consume_token(&Token::SemiColon); + if parser.peek_token().token != Token::EOF { + return Err(SqlUnsupported::MultiStatement.into()); + } + return Ok(SqlAst::Environment(SqlEnvironment { key, value })); + } + let mut stmts = parser.parse_statements()?; if stmts.len() > 1 { return Err(SqlUnsupported::MultiStatement.into()); } diff --git a/crates/testing/tests/environment.rs b/crates/testing/tests/environment.rs new file mode 100644 index 00000000000..7cdf4fecf0e --- /dev/null +++ b/crates/testing/tests/environment.rs @@ -0,0 +1,177 @@ +//! Actual module calls exercise environment ABI, bindings, and snapshot semantics. +use serial_test::serial; +use spacetimedb::host::{FunctionArgs, ModuleHost}; +use spacetimedb_lib::identity::AuthCtx; +use spacetimedb_lib::{bsatn, sats::product, AlgebraicValue, Identity}; +use spacetimedb_testing::modules::{CompilationMode, CompiledModule, DEFAULT_CONFIG}; + +async fn sql(module: &ModuleHost, statement: String) -> Vec { + spacetimedb::sql::execute::run( + module.relational_db().clone(), + statement, + AuthCtx::for_current(Identity::ZERO), + Some(module.info.subscriptions.clone()), + Some(module.clone()), + &mut vec![], + ) + .await + .unwrap() + .rows +} + +async fn set_environment(module: &ModuleHost, key: &str, value: &str) { + sql(module, format!("SET env.{key} = '{}'", value.replace('\'', "''"))).await; +} + +fn exercise_fixture(name: &str) { + CompiledModule::compile(name, CompilationMode::Debug).with_module_async(DEFAULT_CONFIG, |handle| async move { + let module = handle.client.module(); + for (key, expected) in [ + ("MISSING", None), + ("EMPTY", Some("".to_string())), + ("UTF8", Some("héllo 🌍".to_string())), + ("NUL", Some("before\0after".to_string())), + ("MAXIMUM", Some("é".repeat(4096))), + ] { + if let Some(value) = &expected { + set_environment(&module, key, value).await; + } + let args = product![key, expected.clone()]; + let result = module + .call_reducer( + Identity::ZERO, + None, + None, + None, + None, + "expect_environment", + FunctionArgs::Bsatn(bsatn::to_vec(&args).unwrap().into()), + ) + .await; + let result = result + .map_err(anyhow::Error::from) + .and_then(|r| r.outcome.into_result()); + assert!( + result.is_ok(), + "{name} {key}: {result:?}; module log: {}", + handle.read_log(None).await + ); + let read = || FunctionArgs::Bsatn(bsatn::to_vec(&product![key]).unwrap().into()); + let result = module + .call_procedure(Identity::ZERO, None, None, "read_environment", read()) + .await + .result + .unwrap() + .return_val; + assert_eq!(result, AlgebraicValue::from(expected.clone())); + if expected.is_some() { + set_environment(&module, key, "updated").await; + let result = module + .call_procedure(Identity::ZERO, None, None, "read_environment", read()) + .await + .result + .unwrap() + .return_val; + assert_eq!(result, AlgebraicValue::from(Some("updated".to_string()))); + sql(&module, format!("DELETE env.{key}")).await; + let result = module + .call_procedure(Identity::ZERO, None, None, "read_environment", read()) + .await + .result + .unwrap() + .return_val; + assert_eq!(result, AlgebraicValue::from(None::)); + } + } + if name == "environment-test" { + set_environment(&module, "HANDLER", "handler snapshot").await; + let (_, body) = module + .call_http_handler( + module.info.module_def.http_handler_ids_and_defs().next().unwrap().0, + spacetimedb_lib::http::Request { + method: spacetimedb_lib::http::Method::Get, + headers: std::iter::empty().collect(), + timeout: None, + uri: "/environment".into(), + version: spacetimedb_lib::http::Version::Http11, + }, + Default::default(), + ) + .await + .unwrap(); + assert_eq!(&body[..], b"handler snapshot"); + // This view first reads a missing key. Its dependency must survive + // absence, and normal SQL mutations must invalidate its cached row. + let read_view = || "SELECT * FROM environment_value".to_string(); + assert_eq!(sql(&module, read_view()).await, vec![product![None::]]); + set_environment(&module, "WATCHED", "first").await; + assert_eq!( + sql(&module, read_view()).await, + vec![product![Some("first".to_string())]] + ); + set_environment(&module, "WATCHED", "second").await; + assert_eq!( + sql(&module, read_view()).await, + vec![product![Some("second".to_string())]] + ); + sql(&module, "DELETE env.WATCHED".into()).await; + assert_eq!(sql(&module, read_view()).await, vec![product![None::]]); + set_environment(&module, "LIMIT", &"x".repeat(8192)).await; + for _ in 0..2 { + module + .call_reducer( + Identity::ZERO, + None, + None, + None, + None, + "bounded_environment_sources", + FunctionArgs::Nullary, + ) + .await + .unwrap() + .outcome + .into_result() + .unwrap(); + } + } + let args = product!["A=B", None::]; + let result = module + .call_reducer( + Identity::ZERO, + None, + None, + None, + None, + "expect_environment", + FunctionArgs::Bsatn(bsatn::to_vec(&args).unwrap().into()), + ) + .await; + module.exit().await; + assert!(result.is_err() || result.unwrap().outcome.into_result().is_err()); + }); +} + +#[test] +#[serial] +fn rust_environment_reads_are_not_cached_and_preserve_missing_empty_utf8_and_nul() { + exercise_fixture("environment-test"); +} + +#[test] +#[serial] +fn typescript_environment_reads_are_not_cached_and_preserve_missing_empty_utf8_and_nul() { + exercise_fixture("module-test-ts"); +} + +#[test] +#[serial] +fn cpp_environment_reads_are_not_cached_and_preserve_missing_empty_utf8_and_nul() { + exercise_fixture("module-test-cpp"); +} + +#[test] +#[serial] +fn csharp_environment_reads_are_not_cached_and_preserve_missing_empty_utf8_and_nul() { + exercise_fixture("module-test-cs"); +} diff --git a/modules/environment-test/Cargo.toml b/modules/environment-test/Cargo.toml new file mode 100644 index 00000000000..9ae6466b734 --- /dev/null +++ b/modules/environment-test/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "environment-test" +version = "0.0.0" +edition.workspace = true +license-file = "../../LICENSE.txt" +publish = false + +[lib] +crate-type = ["cdylib"] + +[dependencies.spacetimedb] +workspace = true +features = ["unstable"] diff --git a/modules/environment-test/src/lib.rs b/modules/environment-test/src/lib.rs new file mode 100644 index 00000000000..07c8c6aedd8 --- /dev/null +++ b/modules/environment-test/src/lib.rs @@ -0,0 +1,69 @@ +use spacetimedb::{AnonymousViewContext, ProcedureContext, ReducerContext, SpacetimeType}; + +#[spacetimedb::reducer] +pub fn expect_environment(ctx: &ReducerContext, key: String, expected: Option) { + assert_eq!(ctx.env.get(&key), expected); + assert_eq!(ctx.as_read_only().env.get(&key), expected); + assert_eq!(ctx.as_anonymous_read_only().env.get(&key), expected); +} + +#[spacetimedb::procedure] +pub fn read_environment(ctx: &mut ProcedureContext, key: String) -> Option { + let outside = ctx.env.get(&key); + ctx.with_tx(|tx| assert_eq!(tx.env.get(&key), outside)); + outside +} + +#[derive(SpacetimeType)] +pub struct EnvironmentValue { + pub value: Option, +} + +#[spacetimedb::view(accessor = environment_value, public)] +pub fn environment_value(ctx: &AnonymousViewContext) -> Option { + Some(EnvironmentValue { + value: ctx.env.get("WATCHED"), + }) +} + +/// Hand-written ABI callers cannot retain unbounded host allocations. +#[spacetimedb::reducer] +pub fn bounded_environment_sources(_ctx: &ReducerContext) { + use spacetimedb::sys::raw::{self, BytesSource}; + let mut sources = Vec::new(); + for i in 0..=256 { + let mut source = BytesSource::INVALID; + let status = unsafe { raw::env_get(b"LIMIT".as_ptr(), 5, &mut source) }; + if i == 256 { + assert_eq!(status, 9); // NO_SPACE + } else { + assert_eq!(status, 0); + assert!(source != BytesSource::INVALID); + sources.push(source); + } + } + let mut buffer = [0u8; 8192]; + let mut len = buffer.len(); + let status = unsafe { raw::bytes_source_read(sources[0], buffer.as_mut_ptr(), &mut len) }; + assert_eq!(status, -1); + assert_eq!(len, buffer.len()); + let mut source = BytesSource::INVALID; + assert_eq!(unsafe { raw::env_get(b"LIMIT".as_ptr(), 5, &mut source) }, 0); + assert!(source != BytesSource::INVALID); + // The remaining sources are released when this invocation ends. +} + +#[spacetimedb::http::handler] +pub fn handler_environment( + ctx: &mut spacetimedb::http::HandlerContext, + _request: spacetimedb::http::Request, +) -> spacetimedb::http::Response { + let outside = ctx.env.get("HANDLER"); + ctx.with_tx(|tx| assert_eq!(tx.env.get("HANDLER"), outside)); + spacetimedb::http::Response::new(spacetimedb::http::Body::from_bytes(outside.unwrap())) +} + +#[spacetimedb::http::router] +pub fn router() -> spacetimedb::http::Router { + spacetimedb::http::Router::new().get("/environment", handler_environment) +} diff --git a/modules/module-test-cpp/src/lib.cpp b/modules/module-test-cpp/src/lib.cpp index 3a93f0b20ec..1102ef8be5f 100644 --- a/modules/module-test-cpp/src/lib.cpp +++ b/modules/module-test-cpp/src/lib.cpp @@ -720,3 +720,15 @@ SPACETIMEDB_HTTP_HANDLER(get_simple, HandlerContext ctx, HttpRequest request) { SPACETIMEDB_HTTP_ROUTER(router) { return Router().get("/get", get_simple); } + +SPACETIMEDB_REDUCER(expect_environment, ReducerContext ctx, std::string key, std::optional expected) { + if (ctx.env.get(key) != expected) LOG_PANIC("environment value mismatch"); + return Ok(); +} +SPACETIMEDB_PROCEDURE(std::optional, read_environment, ProcedureContext ctx, std::string key) { + const auto outside = ctx.env.get(key); + ctx.with_tx([&](TxContext& tx) { + if (tx.env.get(key) != outside) LOG_PANIC("transaction environment value mismatch"); + }); + return outside; +} diff --git a/modules/module-test-cs/EnvironmentTests.cs b/modules/module-test-cs/EnvironmentTests.cs new file mode 100644 index 00000000000..1798d7bda44 --- /dev/null +++ b/modules/module-test-cs/EnvironmentTests.cs @@ -0,0 +1,25 @@ +#pragma warning disable STDB_UNSTABLE +namespace SpacetimeDB.Modules.ModuleTestCs; + +using SpacetimeDB; + +public static partial class EnvironmentTests +{ + [Reducer] + public static void expect_environment(ReducerContext ctx, string key, string? expected) + { + if (ctx.Env.Get(key) != expected) throw new Exception("environment value mismatch"); + } + + [Procedure] + public static string? read_environment(ProcedureContext ctx, string key) + { + var outside = ctx.Env.Get(key); + ctx.WithTx(tx => + { + if (tx.Env.Get(key) != outside) throw new Exception("transaction environment value mismatch"); + return true; + }); + return outside; + } +} diff --git a/modules/module-test-ts/src/index.ts b/modules/module-test-ts/src/index.ts index 0eba467a7d0..10520d2b72c 100644 --- a/modules/module-test-ts/src/index.ts +++ b/modules/module-test-ts/src/index.ts @@ -552,3 +552,26 @@ export const libHello = spacetimedb.httpHandler((ctx, req) => { export const router = spacetimedb.httpRouter( new Router().get('/get', getSimple).get('/lib-hello', libHello) ); + +// Dedicated environment ABI integration exercised by crates/testing. +export const expect_environment = spacetimedb.reducer( + { key: t.string(), expected: t.option(t.string()) }, + (ctx, { key, expected }) => { + if (ctx.env.get(key) !== (expected ?? null)) { + throw new Error('environment value mismatch'); + } + } +); +export const read_environment = spacetimedb.procedure( + { key: t.string() }, + t.option(t.string()), + (ctx, { key }) => { + const outside = ctx.env.get(key); + ctx.withTx(tx => { + if (tx.env.get(key) !== outside) { + throw new Error('transaction environment value mismatch'); + } + }); + return outside ?? undefined; + } +); From 4acc884d97c35313c8092f22f877bfe13124bcb9 Mon Sep 17 00:00:00 2001 From: Tyler Cloutier Date: Tue, 8 Sep 2026 14:55:28 -0400 Subject: [PATCH 02/34] Extend V10 visibility and expose trusted invocation authority --- Cargo.lock | 7 + Cargo.toml | 1 + crates/bindings-cpp/README.md | 25 +- .../include/spacetimedb/abi/FFI.h | 1 + .../include/spacetimedb/abi/abi.h | 6 +- .../include/spacetimedb/auth_ctx.h | 90 +++-- .../include/spacetimedb/function_visibility.h | 6 + .../internal/autogen/FunctionVisibility.g.h | 2 + .../autogen/RawModuleDefV10Section.g.h | 2 +- .../spacetimedb/internal/v10_builder.h | 5 +- .../include/spacetimedb/jwt_claims.h | 12 +- .../bindings-cpp/include/spacetimedb/macros.h | 10 +- .../include/spacetimedb/procedure_context.h | 19 +- .../bindings-cpp/src/internal/v10_builder.cpp | 46 ++- crates/bindings-cpp/tests/unit/CMakeLists.txt | 13 +- .../tests/unit/environment_unit_tests.cpp | 50 --- .../unit/function_visibility_unit_tests.cpp | 98 +++++ .../tests/unit/hosted_auth_unit_tests.cpp | 137 +++++++ crates/bindings-csharp/Codegen.Tests/Tests.cs | 59 +++ .../diag/snapshots/Module#FFI.verified.cs | 4 +- .../server/snapshots/Module#FFI.verified.cs | 2 +- crates/bindings-csharp/Codegen/Diag.cs | 9 + crates/bindings-csharp/Codegen/Module.cs | 52 ++- crates/bindings-csharp/README.md | 22 +- .../Runtime.Tests/FunctionVisibilityTests.cs | 68 ++++ .../Runtime.Tests/HostedAuthTests.cs | 43 +++ crates/bindings-csharp/Runtime/Attrs.cs | 12 + crates/bindings-csharp/Runtime/AuthCtx.cs | 34 +- .../Internal/Autogen/FunctionVisibility.g.cs | 2 + .../Autogen/RawModuleDefV10Section.g.cs | 3 +- .../bindings-csharp/Runtime/Internal/FFI.cs | 11 + .../Runtime/Internal/Module.cs | 34 +- crates/bindings-csharp/Runtime/JwtClaims.cs | 4 +- crates/bindings-csharp/Runtime/Runtime.csproj | 1 + crates/bindings-csharp/Runtime/bindings.c | 4 + crates/bindings-macro/src/procedure.rs | 37 +- crates/bindings-macro/src/reducer.rs | 89 +++++ crates/bindings-sys/src/lib.rs | 16 +- crates/bindings-typescript/README.md | 20 + .../src/lib/autogen/types.ts | 3 + .../bindings-typescript/src/lib/reducers.ts | 4 +- crates/bindings-typescript/src/lib/schema.ts | 2 + .../src/server/function_visibility.ts | 24 ++ .../bindings-typescript/src/server/index.ts | 1 + .../src/server/procedures.ts | 38 +- .../src/server/reducers.ts | 18 +- .../bindings-typescript/src/server/runtime.ts | 47 +-- .../bindings-typescript/src/server/schema.ts | 10 +- .../bindings-typescript/src/server/sys.d.ts | 5 +- .../tests/__mocks__/spacetime-auth.ts | 2 + .../tests/hosted_auth.test.ts | 303 +++++++++++++++ crates/bindings-typescript/vitest.config.ts | 4 + crates/bindings/src/http.rs | 8 +- crates/bindings/src/lib.rs | 139 +++++-- crates/bindings/src/rt.rs | 20 +- .../tests/pass/function_visibility.rs | 62 +++ crates/bindings/tests/ui/tables.stderr | 16 +- crates/cli/src/subcommands/generate.rs | 2 +- crates/client-api/src/routes/database.rs | 3 +- crates/client-api/src/routes/mcp.rs | 4 +- crates/codegen/src/util.rs | 88 ++++- crates/core/src/host/host_controller.rs | 3 + .../host_controller/invocation_flags_tests.rs | 139 +++++++ crates/core/src/host/instance_env.rs | 11 + crates/core/src/host/mod.rs | 1 + crates/core/src/host/module_host.rs | 16 +- crates/core/src/host/v8/mod.rs | 2 + crates/core/src/host/v8/syscall/common.rs | 1 + crates/core/src/host/v8/syscall/mod.rs | 2 +- crates/core/src/host/v8/syscall/v1.rs | 1 + crates/core/src/host/v8/syscall/v2.rs | 13 + crates/core/src/host/wasm_common.rs | 2 +- .../src/host/wasm_common/module_host_actor.rs | 16 + .../src/host/wasmtime/wasm_instance_env.rs | 8 + .../core/src/host/wasmtime/wasmtime_module.rs | 2 + crates/lib/src/db/raw_def/v10.rs | 224 ++++++++++- crates/schema/src/auto_migrate.rs | 61 +++ crates/schema/src/auto_migrate/formatter.rs | 9 + .../src/auto_migrate/termcolor_formatter.rs | 9 + crates/schema/src/def.rs | 168 +++++++-- crates/schema/src/def/validate/v10.rs | 355 +++++++++++++++++- crates/schema/src/def/validate/v9.rs | 7 +- crates/schema/src/error.rs | 8 + .../src/subcommands/extract_schema.rs | 4 +- crates/testing/tests/invocation_flags.rs | 76 ++++ modules/invocation-flags-test/Cargo.toml | 13 + modules/invocation-flags-test/src/lib.rs | 78 ++++ modules/module-test-ts/src/index.ts | 2 +- modules/module-test/src/lib.rs | 2 +- modules/sdk-test-procedure-ts/src/index.ts | 2 +- modules/sdk-test-procedure/src/lib.rs | 2 +- .../procedure-client/src/test_handlers.rs | 14 +- 92 files changed, 2754 insertions(+), 356 deletions(-) create mode 100644 crates/bindings-cpp/include/spacetimedb/function_visibility.h delete mode 100644 crates/bindings-cpp/tests/unit/environment_unit_tests.cpp create mode 100644 crates/bindings-cpp/tests/unit/function_visibility_unit_tests.cpp create mode 100644 crates/bindings-cpp/tests/unit/hosted_auth_unit_tests.cpp create mode 100644 crates/bindings-csharp/Runtime.Tests/FunctionVisibilityTests.cs create mode 100644 crates/bindings-csharp/Runtime.Tests/HostedAuthTests.cs create mode 100644 crates/bindings-typescript/src/server/function_visibility.ts create mode 100644 crates/bindings-typescript/tests/__mocks__/spacetime-auth.ts create mode 100644 crates/bindings-typescript/tests/hosted_auth.test.ts create mode 100644 crates/bindings/tests/pass/function_visibility.rs create mode 100644 crates/core/src/host/host_controller/invocation_flags_tests.rs create mode 100644 crates/testing/tests/invocation_flags.rs create mode 100644 modules/invocation-flags-test/Cargo.toml create mode 100644 modules/invocation-flags-test/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 9b030bc73ff..65172309213 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3608,6 +3608,13 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "invocation-flags-test" +version = "0.0.0" +dependencies = [ + "spacetimedb", +] + [[package]] name = "ipnet" version = "2.11.0" diff --git a/Cargo.toml b/Cargo.toml index afdde4e3253..ba682e45ba5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -47,6 +47,7 @@ members = [ "modules/perf-test", "modules/module-test", "modules/environment-test", + "modules/invocation-flags-test", "templates/basic-rs/spacetimedb", "templates/chat-console-rs/spacetimedb", "modules/sdk-test", diff --git a/crates/bindings-cpp/README.md b/crates/bindings-cpp/README.md index ef31361c10b..4fbfc2a2d86 100644 --- a/crates/bindings-cpp/README.md +++ b/crates/bindings-cpp/README.md @@ -2,6 +2,30 @@ The SpacetimeDB C++ Module Library provides a modern C++20 API for building SpacetimeDB modules that run inside the database as WebAssembly. +## Function visibility and invocation authentication + +Apply `SPACETIMEDB_FUNCTION_VISIBILITY(name, Public)`, `Private`, or `Internal` +to a reducer or procedure after its definition: + +```cpp +SPACETIMEDB_REDUCER(process_jobs, ReducerContext ctx) { + return Ok(); +} +SPACETIMEDB_FUNCTION_VISIBILITY(process_jobs, Internal); +``` + +Omission means public for ordinary functions and private for scheduled functions. +An explicit choice is preserved when the function is scheduled. Lifecycle +reducers permit only omission or `Internal` and can only run for their host +lifecycle event. Internal functions require verified internal authority. Private +functions also admit the owner, and public functions admit any client. + +`ctx.sender_auth().is_internal()` captures the host's invocation authority. It is +independent of connection and JWT presence, so an internal call can have a JWT. +JWT identity is the verified sender supplied by the host. Procedures preserve +this authentication in `with_tx` and `try_with_tx`. Newly compiled modules emit +schema V10 and advertise `hosted_auth_v1`, requiring a compatible host. + ## Current State This library provides a production-ready C++ bindings for SpacetimeDB with complete type system support: @@ -274,4 +298,3 @@ See the `modules/*-cpp/src/` directory for example modules: ## Contributing This library is part of the SpacetimeDB project. Please see the main repository for contribution guidelines. - diff --git a/crates/bindings-cpp/include/spacetimedb/abi/FFI.h b/crates/bindings-cpp/include/spacetimedb/abi/FFI.h index f3ad213452f..9133eed5863 100644 --- a/crates/bindings-cpp/include/spacetimedb/abi/FFI.h +++ b/crates/bindings-cpp/include/spacetimedb/abi/FFI.h @@ -74,6 +74,7 @@ using ::identity; // ===== JWT ===== using ::get_jwt; using ::env_get; +using ::get_call_auth_flags; // ===== Procedure Transactions ===== using ::procedure_start_mut_tx; diff --git a/crates/bindings-cpp/include/spacetimedb/abi/abi.h b/crates/bindings-cpp/include/spacetimedb/abi/abi.h index 285974cff3c..02791f73126 100644 --- a/crates/bindings-cpp/include/spacetimedb/abi/abi.h +++ b/crates/bindings-cpp/include/spacetimedb/abi/abi.h @@ -39,9 +39,10 @@ #define STDB_IMPORT_10_5(name) \ __attribute__((import_module("spacetime_10.5"), import_name(#name))) extern -// ABI10.6 is reserved for the separate invocation-authority extension. #define STDB_IMPORT_10_7(name) \ __attribute__((import_module("spacetime_10.7"), import_name(#name))) extern +#define STDB_IMPORT_10_6(name) \ + __attribute__((import_module("spacetime_10.6"), import_name(#name))) extern // Import opaque types into global namespace for C compatibility using SpacetimeDB::Status; @@ -65,6 +66,9 @@ extern "C" { STDB_IMPORT_10_7(env_get) Status env_get(const uint8_t* key, uint32_t key_len, BytesSource* out); +// Verified invocation authority. Bit 0 is INTERNAL; JWT presence is independent. +STDB_IMPORT_10_6(get_call_auth_flags) +uint32_t get_call_auth_flags(); // ===== Table and Index Management ===== STDB_IMPORT(table_id_from_name) diff --git a/crates/bindings-cpp/include/spacetimedb/auth_ctx.h b/crates/bindings-cpp/include/spacetimedb/auth_ctx.h index 00a4898e020..a8ae7fc62c9 100644 --- a/crates/bindings-cpp/include/spacetimedb/auth_ctx.h +++ b/crates/bindings-cpp/include/spacetimedb/auth_ctx.h @@ -28,22 +28,25 @@ struct ConnectionId; class AuthCtx { private: bool is_internal_; + std::optional verified_sender_; mutable std::shared_ptr> jwt_; std::function()> jwt_loader_; // Private constructor used by factory methods - AuthCtx(bool is_internal, std::function()> loader); + AuthCtx(bool is_internal, std::function()> loader, + std::optional verified_sender = std::nullopt); + static AuthCtx from_connection_with_flags(ConnectionId connection_id, Identity sender, uint32_t flags); public: /** * @brief Creates an AuthCtx from an optional ConnectionId. * * If the connection_id is present, creates an AuthCtx that will load the JWT. - * If the connection_id is absent, creates an internal AuthCtx. + * Internal authority is captured from the host, independently of connection presence. * * @param connection_id Optional connection ID - * @param sender The identity of the caller (already derived from JWT claims by the host) - * @return An AuthCtx based on the connection_id + * @param sender The verified caller Identity supplied by the host + * @return An AuthCtx with captured invocation authority and lazy JWT loading */ static AuthCtx from_connection_id_opt(std::optional connection_id, Identity sender); @@ -63,11 +66,10 @@ class AuthCtx { * This is primarily used for testing purposes, allowing you to create * an AuthCtx with specific JWT claims without needing a real connection. * - * Note: The Identity must be computed by calling the host function, - * as we cannot compute Blake3 hashes in WASM. + * The Identity must be the verified sender supplied by the host. * * @param jwt_payload The raw JWT payload (JSON claims) - * @param identity The identity derived from the JWT's issuer and subject + * @param identity The verified sender Identity * @return An AuthCtx with the provided JWT */ static AuthCtx from_jwt_payload(std::string jwt_payload, Identity identity); @@ -76,11 +78,10 @@ class AuthCtx { * @brief Creates an AuthCtx that reads the JWT for the given connection ID. * * The JWT will be lazily loaded from the host when first accessed. - * The identity parameter is the sender's identity, already derived from - * JWT claims by the host (using Blake3 hashing). + * The identity parameter is the verified sender supplied by the host. * * @param connection_id The connection ID to load the JWT for - * @param sender The identity of the caller (already derived from JWT claims by the host) + * @param sender The verified sender Identity supplied by the host * @return An AuthCtx that will load the JWT on demand */ static AuthCtx from_connection_id(ConnectionId connection_id, Identity sender); @@ -93,9 +94,9 @@ class AuthCtx { bool is_internal() const { return is_internal_; } /** - * @brief Checks if there is a JWT without loading it. + * @brief Checks if there is a JWT, loading it lazily if necessary. * - * If is_internal() returns true, this will return false. + * Independent of is_internal(). Internal calls can also have a JWT. * * @return true if a JWT is available */ @@ -113,9 +114,8 @@ class AuthCtx { /** * @brief Gets the caller's identity. * - * For internal calls, this returns the database's identity. - * For external calls, this returns the identity derived from the JWT - * (based on the issuer and subject claims). + * Returns the verified sender captured when constructing the context, + * independently of JWT presence or token claims. * * @return The caller's Identity */ @@ -126,16 +126,16 @@ class AuthCtx { // INLINE IMPLEMENTATIONS // ============================================================================ -constexpr uint16_t ERROR_BUFFER_TOO_SMALL = 11; - -inline AuthCtx::AuthCtx(bool is_internal, std::function()> loader) - : is_internal_(is_internal), jwt_loader_(std::move(loader)) {} +inline AuthCtx::AuthCtx(bool is_internal, std::function()> loader, + std::optional verified_sender) + : is_internal_(is_internal), verified_sender_(std::move(verified_sender)), jwt_loader_(std::move(loader)) {} inline AuthCtx AuthCtx::from_connection_id_opt(std::optional connection_id, Identity sender) { + const auto flags = FFI::get_call_auth_flags(); if (connection_id.has_value()) { - return from_connection_id(*connection_id, std::move(sender)); + return from_connection_with_flags(*connection_id, std::move(sender), flags); } else { - return internal(); + return AuthCtx((flags & 1) != 0, []() -> std::optional { return std::nullopt; }, sender); } } @@ -144,13 +144,17 @@ inline AuthCtx AuthCtx::internal() { } inline AuthCtx AuthCtx::from_jwt_payload(std::string jwt_payload, Identity identity) { - return AuthCtx(false, [payload = std::move(jwt_payload), id = std::move(identity)]() mutable -> std::optional { + return AuthCtx(false, [payload = std::move(jwt_payload), id = identity]() mutable -> std::optional { return JwtClaims(std::move(payload), std::move(id)); - }); + }, identity); } inline AuthCtx AuthCtx::from_connection_id(ConnectionId connection_id, Identity sender) { - return AuthCtx(false, [connection_id, sender]() -> std::optional { + return from_connection_with_flags(connection_id, std::move(sender), FFI::get_call_auth_flags()); +} + +inline AuthCtx AuthCtx::from_connection_with_flags(ConnectionId connection_id, Identity sender, uint32_t flags) { + return AuthCtx((flags & 1) != 0, [connection_id, sender]() -> std::optional { // Call the host FFI to get the JWT BytesSource jwt_source; @@ -169,35 +173,24 @@ inline AuthCtx AuthCtx::from_connection_id(ConnectionId connection_id, Identity } // Read the JWT payload from the BytesSource - std::vector buffer; - buffer.resize(4096); // Start with 4KB buffer - - size_t buffer_len = buffer.size(); - int16_t result = bytes_source_read(jwt_source, buffer.data(), &buffer_len); - - while (result == ERROR_BUFFER_TOO_SMALL) { - buffer.resize(buffer.size() * 2); - buffer_len = buffer.size(); - result = bytes_source_read(jwt_source, buffer.data(), &buffer_len); + std::array buffer; + std::string jwt_payload; + for (;;) { + size_t buffer_len = buffer.size(); + const auto result = bytes_source_read(jwt_source, buffer.data(), &buffer_len); + if (result != 0 && result != -1) return std::nullopt; + jwt_payload.append(reinterpret_cast(buffer.data()), buffer_len); + // -1 is successful exhaustion and may include the final payload bytes. + if (result == -1) break; + if (buffer_len == 0) return std::nullopt; } - - if (result < 0) { - return std::nullopt; - } - - // Convert bytes to string - std::string jwt_payload(buffer.begin(), buffer.begin() + buffer_len); - - // Use the provided sender identity (already computed by host from JWT claims) + if (jwt_payload.empty()) return std::nullopt; + // Token claims cannot override the verified sender, including hosted tokens. return JwtClaims(std::move(jwt_payload), sender); - }); + }, sender); } inline bool AuthCtx::has_jwt() const { - if (is_internal_) { - return false; - } - // Load the JWT if not already loaded, then check if it has a value // This ensures has_jwt() and get_jwt() are consistent return get_jwt().has_value(); @@ -211,6 +204,7 @@ inline const std::optional& AuthCtx::get_jwt() const { } inline Identity AuthCtx::get_caller_identity() const { + if (verified_sender_.has_value()) return *verified_sender_; if (is_internal_) { // Return database identity for internal calls std::array identity_bytes; diff --git a/crates/bindings-cpp/include/spacetimedb/function_visibility.h b/crates/bindings-cpp/include/spacetimedb/function_visibility.h new file mode 100644 index 00000000000..9bd36e19e48 --- /dev/null +++ b/crates/bindings-cpp/include/spacetimedb/function_visibility.h @@ -0,0 +1,6 @@ +#pragma once + +namespace SpacetimeDB { +// Omission preserves the host default: Public ordinarily, Private when scheduled. +enum class FunctionVisibility { Public, Private, Internal }; +} diff --git a/crates/bindings-cpp/include/spacetimedb/internal/autogen/FunctionVisibility.g.h b/crates/bindings-cpp/include/spacetimedb/internal/autogen/FunctionVisibility.g.h index 423276de9b4..9795b3bd2d7 100644 --- a/crates/bindings-cpp/include/spacetimedb/internal/autogen/FunctionVisibility.g.h +++ b/crates/bindings-cpp/include/spacetimedb/internal/autogen/FunctionVisibility.g.h @@ -18,5 +18,7 @@ namespace SpacetimeDB::Internal { enum class FunctionVisibility : uint8_t { Private = 0, ClientCallable = 1, + Internal = 2, + ExplicitClientCallable = 3, }; } // namespace SpacetimeDB::Internal diff --git a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDefV10Section.g.h b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDefV10Section.g.h index ea2e4b5ec85..d7002058c59 100644 --- a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDefV10Section.g.h +++ b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDefV10Section.g.h @@ -30,5 +30,5 @@ namespace SpacetimeDB::Internal { -SPACETIMEDB_INTERNAL_TAGGED_ENUM(RawModuleDefV10Section, SpacetimeDB::Internal::Typespace, std::vector, std::vector, std::vector, std::vector, std::vector, std::vector, std::vector, std::vector, SpacetimeDB::Internal::CaseConversionPolicy, SpacetimeDB::Internal::ExplicitNames, std::vector, std::vector, std::vector, std::vector) +SPACETIMEDB_INTERNAL_TAGGED_ENUM(RawModuleDefV10Section, SpacetimeDB::Internal::Typespace, std::vector, std::vector, std::vector, std::vector, std::vector, std::vector, std::vector, std::vector, SpacetimeDB::Internal::CaseConversionPolicy, SpacetimeDB::Internal::ExplicitNames, std::vector, std::vector, std::vector, std::vector, std::vector) } // namespace SpacetimeDB::Internal diff --git a/crates/bindings-cpp/include/spacetimedb/internal/v10_builder.h b/crates/bindings-cpp/include/spacetimedb/internal/v10_builder.h index 235b5e5f680..f398a4eb9c2 100644 --- a/crates/bindings-cpp/include/spacetimedb/internal/v10_builder.h +++ b/crates/bindings-cpp/include/spacetimedb/internal/v10_builder.h @@ -13,6 +13,7 @@ #include #include "../bsatn/bsatn.h" #include "../database.h" +#include "../function_visibility.h" #include "autogen/CaseConversionPolicy.g.h" #include "autogen/ExplicitNameEntry.g.h" #include "autogen/NameMapping.g.h" @@ -49,6 +50,7 @@ void fail_reducer(std::string message); namespace Internal { +// Builds the V10 module definition with explicit function visibility. class V10Builder { public: V10Builder() = default; @@ -437,7 +439,7 @@ class V10Builder { RawReducerDefV10 reducer_def{ reducer_name, ProductType{}, - FunctionVisibility::Private, + FunctionVisibility::Internal, MakeUnitAlgebraicType(), MakeStringAlgebraicType(), }; @@ -646,6 +648,7 @@ class V10Builder { void RegisterExplicitTableName(const std::string& source_name, const std::string& canonical_name); void RegisterExplicitFunctionName(const std::string& source_name, const std::string& canonical_name); + void SetFunctionVisibility(const std::string& source_name, ::SpacetimeDB::FunctionVisibility visibility); void RegisterExplicitIndexName(const std::string& source_name, const std::string& canonical_name); RawModuleDefV10 BuildModuleDef() const; diff --git a/crates/bindings-cpp/include/spacetimedb/jwt_claims.h b/crates/bindings-cpp/include/spacetimedb/jwt_claims.h index cdc9aef511d..6a72e973633 100644 --- a/crates/bindings-cpp/include/spacetimedb/jwt_claims.h +++ b/crates/bindings-cpp/include/spacetimedb/jwt_claims.h @@ -15,8 +15,8 @@ namespace SpacetimeDB { * This class provides lazy parsing of JWT claims, parsing specific fields * on demand. It follows the same pattern as the Rust and C# implementations. * - * The Identity is provided in the constructor because computing it requires - * Blake3 hashing, which is done on the host side. + * The Identity is the verified sender supplied by the host. Token claims + * cannot override it, including for hosted container credentials. */ class JwtClaims { private: @@ -36,11 +36,10 @@ class JwtClaims { /** * @brief Constructs a JwtClaims from a JWT payload and its associated Identity. * - * The Identity must be provided because computing it requires Blake3 hashing, - * which is performed on the host side. + * The Identity must be the verified sender supplied by the host. * * @param jwt_payload The raw JWT payload (JSON claims) - * @param identity The identity derived from the JWT's issuer and subject + * @param identity The verified sender Identity */ JwtClaims(std::string jwt_payload, Identity identity); @@ -71,8 +70,7 @@ class JwtClaims { /** * @brief Returns the identity for these credentials. * - * The identity is based on the 'iss' and 'sub' claims and is computed - * using Blake3 hashing on the host side. + * This is the verified sender supplied by the host, independently of claims. * * @return The identity */ diff --git a/crates/bindings-cpp/include/spacetimedb/macros.h b/crates/bindings-cpp/include/spacetimedb/macros.h index 2807be4333b..b4ac3ba0d9c 100644 --- a/crates/bindings-cpp/include/spacetimedb/macros.h +++ b/crates/bindings-cpp/include/spacetimedb/macros.h @@ -609,6 +609,15 @@ inline std::vector parseParameterNames(const std::string& param_lis // VISIBILITY FILTER MACRO // ============================================================================= +// Apply to a registered reducer or procedure. Runs after function registration; +// lifecycle reducers only accept Internal. Scheduling preserves this choice. +#define SPACETIMEDB_FUNCTION_VISIBILITY(function_name, visibility) \ + extern "C" __attribute__((export_name("__preinit__40_visibility_" #function_name))) \ + void CONCAT(__spacetimedb_function_visibility_, function_name)() { \ + ::SpacetimeDB::Internal::getV10Builder().SetFunctionVisibility( \ + #function_name, ::SpacetimeDB::FunctionVisibility::visibility); \ + } + /** * @brief Set module case conversion policy using a fixed preinit registration symbol. * @@ -917,4 +926,3 @@ inline std::vector parseParameterNames(const std::string& param_lis #endif // SPACETIMEDB_MACROS_H - diff --git a/crates/bindings-cpp/include/spacetimedb/procedure_context.h b/crates/bindings-cpp/include/spacetimedb/procedure_context.h index 35c93c31475..ee459ab980d 100644 --- a/crates/bindings-cpp/include/spacetimedb/procedure_context.h +++ b/crates/bindings-cpp/include/spacetimedb/procedure_context.h @@ -57,6 +57,7 @@ struct ProcedureContext { private: // Caller's identity - who invoked this procedure Identity sender_; + AuthCtx sender_auth_ = AuthCtx::internal(); public: Environment env; @@ -83,7 +84,11 @@ struct ProcedureContext { ProcedureContext() = default; ProcedureContext(Identity s, Timestamp t, ConnectionId conn_id) - : sender_(s), timestamp(t), connection_id(conn_id) {} + : sender_(s), sender_auth_(AuthCtx::from_connection_id_opt( + conn_id.id.low == 0 && conn_id.id.high == 0 ? std::nullopt : std::optional(conn_id), s)), + timestamp(t), connection_id(conn_id) {} + + const AuthCtx& sender_auth() const { return sender_auth_; } Identity sender() const { return sender_; @@ -99,7 +104,7 @@ struct ProcedureContext { * @code * auto module_id = ctx.database_identity(); * std::string url = "http://localhost:3000/v1/database/" + - * module_id.to_hex() + "/schema?version=9"; + * module_id.to_hex_string() + "/schema?version=10"; * @endcode */ Identity database_identity() const { @@ -198,8 +203,9 @@ struct ProcedureContext { auto make_reducer_ctx = [this](Timestamp tx_timestamp) { return ReducerContext( sender(), - std::optional(connection_id), - tx_timestamp + connection_id.id.low == 0 && connection_id.id.high == 0 ? std::nullopt : std::optional(connection_id), + tx_timestamp, + sender_auth_ ); }; return Internal::with_tx(make_reducer_ctx, body); @@ -230,8 +236,9 @@ struct ProcedureContext { auto make_reducer_ctx = [this](Timestamp tx_timestamp) { return ReducerContext( sender(), - std::optional(connection_id), - tx_timestamp + connection_id.id.low == 0 && connection_id.id.high == 0 ? std::nullopt : std::optional(connection_id), + tx_timestamp, + sender_auth_ ); }; return Internal::try_with_tx(make_reducer_ctx, body); diff --git a/crates/bindings-cpp/src/internal/v10_builder.cpp b/crates/bindings-cpp/src/internal/v10_builder.cpp index a931b79a7c2..3298f19da3d 100644 --- a/crates/bindings-cpp/src/internal/v10_builder.cpp +++ b/crates/bindings-cpp/src/internal/v10_builder.cpp @@ -219,6 +219,31 @@ RawConstraintDefV10 V10Builder::CreateUniqueConstraint(const std::string& table_ }; } +void V10Builder::SetFunctionVisibility(const std::string& name, ::SpacetimeDB::FunctionVisibility visibility) { + FunctionVisibility declared; + switch (visibility) { + case ::SpacetimeDB::FunctionVisibility::Public: declared = FunctionVisibility::ExplicitClientCallable; break; + case ::SpacetimeDB::FunctionVisibility::Private: declared = FunctionVisibility::Private; break; + case ::SpacetimeDB::FunctionVisibility::Internal: declared = FunctionVisibility::Internal; break; + default: + SetConstraintRegistrationError("INVALID_FUNCTION_VISIBILITY", "function='" + name + "'"); + return; + } + for (const auto& lifecycle : lifecycle_reducers_) { + if (lifecycle.function_name == name && declared != FunctionVisibility::Internal) { + SetConstraintRegistrationError("INVALID_LIFECYCLE_VISIBILITY", "function='" + name + "' must be Internal"); + return; + } + } + for (auto& reducer : reducers_) { + if (reducer.source_name == name) { reducer.visibility = declared; return; } + } + for (auto& procedure : procedures_) { + if (procedure.source_name == name) { procedure.visibility = declared; return; } + } + SetConstraintRegistrationError("UNKNOWN_FUNCTION_VISIBILITY", "function='" + name + "' is not a reducer or procedure"); +} + RawModuleDefV10 V10Builder::BuildModuleDef() const { RawModuleDefV10 v10_module; @@ -227,27 +252,12 @@ RawModuleDefV10 V10Builder::BuildModuleDef() const { std::vector reducers = reducers_; std::vector procedures = procedures_; - std::unordered_set internal_functions; - for (const auto& lifecycle : lifecycle_reducers_) { - internal_functions.insert(lifecycle.function_name); - } - for (const auto& schedule : schedules_) { - internal_functions.insert(schedule.function_name); - } - for (auto& reducer : reducers) { - if (internal_functions.find(reducer.source_name) != internal_functions.end()) { - reducer.visibility = FunctionVisibility::Private; - } - } - for (auto& procedure : procedures) { - if (internal_functions.find(procedure.source_name) != internal_functions.end()) { - procedure.visibility = FunctionVisibility::Private; - } - } - RawModuleDefV10Section section_typespace; section_typespace.set<0>(typespace_); v10_module.sections.push_back(section_typespace); + RawModuleDefV10Section capabilities; + capabilities.set<15>(std::vector{"hosted_auth_v1"}); + v10_module.sections.push_back(std::move(capabilities)); if (!types.empty()) { RawModuleDefV10Section section_types; diff --git a/crates/bindings-cpp/tests/unit/CMakeLists.txt b/crates/bindings-cpp/tests/unit/CMakeLists.txt index da7b8705e1c..2a540f96f31 100644 --- a/crates/bindings-cpp/tests/unit/CMakeLists.txt +++ b/crates/bindings-cpp/tests/unit/CMakeLists.txt @@ -11,7 +11,18 @@ endif() add_executable(bindings_cpp_unit_tests main.cpp http_unit_tests.cpp - environment_unit_tests.cpp + hosted_auth_unit_tests.cpp + function_visibility_unit_tests.cpp +) + +# Exercise the real module builder without the standalone WASI shims, which +# replace the Node test runner's standard I/O and process lifecycle functions. +target_sources(bindings_cpp_unit_tests PRIVATE + ../../src/internal/Module.cpp + ../../src/internal/AlgebraicType.cpp + ../../src/internal/v9_builder.cpp + ../../src/internal/v10_builder.cpp + ../../src/internal/module_type_registration.cpp ) target_include_directories(bindings_cpp_unit_tests PRIVATE diff --git a/crates/bindings-cpp/tests/unit/environment_unit_tests.cpp b/crates/bindings-cpp/tests/unit/environment_unit_tests.cpp deleted file mode 100644 index c6bd0f58cd8..00000000000 --- a/crates/bindings-cpp/tests/unit/environment_unit_tests.cpp +++ /dev/null @@ -1,50 +0,0 @@ -#include "test_harness.h" -#include "spacetimedb/environment.h" -#include "spacetimedb/bsatn/reader.h" -#include -#include - -using namespace SpacetimeDB; - -namespace { -size_t payload_offset; -std::string payload; -} - -extern "C" Status env_get(const uint8_t* key, uint32_t key_len, BytesSource* out) { - payload_offset = 0; - const std::string name(reinterpret_cast(key), key_len); - *out = BytesSource{name == "MISSING" ? 0u : 1u}; - return Status{0}; -} - -extern "C" int16_t bytes_source_read(BytesSource, uint8_t* out, size_t* len) { - *len = std::min(*len, payload.size() - payload_offset); - std::memcpy(out, payload.data() + payload_offset, *len); - payload_offset += *len; - return payload_offset == payload.size() ? -1 : 0; -} - -extern "C" void console_log(LogLevel, const uint8_t*, size_t, const uint8_t*, size_t, - uint32_t, const uint8_t*, size_t) {} - -TEST_CASE(environment_preserves_missing_empty_and_all_chunks_without_caching) { - Environment env; - ASSERT_TRUE(!env.get("MISSING").has_value()); - ASSERT_EQ(std::string{}, env.get("EMPTY").value()); - payload = std::string(8192, 'x'); - ASSERT_EQ(payload, env.get("LARGE").value()); - payload = std::string("a\0b", 3); - ASSERT_EQ(payload, env.get("NUL").value()); - payload = "updated"; - ASSERT_EQ(payload, env.get("NUL").value()); -} - -TEST_CASE(optional_reader_matches_canonical_bsatn_tags_and_preserves_following_bytes) { - const std::vector bytes{1, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 'a', 0, 'b', 42}; - bsatn::Reader reader(bytes.data(), bytes.size()); - ASSERT_TRUE(!bsatn::deserialize>(reader).has_value()); - ASSERT_EQ(std::string{}, bsatn::deserialize>(reader).value()); - ASSERT_EQ(std::string("a\0b", 3), bsatn::deserialize>(reader).value()); - ASSERT_EQ(uint8_t{42}, reader.read_u8()); -} diff --git a/crates/bindings-cpp/tests/unit/function_visibility_unit_tests.cpp b/crates/bindings-cpp/tests/unit/function_visibility_unit_tests.cpp new file mode 100644 index 00000000000..c9dc94ba3ce --- /dev/null +++ b/crates/bindings-cpp/tests/unit/function_visibility_unit_tests.cpp @@ -0,0 +1,98 @@ +#include "test_harness.h" +#include "spacetimedb/reducer_error.h" +#include "spacetimedb/procedure_context.h" +#include "spacetimedb/internal/v10_builder.h" +#include "spacetimedb/internal/autogen/RawModuleDef.g.h" +#include "spacetimedb/macros.h" + +using namespace SpacetimeDB; +using namespace SpacetimeDB::Internal; + +namespace { +ReducerResult noop(ReducerContext) { return Ok(); } +uint32_t procedure(ProcedureContext) { return 7; } +} + +SPACETIMEDB_FUNCTION_VISIBILITY(visibility_macro_target, Internal); + +TEST_CASE(visibility_macro_applies_after_function_registration) { + auto& builder = getV10Builder(); + builder.RegisterReducer("visibility_macro_target", &noop, {}); + __spacetimedb_function_visibility_visibility_macro_target(); + bool found = false; + for (const auto& section : builder.BuildModuleDef().sections) { + if (section.get_tag() != 3) continue; + for (const auto& reducer : section.get<3>()) { + if (reducer.source_name != "visibility_macro_target") continue; + ASSERT_EQ(SpacetimeDB::Internal::FunctionVisibility::Internal, reducer.visibility); + found = true; + } + } + ASSERT_TRUE(found); +} + +TEST_CASE(v10_retains_explicit_visibility_and_schedule_default) { + V10Builder builder; + builder.RegisterReducer("omitted", &noop, {}); + builder.RegisterReducer("public", &noop, {}); + builder.RegisterReducer("private", &noop, {}); + builder.RegisterReducer("internal", &noop, {}); + builder.SetFunctionVisibility("public", SpacetimeDB::FunctionVisibility::Public); + builder.SetFunctionVisibility("private", SpacetimeDB::FunctionVisibility::Private); + builder.SetFunctionVisibility("internal", SpacetimeDB::FunctionVisibility::Internal); + builder.RegisterSchedule("jobs", 0, "public"); + builder.RegisterSchedule("other_jobs", 0, "omitted"); + builder.RegisterProcedure("procedure", &procedure); + builder.SetFunctionVisibility("procedure", SpacetimeDB::FunctionVisibility::Internal); + + RawModuleDef versioned; + versioned.set<2>(builder.BuildModuleDef()); + std::vector bytes; + bsatn::Writer writer(bytes); + bsatn::serialize(writer, versioned); + ASSERT_EQ(uint8_t{2}, bytes.at(0)); + ASSERT_EQ(uint8_t{2}, versioned.get_tag()); + bool saw_reducers = false, saw_procedure = false, saw_capability = false; + for (const auto& section : versioned.get<2>().sections) { + if (section.get_tag() == 3) { + const auto& reducers = section.get<3>(); + ASSERT_EQ(size_t{4}, reducers.size()); + ASSERT_EQ(SpacetimeDB::Internal::FunctionVisibility::ClientCallable, reducers[0].visibility); + ASSERT_EQ(SpacetimeDB::Internal::FunctionVisibility::ExplicitClientCallable, reducers[1].visibility); + ASSERT_EQ(SpacetimeDB::Internal::FunctionVisibility::Private, reducers[2].visibility); + ASSERT_EQ(SpacetimeDB::Internal::FunctionVisibility::Internal, reducers[3].visibility); + saw_reducers = true; + } else if (section.get_tag() == 4) { + ASSERT_EQ(SpacetimeDB::Internal::FunctionVisibility::Internal, section.get<4>().at(0).visibility); + saw_procedure = true; + } else if (section.get_tag() == 15) { + ASSERT_EQ(std::vector{"hosted_auth_v1"}, section.get<15>()); + saw_capability = true; + } + } + ASSERT_TRUE(saw_reducers && saw_procedure && saw_capability); +} + +TEST_CASE(v10_visibility_extends_enum_without_changing_reducer_field_layout) { + V10Builder builder; + builder.RegisterReducer("r", &noop, {}); + auto reducer = builder.GetReducers().at(0); + for (uint8_t tag = 0; tag <= 3; ++tag) { + reducer.visibility = static_cast(tag); + std::vector bytes; + bsatn::Writer writer(bytes); + bsatn::serialize(writer, reducer); + const std::vector expected{1, 0, 0, 0, 'r', 0, 0, 0, 0, tag, 2, 0, 0, 0, 0, 4}; + ASSERT_EQ(expected, bytes); + const RawProcedureDefV10 procedure_def{ + "p", ProductType{}, reducer.ok_return_type, reducer.visibility, + }; + std::vector procedure_bytes; + bsatn::Writer procedure_writer(procedure_bytes); + bsatn::serialize(procedure_writer, procedure_def); + const std::vector expected_procedure{ + 1, 0, 0, 0, 'p', 0, 0, 0, 0, 2, 0, 0, 0, 0, tag, + }; + ASSERT_EQ(expected_procedure, procedure_bytes); + } +} diff --git a/crates/bindings-cpp/tests/unit/hosted_auth_unit_tests.cpp b/crates/bindings-cpp/tests/unit/hosted_auth_unit_tests.cpp new file mode 100644 index 00000000000..cfa432bf30b --- /dev/null +++ b/crates/bindings-cpp/tests/unit/hosted_auth_unit_tests.cpp @@ -0,0 +1,137 @@ +#include "test_harness.h" +#include "spacetimedb/procedure_context.h" + +#include +#include + +using namespace SpacetimeDB; + +namespace { +uint32_t auth_flags; +size_t flag_reads; +size_t jwt_reads; +size_t payload_offset; +std::string jwt_payload; + +Identity verified_sender() { + std::array bytes{}; + bytes[0] = 42; + return Identity(bytes); +} + +void reset_host(uint32_t flags, std::string payload = {}) { + auth_flags = flags; + flag_reads = jwt_reads = payload_offset = 0; + jwt_payload = std::move(payload); +} +} + +extern "C" uint32_t get_call_auth_flags() { + ++flag_reads; + return auth_flags; +} + +extern "C" Status get_jwt(const uint8_t*, BytesSource* out) { + ++jwt_reads; + payload_offset = 0; + *out = BytesSource{jwt_payload.empty() ? 0u : 1u}; + return Status{0}; +} + +extern "C" Status env_get(const uint8_t* key, uint32_t key_len, BytesSource* out) { + payload_offset = 0; + const std::string name(reinterpret_cast(key), key_len); + if (name == "ERROR") return Status{1}; + *out = BytesSource{name == "MISSING" ? 0u : 1u}; + return Status{0}; +} + +extern "C" int16_t bytes_source_read(BytesSource, uint8_t* out, size_t* len) { + *len = std::min(*len, jwt_payload.size() - payload_offset); + std::memcpy(out, jwt_payload.data() + payload_offset, *len); + payload_offset += *len; + // Successful exhaustion can return the last bytes together with -1. + return payload_offset == jwt_payload.size() ? -1 : 0; +} + +extern "C" void identity(uint8_t* out) { std::memset(out, 0, 32); } +extern "C" Status procedure_start_mut_tx(int64_t* out) { *out = 0; return Status{0}; } +extern "C" Status procedure_commit_mut_tx() { return Status{0}; } +extern "C" Status procedure_abort_mut_tx() { return Status{0}; } +extern "C" void console_log(LogLevel, const uint8_t*, size_t, const uint8_t*, size_t, + uint32_t, const uint8_t*, size_t) {} + +TEST_CASE(authority_without_connection_is_captured_from_host) { + for (uint32_t flags : {0u, 1u}) { + reset_host(flags); + auto ctx = AuthCtx::from_connection_id_opt(std::nullopt, verified_sender()); + auth_flags = flags ^ 1; + ASSERT_EQ(size_t{1}, flag_reads); + ASSERT_EQ(flags == 1, ctx.is_internal()); + ASSERT_TRUE(!ctx.has_jwt()); + ASSERT_EQ(verified_sender(), ctx.get_caller_identity()); + ASSERT_EQ(size_t{0}, jwt_reads); + } +} + +TEST_CASE(internal_call_retains_lazy_jwt_and_verified_identity) { + reset_host(1, R"({"iss":"other","sub":"other","identity":"untrusted"})"); + auto ctx = AuthCtx::from_connection_id(ConnectionId(5), verified_sender()); + auth_flags = 0; + ASSERT_TRUE(ctx.is_internal()); + ASSERT_EQ(size_t{0}, jwt_reads); + ASSERT_TRUE(ctx.has_jwt()); + ASSERT_EQ(verified_sender(), ctx.get_jwt()->get_identity()); + ASSERT_EQ(std::string("other"), ctx.get_jwt()->subject()); + ASSERT_EQ(size_t{1}, jwt_reads); +} + +TEST_CASE(jwt_source_reads_all_chunks_including_final_exhausted_bytes) { + reset_host(0, "{\"padding\":\"" + std::string(8192, 'x') + "\",\"sub\":\"last\"}"); + auto ctx = AuthCtx::from_connection_id(ConnectionId(5), verified_sender()); + ASSERT_TRUE(ctx.has_jwt()); + ASSERT_EQ(std::string("last"), ctx.get_jwt()->subject()); + ASSERT_EQ(jwt_payload.size(), payload_offset); + ASSERT_EQ(size_t{1}, jwt_reads); +} + +TEST_CASE(procedure_transactions_preserve_authority_connection_and_sender) { + for (uint64_t connection : {0u, 5u}) { + reset_host(1, R"({"sub":"worker"})"); + ProcedureContext ctx(verified_sender(), Timestamp::from_micros_since_epoch(0), ConnectionId(connection)); + auth_flags = 0; + ctx.with_tx([&](TxContext& tx) { + ASSERT_TRUE(tx.sender_auth().is_internal()); + ASSERT_EQ(verified_sender(), tx.sender()); + ASSERT_EQ(connection != 0, tx.connection_id.has_value()); + ASSERT_EQ(connection != 0, tx.sender_auth().has_jwt()); + if (connection) ASSERT_EQ(verified_sender(), tx.sender_auth().get_jwt()->get_identity()); + }); + ASSERT_EQ(size_t{1}, flag_reads); + } +} + +TEST_CASE(environment_preserves_missing_empty_and_all_chunks_without_caching) { + Environment env; + reset_host(0); + ASSERT_TRUE(!env.get("MISSING").has_value()); + ASSERT_EQ(std::string{}, env.get("EMPTY").value()); + jwt_payload = std::string(8192, 'x'); + ASSERT_EQ(jwt_payload, env.get("LARGE").value()); + jwt_payload = std::string("a\0b", 3); + ASSERT_EQ(jwt_payload, env.get("NUL").value()); + jwt_payload = "updated"; + ASSERT_EQ(jwt_payload, env.get("NUL").value()); + ProcedureContext procedure(verified_sender(), Timestamp::from_micros_since_epoch(0), ConnectionId(0)); + ASSERT_EQ(jwt_payload, procedure.env.get("VALUE").value()); + procedure.with_tx([&](TxContext& tx) { ASSERT_EQ(jwt_payload, tx.env.get("VALUE").value()); }); +} + +TEST_CASE(optional_reader_matches_canonical_bsatn_tags_and_preserves_following_bytes) { + const std::vector bytes{1, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 'a', 0, 'b', 42}; + bsatn::Reader reader(bytes.data(), bytes.size()); + ASSERT_TRUE(!bsatn::deserialize>(reader).has_value()); + ASSERT_EQ(std::string{}, bsatn::deserialize>(reader).value()); + ASSERT_EQ(std::string("a\0b", 3), bsatn::deserialize>(reader).value()); + ASSERT_EQ(uint8_t{42}, reader.read_u8()); +} diff --git a/crates/bindings-csharp/Codegen.Tests/Tests.cs b/crates/bindings-csharp/Codegen.Tests/Tests.cs index 4133819ef9c..6f0a33fa0e3 100644 --- a/crates/bindings-csharp/Codegen.Tests/Tests.cs +++ b/crates/bindings-csharp/Codegen.Tests/Tests.cs @@ -347,6 +347,65 @@ public static void @params(ProcedureContext ctx) Assert.Empty(GetCompilationErrors(compilationAfterGen)); } + [Fact] + public static async Task ExplicitFunctionVisibilityCompilesAndRejectsExternalLifecycle() + { + var fixture = await Fixture.Compile("server"); + const string source = """ + using SpacetimeDB; + public static partial class VisibilityFunctions + { + [Reducer(Visibility = FunctionVisibility.Public)] + public static void PublicJob(ReducerContext ctx) {} + [Reducer(Visibility = FunctionVisibility.Private)] + public static void PrivateJob(ReducerContext ctx) {} + [Reducer(Visibility = FunctionVisibility.Internal)] + public static void InternalJob(ReducerContext ctx) {} + [Procedure(Visibility = FunctionVisibility.Internal)] + public static int InternalProcedure(ProcedureContext ctx) => 1; + } + """; + var parseOptions = new CSharpParseOptions(fixture.SampleCompilation.LanguageVersion); + var tree = CSharpSyntaxTree.ParseText(source, parseOptions); + var compilation = fixture.SampleCompilation.AddSyntaxTrees(tree); + var driver = CSharpGeneratorDriver.Create( + [ + new SpacetimeDB.Codegen.Type().AsSourceGenerator(), + new SpacetimeDB.Codegen.Module().AsSourceGenerator(), + ], + parseOptions: parseOptions + ); + var result = driver.RunGenerators(compilation).GetRunResult(); + Assert.Empty(result.Diagnostics.Where(d => d.Severity == DiagnosticSeverity.Error)); + Assert.Empty(GetCompilationErrors(compilation.AddSyntaxTrees(result.GeneratedTrees))); + var generated = string.Join("\n", result.GeneratedTrees.Select(t => t.ToString())); + Assert.Contains( + "Visibility: SpacetimeDB.Internal.FunctionVisibility.ExplicitClientCallable", + generated + ); + Assert.Contains("Visibility: SpacetimeDB.Internal.FunctionVisibility.Private", generated); + Assert.Contains("Visibility: SpacetimeDB.Internal.FunctionVisibility.Internal", generated); + + var invalid = CSharpSyntaxTree.ParseText( + """ + using SpacetimeDB; + public static partial class BadVisibility + { + [Reducer(ReducerKind.Init, Visibility = FunctionVisibility.Public)] + public static void InvalidLifecycle(ReducerContext ctx) {} + } + """, + parseOptions + ); + var rejected = driver + .RunGenerators(fixture.SampleCompilation.AddSyntaxTrees(invalid)) + .GetRunResult(); + Assert.Contains( + rejected.Diagnostics, + diagnostic => diagnostic.GetMessage().Contains("Lifecycle reducers only permit") + ); + } + [Fact] public static async Task TestDiagnostics() { diff --git a/crates/bindings-csharp/Codegen.Tests/fixtures/diag/snapshots/Module#FFI.verified.cs b/crates/bindings-csharp/Codegen.Tests/fixtures/diag/snapshots/Module#FFI.verified.cs index 2eb70c4a352..12863525777 100644 --- a/crates/bindings-csharp/Codegen.Tests/fixtures/diag/snapshots/Module#FFI.verified.cs +++ b/crates/bindings-csharp/Codegen.Tests/fixtures/diag/snapshots/Module#FFI.verified.cs @@ -3298,7 +3298,7 @@ SpacetimeDB.BSATN.ITypeRegistrar registrar new( SourceName: nameof(TestDuplicateReducerKind1), Params: [], - Visibility: SpacetimeDB.Internal.FunctionVisibility.ClientCallable, + Visibility: SpacetimeDB.Internal.FunctionVisibility.Internal, OkReturnType: SpacetimeDB.BSATN.AlgebraicType.Unit, ErrReturnType: new SpacetimeDB.BSATN.AlgebraicType.String(default) ); @@ -3319,7 +3319,7 @@ SpacetimeDB.BSATN.ITypeRegistrar registrar new( SourceName: nameof(TestDuplicateReducerKind2), Params: [], - Visibility: SpacetimeDB.Internal.FunctionVisibility.ClientCallable, + Visibility: SpacetimeDB.Internal.FunctionVisibility.Internal, OkReturnType: SpacetimeDB.BSATN.AlgebraicType.Unit, ErrReturnType: new SpacetimeDB.BSATN.AlgebraicType.String(default) ); diff --git a/crates/bindings-csharp/Codegen.Tests/fixtures/server/snapshots/Module#FFI.verified.cs b/crates/bindings-csharp/Codegen.Tests/fixtures/server/snapshots/Module#FFI.verified.cs index 7f196c87a30..17ddaac9371 100644 --- a/crates/bindings-csharp/Codegen.Tests/fixtures/server/snapshots/Module#FFI.verified.cs +++ b/crates/bindings-csharp/Codegen.Tests/fixtures/server/snapshots/Module#FFI.verified.cs @@ -2342,7 +2342,7 @@ SpacetimeDB.BSATN.ITypeRegistrar registrar new( SourceName: nameof(Init), Params: [], - Visibility: SpacetimeDB.Internal.FunctionVisibility.ClientCallable, + Visibility: SpacetimeDB.Internal.FunctionVisibility.Internal, OkReturnType: SpacetimeDB.BSATN.AlgebraicType.Unit, ErrReturnType: new SpacetimeDB.BSATN.AlgebraicType.String(default) ); diff --git a/crates/bindings-csharp/Codegen/Diag.cs b/crates/bindings-csharp/Codegen/Diag.cs index c2374db5b8e..5d6f3259d56 100644 --- a/crates/bindings-csharp/Codegen/Diag.cs +++ b/crates/bindings-csharp/Codegen/Diag.cs @@ -361,4 +361,13 @@ string type $"View '{ctx.method.Identifier}' declares primary key '{ctx.primaryKey}', but its type '{ctx.type}' is not supported for view primary keys.", ctx => ctx.primaryKeySyntax ); + + public static readonly ErrorDescriptor InvalidFunctionVisibility = + new( + group, + "Invalid function visibility", + _ => + $"Visibility must be Default, Public, Private, or Internal. Lifecycle reducers only permit Default or Internal.", + method => method.Identifier + ); } diff --git a/crates/bindings-csharp/Codegen/Module.cs b/crates/bindings-csharp/Codegen/Module.cs index e3f146ad3f2..daec6521008 100644 --- a/crates/bindings-csharp/Codegen/Module.cs +++ b/crates/bindings-csharp/Codegen/Module.cs @@ -1500,13 +1500,46 @@ public static byte[] Invoke( } /// -/// Represents a reducer method declaration in a module. +/// Validates a declared function visibility and maps it to the V10 schema. /// +static class FunctionVisibilityDeclaration +{ + internal static string Resolve( + FunctionVisibility visibility, + bool lifecycle, + MethodDeclarationSyntax method, + DiagReporter diag + ) + { + if ( + ( + lifecycle + && visibility is not (FunctionVisibility.Default or FunctionVisibility.Internal) + ) || !Enum.IsDefined(typeof(FunctionVisibility), visibility) + ) + { + diag.Report(ErrorDescriptor.InvalidFunctionVisibility, method); + return "SpacetimeDB.Internal.FunctionVisibility.Internal"; + } + return visibility switch + { + FunctionVisibility.Public => + "SpacetimeDB.Internal.FunctionVisibility.ExplicitClientCallable", + FunctionVisibility.Private => "SpacetimeDB.Internal.FunctionVisibility.Private", + FunctionVisibility.Internal => "SpacetimeDB.Internal.FunctionVisibility.Internal", + _ => lifecycle + ? "SpacetimeDB.Internal.FunctionVisibility.Internal" + : "SpacetimeDB.Internal.FunctionVisibility.ClientCallable", + }; + } +} + record ReducerDeclaration { public readonly string Name; public readonly string? CanonicalName; public readonly ReducerKind Kind; + public readonly string Visibility; public readonly string FullName; public readonly EquatableArray Args; public readonly Scope Scope; @@ -1545,6 +1578,12 @@ public ReducerDeclaration(GeneratorAttributeSyntaxContext context, DiagReporter } Kind = attr.Kind; + Visibility = FunctionVisibilityDeclaration.Resolve( + attr.Visibility, + Kind != ReducerKind.UserDefined, + methodSyntax, + diag + ); CanonicalName = attr.Name; FullName = SymbolToName(method); Args = new( @@ -1573,7 +1612,7 @@ sealed class {{Identifier}}: SpacetimeDB.Internal.IReducer { public SpacetimeDB.Internal.RawReducerDefV10 MakeReducerDef(SpacetimeDB.BSATN.ITypeRegistrar registrar) => new ( SourceName: nameof({{Identifier}}), Params: [{{MemberDeclaration.GenerateDefs(Args)}}], - Visibility: SpacetimeDB.Internal.FunctionVisibility.ClientCallable, + Visibility: {{Visibility}}, OkReturnType: SpacetimeDB.BSATN.AlgebraicType.Unit, ErrReturnType: new SpacetimeDB.BSATN.AlgebraicType.String(default) ); @@ -1630,6 +1669,7 @@ record ProcedureDeclaration { public readonly string Name; public readonly string? CanonicalName; + public readonly string Visibility; public readonly string FullName; public readonly EquatableArray Args; public readonly Scope Scope; @@ -1646,6 +1686,12 @@ public ProcedureDeclaration(GeneratorAttributeSyntaxContext context, DiagReporte var methodSyntax = (MethodDeclarationSyntax)context.TargetNode; var method = (IMethodSymbol)context.TargetSymbol; var attr = context.Attributes.Single().ParseAs(); + Visibility = FunctionVisibilityDeclaration.Resolve( + attr.Visibility, + false, + methodSyntax, + diag + ); if ( method.Parameters.FirstOrDefault()?.Type @@ -1804,7 +1850,7 @@ sealed class {{{Identifier}}} : SpacetimeDB.Internal.IProcedure { SourceName: nameof({{{Identifier}}}), Params: [{{{MemberDeclaration.GenerateDefs(Args)}}}], ReturnType: {{{returnTypeExpr}}}, - Visibility: SpacetimeDB.Internal.FunctionVisibility.ClientCallable + Visibility: {{{Visibility}}} ); public static byte[] Invoke(BinaryReader reader, SpacetimeDB.Internal.IProcedureContext ctx) { diff --git a/crates/bindings-csharp/README.md b/crates/bindings-csharp/README.md index 289bd570ff0..94158b66fb5 100644 --- a/crates/bindings-csharp/README.md +++ b/crates/bindings-csharp/README.md @@ -6,6 +6,27 @@ See the [C# module library reference](https://spacetimedb.com/docs/modules/c-sha ## Internal documentation +### Function visibility and invocation authentication + +Reducers and procedures can declare `Visibility = FunctionVisibility.Public`, +`Private`, or `Internal` in their attributes. Omission (`Default`) means public +for ordinary functions and private for scheduled functions. An explicit choice +is preserved when the function is scheduled. Lifecycle reducers permit only +omission or `Internal` and can only run for their host lifecycle event. + +Internal functions require verified internal authority. Private functions also +admit the owner, and public functions admit any client. For example: + +```csharp +[Reducer(Visibility = FunctionVisibility.Internal)] +public static void ProcessJobs(ReducerContext ctx) { } +``` + +`ctx.SenderAuth.IsInternal` comes from the host's invocation authority. It is +independent of connection and JWT presence, so an internal call can have a JWT. +JWT identity is the verified sender supplied by the host. Newly compiled modules +emit schema V10 and advertise `hosted_auth_v1`, requiring a compatible host. + These projects contain the SpacetimeDB SATS typesystem, codegen and runtime bindings for SpacetimeDB WebAssembly modules. It also contains serialization code for SpacetimeDB C# clients. @@ -19,4 +40,3 @@ The [`Codegen`](./Codegen/) and [`Runtime`](./Runtime/) libraries are used: - only by C# Modules. They provide all of the functionality needed to write SpacetimeDB modules in C#. See their READMEs for more information. - diff --git a/crates/bindings-csharp/Runtime.Tests/FunctionVisibilityTests.cs b/crates/bindings-csharp/Runtime.Tests/FunctionVisibilityTests.cs new file mode 100644 index 00000000000..d50e6cff7f4 --- /dev/null +++ b/crates/bindings-csharp/Runtime.Tests/FunctionVisibilityTests.cs @@ -0,0 +1,68 @@ +namespace Runtime.Tests; + +using SpacetimeDB.BSATN; +using SpacetimeDB.Internal; + +public class FunctionVisibilityTests +{ + [Theory] + [InlineData(FunctionVisibility.Private, 0)] + [InlineData(FunctionVisibility.ClientCallable, 1)] + [InlineData(FunctionVisibility.Internal, 2)] + [InlineData(FunctionVisibility.ExplicitClientCallable, 3)] + public void V10RetainsVisibilityEnumEncoding(FunctionVisibility visibility, byte tag) + { + var bytes = IStructuralReadWrite.ToBytes( + new SpacetimeDB.BSATN.Enum(), + visibility + ); + Assert.Equal(new byte[] { tag }, bytes); + } + + [Theory] + [InlineData(FunctionVisibility.ExplicitClientCallable)] + [InlineData(FunctionVisibility.ClientCallable)] + [InlineData(FunctionVisibility.Private)] + [InlineData(FunctionVisibility.Internal)] + public void SchedulingPreservesVisibility(FunctionVisibility visibility) + { + var module = new RawModuleDefV10(); + var reducer = new RawReducerDefV10( + "run_job", + [], + visibility, + AlgebraicType.Unit, + new AlgebraicType.String(default) + ); + module.RegisterReducer(reducer, null); + module.RegisterTable( + new RawTableDefV10 { SourceName = "jobs" }, + new RawScheduleDefV10(null, "jobs", 0, "run_job") + ); + var raw = module.BuildModuleDefinition(); + var reducers = Assert.Single(raw.Sections.OfType()); + Assert.Equal(visibility, Assert.Single(reducers.Reducers_).Visibility); + var capabilities = Assert.Single( + raw.Sections.OfType() + ); + Assert.Contains("hosted_auth_v1", capabilities.Capabilities_); + } + + [Theory] + [InlineData(FunctionVisibility.ClientCallable)] + [InlineData(FunctionVisibility.ExplicitClientCallable)] + public void LifecycleRejectsExternalVisibility(FunctionVisibility visibility) + { + var module = new RawModuleDefV10(); + var reducer = new RawReducerDefV10( + "initialize", + [], + visibility, + AlgebraicType.Unit, + new AlgebraicType.String(default) + ); + Assert.Throws( + () => module.RegisterReducer(reducer, Lifecycle.Init) + ); + } +} diff --git a/crates/bindings-csharp/Runtime.Tests/HostedAuthTests.cs b/crates/bindings-csharp/Runtime.Tests/HostedAuthTests.cs new file mode 100644 index 00000000000..ada932fb631 --- /dev/null +++ b/crates/bindings-csharp/Runtime.Tests/HostedAuthTests.cs @@ -0,0 +1,43 @@ +namespace Runtime.Tests; + +using SpacetimeDB; + +public class HostedAuthTests +{ + [Theory] + [InlineData(0u, false)] + [InlineData(1u, true)] + public void NoJwtCallsPreserveVerifiedInternalFlag(uint flags, bool expectedInternal) + { + var auth = AuthCtx.FromVerifiedCall(flags, () => null); + Assert.Equal(expectedInternal, auth.IsInternal); + Assert.False(auth.HasJwt); + Assert.Null(auth.Jwt); + } + + [Fact] + public void InternalCallCanRetainJwtAndVerifiedSenderIdentity() + { + var sender = Identity.FromHexString(new string('a', 64)); + var reads = 0; + var flags = 1u; + var auth = AuthCtx.FromVerifiedCall( + flags, + () => + { + reads++; + return new JwtClaims( + "{\"iss\":\"different-issuer\",\"sub\":\"different-subject\",\"identity\":\"untrusted\"}", + sender + ); + } + ); + flags = 0; + Assert.True(auth.IsInternal); + Assert.Equal(0, reads); + Assert.True(auth.HasJwt); + Assert.Equal(sender, auth.Jwt!.Identity); + Assert.Equal("different-subject", auth.Jwt.Subject); + Assert.Equal(1, reads); + } +} diff --git a/crates/bindings-csharp/Runtime/Attrs.cs b/crates/bindings-csharp/Runtime/Attrs.cs index 3865f925a2a..d214960d4fa 100644 --- a/crates/bindings-csharp/Runtime/Attrs.cs +++ b/crates/bindings-csharp/Runtime/Attrs.cs @@ -199,18 +199,30 @@ public enum ReducerKind ClientDisconnected, } + /// Invocation admission for reducers and procedures. + public enum FunctionVisibility + { + /// Public for ordinary functions, Private for scheduled functions. + Default, + Public, + Private, + Internal, + } + [AttributeUsage(AttributeTargets.Method, Inherited = false)] public sealed class ReducerAttribute(ReducerKind kind = ReducerKind.UserDefined) : Attribute { public ReducerKind Kind => kind; public string? Name { get; init; } + public FunctionVisibility Visibility { get; init; } } [AttributeUsage(AttributeTargets.Method, Inherited = false)] public sealed class ProcedureAttribute() : Attribute { public string? Name { get; init; } + public FunctionVisibility Visibility { get; init; } } [AttributeUsage(AttributeTargets.Method, Inherited = false)] diff --git a/crates/bindings-csharp/Runtime/AuthCtx.cs b/crates/bindings-csharp/Runtime/AuthCtx.cs index c7fcdea47e0..ff34468ad76 100644 --- a/crates/bindings-csharp/Runtime/AuthCtx.cs +++ b/crates/bindings-csharp/Runtime/AuthCtx.cs @@ -16,12 +16,10 @@ private AuthCtx(bool isInternal, Func jwtFactory) } /// - /// Create an AuthCtx for an internal call, with no JWT. + /// Capture verified invocation authority independently from lazy JWT loading. /// - private static AuthCtx Internal() - { - return new AuthCtx(isInternal: true, jwtFactory: () => null); - } + internal static AuthCtx FromVerifiedCall(uint callAuthFlags, Func jwtFactory) => + new(isInternal: (callAuthFlags & 1) != 0, jwtFactory); /// /// Create an AuthCtx by looking up the credentials for a connection id in system tables. @@ -31,20 +29,27 @@ private static AuthCtx Internal() /// public static AuthCtx BuildFromSystemTables(ConnectionId? connectionId, Identity identity) { + // Read synchronously while this invocation is active. Neither connection + // presence nor token claims determine internal authority. + var callAuthFlags = SpacetimeDB.Internal.FFI.get_call_auth_flags(); if (connectionId == null) { - return Internal(); + return FromVerifiedCall(callAuthFlags, () => null); } - return FromConnectionId(connectionId.Value, identity); + return FromConnectionId(connectionId.Value, identity, callAuthFlags); } /// /// Create an AuthCtx that reads JWT for a given connection ID. /// - private static AuthCtx FromConnectionId(ConnectionId connectionId, Identity identity) + private static AuthCtx FromConnectionId( + ConnectionId connectionId, + Identity identity, + uint callAuthFlags + ) { - return new AuthCtx( - isInternal: false, + return FromVerifiedCall( + callAuthFlags, jwtFactory: () => { var result = SpacetimeDB.Internal.FFI.get_jwt(ref connectionId, out var source); @@ -65,23 +70,18 @@ private static AuthCtx FromConnectionId(ConnectionId connectionId, Identity iden } /// - /// True if this reducer was spawned from inside the database. + /// True if the host verified internal authority for this invocation. /// public bool IsInternal => _isInternal; /// /// Check if there is a JWT present. - /// If IsInternal is true, this will be false. + /// Independent of IsInternal. An internal call may also have a JWT. /// public bool HasJwt { get { - if (_isInternal) - { - return false; - } - // At this point we do load the bytes. return _jwtLazy.Value != null; } diff --git a/crates/bindings-csharp/Runtime/Internal/Autogen/FunctionVisibility.g.cs b/crates/bindings-csharp/Runtime/Internal/Autogen/FunctionVisibility.g.cs index 2f9772dd591..29adc856f78 100644 --- a/crates/bindings-csharp/Runtime/Internal/Autogen/FunctionVisibility.g.cs +++ b/crates/bindings-csharp/Runtime/Internal/Autogen/FunctionVisibility.g.cs @@ -12,5 +12,7 @@ public enum FunctionVisibility { Private, ClientCallable, + Internal, + ExplicitClientCallable, } } diff --git a/crates/bindings-csharp/Runtime/Internal/Autogen/RawModuleDefV10Section.g.cs b/crates/bindings-csharp/Runtime/Internal/Autogen/RawModuleDefV10Section.g.cs index 61212c98e89..21f96921e5a 100644 --- a/crates/bindings-csharp/Runtime/Internal/Autogen/RawModuleDefV10Section.g.cs +++ b/crates/bindings-csharp/Runtime/Internal/Autogen/RawModuleDefV10Section.g.cs @@ -23,6 +23,7 @@ public partial record RawModuleDefV10Section : SpacetimeDB.TaggedEnum<( System.Collections.Generic.List HttpHandlers, System.Collections.Generic.List HttpRoutes, System.Collections.Generic.List ViewPrimaryKeys, - System.Collections.Generic.List Submodules + System.Collections.Generic.List Submodules, + System.Collections.Generic.List Capabilities )>; } diff --git a/crates/bindings-csharp/Runtime/Internal/FFI.cs b/crates/bindings-csharp/Runtime/Internal/FFI.cs index f498e6ed8ea..8b2ecca8206 100644 --- a/crates/bindings-csharp/Runtime/Internal/FFI.cs +++ b/crates/bindings-csharp/Runtime/Internal/FFI.cs @@ -109,6 +109,14 @@ internal static partial class FFI #endif ; + const string StdbNamespace10_6 = +#if EXPERIMENTAL_WASM_AOT + "spacetime_10.6" +#else + "bindings" +#endif + ; + const string StdbNamespace10_7 = #if EXPERIMENTAL_WASM_AOT "spacetime_10.7" @@ -123,6 +131,9 @@ public static unsafe partial CheckedStatus env_get( uint keyLen, out BytesSource source ); + [LibraryImport(StdbNamespace10_6)] + public static partial uint get_call_auth_flags(); + [NativeMarshalling(typeof(Marshaller))] public struct CheckedStatus diff --git a/crates/bindings-csharp/Runtime/Internal/Module.cs b/crates/bindings-csharp/Runtime/Internal/Module.cs index 177fddd785a..99b64aa0725 100644 --- a/crates/bindings-csharp/Runtime/Internal/Module.cs +++ b/crates/bindings-csharp/Runtime/Internal/Module.cs @@ -56,13 +56,23 @@ internal AlgebraicType.Ref RegisterType(Func l.FunctionName) - .Concat(scheduleDefs.Select(s => s.FunctionName)) - .ToHashSet(StringComparer.Ordinal); - - foreach (var reducer in reducerDefs) - { - if (internalFunctions.Contains(reducer.SourceName)) - { - reducer.Visibility = FunctionVisibility.Private; - } - } - - foreach (var procedure in procedureDefs) - { - if (internalFunctions.Contains(procedure.SourceName)) - { - procedure.Visibility = FunctionVisibility.Private; - } - } - var sections = new List { new RawModuleDefV10Section.Typespace(typespace), + new RawModuleDefV10Section.Capabilities(["hosted_auth_v1"]), }; if (typeDefs.Count > 0) diff --git a/crates/bindings-csharp/Runtime/JwtClaims.cs b/crates/bindings-csharp/Runtime/JwtClaims.cs index 3ca3e11e029..bbd53cdfd25 100644 --- a/crates/bindings-csharp/Runtime/JwtClaims.cs +++ b/crates/bindings-csharp/Runtime/JwtClaims.cs @@ -16,8 +16,8 @@ public sealed class JwtClaims /// /// Create a JwtClaims from a raw JWT payload (JSON claims) and its associated Identity. /// - /// This only takes an Identity because the Blake3 hash package on nuget wraps rust code. - /// We should not expose this constructor publicly, but it is needed for AuthCtx. + /// Identity is the verified sender provided by the host. Claims cannot + /// override it, including for hosted database credentials. /// internal JwtClaims(string jwt, Identity identity) { diff --git a/crates/bindings-csharp/Runtime/Runtime.csproj b/crates/bindings-csharp/Runtime/Runtime.csproj index 14b2356594e..19e6405b2ec 100644 --- a/crates/bindings-csharp/Runtime/Runtime.csproj +++ b/crates/bindings-csharp/Runtime/Runtime.csproj @@ -52,6 +52,7 @@ + diff --git a/crates/bindings-csharp/Runtime/bindings.c b/crates/bindings-csharp/Runtime/bindings.c index f4d3635a5f2..64bb4497d4a 100644 --- a/crates/bindings-csharp/Runtime/bindings.c +++ b/crates/bindings-csharp/Runtime/bindings.c @@ -134,6 +134,10 @@ IMPORT(Status, datastore_clear, (TableId table_id, uint64_t* count), (table_id, count)); #undef SPACETIME_MODULE_VERSION + +#define SPACETIME_MODULE_VERSION "spacetime_10.6" +IMPORT(uint32_t, get_call_auth_flags, (void), ()); +#undef SPACETIME_MODULE_VERSION #define SPACETIME_MODULE_VERSION "spacetime_10.7" IMPORT(Status, env_get, (const uint8_t* key, uint32_t key_len, BytesSource* source), (key, key_len, source)); diff --git a/crates/bindings-macro/src/procedure.rs b/crates/bindings-macro/src/procedure.rs index 9f76e5b547f..129b32cc2fe 100644 --- a/crates/bindings-macro/src/procedure.rs +++ b/crates/bindings-macro/src/procedure.rs @@ -1,4 +1,5 @@ use crate::reducer::{assert_only_lifetime_generics, extract_typed_args, generate_explicit_names_impl}; +use crate::reducer::{parse_visibility, DeclaredVisibility}; use crate::sym; use crate::util::{check_duplicate, ident_to_litstr, match_meta}; use proc_macro2::TokenStream; @@ -10,12 +11,16 @@ use syn::{ItemFn, LitStr}; pub(crate) struct ProcedureArgs { /// For consistency with reducers: allow specifying a different export name than the Rust function name. name: Option, + visibility: Option, } impl ProcedureArgs { pub(crate) fn parse(input: TokenStream) -> syn::Result { let mut args = Self::default(); syn::meta::parser(|meta| { + if parse_visibility(&meta, &mut args.visibility)? { + return Ok(()); + } match_meta!(match meta { sym::name => { check_duplicate(&args.name, &meta)?; @@ -29,10 +34,11 @@ impl ProcedureArgs { } } -pub(crate) fn procedure_impl(_args: ProcedureArgs, original_function: &ItemFn) -> syn::Result { +pub(crate) fn procedure_impl(args: ProcedureArgs, original_function: &ItemFn) -> syn::Result { let func_name = &original_function.sig.ident; let vis = &original_function.vis; - let explicit_name = _args.name.as_ref(); + let explicit_name = args.name.as_ref(); + let visibility = args.visibility.map(DeclaredVisibility::tokens).into_iter(); let procedure_name = ident_to_litstr(func_name); @@ -117,6 +123,7 @@ pub(crate) fn procedure_impl(_args: ProcedureArgs, original_function: &ItemFn) - /// The name of this function const NAME: &'static str = #procedure_name; + #(const DECLARED_VISIBILITY: Option = Some(#visibility);)* /// The parameter names of this function const ARG_NAMES: &'static [Option<&'static str>] = &[#(#opt_arg_names),*]; @@ -133,3 +140,29 @@ pub(crate) fn procedure_impl(_args: ProcedureArgs, original_function: &ItemFn) - #generate_explicit_names }) } + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn procedure_visibility_rejects_duplicates_and_emits_selection() { + assert!(ProcedureArgs::parse(quote!(private, public)).is_err()); + assert!(ProcedureArgs::parse(quote!(internal, internal)).is_err()); + let function: ItemFn = syn::parse_quote!( + fn example(ctx: &mut ProcedureContext) -> u64 { + 0 + } + ); + for (input, expected) in [ + (quote!(internal), "Internal"), + (quote!(private), "Private"), + (quote!(public), "ClientCallable"), + ] { + let tokens = procedure_impl(ProcedureArgs::parse(input).unwrap(), &function) + .unwrap() + .to_string(); + assert!(tokens.contains("DECLARED_VISIBILITY")); + assert!(tokens.contains(&format!("FunctionVisibility :: {expected}"))); + } + } +} diff --git a/crates/bindings-macro/src/reducer.rs b/crates/bindings-macro/src/reducer.rs index ac261ced35f..3093a51397a 100644 --- a/crates/bindings-macro/src/reducer.rs +++ b/crates/bindings-macro/src/reducer.rs @@ -10,6 +10,44 @@ use syn::{FnArg, Ident, ItemFn, LitStr, PatType}; pub(crate) struct ReducerArgs { name: Option, lifecycle: Option, + visibility: Option, +} + +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum DeclaredVisibility { + Internal, + Private, + Public, +} + +impl DeclaredVisibility { + pub(crate) fn tokens(self) -> TokenStream { + let variant = match self { + Self::Internal => "Internal", + Self::Private => "Private", + Self::Public => "ClientCallable", + }; + let variant = Ident::new(variant, Span::call_site()); + quote!(spacetimedb::rt::FunctionVisibility::#variant) + } +} + +pub(crate) fn parse_visibility( + meta: &syn::meta::ParseNestedMeta<'_>, + visibility: &mut Option, +) -> syn::Result { + let value = if meta.path.is_ident("internal") { + DeclaredVisibility::Internal + } else if meta.path.is_ident("private") { + DeclaredVisibility::Private + } else if meta.path.is_ident("public") { + DeclaredVisibility::Public + } else { + return Ok(false); + }; + check_duplicate_msg(visibility, meta, "already specified a function visibility")?; + *visibility = Some(value); + Ok(true) } enum LifecycleReducer { @@ -37,6 +75,9 @@ impl ReducerArgs { pub(crate) fn parse(input: TokenStream) -> syn::Result { let mut args = Self::default(); syn::meta::parser(|meta| { + if parse_visibility(&meta, &mut args.visibility)? { + return Ok(()); + } let mut set_lifecycle = |kind: fn(Span) -> _| -> syn::Result<()> { check_duplicate_msg(&args.lifecycle, &meta, "already specified a lifecycle reducer kind")?; args.lifecycle = Some(kind(meta.path.span())); @@ -55,6 +96,12 @@ impl ReducerArgs { Ok(()) }) .parse2(input)?; + if args.lifecycle.is_some() && args.visibility.is_some_and(|v| v != DeclaredVisibility::Internal) { + return Err(syn::Error::new( + Span::call_site(), + "lifecycle reducers must have internal visibility", + )); + } Ok(args) } } @@ -101,6 +148,7 @@ pub(crate) fn reducer_impl(args: ReducerArgs, original_function: &ItemFn) -> syn assert_only_lifetime_generics(original_function, "reducers")?; let lifecycle = args.lifecycle.iter().filter_map(|lc| lc.to_lifecycle_value()); + let visibility = args.visibility.map(DeclaredVisibility::tokens).into_iter(); let typed_args = extract_typed_args(original_function)?; @@ -165,6 +213,7 @@ pub(crate) fn reducer_impl(args: ReducerArgs, original_function: &ItemFn) -> syn /// The function kind, which will cause scheduled tables to accept reducers. type FnKind = spacetimedb::rt::FnKindReducer; const NAME: &'static str = #reducer_name; + #(const DECLARED_VISIBILITY: Option = Some(#visibility);)* #(const LIFECYCLE: Option = Some(#lifecycle);)* const ARG_NAMES: &'static [Option<&'static str>] = &[#(#opt_arg_names),*]; const INVOKE: Self::Invoke = #func_name::invoke; @@ -202,3 +251,43 @@ pub(crate) fn generate_explicit_names_impl( } } } + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn visibility_declarations_are_unambiguous() { + for input in [ + quote!(private, public), + quote!(internal, internal), + quote!(init, private), + quote!(public, client_connected), + ] { + assert!(ReducerArgs::parse(input).is_err()); + } + for input in [ + quote!(), + quote!(public), + quote!(private), + quote!(internal), + quote!(init, internal), + ] { + assert!(ReducerArgs::parse(input).is_ok()); + } + } + #[test] + fn rust_item_visibility_does_not_select_database_visibility() { + let function: ItemFn = syn::parse_quote!( + pub fn example(ctx: &ReducerContext) {} + ); + let implicit = reducer_impl(ReducerArgs::parse(quote!()).unwrap(), &function) + .unwrap() + .to_string(); + assert!(!implicit.contains("DECLARED_VISIBILITY")); + let explicit = reducer_impl(ReducerArgs::parse(quote!(internal)).unwrap(), &function) + .unwrap() + .to_string(); + assert!(explicit.contains("DECLARED_VISIBILITY")); + assert!(explicit.contains("FunctionVisibility :: Internal")); + } +} diff --git a/crates/bindings-sys/src/lib.rs b/crates/bindings-sys/src/lib.rs index 2a3bd77b454..235a25d545f 100644 --- a/crates/bindings-sys/src/lib.rs +++ b/crates/bindings-sys/src/lib.rs @@ -883,7 +883,14 @@ pub mod raw { pub fn datastore_clear(table_id: TableId, out: *mut u64) -> u16; } - // ABI10.6 is reserved for the separate invocation-authority extension. + #[link(wasm_import_module = "spacetime_10.6")] + unsafe extern "C" { + /// Authentication flags for the active invocation. Bit 0 is INTERNAL. + /// Read at context construction; neither a missing connection ID nor JWT + /// claims imply internal authority. Unknown bits must be ignored. + pub fn get_call_auth_flags() -> u32; + } + #[link(wasm_import_module = "spacetime_10.7")] unsafe extern "C" { /// Read a UTF-8 environment value. Writes INVALID for a missing key; @@ -1676,3 +1683,10 @@ pub mod procedure { } } } + +/// Read host-verified authentication flags for the active invocation. +/// Bit 0 is INTERNAL; all other bits are reserved. +pub fn get_call_auth_flags() -> u32 { + // SAFETY: no pointers or guest-provided values are passed to the host. + unsafe { raw::get_call_auth_flags() } +} diff --git a/crates/bindings-typescript/README.md b/crates/bindings-typescript/README.md index 48e7cfd1535..82c0b9ece87 100644 --- a/crates/bindings-typescript/README.md +++ b/crates/bindings-typescript/README.md @@ -18,6 +18,26 @@ You can use the package in the browser, using a bundler like vite/parcel/rsbuild ### Usage +#### Module function visibility and invocation authentication + +Reducer and procedure options accept `visibility: 'public'`, `'private'`, or +`'internal'`. For example, `spacetime.reducer({ visibility: 'internal' }, ctx => {})` +declares an internal reducer. Omission means public for ordinary functions and +private for scheduled functions. An explicit choice is preserved when the +function is scheduled. Lifecycle reducers permit only omission or `'internal'` +and can only run for their host lifecycle event. + +Internal functions require verified internal authority. Private functions also +admit the owner, and public functions admit any client. `ctx.senderAuth.isInternal` +captures the host's invocation authority independently of connection and JWT +presence, so an internal call can have a JWT. `ctx.senderAuth.jwt.identity` is the +verified sender supplied by the host. Procedure transactions preserve this +authentication. Newly compiled modules retain schema V10 and advertise +`hosted_auth_v1`. The extended visibility values and capability section require +a compatible host; older V10 definitions retain their existing defaults. + +#### Client SDK + In order to connect to a database you have to generate module bindings for your database. ```ts diff --git a/crates/bindings-typescript/src/lib/autogen/types.ts b/crates/bindings-typescript/src/lib/autogen/types.ts index 3cee51f03d5..8206f799e69 100644 --- a/crates/bindings-typescript/src/lib/autogen/types.ts +++ b/crates/bindings-typescript/src/lib/autogen/types.ts @@ -76,6 +76,8 @@ export type ExplicitNames = __Infer; export const FunctionVisibility = __t.enum('FunctionVisibility', { Private: __t.unit(), ClientCallable: __t.unit(), + Internal: __t.unit(), + ExplicitClientCallable: __t.unit(), }); export type FunctionVisibility = __Infer; @@ -393,6 +395,7 @@ export const RawModuleDefV10Section = __t.enum('RawModuleDefV10Section', { get Submodules() { return __t.array(RawSubmoduleV10); }, + Capabilities: __t.array(__t.string()), }); export type RawModuleDefV10Section = __Infer; diff --git a/crates/bindings-typescript/src/lib/reducers.ts b/crates/bindings-typescript/src/lib/reducers.ts index eaa8de94b55..04c141bc104 100644 --- a/crates/bindings-typescript/src/lib/reducers.ts +++ b/crates/bindings-typescript/src/lib/reducers.ts @@ -61,7 +61,7 @@ export type Reducer = ( * Authentication information for the caller of a reducer. */ export type AuthCtx = Readonly<{ - /** Whether the caller is an internal system process. */ + /** Whether the host verified internal invocation authority. Independent of JWT presence. */ isInternal: boolean; /** Whether the caller has authenticated with a JWT token. */ hasJWT: boolean; @@ -93,7 +93,7 @@ export interface JwtClaims { readonly issuer: string; /** The audience of the JWT token ('aud') */ readonly audience: readonly string[]; - /** The identity associated with the JWT token, which is based on the sub and iss */ + /** The verified sender Identity provided by the host, including hosted credentials. */ readonly identity: Identity; /** The full payload as a JsonObject */ readonly fullPayload: JsonObject; diff --git a/crates/bindings-typescript/src/lib/schema.ts b/crates/bindings-typescript/src/lib/schema.ts index eb3821f6606..762b4f9079a 100644 --- a/crates/bindings-typescript/src/lib/schema.ts +++ b/crates/bindings-typescript/src/lib/schema.ts @@ -200,6 +200,7 @@ export class ModuleContext { lifeCycleReducers: [], httpHandlers: [], httpRoutes: [], + capabilities: ['hosted_auth_v1'], caseConversionPolicy: { tag: 'SnakeCase' }, explicitNames: { entries: [], @@ -221,6 +222,7 @@ export class ModuleContext { const module = this.#moduleDef; push(module.typespace && { tag: 'Typespace', value: module.typespace }); + push({ tag: 'Capabilities', value: module.capabilities }); push(module.types && { tag: 'Types', value: module.types }); push(module.tables && { tag: 'Tables', value: module.tables }); push(module.reducers && { tag: 'Reducers', value: module.reducers }); diff --git a/crates/bindings-typescript/src/server/function_visibility.ts b/crates/bindings-typescript/src/server/function_visibility.ts new file mode 100644 index 00000000000..658fb1dad0d --- /dev/null +++ b/crates/bindings-typescript/src/server/function_visibility.ts @@ -0,0 +1,24 @@ +import { FunctionVisibility as RawFunctionVisibility } from '../lib/autogen/types'; + +/** Internal functions require verified internal authority. Private functions also + * admit the owner. Public functions admit any authenticated client. */ +export type FunctionVisibility = 'public' | 'private' | 'internal'; + +export function rawVisibility( + visibility: FunctionVisibility | undefined +): RawFunctionVisibility { + switch (visibility) { + case undefined: + // Preserve V10's existing context-dependent default, including scheduled + // private functions, without changing the raw definition's field layout. + return RawFunctionVisibility.ClientCallable; + case 'public': + return RawFunctionVisibility.ExplicitClientCallable; + case 'private': + return RawFunctionVisibility.Private; + case 'internal': + return RawFunctionVisibility.Internal; + default: + throw new TypeError('Invalid function visibility'); + } +} diff --git a/crates/bindings-typescript/src/server/index.ts b/crates/bindings-typescript/src/server/index.ts index 3ac3e8f0fbb..fdf3f45a224 100644 --- a/crates/bindings-typescript/src/server/index.ts +++ b/crates/bindings-typescript/src/server/index.ts @@ -10,6 +10,7 @@ export { table } from '../lib/table'; export { SenderError, SpacetimeHostError, errors } from './errors'; export type { Reducer, ReducerCtx, JwtClaims, AuthCtx } from '../lib/reducers'; export type { ReducerExport } from './reducers'; +export type { FunctionVisibility } from './function_visibility'; export { type DbView } from './db_view'; export * from './query'; export type { diff --git a/crates/bindings-typescript/src/server/procedures.ts b/crates/bindings-typescript/src/server/procedures.ts index f4d57416e69..076429dd80b 100644 --- a/crates/bindings-typescript/src/server/procedures.ts +++ b/crates/bindings-typescript/src/server/procedures.ts @@ -5,12 +5,12 @@ import { type Deserializer, type Serializer, } from '../lib/algebraic_type'; -import { FunctionVisibility } from '../lib/autogen/types'; +import { rawVisibility, type FunctionVisibility } from './function_visibility'; import BinaryReader from '../lib/binary_reader'; import BinaryWriter from '../lib/binary_writer'; import type { ConnectionId } from '../lib/connection_id'; import { Identity } from '../lib/identity'; -import type { ParamsObj, ReducerCtx } from '../lib/reducers'; +import type { AuthCtx, ParamsObj, ReducerCtx } from '../lib/reducers'; import { type UntypedSchemaDef } from '../lib/schema'; import type { ScheduleTableForParams } from '../lib/table_schema'; import { Timestamp } from '../lib/timestamp'; @@ -28,6 +28,7 @@ import { makeRandom, type Random } from './rng'; import { assignTxAliasViews, buildProcedureAliasCtxMap, + AuthCtxImpl, callUserFunction, ReducerCtxImpl, runWithTx, @@ -58,21 +59,19 @@ export function makeProcedureExport< ret: Ret, fn: ProcedureFn ): ProcedureExport { - const name = opts?.name; - const procedureExport: ProcedureExport = (...args) => fn(...args); procedureExport[exportContext] = ctx; procedureExport[registerExport] = (ctx, exportName) => { - registerProcedure(ctx, name ?? exportName, params, ret, fn); + registerProcedure(ctx, exportName, params, ret, fn, opts); ctx.functionExports.set( procedureExport as ProcedureExport, - name ?? exportName + exportName ); if (opts?.onSchedule !== undefined) { ctx.pendingSchedules.push({ table: opts.onSchedule, - functionName: name ?? exportName, + functionName: opts.name ?? exportName, }); } }; @@ -90,7 +89,9 @@ export interface ProcedureOpts< Params extends ParamsObj = ParamsObj, Ret extends TypeBuilder = TypeBuilder, > { - name: string; + name?: string; + /** Defaults to public, or private when scheduled. */ + visibility?: FunctionVisibility; onSchedule?: Ret extends ReturnType ? ScheduleTableForParams : never; @@ -116,6 +117,7 @@ export interface ProcedureCtx { readonly identity: Identity; readonly timestamp: Timestamp; readonly connectionId: ConnectionId | null; + readonly senderAuth: AuthCtx; readonly http: HttpClient; readonly random: Random; readonly as: ProcedureAliasViews; @@ -128,12 +130,6 @@ export interface ProcedureCtx { export interface TransactionCtx extends ReducerCtx {} -type ITransactionCtx = TransactionCtx; - -const TransactionCtxImpl = class TransactionCtx - extends ReducerCtxImpl - implements ITransactionCtx {}; - function registerProcedure< S extends UntypedSchemaDef, Params extends ParamsObj, @@ -161,7 +157,7 @@ function registerProcedure< sourceName: exportName, params: paramsType, returnType, - visibility: FunctionVisibility.ClientCallable, + visibility: rawVisibility(opts?.visibility), }); if (opts?.name != null) { @@ -232,6 +228,7 @@ const ProcedureCtxImpl = class ProcedureCtx #dispatches: SubmoduleDispatchInfo[]; #parentPrefix: string; #asViews: object | undefined; + readonly senderAuth: AuthCtx; constructor( readonly sender: Identity, @@ -244,6 +241,11 @@ const ProcedureCtxImpl = class ProcedureCtx this.#dbView = dbView; this.#dispatches = dispatches; this.#parentPrefix = parentPrefix; + this.senderAuth = AuthCtxImpl.fromSystemTables( + connectionId, + sender, + sys.get_call_auth_flags() + ); } get databaseIdentity() { @@ -274,11 +276,13 @@ const ProcedureCtxImpl = class ProcedureCtx const dispatches = this.#dispatches; const parentPrefix = this.#parentPrefix; return runWithTx(timestamp => { - const tx = new TransactionCtxImpl( + const tx = new ReducerCtxImpl( this.sender, timestamp, this.connectionId, - this.#dbView() + this.#dbView(), + {}, + this.senderAuth ); assignTxAliasViews(tx, dispatches, parentPrefix); return tx as unknown as TransactionCtx; diff --git a/crates/bindings-typescript/src/server/reducers.ts b/crates/bindings-typescript/src/server/reducers.ts index ea5f770faf8..25de0c98820 100644 --- a/crates/bindings-typescript/src/server/reducers.ts +++ b/crates/bindings-typescript/src/server/reducers.ts @@ -1,5 +1,6 @@ import { AlgebraicType } from '../lib/algebraic_type'; -import { FunctionVisibility, type Lifecycle } from '../lib/autogen/types'; +import { type Lifecycle } from '../lib/autogen/types'; +import { rawVisibility, type FunctionVisibility } from './function_visibility'; import type { ParamsObj, Reducer } from '../lib/reducers'; import { type UntypedSchemaDef } from '../lib/schema'; import type { ScheduleTableForParams } from '../lib/table_schema'; @@ -19,7 +20,9 @@ export interface ReducerExport< ModuleExport {} export interface ReducerOpts { - name: string; + name?: string; + /** Defaults to public, or private when scheduled. Lifecycle hooks are internal. */ + visibility?: FunctionVisibility; onSchedule?: ScheduleTableForParams; } @@ -84,12 +87,19 @@ export function registerReducer( const ref = ctx.registerTypesRecursively(params); const paramsType = ctx.resolveType(ref).value; const isLifecycle = lifecycle != null; + if ( + isLifecycle && + opts?.visibility != null && + opts.visibility !== 'internal' + ) { + throw new TypeError('Lifecycle reducers only support internal visibility'); + } ctx.moduleDef.reducers.push({ sourceName: exportName, params: paramsType, - //ModuleDef validation code is responsible to mark private reducers - visibility: FunctionVisibility.ClientCallable, + // Keep the legacy default distinct from an explicit public declaration. + visibility: rawVisibility(opts?.visibility), //Hardcoded for now - reducers do not return values yet okReturnType: AlgebraicType.Product({ elements: [] }), errReturnType: AlgebraicType.String, diff --git a/crates/bindings-typescript/src/server/runtime.ts b/crates/bindings-typescript/src/server/runtime.ts index 1fac18ac95b..aaa50ad186d 100644 --- a/crates/bindings-typescript/src/server/runtime.ts +++ b/crates/bindings-typescript/src/server/runtime.ts @@ -1,6 +1,7 @@ import { environment } from './environment'; import * as _syscalls2_0 from 'spacetime:sys@2.0'; import * as _syscalls2_1 from 'spacetime:sys@2.1'; +import * as _syscalls2_2 from 'spacetime:sys@2.2'; import type { ModuleHooks, u128, u16, u256, u32 } from 'spacetime:sys@2.0'; import { @@ -80,7 +81,7 @@ import { HttpRequest, HttpResponse } from '../lib/autogen/types'; const { freeze } = Object; -export const sys = { ..._syscalls2_0, ..._syscalls2_1 }; +export const sys = { ..._syscalls2_0, ..._syscalls2_1, ..._syscalls2_2 }; function requestFromWire(request: HttpRequest, body: Uint8Array): Request { return Request[makeRequest](body, { @@ -125,7 +126,8 @@ class JwtClaimsImpl implements JwtClaims { /** * Creates a new JwtClaims instance. * @param rawPayload The JWT payload as a raw JSON string. - * @param identity The identity for this JWT. We are only taking this because we don't have a blake3 implementation (which we need to compute it). + * @param identity The verified sender Identity supplied by the host. Claims + * cannot override it, including for hosted database credentials. */ constructor( public readonly rawPayload: string, @@ -153,7 +155,7 @@ class JwtClaimsImpl implements JwtClaims { } } -class AuthCtxImpl implements AuthCtx { +export class AuthCtxImpl implements AuthCtx { public readonly isInternal: boolean; // Source of the JWT payload string, if there is one. @@ -199,29 +201,21 @@ class AuthCtxImpl implements AuthCtx { return this._jwtClaims!; } - /** Create a context representing internal (non-user) requests. */ - static internal(): AuthCtx { - return new AuthCtxImpl({ - isInternal: true, - jwtSource: () => null, - senderIdentity: Identity.zero(), - }); - } - /** If there is a connection id, look up the JWT payload from the system tables. */ static fromSystemTables( connectionId: ConnectionId | null, - sender: Identity + sender: Identity, + callAuthFlags: number ): AuthCtx { if (connectionId === null) { return new AuthCtxImpl({ - isInternal: false, + isInternal: (callAuthFlags & 1) !== 0, jwtSource: () => null, senderIdentity: sender, }); } return new AuthCtxImpl({ - isInternal: false, + isInternal: (callAuthFlags & 1) !== 0, jwtSource: () => { const payloadBuf = sys.get_jwt_payload(connectionId.__connection_id__); if (payloadBuf.length === 0) return null; @@ -240,7 +234,7 @@ export const ReducerCtxImpl = class ReducerCtx< > implements IReducerCtx { #identity: Identity | undefined; - #senderAuth: AuthCtx | undefined; + #senderAuth: AuthCtx; #uuidCounter: { value: number } | undefined; #random: Random | undefined; sender: Identity; @@ -255,7 +249,8 @@ export const ReducerCtxImpl = class ReducerCtx< timestamp: Timestamp, connectionId: ConnectionId | null, dbView: DbView, - asViews: object = {} + asViews: object = {}, + senderAuth?: AuthCtx ) { Object.seal(this); this.sender = sender; @@ -263,6 +258,13 @@ export const ReducerCtxImpl = class ReducerCtx< this.connectionId = connectionId; this.db = dbView as unknown as DbView; this.as = asViews as AliasViews; + this.#senderAuth = + senderAuth ?? + AuthCtxImpl.fromSystemTables( + connectionId, + sender, + sys.get_call_auth_flags() + ); } /** Reset the `ReducerCtx` to be used for a new transaction */ @@ -278,7 +280,11 @@ export const ReducerCtxImpl = class ReducerCtx< me.timestamp = timestamp; me.connectionId = connectionId; me.#uuidCounter = undefined; - me.#senderAuth = undefined; + me.#senderAuth = AuthCtxImpl.fromSystemTables( + connectionId, + sender, + sys.get_call_auth_flags() + ); if (dbView !== undefined) { me.db = dbView; } @@ -296,10 +302,7 @@ export const ReducerCtxImpl = class ReducerCtx< } get senderAuth() { - return (this.#senderAuth ??= AuthCtxImpl.fromSystemTables( - this.connectionId, - this.sender - )); + return this.#senderAuth; } get random() { diff --git a/crates/bindings-typescript/src/server/schema.ts b/crates/bindings-typescript/src/server/schema.ts index c399a66d31e..648b5c42fc3 100644 --- a/crates/bindings-typescript/src/server/schema.ts +++ b/crates/bindings-typescript/src/server/schema.ts @@ -405,7 +405,10 @@ export class Schema implements ModuleDefaultExport { case 2: { let arg1; [arg1, fn] = args; - if (typeof arg1.name === 'string') + if ( + typeof arg1.name === 'string' || + typeof arg1.visibility === 'string' + ) opts = arg1 as ReducerOptsWithOptionalName; else params = arg1 as Params; break; @@ -644,7 +647,10 @@ export class Schema implements ModuleDefaultExport { case 3: { let arg1; [arg1, ret, fn] = args; - if (typeof arg1.name === 'string') + if ( + typeof arg1.name === 'string' || + typeof arg1.visibility === 'string' + ) opts = arg1 as ProcedureOptsWithOptionalName; else params = arg1 as Params; break; diff --git a/crates/bindings-typescript/src/server/sys.d.ts b/crates/bindings-typescript/src/server/sys.d.ts index 1f74debd2fc..32addabb9e7 100644 --- a/crates/bindings-typescript/src/server/sys.d.ts +++ b/crates/bindings-typescript/src/server/sys.d.ts @@ -124,7 +124,10 @@ declare module 'spacetime:sys@2.1' { export function datastore_clear(table_id: u32): u64; } -// sys2.2 is reserved for the separate invocation-authority extension. +declare module 'spacetime:sys@2.2' { + /** Verified invocation flags. Bit 0 is INTERNAL; JWT presence is independent. */ + export function get_call_auth_flags(): number; +} declare module 'spacetime:sys@2.3' { /** Null means missing; an empty string is a present value. */ diff --git a/crates/bindings-typescript/tests/__mocks__/spacetime-auth.ts b/crates/bindings-typescript/tests/__mocks__/spacetime-auth.ts new file mode 100644 index 00000000000..940221f4710 --- /dev/null +++ b/crates/bindings-typescript/tests/__mocks__/spacetime-auth.ts @@ -0,0 +1,2 @@ +// Ordinary host calls are external unless a test supplies trusted flags. +export const get_call_auth_flags = (): number => 0; diff --git a/crates/bindings-typescript/tests/hosted_auth.test.ts b/crates/bindings-typescript/tests/hosted_auth.test.ts new file mode 100644 index 00000000000..047aaf2d196 --- /dev/null +++ b/crates/bindings-typescript/tests/hosted_auth.test.ts @@ -0,0 +1,303 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +const host = vi.hoisted(() => ({ + flags: 0, + payload: '', + jwtReads: 0, + flagReads: 0, +})); +vi.mock('spacetime:sys@2.0', () => ({ + moduleHooks: Symbol('moduleHooks'), + identity: () => 1n, + row_iter_bsatn_close: () => {}, + procedure_start_mut_tx: () => 0n, + procedure_commit_mut_tx: () => {}, + procedure_abort_mut_tx: () => {}, + get_jwt_payload: () => { + host.jwtReads++; + return new TextEncoder().encode(host.payload); + }, +})); +vi.mock('spacetime:sys@2.2', () => ({ + get_call_auth_flags: () => { + host.flagReads++; + return host.flags; + }, +})); + +import { ReducerCtxImpl } from '../src/server/runtime'; +import { ConnectionId } from '../src/lib/connection_id'; +import { Identity } from '../src/lib/identity'; +import { Timestamp } from '../src/lib/timestamp'; +import { schema, exportContext, registerExport } from '../src/server/schema'; +import { callProcedure } from '../src/server/procedures'; +import { t } from '../src/lib/type_builders'; +import { + AlgebraicType, + FunctionVisibility, + ProductType, + RawModuleDef, + RawModuleDefV10Section, + RawReducerDefV10, +} from '../src/lib/autogen/types'; +import BinaryReader from '../src/lib/binary_reader'; +import BinaryWriter from '../src/lib/binary_writer'; + +beforeEach(() => { + Object.assign(host, { flags: 0, payload: '', jwtReads: 0, flagReads: 0 }); +}); + +describe('verified invocation authentication', () => { + it.each([0, 1])( + 'preserves flag %s for calls without a connection or JWT', + flags => { + host.flags = flags; + const ctx = new ReducerCtxImpl( + new Identity(1n), + Timestamp.UNIX_EPOCH, + null, + {} + ); + host.flags = flags ^ 1; + expect(host.flagReads).toBe(1); + expect(ctx.senderAuth.isInternal).toBe(Boolean(flags)); + expect(ctx.senderAuth.hasJWT).toBe(false); + expect(ctx.senderAuth.jwt).toBeNull(); + expect(host.jwtReads).toBe(0); + } + ); + + it('retains an internal connection and JWT independently, using the verified sender Identity', () => { + host.flags = 1; + host.payload = JSON.stringify({ + iss: 'unrelated-issuer', + sub: 'unrelated-subject', + identity: 'untrusted-claim', + }); + const sender = new Identity(123n); + const connection = new ConnectionId(7n); + const ctx = new ReducerCtxImpl( + sender, + Timestamp.UNIX_EPOCH, + connection, + {} + ); + host.flags = 0; + expect(ctx.connectionId).toBe(connection); + expect(ctx.senderAuth.isInternal).toBe(true); + expect(host.jwtReads).toBe(0); + expect(ctx.senderAuth.hasJWT).toBe(true); + expect(ctx.senderAuth.jwt?.identity).toBe(sender); + expect(ctx.senderAuth.jwt?.subject).toBe('unrelated-subject'); + expect(host.jwtReads).toBe(1); + }); + + it('refreshes captured flags and sender when a cached reducer context is reused', () => { + host.flags = 1; + const ctx = new ReducerCtxImpl( + new Identity(1n), + Timestamp.UNIX_EPOCH, + null, + {} + ); + const firstAuth = ctx.senderAuth; + host.flags = 0; + ReducerCtxImpl.reset( + ctx, + new Identity(2n), + Timestamp.UNIX_EPOCH, + new ConnectionId(8n) + ); + host.flags = 1; + expect(firstAuth.isInternal).toBe(true); + expect(ctx.senderAuth.isInternal).toBe(false); + expect(ctx.senderAuth.hasJWT).toBe(false); + }); + + it('preserves procedure auth inside a transaction after the host flags change', () => { + host.flags = 1; + const module = schema({}); + const proc = module.procedure(t.unit(), ctx => { + host.flags = 0; + ctx.withTx(tx => { + expect(tx.senderAuth).toBe(ctx.senderAuth); + expect(tx.senderAuth.isInternal).toBe(true); + expect(tx.connectionId).toBe(ctx.connectionId); + }); + return {}; + }); + const inner = proc[exportContext]!; + proc[registerExport](inner, 'procedure_auth'); + callProcedure( + inner.procedures, + 0, + new Identity(9n), + new ConnectionId(8n), + Timestamp.UNIX_EPOCH, + new Uint8Array(), + () => ({}) + ); + expect(host.flagReads).toBe(1); + }); +}); + +describe('V10 explicit function visibility', () => { + it('preserves existing visibility tags and appends the new variants and capability section', () => { + const legacyVisibility = t.enum('LegacyFunctionVisibility', { + Private: t.unit(), + ClientCallable: t.unit(), + }); + const variants = [ + FunctionVisibility.Private, + FunctionVisibility.ClientCallable, + FunctionVisibility.Internal, + FunctionVisibility.ExplicitClientCallable, + ]; + for (const [tag, visibility] of variants.entries()) { + const writer = new BinaryWriter(8); + FunctionVisibility.serialize(writer, visibility); + expect([...writer.getBuffer()]).toEqual([tag]); + const reader = new BinaryReader(writer.getBuffer()); + if (tag < 2) { + expect(legacyVisibility.deserialize(reader).tag).toBe(visibility.tag); + } + } + const writer = new BinaryWriter(8); + RawModuleDefV10Section.serialize(writer, { + tag: 'Capabilities', + value: [], + }); + expect([...writer.getBuffer()]).toEqual([15, 0, 0, 0, 0]); + }); + + it('retains the V10 reducer field layout without an optional visibility wrapper', () => { + const module = schema({}); + const reducer = module.reducer({ visibility: 'public' }, () => {}); + const inner = reducer[exportContext]!; + reducer[registerExport](inner, 'public_reducer'); + const definition = inner.moduleDef.reducers[0]; + const writer = new BinaryWriter(128); + RawReducerDefV10.serialize(writer, definition); + const expected = new BinaryWriter(128); + expected.writeString(definition.sourceName); + ProductType.serialize(expected, definition.params); + expected.writeByte(3); + AlgebraicType.serialize(expected, definition.okReturnType); + AlgebraicType.serialize(expected, definition.errReturnType); + expect(writer.getBuffer()).toEqual(expected.getBuffer()); + }); + + it('serializes omission separately from explicit visibility and advertises hosted auth', () => { + const module = schema({}); + const omitted = module.reducer(() => {}); + const explicitlyPublic = module.reducer({ visibility: 'public' }, () => {}); + const privateReducer = module.reducer({ visibility: 'private' }, () => {}); + const internalReducer = module.reducer( + { visibility: 'internal' }, + () => {} + ); + const inner = omitted[exportContext]!; + for (const [name, reducer] of Object.entries({ + omitted, + explicitlyPublic, + privateReducer, + internalReducer, + })) { + reducer[registerExport](inner, name); + } + // Being scheduled must not erase a public choice or manufacture an explicit + // choice for the default. The host resolves the latter to Private. + for (const name of [ + 'omitted', + 'explicitlyPublic', + 'privateReducer', + 'internalReducer', + ]) { + inner.moduleDef.schedules.push({ + sourceName: undefined, + tableName: `jobs_${name}`, + scheduleAtCol: 0, + functionName: name, + }); + } + const raw = RawModuleDef.V10(inner.rawModuleDefV10()); + const writer = new BinaryWriter(128); + RawModuleDef.serialize(writer, raw); + expect(writer.getBuffer()[0]).toBe(2); + const decoded = RawModuleDef.deserialize( + new BinaryReader(writer.getBuffer()) + ); + const roundTrip = new BinaryWriter(128); + RawModuleDef.serialize(roundTrip, decoded); + expect(roundTrip.getBuffer()).toEqual(writer.getBuffer()); + expect(decoded.tag).toBe('V10'); + if (decoded.tag !== 'V10') throw new Error('Expected V10'); + const reducers = decoded.value.sections.find( + section => section.tag === 'Reducers' + ); + expect(reducers?.value.map(reducer => reducer.visibility.tag)).toEqual([ + 'ClientCallable', + 'ExplicitClientCallable', + 'Private', + 'Internal', + ]); + expect( + inner.moduleDef.reducers.map(reducer => reducer.visibility.tag) + ).toEqual([ + 'ClientCallable', + 'ExplicitClientCallable', + 'Private', + 'Internal', + ]); + expect(inner.moduleDef.capabilities).toEqual(['hosted_auth_v1']); + }); + + it('retains procedure names and explicit visibility, including a visibility parameter', () => { + const module = schema({}); + const proc = module.procedure( + { name: 'public_name', visibility: 'internal' }, + t.unit(), + () => ({}) + ); + const reducer = module.reducer({ visibility: t.string() }, () => {}); + const inner = proc[exportContext]!; + proc[registerExport](inner, 'source_name'); + reducer[registerExport](inner, 'accept_visibility'); + expect(inner.moduleDef.procedures[0].sourceName).toBe('source_name'); + expect(inner.moduleDef.procedures[0].visibility.tag).toBe('Internal'); + expect(inner.moduleDef.explicitNames.entries).toContainEqual({ + tag: 'Function', + value: { sourceName: 'source_name', canonicalName: 'public_name' }, + }); + expect(inner.moduleDef.reducers[0].params.elements[0].name).toBe( + 'visibility' + ); + }); + + it.each(['private', 'public'] as const)( + 'rejects explicit %s lifecycle declarations', + visibility => { + const module = schema({}); + const invalid = module.init({ visibility }, () => {}); + expect(() => + invalid[registerExport](invalid[exportContext]!, 'invalid_init') + ).toThrow('Lifecycle reducers only support internal visibility'); + } + ); + + it.each([undefined, 'internal'] as const)( + 'preserves permitted lifecycle declaration %s for host event dispatch', + visibility => { + const module = schema({}); + const valid = module.init({ visibility }, () => {}); + valid[registerExport](valid[exportContext]!, 'valid_init'); + const inner = valid[exportContext]!; + expect(inner.moduleDef.reducers[0].visibility.tag).toBe( + visibility === undefined ? 'ClientCallable' : 'Internal' + ); + expect(inner.moduleDef.lifeCycleReducers).toEqual([ + { lifecycleSpec: { tag: 'Init' }, functionName: 'valid_init' }, + ]); + } + ); +}); diff --git a/crates/bindings-typescript/vitest.config.ts b/crates/bindings-typescript/vitest.config.ts index 7cfc858d47c..9f80a60db65 100644 --- a/crates/bindings-typescript/vitest.config.ts +++ b/crates/bindings-typescript/vitest.config.ts @@ -14,6 +14,10 @@ export default defineConfig({ alias: [ { find: 'spacetime:sys@2.0', replacement: sysMock }, { find: 'spacetime:sys@2.1', replacement: sysMock }, + { + find: 'spacetime:sys@2.2', + replacement: fileURLToPath(new URL('./tests/__mocks__/spacetime-auth.ts', import.meta.url)), + }, ], }, test: { diff --git a/crates/bindings/src/http.rs b/crates/bindings/src/http.rs index 3638d35f9e2..1ad7a0edef5 100644 --- a/crates/bindings/src/http.rs +++ b/crates/bindings/src/http.rs @@ -11,7 +11,7 @@ use crate::IterBuf; #[cfg(all(feature = "unstable", feature = "rand08"))] use crate::StdbRng; #[cfg(feature = "unstable")] -use crate::{try_with_tx, with_tx, Timestamp, TxContext}; +use crate::{try_with_tx, with_tx, AuthCtx, Timestamp, TxContext}; use bytes::Bytes; #[cfg(all(feature = "rand08", feature = "unstable"))] use rand08::RngCore; @@ -106,6 +106,7 @@ pub struct HandlerContext { /// Methods for performing HTTP requests. pub http: HttpClient, + sender_auth: AuthCtx, #[cfg(feature = "rand08")] pub(crate) rng: OnceCell, @@ -123,6 +124,7 @@ impl HandlerContext { env: crate::Environment::default(), timestamp, http: HttpClient {}, + sender_auth: AuthCtx::from_invocation(Identity::ZERO, None), #[cfg(feature = "rand08")] rng: OnceCell::new(), #[cfg(feature = "rand08")] @@ -143,12 +145,12 @@ impl HandlerContext { /// Acquire a mutable transaction and execute `body` with read-write access. pub fn with_tx(&mut self, body: impl Fn(&TxContext) -> T) -> T { - with_tx(body, Identity::ZERO, None) + with_tx(Identity::ZERO, None, &self.sender_auth, body) } /// Acquire a mutable transaction and execute `body` with read-write access. pub fn try_with_tx(&mut self, body: impl Fn(&TxContext) -> Result) -> Result { - try_with_tx(body, Identity::ZERO, None) + try_with_tx(Identity::ZERO, None, &self.sender_auth, body) } /// Create a new random [`Uuid`] `v4` using the built-in RNG. diff --git a/crates/bindings/src/lib.rs b/crates/bindings/src/lib.rs index d1548bca052..b234c6c5c27 100644 --- a/crates/bindings/src/lib.rs +++ b/crates/bindings/src/lib.rs @@ -1081,13 +1081,24 @@ impl ReducerContext { #[doc(hidden)] fn new(db: Local, sender: Identity, connection_id: Option, timestamp: Timestamp) -> Self { + let sender_auth = AuthCtx::from_invocation(sender, connection_id); + Self::new_with_auth(db, sender, connection_id, timestamp, sender_auth) + } + + fn new_with_auth( + db: Local, + sender: Identity, + connection_id: Option, + timestamp: Timestamp, + sender_auth: AuthCtx, + ) -> Self { Self { env: Environment::default(), db, sender, timestamp, connection_id, - sender_auth: AuthCtx::from_connection_id_opt(connection_id), + sender_auth, #[cfg(feature = "rand08")] rng: std::cell::OnceCell::new(), #[cfg(feature = "rand08")] @@ -1213,13 +1224,11 @@ impl Deref for TxContext { } } -/// We need to passthrough identity and connection_id because procedures can be invoked by users. -/// For [HttpContext] this is always anonymous ([Identity::ZERO]). -/// Construct the inner [ReducerContext] with the appropriate caller information. fn try_with_tx( - body: impl Fn(&TxContext) -> Result, - identity: Identity, + sender: Identity, connection_id: Option, + sender_auth: &AuthCtx, + body: impl Fn(&TxContext) -> Result, ) -> Result { let abort = || { crate::sys::procedure::procedure_abort_mut_tx() @@ -1231,7 +1240,8 @@ fn try_with_tx( .expect("holding `&mut HandlerContext`, so should not be in a tx already; called manually elsewhere?"); let timestamp = Timestamp::from_micros_since_unix_epoch(timestamp); - let tx = ReducerContext::new(crate::Local {}, identity, connection_id, timestamp); + // Every retry retains the original invocation's identity and authority. + let tx = ReducerContext::new_with_auth(crate::Local {}, sender, connection_id, timestamp, sender_auth.clone()); let tx = TxContext(tx); struct DoOnDrop(F); @@ -1264,9 +1274,14 @@ fn try_with_tx( res } -fn with_tx(body: impl Fn(&TxContext) -> T, identity: Identity, connection_id: Option) -> T { +fn with_tx( + sender: Identity, + connection_id: Option, + sender_auth: &AuthCtx, + body: impl Fn(&TxContext) -> T, +) -> T { use core::convert::Infallible; - match try_with_tx::(|tx| Ok(body(tx)), identity, connection_id) { + match try_with_tx::(sender, connection_id, sender_auth, |tx| Ok(body(tx))) { Ok(v) => v, Err(e) => match e {}, } @@ -1292,6 +1307,7 @@ pub struct ProcedureContext { /// /// Will be `None` for certain scheduled procedures. connection_id: Option, + sender_auth: AuthCtx, /// Methods for performing HTTP requests. pub http: crate::http::HttpClient, @@ -1314,6 +1330,7 @@ impl ProcedureContext { timestamp, connection_id, env: Environment::default(), + sender_auth: AuthCtx::from_invocation(sender, connection_id), http: http::HttpClient {}, #[cfg(feature = "rand08")] rng: std::cell::OnceCell::new(), @@ -1322,6 +1339,11 @@ impl ProcedureContext { } } + /// Host-verified authentication for this invocation, retained in transactions. + pub fn sender_auth(&self) -> &AuthCtx { + &self.sender_auth + } + /// The `Identity` of the client that invoked the procedure. pub fn sender(&self) -> Identity { self.sender @@ -1407,7 +1429,7 @@ impl ProcedureContext { /// callers should avoid writing to any captured mutable state within `body`, /// This includes interior mutability through types like [`std::cell::Cell`]. pub fn with_tx(&mut self, body: impl Fn(&TxContext) -> T) -> T { - with_tx(body, self.sender(), self.connection_id()) + with_tx(self.sender, self.connection_id, &self.sender_auth, body) } /// Acquire a mutable transaction @@ -1440,7 +1462,7 @@ impl ProcedureContext { /// callers should avoid writing to any captured mutable state within `body`, /// This includes interior mutability through types like [`std::cell::Cell`]. pub fn try_with_tx(&mut self, body: impl Fn(&TxContext) -> Result) -> Result { - try_with_tx(body, self.sender(), self.connection_id()) + try_with_tx(self.sender, self.connection_id, &self.sender_auth, body) } /// Create a new random [`Uuid`] `v4` using the built-in RNG. @@ -1873,6 +1895,7 @@ impl CtxWithHttp for ProcedureContext { /// [JWT]: https://en.wikipedia.org/wiki/JSON_Web_Token #[non_exhaustive] pub struct JwtClaims { + identity: Identity, payload: String, parsed: OnceCell, audience: OnceCell>, @@ -1888,10 +1911,16 @@ pub struct AuthCtx { } impl AuthCtx { - /// Creates an [`AuthCtx`] both for cases where there's a [`ConnectionId`] - /// and for when there isn't. - fn from_connection_id_opt(conn_id: Option) -> Self { - conn_id.map(Self::from_connection_id).unwrap_or_else(Self::internal) + /// Capture host authority immediately. JWT loading remains independent and lazy. + fn from_invocation(sender: Identity, connection_id: Option) -> Self { + let flags = spacetimedb_bindings_sys::get_call_auth_flags(); + Self::from_host_auth(sender, flags, move || connection_id.and_then(rt::get_jwt)) + } + + fn from_host_auth(sender: Identity, flags: u32, jwt_fn: impl FnOnce() -> Option + 'static) -> Self { + Self::new(flags & 1 != 0, move || { + jwt_fn().map(|payload| JwtClaims::new(payload, sender)) + }) } fn new(is_internal: bool, jwt_fn: impl FnOnce() -> Option + 'static) -> Self { @@ -1914,14 +1943,18 @@ impl AuthCtx { /// /// [JWT]: https://en.wikipedia.org/wiki/JSON_Web_Token pub fn from_jwt_payload(jwt_payload: String) -> AuthCtx { - Self::new(false, move || Some(JwtClaims::new(jwt_payload))) + let parsed: serde_json::Value = serde_json::from_str(&jwt_payload).expect("invalid test JWT payload"); + let sender = Identity::from_claims( + parsed["iss"].as_str().expect("missing test issuer"), + parsed["sub"].as_str().expect("missing test subject"), + ); + Self::from_jwt_payload_for_sender(jwt_payload, sender, false) } - /// Creates an [`AuthCtx`] that reads the [JWT] for the given connection id. - /// - /// [JWT]: https://en.wikipedia.org/wiki/JSON_Web_Token - fn from_connection_id(connection_id: ConnectionId) -> AuthCtx { - Self::new(false, move || rt::get_jwt(connection_id).map(JwtClaims::new)) + /// Create test authentication with an explicit effective sender and authority. + /// Production contexts receive these values from the host, not JWT claims. + pub fn from_jwt_payload_for_sender(jwt_payload: String, sender: Identity, is_internal: bool) -> AuthCtx { + Self::new(is_internal, move || Some(JwtClaims::new(jwt_payload, sender))) } /// Returns whether this reducer was spawned from inside the database. @@ -1929,8 +1962,8 @@ impl AuthCtx { self.is_internal } - /// Checks if there is a [JWT] without loading it. - /// If [`AuthCtx::is_internal`] returns true, this will return false. + /// Returns whether this invocation has a [JWT]. Internal invocations may + /// also carry a JWT; internal authority and credential presence are independent. /// /// [JWT]: https://en.wikipedia.org/wiki/JSON_Web_Token pub fn has_jwt(&self) -> bool { @@ -1946,8 +1979,9 @@ impl AuthCtx { } impl JwtClaims { - fn new(jwt: String) -> Self { + fn new(jwt: String, identity: Identity) -> Self { Self { + identity, payload: jwt, parsed: OnceCell::new(), audience: OnceCell::new(), @@ -1989,10 +2023,10 @@ impl JwtClaims { self.audience.get_or_init(|| self.extract_audience()) } - /// Returns the identity for these credentials, which is - /// based on the iss and sub claims. + /// The effective sender verified by the host for this invocation. + /// Hosted database credentials need not derive this Identity from iss/sub. pub fn identity(&self) -> Identity { - Identity::from_claims(self.issuer(), self.subject()) + self.identity } /// Get the whole JWT payload as a json string. @@ -2145,3 +2179,54 @@ mod tests { assert_eq!(audience, &["my-project-id".to_string()]); } } + +#[cfg(test)] +mod hosted_auth_tests { + use super::*; + + #[test] + fn host_authority_is_independent_of_jwt_presence() { + for internal in [false, true] { + for has_jwt in [false, true] { + let flags = u32::from(internal) | (1 << 31); + let auth = AuthCtx::from_host_auth(Identity::ONE, flags, move || { + has_jwt.then(|| r#"{"iss":"hosted","sub":"generation","hex_identity":"untrusted"}"#.to_string()) + }); + assert_eq!(auth.is_internal(), internal); + assert_eq!(auth.has_jwt(), has_jwt); + if let Some(jwt) = auth.jwt() { + assert_eq!(jwt.identity(), Identity::ONE); + assert_ne!(jwt.identity(), Identity::from_claims(jwt.issuer(), jwt.subject())); + } + } + } + } + + #[test] + fn verified_sender_wins_over_signed_payload_identity_claims() { + let payload = format!( + r#"{{"iss":"hosted","sub":"generation","hex_identity":"{}"}}"#, + Identity::ZERO.to_hex() + ); + let auth = AuthCtx::from_host_auth(Identity::ONE, 1, move || Some(payload)); + assert_eq!(auth.jwt().unwrap().identity(), Identity::ONE); + assert!(auth.is_internal()); + assert!(auth.has_jwt()); + } + + #[test] + fn transaction_context_preserves_identity_connection_and_authentication() { + let sender = Identity::ONE; + let connection = Some(ConnectionId::from_u128(123)); + let auth = AuthCtx::from_host_auth(sender, 1, || Some(r#"{"iss":"hosted","sub":"generation"}"#.into())); + // Procedure transactions/retries call this same constructor with the + // original captured AuthCtx rather than manufacturing internal authority. + for timestamp in [Timestamp::UNIX_EPOCH, Timestamp::from_micros_since_unix_epoch(1)] { + let context = ReducerContext::new_with_auth(Local {}, sender, connection, timestamp, auth.clone()); + assert_eq!(context.sender(), sender); + assert_eq!(context.connection_id(), connection); + assert!(context.sender_auth().is_internal()); + assert_eq!(context.sender_auth().jwt().unwrap().identity(), sender); + } + } +} diff --git a/crates/bindings/src/rt.rs b/crates/bindings/src/rt.rs index b09c998f5c3..3ee8731225e 100644 --- a/crates/bindings/src/rt.rs +++ b/crates/bindings/src/rt.rs @@ -1,5 +1,7 @@ #![deny(unsafe_op_in_unsafe_fn)] +pub use spacetimedb_lib::db::raw_def::v10::FunctionVisibility; + use crate::query_builder::{FromWhere, HasCols, LeftSemiJoin, RawQuery, RightSemiJoin, Table as QbTable}; use crate::table::IndexAlgo; use crate::{sys, AnonymousViewContext, IterBuf, ReducerContext, ReducerResult, SpacetimeType, Table, ViewContext}; @@ -159,6 +161,9 @@ pub trait FnInfo: ExplicitNames { /// The lifecycle of the function, if there is one. const LIFECYCLE: Option = None; + /// Explicit SpacetimeDB visibility; Rust item visibility is independent. + const DECLARED_VISIBILITY: Option = None; + /// A description of the parameter names of the function. const ARG_NAMES: &'static [Option<&'static str>]; @@ -800,9 +805,13 @@ pub fn register_reducer<'a, A: Args<'a>, I: FnInfo>(_: impl register_describer(|module| { let params = A::schema::(&mut module.inner); if let Some(lifecycle) = I::LIFECYCLE { - module.inner.add_lifecycle_reducer(lifecycle, I::NAME, params); + module + .inner + .add_lifecycle_reducer_with_visibility(lifecycle, I::NAME, params, I::DECLARED_VISIBILITY); } else { - module.inner.add_reducer(I::NAME, params); + module + .inner + .add_reducer_with_visibility(I::NAME, params, I::DECLARED_VISIBILITY); } module.reducers.push(I::INVOKE); @@ -819,7 +828,9 @@ where register_describer(|module| { let params = A::schema::(&mut module.inner); let ret_ty = ::make_type(&mut module.inner); - module.inner.add_procedure(I::NAME, params, ret_ty); + module + .inner + .add_procedure_with_visibility(I::NAME, params, ret_ty, I::DECLARED_VISIBILITY); module.procedures.push(I::INVOKE); module.inner.add_explicit_names(I::explicit_names()); @@ -982,6 +993,9 @@ extern "C" fn __describe_module__(description: BytesSink) { describer(&mut module) } + // These bindings capture host flags and preserve the verified sender in JWT claims. + module.inner.add_capability("hosted_auth_v1"); + // Serialize the module to bsatn. let module_def = module.inner.finish(); let module_def = RawModuleDef::V10(module_def); diff --git a/crates/bindings/tests/pass/function_visibility.rs b/crates/bindings/tests/pass/function_visibility.rs new file mode 100644 index 00000000000..7f53af5717f --- /dev/null +++ b/crates/bindings/tests/pass/function_visibility.rs @@ -0,0 +1,62 @@ +#![deny(warnings)] + +use spacetimedb::rt::{FnInfo, FunctionVisibility}; +use spacetimedb::{ProcedureContext, ReducerContext}; + +#[spacetimedb::reducer(internal)] +pub fn internal_reducer(_ctx: &ReducerContext) {} + +#[spacetimedb::reducer(private)] +fn private_reducer(_ctx: &ReducerContext) {} + +#[spacetimedb::reducer(public)] +fn public_reducer(_ctx: &ReducerContext) {} + +#[spacetimedb::reducer(init, internal)] +fn initialize(_ctx: &ReducerContext) {} + +#[spacetimedb::procedure(internal)] +fn internal_procedure(_ctx: &mut ProcedureContext) -> u64 { + 0 +} + +#[spacetimedb::procedure(private)] +fn private_procedure(_ctx: &mut ProcedureContext) -> u64 { + 0 +} + +#[spacetimedb::procedure(public)] +fn public_procedure(_ctx: &mut ProcedureContext) -> u64 { + 0 +} + +fn main() { + assert!(matches!( + internal_reducer::DECLARED_VISIBILITY, + Some(FunctionVisibility::Internal) + )); + assert!(matches!( + private_reducer::DECLARED_VISIBILITY, + Some(FunctionVisibility::Private) + )); + assert!(matches!( + public_reducer::DECLARED_VISIBILITY, + Some(FunctionVisibility::ClientCallable) + )); + assert!(matches!( + initialize::DECLARED_VISIBILITY, + Some(FunctionVisibility::Internal) + )); + assert!(matches!( + internal_procedure::DECLARED_VISIBILITY, + Some(FunctionVisibility::Internal) + )); + assert!(matches!( + private_procedure::DECLARED_VISIBILITY, + Some(FunctionVisibility::Private) + )); + assert!(matches!( + public_procedure::DECLARED_VISIBILITY, + Some(FunctionVisibility::ClientCallable) + )); +} diff --git a/crates/bindings/tests/ui/tables.stderr b/crates/bindings/tests/ui/tables.stderr index 7609d9ba378..18b61f49224 100644 --- a/crates/bindings/tests/ui/tables.stderr +++ b/crates/bindings/tests/ui/tables.stderr @@ -209,13 +209,13 @@ error[E0277]: `&'a Alpha` cannot appear as an argument to an index filtering ope = note: The allowed set of types are limited to integers, bool, strings, `Identity`, `Uuid`, `Timestamp`, `ConnectionId`, `Hash` and no-payload enums which derive `SpacetimeType`, = help: the following other types implement trait `FilterableValue`: &ConnectionId + &ContainerMode &FunctionVisibility &Identity &Lifecycle - &TableAccess - &TableType - &bool - ðnum::int::I256 + &PortExposure + &PortProtocol + &RestartPolicy and $N others note: required by a bound in `UniqueColumn::::ColType, Col>::find` --> src/table.rs @@ -241,13 +241,13 @@ help: the trait `FilterableValue` is not implemented for `Alpha` | ^^^^^^^^^^^^ = help: the following other types implement trait `FilterableValue`: &ConnectionId + &ContainerMode &FunctionVisibility &Identity &Lifecycle - &TableAccess - &TableType - &bool - ðnum::int::I256 + &PortExposure + &PortProtocol + &RestartPolicy and $N others = note: required for `Alpha` to implement `IndexScanRangeBounds<(Alpha,), SingleBound>` note: required by a bound in `RangedIndex::::filter` diff --git a/crates/cli/src/subcommands/generate.rs b/crates/cli/src/subcommands/generate.rs index e312bf65ea9..56016694d32 100644 --- a/crates/cli/src/subcommands/generate.rs +++ b/crates/cli/src/subcommands/generate.rs @@ -267,7 +267,7 @@ pub fn cli() -> clap::Command { .long("include-private") .action(SetTrue) .default_value("false") - .help("Include private tables and functions in generated code (types are always included)."), + .help("Include private tables and private/internal non-lifecycle functions (types are always included)."), ) .arg(common_args::yes()) .arg( diff --git a/crates/client-api/src/routes/database.rs b/crates/client-api/src/routes/database.rs index f170f9b290e..9d3561b4542 100644 --- a/crates/client-api/src/routes/database.rs +++ b/crates/client-api/src/routes/database.rs @@ -533,7 +533,8 @@ where let module_def = &module.info.module_def; let response_json = match version { SchemaVersion::V9 => { - let raw = RawModuleDefV9::from(module_def.as_ref().clone()); + let raw = RawModuleDefV9::try_from(module_def.as_ref().clone()) + .map_err(|err| bad_request(err.to_string().into()))?; axum::Json(sats::serde::SerdeWrapper(raw)).into_response() } SchemaVersion::V10 => { diff --git a/crates/client-api/src/routes/mcp.rs b/crates/client-api/src/routes/mcp.rs index 2a5c2c156ec..b94f6739db3 100644 --- a/crates/client-api/src/routes/mcp.rs +++ b/crates/client-api/src/routes/mcp.rs @@ -9,7 +9,7 @@ use spacetimedb::auth::identity::ConnectionAuthCtx; use spacetimedb::host::{FunctionArgs, ReducerOutcome}; use spacetimedb::identity::Identity; use spacetimedb::messages::control_db::Database; -use spacetimedb_lib::db::raw_def::v9::RawModuleDefV9; +use spacetimedb_lib::db::raw_def::v10::RawModuleDefV10; use spacetimedb_lib::sats; use super::database::{ @@ -350,7 +350,7 @@ where let database = target.resolve(ctx, addressed).await?; let leader = find_database_leader(ctx, &database).await?; let module = leader.wait_for_module(MODULE_WAIT_TIMEOUT).await.map_err(log_and_500)?; - let raw = RawModuleDefV9::from(module.info.module_def.as_ref().clone()); + let raw = RawModuleDefV10::from(module.info.module_def.as_ref().clone()); let json = serde_json::to_string(&sats::serde::SerdeWrapper(raw)).map_err(log_and_500)?; Ok(json) } diff --git a/crates/codegen/src/util.rs b/crates/codegen/src/util.rs index 5a62afdd06f..fdb0e7fcce9 100644 --- a/crates/codegen/src/util.rs +++ b/crates/codegen/src/util.rs @@ -10,8 +10,8 @@ use convert_case::{Case, Casing}; use itertools::Itertools; use spacetimedb_lib::db::raw_def::v9::TableAccess; use spacetimedb_lib::sats::layout::PrimitiveType; +use spacetimedb_lib::sats::AlgebraicTypeRef; use spacetimedb_lib::version; -use spacetimedb_lib::{db::raw_def::v9::Lifecycle, sats::AlgebraicTypeRef}; use spacetimedb_primitives::ColList; use spacetimedb_schema::{def::ViewDef, type_for_generate::ProductTypeDef}; use spacetimedb_schema::{ @@ -99,31 +99,20 @@ pub(super) fn is_reducer_invokable(reducer: &ReducerDef) -> bool { reducer.lifecycle.is_none() } -/// Iterate over all the [`ReducerDef`]s defined by the module, in alphabetical order by name. -/// -/// Skipping the `init` reducer and internal [`FunctionVisibiity::Internal`] reducers because -/// they should not be directly invokable. -/// Sorting is not necessary for reducers because they are already stored in an IndexMap. +/// Non-lifecycle reducer entry points in declaration order. Default clients see +/// only public functions; IncludePrivate adds Private and Internal methods. pub(super) fn iter_reducers(module: &ModuleDef, visibility: CodegenVisibility) -> impl Iterator { module .reducers() - // `RawModuleDefV10` already marks all lifecycle reducers as private, but we keep - // this filter for backward compatibility with older versions where `init` - // reducers were not private. - .filter(|reducer| reducer.lifecycle != Some(Lifecycle::Init)) - // Prior to `RawModuleDefV10`, all reducers were public by default. Filtering out - // internal reducers here does not break SDKs built against older versions. + .filter(|reducer| reducer.lifecycle.is_none()) .filter(move |reducer| match visibility { CodegenVisibility::IncludePrivate => true, - CodegenVisibility::OnlyPublic => !reducer.visibility.is_private(), + CodegenVisibility::OnlyPublic => reducer.visibility.is_client_callable(), }) } -/// Iterate over all the [`ProcedureDef`]s defined by the module, in alphabetical order by name. -/// -/// Skipping internal [`FunctionVisibiity::Internal`] procedures because they should not be -/// directly invokable. -/// Sorting is necessary to have deterministic reproducible codegen. +/// Procedure entry points in alphabetical order. Default clients see only Public +/// functions; IncludePrivate also generates Private and Internal methods. pub(super) fn iter_procedures( module: &ModuleDef, visibility: CodegenVisibility, @@ -133,7 +122,7 @@ pub(super) fn iter_procedures( .sorted_by_key(|procedure| &procedure.name) .filter(move |procedure| match visibility { CodegenVisibility::IncludePrivate => true, - CodegenVisibility::OnlyPublic => !procedure.visibility.is_private(), + CodegenVisibility::OnlyPublic => procedure.visibility.is_client_callable(), }) } @@ -223,3 +212,64 @@ pub(super) fn iter_constraints(table: &TableDef) -> impl Iterator impl Iterator { module.types().sorted_by_key(|table| &table.accessor_name) } + +#[cfg(test)] +mod visibility_tests { + use super::*; + use spacetimedb_lib::db::raw_def::{ + v10::{FunctionVisibility, RawModuleDefV10Builder}, + v9::Lifecycle, + }; + use spacetimedb_lib::{AlgebraicType, ProductType}; + + #[test] + fn public_codegen_excludes_internal_private_and_every_lifecycle() { + let mut builder = RawModuleDefV10Builder::new(); + builder.add_reducer("ordinary", ProductType::unit()); + for (name, visibility) in [ + ("public_function", FunctionVisibility::ClientCallable), + ("private_function", FunctionVisibility::Private), + ("internal_function", FunctionVisibility::Internal), + ] { + builder.add_reducer_with_visibility(name, ProductType::unit(), Some(visibility)); + builder.add_procedure_with_visibility( + format!("{name}_procedure"), + ProductType::unit(), + AlgebraicType::unit(), + Some(visibility), + ); + } + for (name, lifecycle) in [ + ("init", Lifecycle::Init), + ("connect", Lifecycle::OnConnect), + ("disconnect", Lifecycle::OnDisconnect), + ] { + builder.add_lifecycle_reducer(lifecycle, name, ProductType::unit()); + } + let module: ModuleDef = builder.finish().try_into().unwrap(); + let names = |visibility| { + iter_reducers(&module, visibility) + .map(|r| &r.name[..]) + .collect::>() + }; + assert_eq!(names(CodegenVisibility::OnlyPublic), ["ordinary", "public_function"]); + assert_eq!( + names(CodegenVisibility::IncludePrivate), + ["ordinary", "public_function", "private_function", "internal_function"] + ); + let names = |visibility| { + iter_procedures(&module, visibility) + .map(|p| &p.name[..]) + .collect::>() + }; + assert_eq!(names(CodegenVisibility::OnlyPublic), ["public_function_procedure"]); + assert_eq!( + names(CodegenVisibility::IncludePrivate), + [ + "internal_function_procedure", + "private_function_procedure", + "public_function_procedure" + ] + ); + } +} diff --git a/crates/core/src/host/host_controller.rs b/crates/core/src/host/host_controller.rs index ff017b733c8..fad3d0a1e7d 100644 --- a/crates/core/src/host/host_controller.rs +++ b/crates/core/src/host/host_controller.rs @@ -55,6 +55,9 @@ use tokio::sync::{watch, OwnedRwLockReadGuard, OwnedRwLockWriteGuard, RwLock as use tokio::time::error::Elapsed; use tokio::time::{interval_at, timeout, Instant}; +#[cfg(test)] +mod invocation_flags_tests; + // TODO: // // - [db::Config] should be per-[Database] diff --git a/crates/core/src/host/host_controller/invocation_flags_tests.rs b/crates/core/src/host/host_controller/invocation_flags_tests.rs new file mode 100644 index 00000000000..a451cf00dc7 --- /dev/null +++ b/crates/core/src/host/host_controller/invocation_flags_tests.rs @@ -0,0 +1,139 @@ +//! Run actual V8 hosts without a network or external service. +use super::*; +use crate::db::persistence::LocalPersistenceProvider; +use crate::host::module_host::CallProcedureParams; +use crate::host::{ArgsTuple, FunctionArgs}; +use spacetimedb_lib::db::raw_def::{v10::FunctionVisibility, v10::RawModuleDefV10Builder, v9::Lifecycle}; +use spacetimedb_paths::FromPathUnchecked; +use spacetimedb_primitives::ProcedureId; +use spacetimedb_sats::{AlgebraicType, ProductType}; + +fn program() -> Program { + let mut schema = RawModuleDefV10Builder::new(); + schema.add_lifecycle_reducer(Lifecycle::Init, "init", ProductType::unit()); + schema.add_reducer("external", ProductType::unit()); + schema.add_reducer_with_visibility("internal", ProductType::unit(), Some(FunctionVisibility::Internal)); + schema.add_reducer_with_visibility("private", ProductType::unit(), Some(FunctionVisibility::Private)); + schema.add_procedure("external_procedure", ProductType::unit(), AlgebraicType::U8); + schema.add_procedure_with_visibility( + "internal_procedure", + ProductType::unit(), + AlgebraicType::U8, + Some(FunctionVisibility::Internal), + ); + let schema = spacetimedb_lib::bsatn::to_vec(&spacetimedb_lib::RawModuleDef::V10(schema.finish())).unwrap(); + Program::from_bytes( + ModuleKind::JS, + format!( + r#" + import {{ register_hooks }} from "spacetime:sys@1.0"; + import {{ register_hooks as register_procedures }} from "spacetime:sys@1.2"; + import {{ get_call_auth_flags }} from "spacetime:sys@2.2"; + register_hooks({{ + __describe_module__: function() {{ return new Uint8Array({schema:?}); }}, + __call_reducer__: function(id) {{ + const expected = id === 0 ? 1 : 0; + if (get_call_auth_flags() !== expected) {{ throw new Error("incorrect invocation flags"); }} + return {{ tag: "ok" }}; + }}, + }}); + register_procedures({{ __call_procedure__: function() {{ + return new Uint8Array([get_call_auth_flags()]); + }} }}); + "# + ) + .into_bytes(), + ) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn invocation_flags_are_host_owned_and_internal_visibility_is_enforced() { + let directory = tempfile::tempdir().unwrap(); + let data = Arc::new(ServerDataDir::from_path_unchecked(directory.path().to_owned())); + let program = program(); + let initial = program.clone(); + let storage = move |hash| { + let program = initial.clone(); + async move { Ok((program.hash == hash).then_some(program.bytes)) } + }; + let controller = HostController::new( + data.clone(), + db::Config { + storage: db::Storage::Memory, + page_pool_max_size: None, + }, + HostRuntimeConfig::default(), + Arc::new(storage), + Arc::new(NullEnergyMonitor), + Arc::new(LocalPersistenceProvider::new(data)), + JobCores::without_pinned_cores(), + ); + let database = Database { + id: 0xab10, + database_identity: Identity::from_u256(0xab10u64.into()), + owner_identity: Identity::ONE, + host_type: HostType::Js, + initial_program: program.hash, + }; + // The init reducer itself asserts flags=1, so successful construction also + // verifies the real host-to-JS syscall path for a trusted lifecycle call. + let module = controller + .get_or_launch_module_host(database.clone(), database.id) + .await + .unwrap(); + for sender in [database.owner_identity, database.database_identity, Identity::ZERO] { + module + .call_reducer(sender, None, None, None, None, "external", FunctionArgs::Nullary) + .await + .unwrap() + .outcome + .into_result() + .unwrap(); + for name in ["internal", "init"] { + assert!(module + .call_reducer(sender, None, None, None, None, name, FunctionArgs::Nullary) + .await + .is_err()); + } + let result = module + .call_procedure(sender, None, None, "external_procedure", FunctionArgs::Nullary) + .await; + assert_eq!(result.result.unwrap().return_val, AlgebraicValue::U8(0)); + assert!(module + .call_procedure(sender, None, None, "internal_procedure", FunctionArgs::Nullary) + .await + .result + .is_err()); + assert_eq!( + module + .call_reducer(sender, None, None, None, None, "private", FunctionArgs::Nullary) + .await + .is_ok(), + sender == database.owner_identity, + ); + } + // Trusted host work uses its explicit constructor. An ordinary later call + // observes zero again even if the procedure instance is reused. + let result = module + .call_procedure_with_params( + "internal_procedure", + CallProcedureParams::from_system( + Timestamp::now(), + database.database_identity, + ProcedureId(1), + ArgsTuple::nullary(), + ), + ) + .await + .unwrap(); + assert_eq!(result.result.unwrap().return_val, AlgebraicValue::U8(1)); + let result = module + .call_procedure(Identity::ONE, None, None, "external_procedure", FunctionArgs::Nullary) + .await; + assert_eq!(result.result.unwrap().return_val, AlgebraicValue::U8(0)); + drop(module); + controller + .exit_module_host(database.id, Duration::from_secs(5)) + .await + .unwrap(); +} diff --git a/crates/core/src/host/instance_env.rs b/crates/core/src/host/instance_env.rs index 1bfb4f7f7c7..42262c991c0 100644 --- a/crates/core/src/host/instance_env.rs +++ b/crates/core/src/host/instance_env.rs @@ -54,6 +54,7 @@ pub struct InstanceEnv { in_anon_tx: bool, /// A procedure's last known transaction offset. procedure_last_tx_offset: Option, + call_auth_flags: u32, } /// `InstanceEnv` needs to be `Send` because it is created on the host thread @@ -238,6 +239,7 @@ impl InstanceEnv { func_name: None, in_anon_tx: false, procedure_last_tx_offset: None, + call_auth_flags: 0, } } @@ -252,6 +254,15 @@ impl InstanceEnv { self.start_instant = Instant::now(); self.func_type = func_type; self.func_name = Some(name); + self.call_auth_flags = 0; + } + + pub(crate) fn set_call_auth_flags(&mut self, flags: u32) { + self.call_auth_flags = flags; + } + + pub(crate) fn get_call_auth_flags(&self) -> u32 { + self.call_auth_flags } /// Returns the name of the most recent reducer to be run in this environment, diff --git a/crates/core/src/host/mod.rs b/crates/core/src/host/mod.rs index f28a515c910..06e19c367b9 100644 --- a/crates/core/src/host/mod.rs +++ b/crates/core/src/host/mod.rs @@ -192,6 +192,7 @@ pub enum AbiCall { JwtLength, GetJwt, EnvGet, + GetCallAuthFlags, VolatileNonatomicScheduleImmediate, diff --git a/crates/core/src/host/module_host.rs b/crates/core/src/host/module_host.rs index 25eb09e6382..4089b88ec5d 100644 --- a/crates/core/src/host/module_host.rs +++ b/crates/core/src/host/module_host.rs @@ -812,6 +812,7 @@ pub struct CallReducerParams { pub timestamp: Timestamp, pub caller_identity: Identity, pub caller_connection_id: ConnectionId, + pub(crate) call_auth_flags: u32, pub client: Option>, pub request_id: Option, pub timer: Option, @@ -832,6 +833,7 @@ impl CallReducerParams { timestamp, caller_identity, caller_connection_id: ConnectionId::ZERO, + call_auth_flags: 1, client: None, request_id: None, timer: None, @@ -1211,6 +1213,7 @@ pub struct CallProcedureParams { pub timestamp: Timestamp, pub caller_identity: Identity, pub caller_connection_id: ConnectionId, + pub(crate) call_auth_flags: u32, pub timer: Option, pub procedure_id: ProcedureId, pub args: ArgsTuple, @@ -1229,6 +1232,7 @@ impl CallProcedureParams { timestamp, caller_identity, caller_connection_id: ConnectionId::ZERO, + call_auth_flags: 1, timer: None, procedure_id, args, @@ -2299,6 +2303,7 @@ impl ModuleHost { timestamp: Timestamp::now(), caller_identity, caller_connection_id, + call_auth_flags: 0, client, request_id, timer, @@ -2326,7 +2331,10 @@ impl ModuleHost { return Err(ReducerCallError::LifecycleReducer(lifecycle)); } - if reducer_def.visibility.is_private() && !self.is_database_owner(caller_identity) { + if !reducer_def + .visibility + .allows_invocation(false, self.is_database_owner(caller_identity)) + { return Err(ReducerCallError::NoSuchReducer); } @@ -2836,7 +2844,10 @@ impl ModuleHost { .procedure_by_name_with_module(procedure_name) .ok_or(ProcedureCallError::NoSuchProcedure)?; - if procedure_def.visibility.is_private() && !self.is_database_owner(caller_identity) { + if !procedure_def + .visibility + .allows_invocation(false, self.is_database_owner(caller_identity)) + { return Err(ProcedureCallError::NoSuchProcedure); } @@ -2851,6 +2862,7 @@ impl ModuleHost { timestamp: Timestamp::now(), caller_identity, caller_connection_id, + call_auth_flags: 0, timer, procedure_id, args, diff --git a/crates/core/src/host/v8/mod.rs b/crates/core/src/host/v8/mod.rs index 6f357962bbb..564e7db862d 100644 --- a/crates/core/src/host/v8/mod.rs +++ b/crates/core/src/host/v8/mod.rs @@ -1988,6 +1988,7 @@ where // Start the timer. // We'd like this tightly around `call`. env.start_funcall(op.name().clone(), op.timestamp(), op.call_type()); + env.instance_env.set_call_auth_flags(op.call_auth_flags()); // Wrap the call in `TryCatch`. // @@ -2113,6 +2114,7 @@ mod test { name: &ReducerName::for_test("foobar"), caller_identity: &Identity::ONE, caller_connection_id: &ConnectionId::ZERO, + call_auth_flags: 0, timestamp: Timestamp::from_micros_since_unix_epoch(24), args: &ArgsTuple::nullary(), }; diff --git a/crates/core/src/host/v8/syscall/common.rs b/crates/core/src/host/v8/syscall/common.rs index 52b78463def..29aedea9b7b 100644 --- a/crates/core/src/host/v8/syscall/common.rs +++ b/crates/core/src/host/v8/syscall/common.rs @@ -44,6 +44,7 @@ pub fn call_call_procedure( name: _, caller_identity: sender, caller_connection_id: connection_id, + call_auth_flags: _, timestamp, arg_bytes: procedure_args, } = op; diff --git a/crates/core/src/host/v8/syscall/mod.rs b/crates/core/src/host/v8/syscall/mod.rs index 029d5836282..2d5e74eb862 100644 --- a/crates/core/src/host/v8/syscall/mod.rs +++ b/crates/core/src/host/v8/syscall/mod.rs @@ -62,8 +62,8 @@ fn resolve_sys_module_inner<'scope>( (1, 3) => Ok(v1::sys_v1_3(scope)), (2, 0) => Ok(v2::sys_v2_0(scope)), (2, 1) => Ok(v2::sys_v2_1(scope)), - // sys2.2 is reserved for invocation authority. (2, 3) => Ok(v2::sys_v2_3(scope)), + (2, 2) => Ok(v2::sys_v2_2(scope)), _ => Err(TypeError(format!( "Could not import {spec:?}, likely because this module was built for a newer version of SpacetimeDB.\n\ It requires sys module v{major}.{minor}, but that version is not supported by the database." diff --git a/crates/core/src/host/v8/syscall/v1.rs b/crates/core/src/host/v8/syscall/v1.rs index f3aea9f6c52..4d6783465df 100644 --- a/crates/core/src/host/v8/syscall/v1.rs +++ b/crates/core/src/host/v8/syscall/v1.rs @@ -495,6 +495,7 @@ pub(super) fn call_call_reducer( name: _, caller_identity: sender, caller_connection_id: conn_id, + call_auth_flags: _, timestamp, args: reducer_args, } = op; diff --git a/crates/core/src/host/v8/syscall/v2.rs b/crates/core/src/host/v8/syscall/v2.rs index 8fb376ec7af..95783ddddb6 100644 --- a/crates/core/src/host/v8/syscall/v2.rs +++ b/crates/core/src/host/v8/syscall/v2.rs @@ -169,6 +169,18 @@ pub(super) fn sys_v2_1<'scope>(scope: &mut PinScope<'scope, '_>) -> Local<'scope ) } +pub(super) fn sys_v2_2<'scope>(scope: &mut PinScope<'scope, '_>) -> Local<'scope, Module> { + create_synthetic_module!( + scope, + "spacetime:sys@2.2", + (with_sys_result, AbiCall::GetCallAuthFlags, get_call_auth_flags), + ) +} + +fn get_call_auth_flags(scope: &mut PinScope<'_, '_>, _args: FunctionCallbackArguments<'_>) -> SysCallResult { + Ok(get_env(scope)?.instance_env.get_call_auth_flags()) +} + pub(super) fn sys_v2_3<'scope>(scope: &mut PinScope<'scope, '_>) -> Local<'scope, Module> { create_synthetic_module!(scope, "spacetime:sys@2.3", (with_sys_result, AbiCall::EnvGet, env_get),) } @@ -467,6 +479,7 @@ pub(super) fn call_call_reducer<'scope>( name: _, caller_identity: sender, caller_connection_id: conn_id, + call_auth_flags: _, timestamp, args: reducer_args, } = op; diff --git a/crates/core/src/host/wasm_common.rs b/crates/core/src/host/wasm_common.rs index dc8baa44227..db0535fca59 100644 --- a/crates/core/src/host/wasm_common.rs +++ b/crates/core/src/host/wasm_common.rs @@ -444,8 +444,8 @@ macro_rules! abi_funcs { "spacetime_10.4"::datastore_delete_by_index_scan_point_bsatn, "spacetime_10.5"::datastore_clear, - // ABI10.6 is reserved for invocation authority. "spacetime_10.7"::env_get, + "spacetime_10.6"::get_call_auth_flags, } $link_async! { diff --git a/crates/core/src/host/wasm_common/module_host_actor.rs b/crates/core/src/host/wasm_common/module_host_actor.rs index aee984fe3a5..95df05f3616 100644 --- a/crates/core/src/host/wasm_common/module_host_actor.rs +++ b/crates/core/src/host/wasm_common/module_host_actor.rs @@ -782,6 +782,7 @@ impl InstanceCommon { timestamp, caller_identity, caller_connection_id, + call_auth_flags, timer, procedure_id, args, @@ -801,6 +802,7 @@ impl InstanceCommon { name: procedure_name.clone().into(), caller_identity, caller_connection_id, + call_auth_flags, timestamp, arg_bytes: args.get_bsatn().clone(), }; @@ -965,6 +967,7 @@ impl InstanceCommon { timestamp, caller_identity, caller_connection_id, + call_auth_flags, client, request_id, reducer_id, @@ -988,6 +991,7 @@ impl InstanceCommon { name: reducer_name, caller_identity: &caller_identity, caller_connection_id: &caller_connection_id, + call_auth_flags, timestamp, args: &args, }; @@ -1853,6 +1857,9 @@ pub trait InstanceOp { fn name(&self) -> &NamespacedIdentifier; fn timestamp(&self) -> Timestamp; fn call_type(&self) -> FuncCallType; + fn call_auth_flags(&self) -> u32 { + 0 + } } /// Describes a view call in a cheaply shareable way. @@ -1913,6 +1920,7 @@ pub struct ReducerOp<'a> { pub name: &'a ReducerName, pub caller_identity: &'a Identity, pub caller_connection_id: &'a ConnectionId, + pub call_auth_flags: u32, pub timestamp: Timestamp, /// The arguments passed to the reducer. pub args: &'a ArgsTuple, @@ -1928,6 +1936,9 @@ impl InstanceOp for ReducerOp<'_> { fn call_type(&self) -> FuncCallType { FuncCallType::Reducer } + fn call_auth_flags(&self) -> u32 { + self.call_auth_flags + } } impl From> for execution_context::ReducerContext { @@ -1937,6 +1948,7 @@ impl From> for execution_context::ReducerContext { name, caller_identity, caller_connection_id, + call_auth_flags: _, timestamp, args, }: ReducerOp<'_>, @@ -1958,6 +1970,7 @@ pub struct ProcedureOp { pub name: NamespacedIdentifier, pub caller_identity: Identity, pub caller_connection_id: ConnectionId, + pub call_auth_flags: u32, pub timestamp: Timestamp, pub arg_bytes: Bytes, } @@ -1972,6 +1985,9 @@ impl InstanceOp for ProcedureOp { fn call_type(&self) -> FuncCallType { FuncCallType::Procedure } + fn call_auth_flags(&self) -> u32 { + self.call_auth_flags + } } /// Describes an HTTP handler call in a cheaply shareable way. diff --git a/crates/core/src/host/wasmtime/wasm_instance_env.rs b/crates/core/src/host/wasmtime/wasm_instance_env.rs index 23d33983440..c35514864da 100644 --- a/crates/core/src/host/wasmtime/wasm_instance_env.rs +++ b/crates/core/src/host/wasmtime/wasm_instance_env.rs @@ -338,6 +338,14 @@ impl WasmInstanceEnv { self.bytes_sinks.remove(&sink).unwrap_or_default() } + pub fn get_call_auth_flags(caller: Caller<'_, Self>) -> u32 { + caller.data().instance_env.get_call_auth_flags() + } + + pub(crate) fn set_call_auth_flags(&mut self, flags: u32) { + self.instance_env.set_call_auth_flags(flags); + } + /// Signal to this `WasmInstanceEnv` that a reducer or procedure call is beginning. /// /// Returns the handle used by reducers and procedures to read from `args` diff --git a/crates/core/src/host/wasmtime/wasmtime_module.rs b/crates/core/src/host/wasmtime/wasmtime_module.rs index a5316ceb95c..57ed3416ffa 100644 --- a/crates/core/src/host/wasmtime/wasmtime_module.rs +++ b/crates/core/src/host/wasmtime/wasmtime_module.rs @@ -647,6 +647,7 @@ impl module_host_actor::WasmInstance for WasmtimeInstance { store .data_mut() .start_funcall(reducer_name, args_bytes, op.timestamp, op.call_type()); + store.data_mut().set_call_auth_flags(op.call_auth_flags); let call_result = call_sync_typed_func( &self.call_reducer, @@ -770,6 +771,7 @@ impl module_host_actor::WasmInstance for WasmtimeInstance { store .data_mut() .start_funcall(op.name().clone(), op.arg_bytes, op.timestamp, FuncCallType::Procedure); + store.data_mut().set_call_auth_flags(op.call_auth_flags); let Some(call_procedure) = self.call_procedure.as_ref() else { let res = module_host_actor::ProcedureExecuteResult { diff --git a/crates/lib/src/db/raw_def/v10.rs b/crates/lib/src/db/raw_def/v10.rs index b21d8470b84..38971f84001 100644 --- a/crates/lib/src/db/raw_def/v10.rs +++ b/crates/lib/src/db/raw_def/v10.rs @@ -101,6 +101,9 @@ pub enum RawModuleDefV10Section { /// Submodules, keyed by the namespace they are registered under. Submodules(Vec), + /// Module bindings capabilities, independent of function visibility. + /// Older hosts reject this section instead of silently ignoring its requirements. + Capabilities(Vec), } #[derive(Debug, Clone, SpacetimeType)] @@ -331,6 +334,9 @@ pub struct RawReducerDefV10 { } /// The visibility of a function (reducer or procedure). +/// +/// New variants MUST be appended to preserve existing BSATN tags. Older hosts +/// reject unknown tags, so new restrictions cannot be silently discarded. #[derive(Debug, Copy, Clone, SpacetimeType)] #[sats(crate = crate)] #[cfg_attr(feature = "test", derive(PartialEq, Eq, PartialOrd, Ord))] @@ -340,11 +346,31 @@ pub enum FunctionVisibility { /// Still callable by the module owner, collaborators, /// and internal module code. /// - /// Enabled for lifecycle reducers and scheduled functions by default. + /// The default for scheduled functions. Older lifecycle definitions also use + /// this tag; lifecycle assignments always enforce host-event-only invocation. Private, - /// Callable from client code. + /// Callable from client code, with the historical contextual defaults. + /// Scheduled functions become Private; lifecycle reducers remain host event handlers. ClientCallable, + + /// Callable only by a host-verified internal invocation. + Internal, + + /// Explicitly callable from client code, including when scheduled. + /// This separate tag preserves the meaning of existing ClientCallable definitions. + ExplicitClientCallable, +} + +impl FunctionVisibility { + /// Encode a source declaration without changing historical contextual defaults. + pub fn from_declaration(declared: Option, default: Self) -> Self { + match declared { + Some(Self::ClientCallable | Self::ExplicitClientCallable) => Self::ExplicitClientCallable, + Some(visibility) => visibility, + None => default, + } + } } /// A schedule definition. @@ -1092,10 +1118,20 @@ impl RawModuleDefV10Builder { /// This is because `SpacetimeType` is not implemented for `ReducerContext`, /// so it can never act like an ordinary argument.) pub fn add_reducer(&mut self, source_name: impl Into, params: ProductType) { + self.add_reducer_with_visibility(source_name, params, None); + } + + /// Add a reducer with an optional explicit visibility declaration. + pub fn add_reducer_with_visibility( + &mut self, + source_name: impl Into, + params: ProductType, + visibility: Option, + ) { self.reducers_mut().push(RawReducerDefV10 { source_name: source_name.into(), params, - visibility: FunctionVisibility::ClientCallable, + visibility: FunctionVisibility::from_declaration(visibility, FunctionVisibility::ClientCallable), ok_return_type: reducer_default_ok_return_type(), err_return_type: reducer_default_err_return_type(), }); @@ -1116,12 +1152,23 @@ impl RawModuleDefV10Builder { source_name: impl Into, params: ProductType, return_type: AlgebraicType, + ) { + self.add_procedure_with_visibility(source_name, params, return_type, None); + } + + /// Add a procedure with an optional explicit visibility declaration. + pub fn add_procedure_with_visibility( + &mut self, + source_name: impl Into, + params: ProductType, + return_type: AlgebraicType, + visibility: Option, ) { self.procedures_mut().push(RawProcedureDefV10 { source_name: source_name.into(), params, return_type, - visibility: FunctionVisibility::ClientCallable, + visibility: FunctionVisibility::from_declaration(visibility, FunctionVisibility::ClientCallable), }) } @@ -1165,6 +1212,19 @@ impl RawModuleDefV10Builder { lifecycle_spec: Lifecycle, function_name: impl Into, params: ProductType, + ) { + self.add_lifecycle_reducer_with_visibility(lifecycle_spec, function_name, params, None); + } + + /// Add a lifecycle reducer with an optional visibility declaration. + /// Source bindings must reject explicit Private or public lifecycle annotations. + /// The raw Private tag remains accepted for compatibility with existing modules. + pub fn add_lifecycle_reducer_with_visibility( + &mut self, + lifecycle_spec: Lifecycle, + function_name: impl Into, + params: ProductType, + visibility: Option, ) { let function_name = function_name.into(); self.lifecycle_reducers_mut().push(RawLifeCycleReducerDefV10 { @@ -1175,7 +1235,7 @@ impl RawModuleDefV10Builder { self.reducers_mut().push(RawReducerDefV10 { source_name: function_name, params, - visibility: FunctionVisibility::Private, + visibility: FunctionVisibility::from_declaration(visibility, FunctionVisibility::Private), ok_return_type: reducer_default_ok_return_type(), err_return_type: reducer_default_err_return_type(), }); @@ -1201,6 +1261,22 @@ impl RawModuleDefV10Builder { }); } + /// Declare a module bindings capability. + pub fn add_capability(&mut self, capability: impl Into) { + if let Some(RawModuleDefV10Section::Capabilities(names)) = self + .module + .sections + .iter_mut() + .find(|section| matches!(section, RawModuleDefV10Section::Capabilities(_))) + { + names.push(capability.into()); + } else { + self.module + .sections + .push(RawModuleDefV10Section::Capabilities(vec![capability.into()])); + } + } + /// Add a row-level security policy to the module. /// /// The `sql` expression should be a valid SQL expression that will be used to filter rows. @@ -1483,3 +1559,141 @@ impl RawTableDefBuilderV10<'_> { .map(|i| ColId(i as u16)) } } + +#[cfg(test)] +mod compatibility_tests { + use super::*; + use crate::{bsatn, RawModuleDef}; + + // Frozen pre-extension wire types. Do not replace the visibility, function, + // or section definitions below with their current counterparts. + #[derive(SpacetimeType)] + #[sats(crate = crate)] + enum LegacyVisibility { + Private, + ClientCallable, + } + + #[derive(SpacetimeType)] + #[sats(crate = crate)] + struct LegacyReducer { + source_name: RawIdentifier, + params: ProductType, + visibility: LegacyVisibility, + ok_return_type: AlgebraicType, + err_return_type: AlgebraicType, + } + + #[derive(SpacetimeType)] + #[sats(crate = crate)] + struct LegacyProcedure { + source_name: RawIdentifier, + params: ProductType, + return_type: AlgebraicType, + visibility: LegacyVisibility, + } + + #[derive(SpacetimeType)] + #[sats(crate = crate)] + enum LegacySection { + Typespace(Typespace), + Types(Vec), + Tables(Vec), + Reducers(Vec), + Procedures(Vec), + Views(Vec), + Schedules(Vec), + LifeCycleReducers(Vec), + RowLevelSecurity(Vec), + CaseConversionPolicy(CaseConversionPolicy), + ExplicitNames(ExplicitNames), + HttpHandlers(Vec), + HttpRoutes(Vec), + } + + #[derive(SpacetimeType)] + #[sats(crate = crate)] + struct LegacyV10 { + sections: Vec, + } + + #[derive(SpacetimeType)] + #[sats(crate = crate)] + enum LegacyModule { + V8BackCompat(crate::RawModuleDefV8), + V9(super::super::v9::RawModuleDefV9), + V10(LegacyV10), + } + + #[test] + fn existing_v10_wire_tags_and_function_products_are_unchanged() { + for (visibility, expected) in [ + (FunctionVisibility::Private, 0), + (FunctionVisibility::ClientCallable, 1), + (FunctionVisibility::Internal, 2), + (FunctionVisibility::ExplicitClientCallable, 3), + ] { + assert_eq!(bsatn::to_vec(&visibility).unwrap(), [expected]); + } + let legacy = LegacyModule::V10(LegacyV10 { + sections: vec![ + LegacySection::Reducers(vec![LegacyReducer { + source_name: "run".into(), + params: ProductType::unit(), + visibility: LegacyVisibility::ClientCallable, + ok_return_type: reducer_default_ok_return_type(), + err_return_type: reducer_default_err_return_type(), + }]), + LegacySection::Procedures(vec![LegacyProcedure { + source_name: "read".into(), + params: ProductType::unit(), + return_type: AlgebraicType::U64, + visibility: LegacyVisibility::Private, + }]), + ], + }); + let bytes = bsatn::to_vec(&legacy).unwrap(); + assert_eq!(bytes[0], 2); + let current: RawModuleDef = bsatn::from_slice(&bytes).unwrap(); + assert_eq!(bsatn::to_vec(¤t).unwrap(), bytes); + let frozen: LegacyModule = bsatn::from_slice(&bsatn::to_vec(¤t).unwrap()).unwrap(); + assert_eq!(bsatn::to_vec(&frozen).unwrap(), bytes); + + assert_eq!( + bsatn::to_vec(&RawModuleDefV10Section::HttpRoutes(vec![])).unwrap(), + [12, 0, 0, 0, 0] + ); + assert_eq!( + bsatn::to_vec(&RawModuleDefV10Section::Capabilities(vec![])).unwrap(), + [15, 0, 0, 0, 0] + ); + } + + #[test] + fn older_hosts_reject_new_visibility_and_capabilities() { + for visibility in [FunctionVisibility::Internal, FunctionVisibility::ExplicitClientCallable] { + for procedure in [false, true] { + let mut builder = RawModuleDefV10Builder::new(); + if procedure { + builder.add_procedure_with_visibility( + "run", + ProductType::unit(), + AlgebraicType::unit(), + Some(visibility), + ); + } else { + builder.add_reducer_with_visibility("run", ProductType::unit(), Some(visibility)); + } + let bytes = bsatn::to_vec(&RawModuleDef::V10(builder.finish())).unwrap(); + assert_eq!(bytes[0], 2); + assert!(bsatn::from_slice::(&bytes).is_err()); + assert!(bsatn::from_slice::(&bytes).is_ok()); + } + } + let mut builder = RawModuleDefV10Builder::new(); + builder.add_capability("hosted_auth_v1"); + let bytes = bsatn::to_vec(&RawModuleDef::V10(builder.finish())).unwrap(); + assert!(bsatn::from_slice::(&bytes).is_err()); + assert!(bsatn::from_slice::(&bytes).is_ok()); + } +} diff --git a/crates/schema/src/auto_migrate.rs b/crates/schema/src/auto_migrate.rs index 97fff428830..db728fcd250 100644 --- a/crates/schema/src/auto_migrate.rs +++ b/crates/schema/src/auto_migrate.rs @@ -220,6 +220,31 @@ pub struct AutoMigratePlan<'def> { } impl AutoMigratePlan<'_> { + /// Function authority changes include every namespace in the published API. + pub fn function_visibility_changes( + &self, + ) -> impl Iterator { + let reducers = self + .old + .all_reducers_with_prefix() + .into_iter() + .filter(|(_, _, old)| old.lifecycle.is_none()) + .filter_map(|(prefix, _, old)| { + let name = format!("{prefix}{}", old.name); + let (_, new) = self.new.reducer_by_name(&name)?; + (old.visibility != new.visibility).then_some((name, &old.visibility, &new.visibility)) + }); + let procedures = self + .old + .all_procedures_with_prefix() + .into_iter() + .filter_map(|(prefix, _, old)| { + let name = format!("{prefix}{}", old.name); + let (_, new) = self.new.procedure_by_name(&name)?; + (old.visibility != new.visibility).then_some((name, &old.visibility, &new.visibility)) + }); + reducers.chain(procedures) + } fn any_step(&self, f: impl Fn(&AutoMigrateStep<'_>) -> bool) -> bool { self.steps.iter().any(f) } @@ -493,6 +518,15 @@ pub fn ponder_auto_migrate<'def>(old: &'def ModuleDef, new: &'def ModuleDef) -> prechecks: Vec::new(), }; + let restricts_function_access = plan.function_visibility_changes().any(|(_, old, new)| { + [false, true] + .into_iter() + .any(|owner| old.allows_invocation(false, owner) && !new.allows_invocation(false, owner)) + }); + if restricts_function_access { + plan.ensure_disconnect_all_users(); + } + let views_ok = auto_migrate_views(&mut plan); let tables_ok = auto_migrate_tables(&mut plan); @@ -2909,6 +2943,33 @@ mod tests { raw.try_into().expect("should be a valid module definition") } + #[test] + fn submodule_visibility_restrictions_disconnect_and_report_qualified_names() { + use spacetimedb_lib::db::raw_def::v10::FunctionVisibility as RawVisibility; + let module = |visibility| { + create_module_def_with_submodules( + |_| {}, + vec![make_submodule("lib", |builder| { + builder.add_reducer_with_visibility("job", ProductType::unit(), Some(visibility)); + builder.add_procedure_with_visibility( + "read", + ProductType::unit(), + AlgebraicType::U8, + Some(visibility), + ); + })], + ) + }; + let old = module(RawVisibility::ExplicitClientCallable); + let restricted = module(RawVisibility::Internal); + let plan = ponder_auto_migrate(&old, &restricted).unwrap(); + assert!(plan.steps.contains(&AutoMigrateStep::DisconnectAllUsers)); + let names: Vec<_> = plan.function_visibility_changes().map(|(name, _, _)| name).collect(); + assert_eq!(names, ["lib.job", "lib.read"]); + let relaxed = ponder_auto_migrate(&restricted, &old).unwrap(); + assert!(!relaxed.steps.contains(&AutoMigrateStep::DisconnectAllUsers)); + } + #[test] fn submodule_table_unchanged() { let submodule = || { diff --git a/crates/schema/src/auto_migrate/formatter.rs b/crates/schema/src/auto_migrate/formatter.rs index 7d079f04a62..6c684121ca8 100644 --- a/crates/schema/src/auto_migrate/formatter.rs +++ b/crates/schema/src/auto_migrate/formatter.rs @@ -20,6 +20,9 @@ use thiserror::Error; pub fn format_plan(f: &mut F, plan: &AutoMigratePlan) -> Result<(), FormattingErrors> { f.format_header()?; + for (name, old, new) in plan.function_visibility_changes() { + f.format_function_visibility(&name, old, new)?; + } for step in &plan.steps { format_step(f, step, plan)?; @@ -180,6 +183,12 @@ pub enum Action { /// It allows for different implementations, such as ANSI formatting or plain text formatting. pub trait MigrationFormatter { fn format_header(&mut self) -> io::Result<()>; + fn format_function_visibility( + &mut self, + name: &str, + old: &crate::def::FunctionVisibility, + new: &crate::def::FunctionVisibility, + ) -> io::Result<()>; fn format_add_table(&mut self, table_info: &TableInfo) -> io::Result<()>; fn format_remove_table(&mut self, table_name: &NamespacedIdentifier) -> io::Result<()>; fn format_view(&mut self, view_info: &ViewInfo, action: Action) -> io::Result<()>; diff --git a/crates/schema/src/auto_migrate/termcolor_formatter.rs b/crates/schema/src/auto_migrate/termcolor_formatter.rs index 811c04b1860..ab27935cddf 100644 --- a/crates/schema/src/auto_migrate/termcolor_formatter.rs +++ b/crates/schema/src/auto_migrate/termcolor_formatter.rs @@ -157,6 +157,15 @@ impl TermColorFormatter { } impl MigrationFormatter for TermColorFormatter { + fn format_function_visibility( + &mut self, + name: &str, + old: &crate::def::FunctionVisibility, + new: &crate::def::FunctionVisibility, + ) -> io::Result<()> { + self.write_bullet(&format!("Function {name} visibility: {old} -> {new}")) + } + fn format_header(&mut self) -> io::Result<()> { let line = "━".repeat(60); self.write_line(&line)?; diff --git a/crates/schema/src/def.rs b/crates/schema/src/def.rs index 521bcb35aa9..3e2dd67762e 100644 --- a/crates/schema/src/def.rs +++ b/crates/schema/src/def.rs @@ -15,7 +15,7 @@ //! After validation, a `ModuleDef` can be converted to the `*Schema` types in `crate::schema` for use in the database. //! (Eventually, we may unify these types...) -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; use std::fmt::{self, Debug, Write}; use std::hash::Hash; @@ -179,6 +179,8 @@ pub struct ModuleDef { /// Submodules, keyed by the namespace they are registered under. submodules: IndexMap, + /// Validated module bindings capabilities. Legacy modules have none. + capabilities: BTreeSet, } #[derive(Debug, Clone, Copy, Eq, PartialEq)] @@ -190,6 +192,14 @@ pub enum RawModuleDefVersion { } impl ModuleDef { + pub fn supports_hosted_auth_v1(&self) -> bool { + self.capabilities.contains(&RawIdentifier::new("hosted_auth_v1")) + } + + pub fn capabilities(&self) -> impl Iterator { + self.capabilities.iter() + } + /// The raw module definition version this module was authored under. pub fn raw_module_def_version(&self) -> RawModuleDefVersion { self.raw_module_def_version @@ -205,6 +215,21 @@ impl ModuleDef { self.tables.values() } + /// The row type of a table or view, addressed by its canonical name. + pub fn type_ref_for_table_like(&self, name: &str) -> Option { + self.table(name) + .map(|table| table.product_type_ref) + .or_else(|| self.view(name).map(|view| view.product_type_ref)) + } + + /// Serialize without reinterpreting the definition's original version semantics. + pub fn into_raw(self) -> RawModuleDef { + match self.raw_module_def_version { + RawModuleDefVersion::V9OrEarlier => RawModuleDef::V9(self.try_into().expect("same-version conversion")), + RawModuleDefVersion::V10 => RawModuleDef::V10(self.into()), + } + } + /// The indexes of the module definition. pub fn indexes(&self) -> impl Iterator { self.tables().flat_map(|table| table.indexes.values()) @@ -944,7 +969,7 @@ impl TryFrom for ModuleDef { RawModuleDef::V8BackCompat(v8_mod) => Self::try_from(v8_mod), RawModuleDef::V9(v9_mod) => Self::try_from(v9_mod), RawModuleDef::V10(v10_mod) => Self::try_from(v10_mod), - _ => unimplemented!(), + _ => Err(crate::error::ValidationError::UnsupportedModuleVersion.into()), } } } @@ -964,11 +989,14 @@ impl TryFrom for ModuleDef { validate::v9::validate(v9_mod) } } -/// Note: this conversion is lossy for modules with submodules. `RawModuleDefV9` has no -/// submodule representation, so submodules (and everything defined in them) are dropped. -/// Callers serving V9 to old clients should be aware those clients see a partial module. -impl From for RawModuleDefV9 { - fn from(val: ModuleDef) -> Self { +impl TryFrom for RawModuleDefV9 { + type Error = SchemaConversionError; + fn try_from(val: ModuleDef) -> Result { + if val.raw_module_def_version != RawModuleDefVersion::V9OrEarlier { + return Err(SchemaConversionError { + target: RawModuleDefVersion::V9OrEarlier, + }); + } let ModuleDef { path: _, tables, @@ -986,6 +1014,7 @@ impl From for RawModuleDefV9 { http_routes: _, raw_module_def_version: _, submodules: _, + capabilities: _, } = val; // Extract column defaults from tables before consuming tables @@ -1004,18 +1033,26 @@ impl From for RawModuleDefV9 { }) .collect(); - RawModuleDefV9 { + let raw_reducers = reducers + .into_values() + .map(TryInto::try_into) + .collect::>()?; + let raw_procedures = procedures + .into_values() + .map(TryInto::try_into) + .collect::, _>>()?; + Ok(RawModuleDefV9 { tables: to_raw(tables), - reducers: reducers.into_iter().map(|(_, def)| def.into()).collect(), + reducers: raw_reducers, types: to_raw(types), misc_exports: column_defaults .into_iter() - .chain(procedures.into_iter().map(|(_, def)| def.into())) + .chain(raw_procedures) .chain(views.into_iter().map(|(_, def)| def.into())) .collect(), typespace, row_level_security: row_level_security_raw.into_iter().map(|(_, def)| def).collect(), - } + }) } } @@ -1046,6 +1083,7 @@ impl From for RawModuleDefV10 { http_routes, raw_module_def_version: _, submodules, + capabilities, } = val; let mut sections = Vec::new(); @@ -1106,7 +1144,15 @@ impl From for RawModuleDefV10 { RawIdentifier::from(rd.accessor_name.clone()), RawIdentifier::from(rd.name.local().clone()), ); - rd.into() + let public_scheduled = rd.visibility.is_client_callable() + && schedules + .iter() + .any(|schedule| schedule.function_name == RawIdentifier::from(rd.name.clone())); + let mut raw: RawReducerDefV10 = rd.into(); + if public_scheduled { + raw.visibility = RawFunctionVisibility::ExplicitClientCallable; + } + raw }) .collect(); if !raw_reducers.is_empty() { @@ -1121,7 +1167,15 @@ impl From for RawModuleDefV10 { RawIdentifier::from(pd.accessor_name.clone()), RawIdentifier::from(pd.name.clone()), ); - pd.into() + let public_scheduled = pd.visibility.is_client_callable() + && schedules + .iter() + .any(|schedule| schedule.function_name == RawIdentifier::from(pd.name.clone())); + let mut raw: RawProcedureDefV10 = pd.into(); + if public_scheduled { + raw.visibility = RawFunctionVisibility::ExplicitClientCallable; + } + raw }) .collect(); if !raw_procedures.is_empty() { @@ -1216,6 +1270,9 @@ impl From for RawModuleDefV10 { sections.push(RawModuleDefV10Section::Submodules(submodules)); } + if !capabilities.is_empty() { + sections.push(RawModuleDefV10Section::Capabilities(capabilities.into_iter().collect())); + } RawModuleDefV10 { sections } } } @@ -2279,9 +2336,36 @@ pub enum FunctionVisibility { /// Callable from client code. ClientCallable, + + /// Callable only by a host-verified internal invocation. + Internal, +} + +impl fmt::Display for FunctionVisibility { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(match self { + Self::Private => "Private", + Self::ClientCallable => "Public", + Self::Internal => "Internal", + }) + } } impl FunctionVisibility { + pub fn is_client_callable(&self) -> bool { + matches!(self, Self::ClientCallable) + } + pub fn is_internal(&self) -> bool { + matches!(self, Self::Internal) + } + /// Lifecycle event dispatch is a separate restriction from this predicate. + pub fn allows_invocation(&self, is_internal: bool, is_authorized_private_caller: bool) -> bool { + match self { + Self::Internal => is_internal, + Self::Private => is_internal || is_authorized_private_caller, + Self::ClientCallable => true, + } + } pub fn is_private(&self) -> bool { matches!(self, FunctionVisibility::Private) } @@ -2292,16 +2376,26 @@ impl From for FunctionVisibility { fn from(val: RawFunctionVisibility) -> Self { match val { RawFunctionVisibility::Private => FunctionVisibility::Private, - RawFunctionVisibility::ClientCallable => FunctionVisibility::ClientCallable, + RawFunctionVisibility::ClientCallable | RawFunctionVisibility::ExplicitClientCallable => { + FunctionVisibility::ClientCallable + } + RawFunctionVisibility::Internal => FunctionVisibility::Internal, } } } +#[derive(Debug, Clone, thiserror::Error)] +#[error("schema cannot be represented as {target:?} without losing function visibility or source-version semantics; request schema version 10")] +pub struct SchemaConversionError { + pub target: RawModuleDefVersion, +} + impl From for RawFunctionVisibility { fn from(val: FunctionVisibility) -> Self { match val { - FunctionVisibility::Private => RawFunctionVisibility::Private, - FunctionVisibility::ClientCallable => RawFunctionVisibility::ClientCallable, + FunctionVisibility::Private => Self::Private, + FunctionVisibility::ClientCallable => Self::ClientCallable, + FunctionVisibility::Internal => Self::Internal, } } } @@ -2345,22 +2439,33 @@ pub struct ReducerDef { pub err_return_type: AlgebraicType, } -impl From for RawReducerDefV9 { - fn from(val: ReducerDef) -> Self { - RawReducerDefV9 { +impl TryFrom for RawReducerDefV9 { + type Error = SchemaConversionError; + fn try_from(val: ReducerDef) -> Result { + if val.lifecycle.is_none() && !val.visibility.is_client_callable() { + return Err(SchemaConversionError { + target: RawModuleDefVersion::V9OrEarlier, + }); + } + Ok(RawReducerDefV9 { name: val.name.into(), params: val.params, lifecycle: val.lifecycle, - } + }) } } impl From for RawReducerDefV10 { fn from(val: ReducerDef) -> Self { + let visibility = if val.lifecycle.is_some() { + RawFunctionVisibility::Private + } else { + val.visibility.into() + }; RawReducerDefV10 { source_name: val.accessor_name.into(), params: val.params, - visibility: val.visibility.into(), + visibility, ok_return_type: val.ok_return_type, err_return_type: val.err_return_type, } @@ -2423,13 +2528,19 @@ pub struct HttpRouteDef { pub path: Box, } -impl From for RawProcedureDefV9 { - fn from(val: ProcedureDef) -> Self { - RawProcedureDefV9 { +impl TryFrom for RawProcedureDefV9 { + type Error = SchemaConversionError; + fn try_from(val: ProcedureDef) -> Result { + if !val.visibility.is_client_callable() { + return Err(SchemaConversionError { + target: RawModuleDefVersion::V9OrEarlier, + }); + } + Ok(RawProcedureDefV9 { name: val.name.into(), params: val.params, return_type: val.return_type, - } + }) } } @@ -2444,9 +2555,10 @@ impl From for RawProcedureDefV10 { } } -impl From for RawMiscModuleExportV9 { - fn from(def: ProcedureDef) -> Self { - Self::Procedure(def.into()) +impl TryFrom for RawMiscModuleExportV9 { + type Error = SchemaConversionError; + fn try_from(def: ProcedureDef) -> Result { + Ok(Self::Procedure(def.try_into()?)) } } diff --git a/crates/schema/src/def/validate/v10.rs b/crates/schema/src/def/validate/v10.rs index d6303f3a81c..416d8b415ed 100644 --- a/crates/schema/src/def/validate/v10.rs +++ b/crates/schema/src/def/validate/v10.rs @@ -78,6 +78,47 @@ impl From for ValidationCase { /// Validate a `RawModuleDefV10` and convert it into a `ModuleDef`, /// or return a stream of errors if the definition is invalid. pub fn validate(def: RawModuleDefV10) -> Result { + let mut seen_capabilities = false; + let mut capabilities = std::collections::BTreeSet::new(); + for section in &def.sections { + if let RawModuleDefV10Section::Capabilities(names) = section { + if seen_capabilities { + return Err(ValidationError::DuplicateModuleSection { + section: "Capabilities".into(), + } + .into()); + } + seen_capabilities = true; + if names.len() > 32 { + return Err(ValidationError::InvalidModuleCapabilities.into()); + } + for name in names { + if name.is_empty() + || name.len() > 64 + || !name + .bytes() + .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'_') + || !capabilities.insert(name.clone()) + { + return Err(ValidationError::InvalidModuleCapabilities.into()); + } + } + } + } + // Retain the raw distinction until schedules are attached. Tag 1 has the + // historical contextual default; tag 3 is an explicit public declaration. + let raw_visibility: HashMap<_, _> = def + .reducers() + .into_iter() + .flatten() + .map(|function| (function.source_name.clone(), function.visibility)) + .chain( + def.procedures() + .into_iter() + .flatten() + .map(|function| (function.source_name.clone(), function.visibility)), + ) + .collect(); let mut typespace = def.typespace().cloned().unwrap_or_else(|| Typespace::EMPTY.clone()); let known_type_definitions = def.types().into_iter().flatten().map(|def| def.ty); let case_policy = def.case_conversion_policy().into(); @@ -276,7 +317,12 @@ pub fn validate(def: RawModuleDefV10) -> Result { attach_schedules_to_tables(&mut tables, schedules)?; check_scheduled_functions_exist(&mut tables, &reducers, &procedures)?; - change_scheduled_functions_and_lifetimes_visibility(&tables, &mut reducers, &mut procedures)?; + change_scheduled_functions_and_lifetimes_visibility( + &tables, + &mut reducers, + &mut procedures, + &raw_visibility, + )?; attach_view_primary_keys(&mut views, view_primary_keys)?; assign_query_view_primary_keys(&tables, &mut views); @@ -320,6 +366,7 @@ pub fn validate(def: RawModuleDefV10) -> Result { procedures, http_handlers, http_routes, + capabilities, raw_module_def_version: RawModuleDefVersion::V10, submodules, }; @@ -383,12 +430,13 @@ fn validate_submodules(submodules: Vec) -> Result, reducers: &mut IndexMap, procedures: &mut IndexMap, + raw_visibility: &HashMap, ) -> Result<()> { for sched_def in tables.iter().filter_map(|(_, t)| t.schedule.as_ref()) { match sched_def.function_kind { @@ -400,7 +448,12 @@ fn change_scheduled_functions_and_lifetimes_visibility( } })?; - def.visibility = crate::def::FunctionVisibility::Private; + if matches!( + raw_visibility.get(&RawIdentifier::from(def.accessor_name.clone())), + Some(RawFunctionVisibility::ClientCallable) + ) { + def.visibility = crate::def::FunctionVisibility::Private; + } } FunctionKind::Procedure => { @@ -411,7 +464,12 @@ fn change_scheduled_functions_and_lifetimes_visibility( } })?; - def.visibility = crate::def::FunctionVisibility::Private; + if matches!( + raw_visibility.get(&RawIdentifier::from(def.accessor_name.clone())), + Some(RawFunctionVisibility::ClientCallable) + ) { + def.visibility = crate::def::FunctionVisibility::Private; + } } FunctionKind::Unknown => {} @@ -420,7 +478,16 @@ fn change_scheduled_functions_and_lifetimes_visibility( for red_def in reducers.iter_mut().map(|(_, r)| r) { if red_def.lifecycle.is_some() { - red_def.visibility = crate::def::FunctionVisibility::Private; + if matches!( + raw_visibility.get(&RawIdentifier::from(red_def.accessor_name.clone())), + Some(RawFunctionVisibility::ExplicitClientCallable) + ) { + return Err(ValidationError::InvalidLifecycleVisibility { + function: red_def.accessor_name.clone().into(), + } + .into()); + } + red_def.visibility = crate::def::FunctionVisibility::Internal; } } @@ -1460,7 +1527,7 @@ mod tests { def.reducers[&check_deliveries_name].visibility, FunctionVisibility::Private, ); - assert_eq!(def.reducers[&init_name].visibility, FunctionVisibility::Private); + assert_eq!(def.reducers[&init_name].visibility, FunctionVisibility::Internal); assert_eq!( def.reducers[&extra_reducer_name].visibility, FunctionVisibility::ClientCallable @@ -2796,3 +2863,277 @@ mod tests { }); } } + +#[cfg(test)] +mod visibility_tests { + use super::*; + use crate::def::FunctionVisibility; + use spacetimedb_lib::db::raw_def::v10; + use spacetimedb_lib::{db::raw_def::v9, RawModuleDef, ScheduleAt}; + use spacetimedb_sats::{AlgebraicType, ProductType}; + use v10::{FunctionVisibility as Declared, RawModuleDefV10Builder}; + + fn scheduled_module(visibility: Option, procedure: bool) -> ModuleDef { + let mut builder = RawModuleDefV10Builder::new(); + let at = builder.add_type::(); + let row = builder + .build_table_with_new_type( + "Jobs", + ProductType::from([("id", AlgebraicType::U64), ("at", at)]), + true, + ) + .with_auto_inc_primary_key(0) + .with_index_no_accessor_name(v9::btree(0), "jobs_id_idx") + .finish(); + let params = ProductType::from([("job", AlgebraicType::Ref(row))]); + if procedure { + builder.add_procedure_with_visibility("run_job", params, AlgebraicType::unit(), visibility); + } else { + builder.add_reducer_with_visibility("run_job", params, visibility); + } + builder.add_schedule("Jobs", 1, "run_job"); + builder.finish().try_into().unwrap() + } + + #[test] + fn explicit_scheduled_visibility_overrides_the_private_default() { + for procedure in [false, true] { + for (selection, expected) in [ + (None, FunctionVisibility::Private), + (Some(Declared::Private), FunctionVisibility::Private), + (Some(Declared::Internal), FunctionVisibility::Internal), + (Some(Declared::ClientCallable), FunctionVisibility::ClientCallable), + ] { + let module = scheduled_module(selection, procedure); + let visibility = if procedure { + &module.procedure("run_job").unwrap().visibility + } else { + &module.reducer("run_job").unwrap().visibility + }; + assert_eq!(visibility, &expected); + assert_eq!(module.raw_module_def_version(), RawModuleDefVersion::V10); + } + } + } + + #[test] + fn ordinary_defaults_and_lifecycle_restrictions() { + let mut builder = RawModuleDefV10Builder::new(); + builder.add_reducer("ordinary", ProductType::unit()); + builder.add_procedure("ordinary_procedure", ProductType::unit(), AlgebraicType::unit()); + builder.add_lifecycle_reducer(v9::Lifecycle::Init, "initialize", ProductType::unit()); + let module: ModuleDef = builder.finish().try_into().unwrap(); + assert!(module.reducer("ordinary").unwrap().visibility.is_client_callable()); + assert!(module + .procedure("ordinary_procedure") + .unwrap() + .visibility + .is_client_callable()); + assert!(module.reducer("initialize").unwrap().visibility.is_internal()); + let exported: RawModuleDefV10 = module.into(); + assert!(exported + .reducers() + .into_iter() + .flatten() + .all(|function| matches!(function.visibility, Declared::ClientCallable | Declared::Private))); + assert!(exported + .procedures() + .into_iter() + .flatten() + .all(|function| matches!(function.visibility, Declared::ClientCallable))); + for selection in [Declared::ClientCallable, Declared::ExplicitClientCallable] { + let mut builder = RawModuleDefV10Builder::new(); + builder.add_lifecycle_reducer_with_visibility( + v9::Lifecycle::Init, + "initialize", + ProductType::unit(), + Some(selection), + ); + assert!(ModuleDef::try_from(builder.finish()) + .unwrap_err() + .to_string() + .contains("must have Internal visibility")); + } + let mut builder = RawModuleDefV10Builder::new(); + builder.add_lifecycle_reducer_with_visibility( + v9::Lifecycle::Init, + "initialize", + ProductType::unit(), + Some(Declared::Internal), + ); + assert!(ModuleDef::try_from(builder.finish()).is_ok()); + } + + #[test] + fn duplicate_definitions_sections_and_lifecycles_are_rejected() { + let mut builder = RawModuleDefV10Builder::new(); + builder.add_reducer("same", ProductType::unit()); + builder.add_procedure("same", ProductType::unit(), AlgebraicType::unit()); + assert!(ModuleDef::try_from(builder.finish()).is_err()); + let raw = v10::RawModuleDefV10 { + sections: vec![ + v10::RawModuleDefV10Section::Capabilities(vec![]), + v10::RawModuleDefV10Section::Capabilities(vec![]), + ], + }; + assert!(ModuleDef::try_from(raw) + .unwrap_err() + .to_string() + .contains("repeated V10 section")); + let mut builder = RawModuleDefV10Builder::new(); + builder.add_lifecycle_reducer(v9::Lifecycle::Init, "a", ProductType::unit()); + builder.add_lifecycle_reducer(v9::Lifecycle::Init, "b", ProductType::unit()); + assert!(ModuleDef::try_from(builder.finish()).is_err()); + } + + #[test] + fn resolved_v10_roundtrips_without_reapplying_defaults_and_rejects_v9_exports() { + for procedure in [false, true] { + for selection in [ + None, + Some(Declared::Private), + Some(Declared::Internal), + Some(Declared::ClientCallable), + ] { + let module = scheduled_module(selection, procedure); + assert!(v9::RawModuleDefV9::try_from(module.clone()).is_err()); + let RawModuleDef::V10(raw) = module.clone().into_raw() else { + panic!("lost source version") + }; + if matches!(selection, Some(Declared::ClientCallable)) { + assert!(raw + .reducers() + .into_iter() + .flatten() + .map(|function| &function.visibility) + .chain( + raw.procedures() + .into_iter() + .flatten() + .map(|function| &function.visibility) + ) + .all(|visibility| matches!(visibility, Declared::ExplicitClientCallable))); + } + let bytes = spacetimedb_lib::bsatn::to_vec(&RawModuleDef::V10(raw)).unwrap(); + let roundtrip: RawModuleDef = spacetimedb_lib::bsatn::from_slice(&bytes).unwrap(); + let roundtrip: ModuleDef = roundtrip.try_into().unwrap(); + if procedure { + assert_eq!( + roundtrip.procedure("run_job").unwrap().visibility, + module.procedure("run_job").unwrap().visibility + ); + } else { + assert_eq!( + roundtrip.reducer("run_job").unwrap().visibility, + module.reducer("run_job").unwrap().visibility + ); + } + assert_eq!(roundtrip.raw_module_def_version(), RawModuleDefVersion::V10); + } + } + } + + #[test] + fn legacy_v9_schedules_stay_public_and_v10_schedules_stay_private() { + let mut builder = v9::RawModuleDefV9Builder::new(); + let at = builder.add_type::(); + let row = builder + .build_table_with_new_type( + "jobs", + ProductType::from([("id", AlgebraicType::U64), ("at", at)]), + true, + ) + .with_auto_inc_primary_key(0) + .with_index(v9::btree(0), "jobs_id_idx") + .with_schedule("run_job", 1) + .finish(); + builder.add_reducer("run_job", ProductType::from([("job", row.into())]), None); + let v9: ModuleDef = builder.finish().try_into().unwrap(); + assert!(v9.reducer("run_job").unwrap().visibility.is_client_callable()); + let upgraded: RawModuleDefV10 = v9.clone().into(); + assert!(matches!( + upgraded.reducers().unwrap()[0].visibility, + Declared::ExplicitClientCallable + )); + let upgraded: ModuleDef = upgraded.try_into().unwrap(); + assert!(upgraded.reducer("run_job").unwrap().visibility.is_client_callable()); + assert!(matches!(v9.into_raw(), RawModuleDef::V9(_))); + + let mut builder = v10::RawModuleDefV10Builder::new(); + let at = builder.add_type::(); + let row = builder + .build_table_with_new_type( + "jobs", + ProductType::from([("id", AlgebraicType::U64), ("at", at)]), + true, + ) + .with_auto_inc_primary_key(0) + .with_index_no_accessor_name(v9::btree(0), "jobs_id_idx") + .finish(); + builder.add_reducer("run_job", ProductType::from([("job", row.into())])); + builder.add_schedule("jobs", 1, "run_job"); + let v10: ModuleDef = builder.finish().try_into().unwrap(); + assert!(v10.reducer("run_job").unwrap().visibility.is_private()); + assert!(matches!(v10.into_raw(), RawModuleDef::V10(_))); + } + + #[test] + fn capabilities_are_explicit_bounded_and_preserved() { + let bare: ModuleDef = RawModuleDefV10Builder::new().finish().try_into().unwrap(); + assert!(!bare.supports_hosted_auth_v1()); + let mut builder = RawModuleDefV10Builder::new(); + builder.add_capability("hosted_auth_v1"); + let module: ModuleDef = builder.finish().try_into().unwrap(); + assert!(module.supports_hosted_auth_v1()); + let reloaded: ModuleDef = module.into_raw().try_into().unwrap(); + assert!(reloaded.supports_hosted_auth_v1()); + for names in [ + vec!["".to_string()], + vec!["Uppercase".to_string()], + vec!["with-dash".to_string()], + vec!["a".repeat(65)], + vec!["duplicate".to_string(); 2], + (0..33).map(|i| format!("cap_{i}")).collect(), + ] { + let mut builder = RawModuleDefV10Builder::new(); + for name in names { + builder.add_capability(name); + } + assert!(ModuleDef::try_from(builder.finish()).is_err()); + } + } + + #[test] + fn narrowing_function_visibility_is_a_reported_client_break() { + let module = |visibility| { + let mut builder = RawModuleDefV10Builder::new(); + builder.add_reducer_with_visibility("run_now", ProductType::unit(), Some(visibility)); + ModuleDef::try_from(builder.finish()).unwrap() + }; + let public = module(Declared::ClientCallable); + let internal = module(Declared::Internal); + let plan = crate::auto_migrate::ponder_migrate(&public, &internal).unwrap(); + assert!(plan.breaks_client()); + let display = plan + .pretty_print(crate::auto_migrate::PrettyPrintStyle::NoColor) + .unwrap(); + assert!(display.contains("run_now")); + assert!(display.contains("Internal")); + assert!(!crate::auto_migrate::ponder_migrate(&internal, &public) + .unwrap() + .breaks_client()); + } + + #[test] + fn visibility_authority_is_cumulative_without_elevating_the_owner() { + for (visibility, external, owner, internal) in [ + (FunctionVisibility::Internal, false, false, true), + (FunctionVisibility::Private, false, true, true), + (FunctionVisibility::ClientCallable, true, true, true), + ] { + assert_eq!(visibility.allows_invocation(false, false), external); + assert_eq!(visibility.allows_invocation(false, true), owner); + assert_eq!(visibility.allows_invocation(true, false), internal); + } + } +} diff --git a/crates/schema/src/def/validate/v9.rs b/crates/schema/src/def/validate/v9.rs index 8de4927afa2..c1c91fa4d4f 100644 --- a/crates/schema/src/def/validate/v9.rs +++ b/crates/schema/src/def/validate/v9.rs @@ -169,6 +169,7 @@ pub fn validate(def: RawModuleDefV9) -> Result { procedures, http_handlers: IndexMap::new(), http_routes: Vec::new(), + capabilities: Default::default(), raw_module_def_version: RawModuleDefVersion::V9OrEarlier, submodules: IndexMap::new(), }; @@ -389,7 +390,11 @@ impl ModuleValidatorV9<'_> { recursive: false, // A ProductTypeDef not stored in a Typespace cannot be recursive. }, lifecycle, - visibility: FunctionVisibility::ClientCallable, + visibility: if lifecycle.is_some() { + FunctionVisibility::Internal + } else { + FunctionVisibility::ClientCallable + }, ok_return_type: reducer_default_ok_return_type(), err_return_type: reducer_default_err_return_type(), }; diff --git a/crates/schema/src/error.rs b/crates/schema/src/error.rs index e9408a35482..434f44ba727 100644 --- a/crates/schema/src/error.rs +++ b/crates/schema/src/error.rs @@ -22,6 +22,14 @@ pub type ValidationErrors = ErrorStream; #[derive(thiserror::Error, Debug, PartialOrd, Ord, PartialEq, Eq)] #[non_exhaustive] pub enum ValidationError { + #[error("unsupported module definition version")] + UnsupportedModuleVersion, + #[error("invalid module capabilities: at most 32 unique names of 1..64 lowercase ASCII letters, digits or underscores are allowed")] + InvalidModuleCapabilities, + #[error("lifecycle reducer `{function}` must have Internal visibility")] + InvalidLifecycleVisibility { function: RawIdentifier }, + #[error("module contains repeated V10 section `{section}`")] + DuplicateModuleSection { section: String }, #[error("name `{name}` is used for multiple entities")] DuplicateName { name: RawIdentifier }, #[error("name `{name}` is used for multiple types")] diff --git a/crates/standalone/src/subcommands/extract_schema.rs b/crates/standalone/src/subcommands/extract_schema.rs index efc77960195..874f213c183 100644 --- a/crates/standalone/src/subcommands/extract_schema.rs +++ b/crates/standalone/src/subcommands/extract_schema.rs @@ -4,7 +4,7 @@ use anyhow::Context; use clap::{ArgMatches, CommandFactory, FromArgMatches}; use spacetimedb::host::extract_schema; use spacetimedb::messages::control_db; -use spacetimedb_lib::{db::raw_def::v10::RawModuleDefV10, sats, RawModuleDef}; +use spacetimedb_lib::sats; /// Extracts the module schema from a local module file. /// WARNING: This command is UNSTABLE and subject to breaking changes. @@ -67,7 +67,7 @@ pub async fn exec(args: &ArgMatches) -> anyhow::Result<()> { let module_def = extract_schema(program_bytes.into(), host_type.into()).await?; - let raw_def = RawModuleDef::V10(RawModuleDefV10::from(module_def)); + let raw_def = module_def.into_raw(); serde_json::to_writer(std::io::stdout().lock(), &sats::serde::SerdeWrapper(raw_def))?; diff --git a/crates/testing/tests/invocation_flags.rs b/crates/testing/tests/invocation_flags.rs new file mode 100644 index 00000000000..16581de6662 --- /dev/null +++ b/crates/testing/tests/invocation_flags.rs @@ -0,0 +1,76 @@ +//! Exercise real Rust Wasm bindings and host admission without a server endpoint. +use serial_test::serial; +use spacetimedb::host::FunctionArgs; +use spacetimedb_lib::{AlgebraicValue, Identity}; +use spacetimedb_testing::modules::{CompilationMode, CompiledModule, DEFAULT_CONFIG}; +use std::time::Duration; + +#[test] +#[serial] +fn wasm_invocation_flags_do_not_infer_authority_from_identity_or_connection_absence() { + CompiledModule::compile("invocation-flags-test", CompilationMode::Debug).with_module_async( + DEFAULT_CONFIG, + |handle| async move { + let module = handle.client.module(); + for sender in [Identity::ZERO, Identity::ONE, handle.db_identity] { + module + .call_reducer(sender, None, None, None, None, "external", FunctionArgs::Nullary) + .await + .unwrap() + .outcome + .into_result() + .unwrap(); + for name in ["internal", "init", "scheduled"] { + assert!(module + .call_reducer(sender, None, None, None, None, name, FunctionArgs::Nullary) + .await + .is_err()); + } + let result = module + .call_procedure(sender, None, None, "external_procedure", FunctionArgs::Nullary) + .await; + assert_eq!(result.result.unwrap().return_val, AlgebraicValue::Bool(true)); + assert!(module + .call_procedure(sender, None, None, "internal_procedure", FunctionArgs::Nullary) + .await + .result + .is_err()); + assert_eq!( + module + .call_reducer(sender, None, None, None, None, "private", FunctionArgs::Nullary) + .await + .is_ok(), + sender == Identity::ZERO, + ); + } + module + .call_reducer( + Identity::ZERO, + None, + None, + None, + None, + "schedule", + FunctionArgs::Nullary, + ) + .await + .unwrap() + .outcome + .into_result() + .unwrap(); + tokio::time::timeout(Duration::from_secs(5), async { + loop { + let result = module + .call_procedure(Identity::ZERO, None, None, "scheduled_finished", FunctionArgs::Nullary) + .await; + if result.result.unwrap().return_val == AlgebraicValue::Bool(true) { + break; + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + }) + .await + .expect("the real scheduled reducer did not observe trusted internal authority"); + }, + ); +} diff --git a/modules/invocation-flags-test/Cargo.toml b/modules/invocation-flags-test/Cargo.toml new file mode 100644 index 00000000000..8b34d4125a9 --- /dev/null +++ b/modules/invocation-flags-test/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "invocation-flags-test" +version = "0.0.0" +edition.workspace = true +license-file = "../../LICENSE.txt" +publish = false + +[lib] +crate-type = ["cdylib"] + +[dependencies.spacetimedb] +workspace = true +features = ["unstable"] diff --git a/modules/invocation-flags-test/src/lib.rs b/modules/invocation-flags-test/src/lib.rs new file mode 100644 index 00000000000..9621ef95f59 --- /dev/null +++ b/modules/invocation-flags-test/src/lib.rs @@ -0,0 +1,78 @@ +//! Real Wasm fixture for generic host invocation authority. +use spacetimedb::{ProcedureContext, ReducerContext, Table}; + +#[spacetimedb::reducer(init)] +pub fn init(ctx: &ReducerContext) { + assert!(ctx.sender_auth().is_internal()); + assert!(!ctx.sender_auth().has_jwt()); +} + +#[spacetimedb::reducer] +pub fn external(ctx: &ReducerContext) { + assert!(!ctx.sender_auth().is_internal()); + assert_eq!(ctx.connection_id(), None); + assert!(!ctx.sender_auth().has_jwt()); +} + +#[spacetimedb::reducer(internal)] +pub fn internal(ctx: &ReducerContext) { + assert!(ctx.sender_auth().is_internal()); +} + +#[spacetimedb::reducer(private)] +pub fn private(_ctx: &ReducerContext) {} + +#[spacetimedb::procedure] +pub fn external_procedure(ctx: &mut ProcedureContext) -> bool { + assert!(!ctx.sender_auth().is_internal()); + let sender = ctx.sender(); + let connection = ctx.connection_id(); + ctx.with_tx(|tx| { + assert!(!tx.sender_auth().is_internal()); + assert_eq!(tx.sender(), sender); + assert_eq!(tx.connection_id(), connection); + }); + true +} + +#[spacetimedb::procedure(internal)] +pub fn internal_procedure(ctx: &mut ProcedureContext) -> bool { + assert!(ctx.sender_auth().is_internal()); + true +} + +#[spacetimedb::table(accessor = jobs, scheduled(scheduled))] +pub struct Job { + #[primary_key] + #[auto_inc] + id: u64, + scheduled_at: spacetimedb::ScheduleAt, +} + +#[spacetimedb::table(accessor = finished)] +pub struct Finished { + #[primary_key] + id: u64, +} + +#[spacetimedb::reducer] +pub fn schedule(ctx: &ReducerContext) { + ctx.db.jobs().insert(Job { + id: 0, + scheduled_at: ctx.timestamp.into(), + }); +} + +#[spacetimedb::reducer(internal)] +pub fn scheduled(ctx: &ReducerContext, job: Job) { + assert!(ctx.sender_auth().is_internal()); + assert_eq!(ctx.sender(), ctx.database_identity()); + assert_eq!(ctx.connection_id(), None); + assert!(!ctx.sender_auth().has_jwt()); + ctx.db.finished().insert(Finished { id: job.id }); +} + +#[spacetimedb::procedure] +pub fn scheduled_finished(ctx: &mut ProcedureContext) -> bool { + ctx.with_tx(|tx| tx.db.finished().iter().next().is_some()) +} diff --git a/modules/module-test-ts/src/index.ts b/modules/module-test-ts/src/index.ts index 10520d2b72c..46c3116e9e7 100644 --- a/modules/module-test-ts/src/index.ts +++ b/modules/module-test-ts/src/index.ts @@ -516,7 +516,7 @@ export const getMySchemaViaHttp = spacetimedb.procedure(t.string(), ctx => { const module_identity = ctx.databaseIdentity; try { const response = ctx.http.fetch( - `http://localhost:3000/v1/database/${module_identity}/schema?version=9` + `http://localhost:3000/v1/database/${module_identity}/schema?version=10` ); return response.text(); } catch (e) { diff --git a/modules/module-test/src/lib.rs b/modules/module-test/src/lib.rs index fc1851b21b0..dfdc27e8e9d 100644 --- a/modules/module-test/src/lib.rs +++ b/modules/module-test/src/lib.rs @@ -546,7 +546,7 @@ fn with_tx(ctx: &mut ProcedureContext) { fn get_my_schema_via_http(ctx: &mut ProcedureContext) -> String { let module_identity = ctx.database_identity(); match ctx.http.get(format!( - "http://localhost:3000/v1/database/{module_identity}/schema?version=9" + "http://localhost:3000/v1/database/{module_identity}/schema?version=10" )) { Ok(result) => result.into_body().into_string_lossy(), Err(e) => format!("{e}"), diff --git a/modules/sdk-test-procedure-ts/src/index.ts b/modules/sdk-test-procedure-ts/src/index.ts index 1885eafd156..76efe296c17 100644 --- a/modules/sdk-test-procedure-ts/src/index.ts +++ b/modules/sdk-test-procedure-ts/src/index.ts @@ -97,7 +97,7 @@ export const read_my_schema = spacetimedb.procedure( const module_identity = ctx.databaseIdentity; const base_url = server_url.replace(/\/+$/, ''); const response = ctx.http.fetch( - `${base_url}/v1/database/${module_identity}/schema?version=9` + `${base_url}/v1/database/${module_identity}/schema?version=10` ); return response.text(); } diff --git a/modules/sdk-test-procedure/src/lib.rs b/modules/sdk-test-procedure/src/lib.rs index 5eb2f848ad5..c9af396f4f2 100644 --- a/modules/sdk-test-procedure/src/lib.rs +++ b/modules/sdk-test-procedure/src/lib.rs @@ -46,7 +46,7 @@ fn read_my_schema(ctx: &mut ProcedureContext, server_url: String) -> String { let server_url = server_url.trim_end_matches('/'); match ctx .http - .get(format!("{server_url}/v1/database/{module_identity}/schema?version=9")) + .get(format!("{server_url}/v1/database/{module_identity}/schema?version=10")) { Ok(result) => result.into_body().into_string_lossy(), Err(e) => panic!("{e}"), diff --git a/sdks/rust/tests/procedure-client/src/test_handlers.rs b/sdks/rust/tests/procedure-client/src/test_handlers.rs index fdfc417cd9b..e81cf5b3724 100644 --- a/sdks/rust/tests/procedure-client/src/test_handlers.rs +++ b/sdks/rust/tests/procedure-client/src/test_handlers.rs @@ -1,7 +1,7 @@ use crate::module_bindings::*; use anyhow::Context; use core::time::Duration; -use spacetimedb_lib::db::raw_def::v9::{RawMiscModuleExportV9, RawModuleDefV9}; +use spacetimedb_lib::db::raw_def::v10::{RawModuleDefV10, RawModuleDefV10Section}; use spacetimedb_sdk::{DbConnectionBuilder, DbContext, Table}; use test_counter::{server_url, TestCounter}; @@ -247,7 +247,7 @@ async fn exec_insert_with_tx_rollback(db_name: &str) { /// Test that a procedure can perform an HTTP request and return a string derived from the response. /// /// Invoke the procedure `read_my_schema`, -/// which does an HTTP request to the `/database/schema` route and returns a JSON-ified [`RawModuleDefV9`], +/// which does an HTTP request to the `/database/schema` route and returns a JSON-ified [`RawModuleDefV10`], /// then (in the client) deserialize the response and assert that it contains a description of that procedure. async fn exec_procedure_http_ok(db_name: &str) { let test_counter = TestCounter::new(); @@ -262,12 +262,14 @@ async fn exec_procedure_http_ok(db_name: &str) { #[allow(clippy::redundant_closure_call)] (|| { anyhow::ensure!(res.is_ok(), "Expected Ok result but got {res:?}"); - let module_def: RawModuleDefV9 = spacetimedb_lib::de::serde::deserialize_from( + let module_def: RawModuleDefV10 = spacetimedb_lib::de::serde::deserialize_from( &mut serde_json::Deserializer::from_str(&res.unwrap()), )?; - anyhow::ensure!(module_def.misc_exports.iter().any(|misc_export| { - if let RawMiscModuleExportV9::Procedure(procedure_def) = misc_export { - &*procedure_def.name == "read_my_schema" + anyhow::ensure!(module_def.sections.iter().any(|section| { + if let RawModuleDefV10Section::Procedures(procedures) = section { + procedures + .iter() + .any(|procedure| &*procedure.source_name == "read_my_schema") } else { false } From 8ef580d2033a904ad2185672402f3a51534a04e3 Mon Sep 17 00:00:00 2001 From: Tyler Cloutier Date: Tue, 8 Sep 2026 15:17:54 -0400 Subject: [PATCH 03/34] Preserve qualified reducer names and complete stacked binding tests --- .../tests/__mocks__/spacetime-environment.ts | 2 ++ crates/bindings-typescript/vitest.config.ts | 10 +++++++++- crates/schema/src/auto_migrate.rs | 4 ++-- 3 files changed, 13 insertions(+), 3 deletions(-) create mode 100644 crates/bindings-typescript/tests/__mocks__/spacetime-environment.ts diff --git a/crates/bindings-typescript/tests/__mocks__/spacetime-environment.ts b/crates/bindings-typescript/tests/__mocks__/spacetime-environment.ts new file mode 100644 index 00000000000..5202167f3f7 --- /dev/null +++ b/crates/bindings-typescript/tests/__mocks__/spacetime-environment.ts @@ -0,0 +1,2 @@ +// Tests have no environment unless a fixture supplies one. +export const env_get = (_name: string): string | null => null; diff --git a/crates/bindings-typescript/vitest.config.ts b/crates/bindings-typescript/vitest.config.ts index 9f80a60db65..3676b551d58 100644 --- a/crates/bindings-typescript/vitest.config.ts +++ b/crates/bindings-typescript/vitest.config.ts @@ -16,7 +16,15 @@ export default defineConfig({ { find: 'spacetime:sys@2.1', replacement: sysMock }, { find: 'spacetime:sys@2.2', - replacement: fileURLToPath(new URL('./tests/__mocks__/spacetime-auth.ts', import.meta.url)), + replacement: fileURLToPath( + new URL('./tests/__mocks__/spacetime-auth.ts', import.meta.url) + ), + }, + { + find: 'spacetime:sys@2.3', + replacement: fileURLToPath( + new URL('./tests/__mocks__/spacetime-environment.ts', import.meta.url) + ), }, ], }, diff --git a/crates/schema/src/auto_migrate.rs b/crates/schema/src/auto_migrate.rs index db728fcd250..4a52e079670 100644 --- a/crates/schema/src/auto_migrate.rs +++ b/crates/schema/src/auto_migrate.rs @@ -229,8 +229,8 @@ impl AutoMigratePlan<'_> { .all_reducers_with_prefix() .into_iter() .filter(|(_, _, old)| old.lifecycle.is_none()) - .filter_map(|(prefix, _, old)| { - let name = format!("{prefix}{}", old.name); + .filter_map(|(_, _, old)| { + let name = old.name.to_string(); let (_, new) = self.new.reducer_by_name(&name)?; (old.visibility != new.visibility).then_some((name, &old.visibility, &new.visibility)) }); From 203a022da3052b0f7cf5f286d94d69b4b130154a Mon Sep 17 00:00:00 2001 From: Tyler Cloutier Date: Tue, 8 Sep 2026 15:20:42 -0400 Subject: [PATCH 04/34] Adapt invocation authority host fixture to current engine construction --- crates/core/src/host/host_controller/invocation_flags_tests.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/core/src/host/host_controller/invocation_flags_tests.rs b/crates/core/src/host/host_controller/invocation_flags_tests.rs index a451cf00dc7..b438ef187e7 100644 --- a/crates/core/src/host/host_controller/invocation_flags_tests.rs +++ b/crates/core/src/host/host_controller/invocation_flags_tests.rs @@ -65,6 +65,7 @@ async fn invocation_flags_are_host_owned_and_internal_visibility_is_enforced() { HostRuntimeConfig::default(), Arc::new(storage), Arc::new(NullEnergyMonitor), + Arc::new(()), Arc::new(LocalPersistenceProvider::new(data)), JobCores::without_pinned_cores(), ); @@ -74,6 +75,7 @@ async fn invocation_flags_are_host_owned_and_internal_visibility_is_enforced() { owner_identity: Identity::ONE, host_type: HostType::Js, initial_program: program.hash, + bootstrap_generation: 0, }; // The init reducer itself asserts flags=1, so successful construction also // verifies the real host-to-JS syscall path for a trusted lifecycle call. From 55ef08da2c406ca5002b8ffe55074e7861df1aa6 Mon Sep 17 00:00:00 2001 From: Tyler Cloutier Date: Tue, 8 Sep 2026 15:26:53 -0400 Subject: [PATCH 05/34] Retain current compiler feature flags in visibility codegen regression --- Cargo.lock | 2 +- crates/bindings-csharp/Codegen.Tests/Tests.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 65172309213..0020e4b5c2b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2302,7 +2302,7 @@ dependencies = [ name = "environment-test" version = "0.0.0" dependencies = [ - "spacetimedb 2.10.0", + "spacetimedb", ] [[package]] diff --git a/crates/bindings-csharp/Codegen.Tests/Tests.cs b/crates/bindings-csharp/Codegen.Tests/Tests.cs index 6f0a33fa0e3..3f569ba00b4 100644 --- a/crates/bindings-csharp/Codegen.Tests/Tests.cs +++ b/crates/bindings-csharp/Codegen.Tests/Tests.cs @@ -365,7 +365,7 @@ public static void InternalJob(ReducerContext ctx) {} public static int InternalProcedure(ProcedureContext ctx) => 1; } """; - var parseOptions = new CSharpParseOptions(fixture.SampleCompilation.LanguageVersion); + var parseOptions = fixture.ParseOptions; var tree = CSharpSyntaxTree.ParseText(source, parseOptions); var compilation = fixture.SampleCompilation.AddSyntaxTrees(tree); var driver = CSharpGeneratorDriver.Create( From 4613001fad5b6f4cf8ce44ed1a473746ea8883b3 Mon Sep 17 00:00:00 2001 From: Tyler Cloutier Date: Tue, 8 Sep 2026 19:23:28 -0400 Subject: [PATCH 06/34] Implement typed publish-only database environments --- Cargo.lock | 2 +- crates/bindings-cpp/CMakeLists.txt | 27 + crates/bindings-cpp/README.md | 48 ++ .../include/spacetimedb/environment.h | 75 ++- .../spacetimedb/environment_declaration.h | 153 ++++++ .../spacetimedb/environment_prelude.h.in | 7 + .../autogen/EnvironmentConstraint.g.h | 7 + .../autogen/EnvironmentDeclaration.g.h | 15 + .../autogen/RawModuleDefV10Section.g.h | 3 +- crates/bindings-cpp/src/abi/wasi_shims.cpp | 17 +- .../bindings-cpp/src/internal/v10_builder.cpp | 5 + .../tests/environment/CMakeLists.txt | 15 + .../tests/environment/declarations.h | 11 + .../tests/environment/fallback.cpp | 8 + .../bindings-cpp/tests/environment/main.cpp | 56 +++ .../bindings-cpp/tests/environment/other.cpp | 6 + .../Codegen.Tests/EnvironmentTests.cs | 137 ++++++ crates/bindings-csharp/Codegen.Tests/Tests.cs | 11 +- .../diag/snapshots/Module#FFI.verified.cs | 12 +- .../snapshots/Module#FFI.verified.cs | 12 +- .../server/snapshots/Module#FFI.verified.cs | 12 +- crates/bindings-csharp/Codegen/Environment.cs | 158 ++++++ crates/bindings-csharp/Codegen/Module.cs | 10 +- crates/bindings-csharp/README.md | 27 + crates/bindings-csharp/Runtime/Attrs.cs | 11 + .../Autogen/EnvironmentConstraint.g.cs | 11 + .../Autogen/EnvironmentDeclaration.g.cs | 29 ++ .../Autogen/RawModuleDefV10Section.g.cs | 3 +- .../bindings-csharp/Runtime/Internal/FFI.cs | 3 +- .../Runtime/Internal/Module.cs | 6 + .../Runtime/build/SpacetimeDB.Runtime.targets | 3 + crates/bindings-macro/src/environment.rs | 252 ++++++++++ crates/bindings-macro/src/lib.rs | 7 + crates/bindings-typescript/README.md | 26 + .../src/lib/autogen/types.ts | 29 ++ .../src/lib/environment.ts | 59 ++- .../bindings-typescript/src/lib/reducers.ts | 4 +- crates/bindings-typescript/src/lib/schema.ts | 2 + .../src/server/environment.ts | 86 +++- .../src/server/http_handlers.ts | 4 +- .../src/server/procedures.ts | 6 +- .../bindings-typescript/src/server/runtime.ts | 6 +- .../bindings-typescript/src/server/schema.ts | 29 +- .../bindings-typescript/src/server/views.ts | 6 +- .../tests/__mocks__/spacetime-sys.ts | 4 + .../tests/environment.test.ts | 149 ++++++ crates/bindings-typescript/vitest.config.ts | 1 + crates/bindings/src/lib.rs | 29 +- crates/bindings/src/rt.rs | 9 + crates/bindings/tests/environment.rs | 4 + crates/bindings/tests/pass/environment.rs | 24 + crates/cli/src/lib.rs | 2 + crates/cli/src/spacetime_config.rs | 57 ++- .../cli/src/spacetime_config/environment.rs | 286 +++++++++++ crates/cli/src/subcommands/env.rs | 284 +++++++++++ crates/cli/src/subcommands/mod.rs | 1 + crates/cli/src/subcommands/publish.rs | 41 +- .../src/subcommands/publish/environment.rs | 180 +++++++ .../subcommands/publish/environment/tests.rs | 307 ++++++++++++ crates/client-api-messages/src/lib.rs | 2 + crates/client-api-messages/src/publish.rs | 117 +++++ crates/client-api/src/lib.rs | 23 + crates/client-api/src/routes/database.rs | 24 +- .../routes/database/publish_environment.rs | 139 ++++++ crates/core/src/db/environment.rs | 139 ++++-- crates/core/src/host/host_controller.rs | 236 ++++++--- crates/core/src/host/instance_env.rs | 207 +++++++- crates/core/src/host/mod.rs | 4 +- crates/core/src/host/module_host.rs | 169 ++++++- crates/core/src/host/v8/mod.rs | 49 +- crates/core/src/host/v8/syscall/common.rs | 8 +- .../src/host/wasm_common/module_host_actor.rs | 433 ++++++++++++++--- .../src/host/wasmtime/wasm_instance_env.rs | 14 +- .../core/src/host/wasmtime/wasmtime_module.rs | 4 +- crates/core/src/sql/execute.rs | 97 ++-- crates/expr/src/errors.rs | 4 - crates/expr/src/statement.rs | 21 - crates/lib/src/db/raw_def/v10.rs | 24 + crates/lib/src/environment.rs | 321 ++++++++++++ crates/query/src/lib.rs | 2 +- crates/schema/src/def.rs | 36 +- crates/schema/src/def/validate/v10.rs | 110 ++++- crates/schema/src/def/validate/v9.rs | 2 + crates/schema/src/error.rs | 8 + crates/smoketests/modules/Cargo.lock | 7 + crates/smoketests/modules/Cargo.toml | 1 + .../modules/environment-publish/Cargo.toml | 11 + .../modules/environment-publish/src/lib.rs | 46 ++ .../tests/standalone/cli/environment.rs | 439 +++++++++++++++++ crates/smoketests/tests/standalone/cli/mod.rs | 1 + crates/sql-parser/src/ast/sql.rs | 9 - crates/sql-parser/src/parser/sql.rs | 35 +- crates/standalone/src/control_db.rs | 50 +- .../standalone/src/control_db/environment.rs | 316 ++++++++++++ crates/standalone/src/environment_tests.rs | 190 ++++++++ crates/standalone/src/lib.rs | 459 +++++++++++------ crates/testing/src/modules.rs | 85 +++- crates/testing/tests/environment.rs | 460 +++++++++++++++--- .../00300-spacetime-publish.md | 6 + .../00700-environment-variables.md | 235 +++++++++ .../00100-cli-reference.md | 66 ++- docs/docusaurus.config.ts | 2 + modules/environment-test/src/lib.rs | 60 ++- modules/module-test-cpp/CMakeLists.txt | 3 + modules/module-test-cpp/environment.h | 9 + modules/module-test-cpp/src/lib.cpp | 2 + modules/module-test-cs/EnvironmentTests.cs | 11 + .../module-test-ts/src/environment_sys.d.ts | 4 + modules/module-test-ts/src/index.ts | 26 +- modules/module-test-ts/src/lib_submodule.ts | 36 ++ 110 files changed, 6836 insertions(+), 720 deletions(-) create mode 100644 crates/bindings-cpp/include/spacetimedb/environment_declaration.h create mode 100644 crates/bindings-cpp/include/spacetimedb/environment_prelude.h.in create mode 100644 crates/bindings-cpp/include/spacetimedb/internal/autogen/EnvironmentConstraint.g.h create mode 100644 crates/bindings-cpp/include/spacetimedb/internal/autogen/EnvironmentDeclaration.g.h create mode 100644 crates/bindings-cpp/tests/environment/CMakeLists.txt create mode 100644 crates/bindings-cpp/tests/environment/declarations.h create mode 100644 crates/bindings-cpp/tests/environment/fallback.cpp create mode 100644 crates/bindings-cpp/tests/environment/main.cpp create mode 100644 crates/bindings-cpp/tests/environment/other.cpp create mode 100644 crates/bindings-csharp/Codegen.Tests/EnvironmentTests.cs create mode 100644 crates/bindings-csharp/Codegen/Environment.cs create mode 100644 crates/bindings-csharp/Runtime/Internal/Autogen/EnvironmentConstraint.g.cs create mode 100644 crates/bindings-csharp/Runtime/Internal/Autogen/EnvironmentDeclaration.g.cs create mode 100644 crates/bindings-macro/src/environment.rs create mode 100644 crates/bindings-typescript/tests/environment.test.ts create mode 100644 crates/bindings/tests/environment.rs create mode 100644 crates/bindings/tests/pass/environment.rs create mode 100644 crates/cli/src/spacetime_config/environment.rs create mode 100644 crates/cli/src/subcommands/env.rs create mode 100644 crates/cli/src/subcommands/publish/environment.rs create mode 100644 crates/cli/src/subcommands/publish/environment/tests.rs create mode 100644 crates/client-api-messages/src/publish.rs create mode 100644 crates/client-api/src/routes/database/publish_environment.rs create mode 100644 crates/smoketests/modules/environment-publish/Cargo.toml create mode 100644 crates/smoketests/modules/environment-publish/src/lib.rs create mode 100644 crates/smoketests/tests/standalone/cli/environment.rs create mode 100644 crates/standalone/src/control_db/environment.rs create mode 100644 crates/standalone/src/environment_tests.rs create mode 100644 docs/docs/00200-core-concepts/00100-databases/00700-environment-variables.md create mode 100644 modules/module-test-cpp/environment.h create mode 100644 modules/module-test-ts/src/environment_sys.d.ts diff --git a/Cargo.lock b/Cargo.lock index 9b030bc73ff..384c70585bd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2302,7 +2302,7 @@ dependencies = [ name = "environment-test" version = "0.0.0" dependencies = [ - "spacetimedb 2.10.0", + "spacetimedb", ] [[package]] diff --git a/crates/bindings-cpp/CMakeLists.txt b/crates/bindings-cpp/CMakeLists.txt index dc5d1433fce..695c91f5b5f 100644 --- a/crates/bindings-cpp/CMakeLists.txt +++ b/crates/bindings-cpp/CMakeLists.txt @@ -28,6 +28,33 @@ add_library(spacetimedb::spacetimedb_cpp_library ALIAS spacetimedb_cpp_library) target_sources(spacetimedb_cpp_library PRIVATE ${LIBRARY_SOURCES}) +# A declared Environment adds named methods to the context member type. Every +# SDK and module translation unit must see the same prelude before any context. +if(NOT DEFINED SPACETIMEDB_ENV_HEADER) + set(SPACETIMEDB_ENV_HEADER "" CACHE FILEPATH "Absolute module environment declaration header") +endif() +if(SPACETIMEDB_ENV_HEADER) + if(NOT IS_ABSOLUTE "${SPACETIMEDB_ENV_HEADER}" OR NOT EXISTS "${SPACETIMEDB_ENV_HEADER}") + message(FATAL_ERROR "SPACETIMEDB_ENV_HEADER must name an existing absolute declaration header") + endif() + # The standalone WASI ABI shims have no context or Environment type. Their + # hand-written ABI declarations must not include the C++ standard library's + # WASI declarations through the module prelude. + set_source_files_properties(src/abi/wasi_shims.cpp PROPERTIES + COMPILE_DEFINITIONS SPACETIMEDB_WASI_SHIMS=1) + configure_file( + ${CMAKE_CURRENT_SOURCE_DIR}/include/spacetimedb/environment_prelude.h.in + ${CMAKE_CURRENT_BINARY_DIR}/include/spacetimedb/environment_prelude.h + @ONLY) + set(SPACETIMEDB_ENV_PRELUDE "${CMAKE_CURRENT_BINARY_DIR}/include/spacetimedb/environment_prelude.h") + target_compile_definitions(spacetimedb_cpp_library PUBLIC SPACETIMEDB_ENV_HEADER_ACTIVE=1) + if(MSVC) + target_compile_options(spacetimedb_cpp_library PUBLIC "/FI${SPACETIMEDB_ENV_PRELUDE}") + else() + target_compile_options(spacetimedb_cpp_library PUBLIC -include "${SPACETIMEDB_ENV_PRELUDE}") + endif() +endif() + # Require C++20 for consumers of this library without forcing global flags target_compile_features(spacetimedb_cpp_library PUBLIC cxx_std_20) target_compile_definitions(spacetimedb_cpp_library PRIVATE SPACETIMEDB_UNSTABLE_FEATURES) diff --git a/crates/bindings-cpp/README.md b/crates/bindings-cpp/README.md index ef31361c10b..155fafedfda 100644 --- a/crates/bindings-cpp/README.md +++ b/crates/bindings-cpp/README.md @@ -275,3 +275,51 @@ See the `modules/*-cpp/src/` directory for example modules: This library is part of the SpacetimeDB project. Please see the main repository for contribution guidelines. + +### Declared environment + +Declare the complete environment schema in a dedicated header, before other SDK +includes. Values are supplied at publish time and are never embedded in this +header. For example, `environment.h`: + +```cpp +#pragma once +#include +SPACETIMEDB_ENV( + (API_URL, std::string), + (MODE, std::string, ("prod", "dev")), + (LOG_LEVEL, std::optional, ("info", "debug")) +) +``` + +Select it in your project's `CMakeLists.txt` **before** adding the SDK directory: + +```cmake +set(SPACETIMEDB_ENV_HEADER "${CMAKE_CURRENT_SOURCE_DIR}/environment.h") +add_subdirectory(path/to/bindings-cpp sdk) +add_executable(my_module src/lib.cpp) +target_link_libraries(my_module PRIVATE spacetimedb_cpp_library) +``` + +The SDK's CMake target includes the declaration header before every SDK and module +translation unit that uses contexts. The standalone WASI ABI shim is excluded +because it has no SDK context types. This keeps the context type consistent; +including the header in only one source file is insufficient. The normal module +source can then use the usual umbrella include: + +```cpp +#include +void example(SpacetimeDB::ReducerContext ctx) { + std::string mode = ctx.env.MODE(); + std::optional level = ctx.env.LOG_LEVEL(); +} +``` + +An empty `SPACETIMEDB_ENV()` or an omitted declaration selects an empty schema. +`get`, C++ keywords, and names colliding with the accessor type do not create named +methods; use the checked `ctx.env.get("get")` form for such declared names. Unknown +keys fail at runtime, including when the module declares no environment variables. +Only the host's root module entry may read environment values. Ordinary C++ helper +calls retain their caller's host scope. Values are private, durable database configuration for secrets and other settings. +Database owners and authorized collaborators can read them; module code can expose +them through its own outputs. diff --git a/crates/bindings-cpp/include/spacetimedb/environment.h b/crates/bindings-cpp/include/spacetimedb/environment.h index f2df08234d0..06020dcee2f 100644 --- a/crates/bindings-cpp/include/spacetimedb/environment.h +++ b/crates/bindings-cpp/include/spacetimedb/environment.h @@ -6,11 +6,14 @@ #include #include #include +#include +#include +#include namespace SpacetimeDB { /// Read-only database environment. Reads use the current transaction, or a /// short snapshot in a procedure outside a transaction. Values are not cached. -class Environment { +class EnvironmentBase { public: std::optional get(std::string_view key) const { if (key.empty() || key.size() > 256) LOG_PANIC("invalid environment variable name"); @@ -30,5 +33,75 @@ class Environment { } } }; + +namespace Internal { +inline std::vector& environment_declarations() { + static std::vector declarations; + return declarations; +} + +template +inline constexpr bool environment_string = std::is_same_v || std::is_same_v>; + +template +T read_environment(std::string_view key) { + static_assert(environment_string, "Environment declarations require string or optional"); + auto value = EnvironmentBase{}.get(key); + if constexpr (std::is_same_v) { + if (!value) LOG_PANIC("required environment value is absent"); + return std::move(*value); + } else { return value; } +} + +template +EnvironmentDeclaration declare_environment(std::string name) { + static_assert(environment_string, "Environment declarations require string or optional"); + EnvironmentConstraint constraint; + constraint.set<0>(std::monostate{}); + return {std::move(name), std::move(constraint), std::is_same_v>}; +} + +// Preserve the complete source literal, including embedded NUL bytes. Implicit +// conversion through std::string(const char*) would truncate those constraints. +struct EnvironmentLiteral { + std::string value; + template + EnvironmentLiteral(const char (&text)[N]) : value(text, N - 1) {} + EnvironmentLiteral(std::string text) : value(std::move(text)) {} +}; + +template +EnvironmentDeclaration declare_environment(std::string name, std::initializer_list allowed) { + auto declaration = declare_environment(std::move(name)); + if (allowed.size() == 0) LOG_PANIC("environment literal union cannot be empty"); + std::unordered_set unique; + std::vector values; + values.reserve(allowed.size()); + for (const auto& literal : allowed) { + const auto& value = literal.value; + if (value.size() > 8192 || !unique.insert(value).second) LOG_PANIC("invalid environment literal union"); + values.push_back(value); + } + if (values.size() == 1) declaration.constraint.template set<1>(std::move(values.front())); + else declaration.constraint.template set<2>(std::move(values)); + return declaration; +} + +inline void validate_environment_declarations() { + const auto& declarations = environment_declarations(); + if (declarations.size() > 256) LOG_PANIC("too many environment declarations"); + std::unordered_set keys; + const auto initial = [](char c) { return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || c == '_'; }; + for (const auto& declaration : declarations) { + const auto& key = declaration.name; + if (key.empty() || key.size() > 256 || !initial(key[0]) || !keys.insert(key).second) LOG_PANIC("invalid environment declaration name"); + for (char c : key) if (!initial(c) && !(c >= '0' && c <= '9')) LOG_PANIC("invalid environment declaration name"); + } +} +} + +#ifndef SPACETIMEDB_ENV_DECLARATION +class Environment : public EnvironmentBase {}; +#endif } #endif diff --git a/crates/bindings-cpp/include/spacetimedb/environment_declaration.h b/crates/bindings-cpp/include/spacetimedb/environment_declaration.h new file mode 100644 index 00000000000..b3df8028d5b --- /dev/null +++ b/crates/bindings-cpp/include/spacetimedb/environment_declaration.h @@ -0,0 +1,153 @@ +#pragma once +// Put SPACETIMEDB_ENV in a module declaration header selected by CMake's +// SPACETIMEDB_ENV_HEADER, before including any SDK context or umbrella header. +#ifndef SPACETIMEDB_ENV_HEADER_ACTIVE +#error "Configure SPACETIMEDB_ENV_HEADER before adding the SDK CMake directory" +#endif +#ifdef SPACETIMEDB_ENVIRONMENT_H +#error "The environment declaration header must precede every SDK include" +#endif +#define SPACETIMEDB_ENV_DECLARATION 1 +#include + +#define STDB_ENV_CAT_I(a, b) a##b +#define STDB_ENV_CAT(a, b) STDB_ENV_CAT_I(a, b) +#define STDB_ENV_SECOND(a, b, ...) b +#define STDB_ENV_PROBE() ignored, 1 +#define STDB_ENV_CHECK(...) STDB_ENV_SECOND(__VA_ARGS__, 0) +#define STDB_ENV_RESERVED(name) STDB_ENV_CHECK(STDB_ENV_CAT(STDB_ENV_RESERVED_, name)) +#define STDB_ENV_KEEP_0(...) __VA_ARGS__ +#define STDB_ENV_KEEP_1(...) +#define STDB_ENV_RESERVED_alignas STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_alignof STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_and STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_and_eq STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_asm STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_atomic_cancel STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_atomic_commit STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_atomic_noexcept STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_auto STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_bitand STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_bitor STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_bool STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_break STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_case STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_catch STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_char STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_char8_t STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_char16_t STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_char32_t STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_class STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_compl STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_concept STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_const STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_consteval STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_constexpr STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_constinit STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_const_cast STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_continue STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_co_await STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_co_return STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_co_yield STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_decltype STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_default STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_delete STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_do STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_double STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_dynamic_cast STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_else STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_enum STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_explicit STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_export STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_extern STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_false STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_float STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_for STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_friend STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_goto STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_if STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_inline STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_int STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_long STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_mutable STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_namespace STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_new STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_noexcept STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_not STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_not_eq STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_nullptr STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_operator STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_or STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_or_eq STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_private STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_protected STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_public STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_reflexpr STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_register STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_reinterpret_cast STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_requires STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_return STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_short STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_signed STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_sizeof STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_static STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_static_assert STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_static_cast STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_struct STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_switch STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_synchronized STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_template STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_this STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_thread_local STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_throw STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_true STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_try STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_typedef STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_typeid STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_typename STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_union STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_unsigned STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_using STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_virtual STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_void STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_volatile STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_wchar_t STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_while STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_xor STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_xor_eq STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_get STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_Environment STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_EnvironmentBase STDB_ENV_PROBE() + +#define STDB_ENV_PARENS () +#define STDB_ENV_EVAL1(...) __VA_ARGS__ +#define STDB_ENV_EVAL2(...) STDB_ENV_EVAL1(STDB_ENV_EVAL1(STDB_ENV_EVAL1(STDB_ENV_EVAL1(__VA_ARGS__)))) +#define STDB_ENV_EVAL3(...) STDB_ENV_EVAL2(STDB_ENV_EVAL2(STDB_ENV_EVAL2(STDB_ENV_EVAL2(__VA_ARGS__)))) +#define STDB_ENV_EVAL4(...) STDB_ENV_EVAL3(STDB_ENV_EVAL3(STDB_ENV_EVAL3(STDB_ENV_EVAL3(__VA_ARGS__)))) +#define STDB_ENV_EVAL(...) STDB_ENV_EVAL4(STDB_ENV_EVAL4(STDB_ENV_EVAL4(STDB_ENV_EVAL4(__VA_ARGS__)))) +#define STDB_ENV_EACH(macro, ...) __VA_OPT__(STDB_ENV_EVAL(STDB_ENV_EACH_I(macro, __VA_ARGS__))) +#define STDB_ENV_EACH_I(macro, tuple, ...) macro tuple __VA_OPT__(STDB_ENV_AGAIN STDB_ENV_PARENS (macro, __VA_ARGS__)) +#define STDB_ENV_AGAIN() STDB_ENV_EACH_I +#define STDB_ENV_UNPAREN(...) __VA_ARGS__ +#define STDB_ENV_MEMBER(name, type, ...) \ + STDB_ENV_CAT(STDB_ENV_KEEP_, STDB_ENV_RESERVED(name))( \ + type name() const { return ::SpacetimeDB::Internal::read_environment(#name); } \ + ) +#define STDB_ENV_METADATA(name, type, ...) \ + ::SpacetimeDB::Internal::declare_environment(#name __VA_OPT__(, { STDB_ENV_UNPAREN __VA_ARGS__ })), + +/// Declare the complete schema, never live values. One declaration per module. +/// Empty SPACETIMEDB_ENV() is supported. Reserved names retain generic get(). +#define SPACETIMEDB_ENV(...) \ + namespace SpacetimeDB { \ + class Environment : public EnvironmentBase { \ + public: STDB_ENV_EACH(STDB_ENV_MEMBER, __VA_ARGS__) \ + }; \ + namespace Internal { \ + inline const bool environment_schema_registered = [] { \ + environment_declarations() = { STDB_ENV_EACH(STDB_ENV_METADATA, __VA_ARGS__) }; \ + validate_environment_declarations(); \ + return true; \ + }(); \ + } \ + } diff --git a/crates/bindings-cpp/include/spacetimedb/environment_prelude.h.in b/crates/bindings-cpp/include/spacetimedb/environment_prelude.h.in new file mode 100644 index 00000000000..4d812bb6d05 --- /dev/null +++ b/crates/bindings-cpp/include/spacetimedb/environment_prelude.h.in @@ -0,0 +1,7 @@ +#pragma once + +// The ABI shim translation unit has no SDK contexts. All context-bearing SDK +// and module translation units see the same declared Environment definition. +#ifndef SPACETIMEDB_WASI_SHIMS +#include "@SPACETIMEDB_ENV_HEADER@" +#endif diff --git a/crates/bindings-cpp/include/spacetimedb/internal/autogen/EnvironmentConstraint.g.h b/crates/bindings-cpp/include/spacetimedb/internal/autogen/EnvironmentConstraint.g.h new file mode 100644 index 00000000000..0b2e45c8ae6 --- /dev/null +++ b/crates/bindings-cpp/include/spacetimedb/internal/autogen/EnvironmentConstraint.g.h @@ -0,0 +1,7 @@ +#pragma once +#include "../autogen_base.h" +#include +#include +namespace SpacetimeDB::Internal { +SPACETIMEDB_INTERNAL_TAGGED_ENUM(EnvironmentConstraint, std::monostate, std::string, std::vector) +} diff --git a/crates/bindings-cpp/include/spacetimedb/internal/autogen/EnvironmentDeclaration.g.h b/crates/bindings-cpp/include/spacetimedb/internal/autogen/EnvironmentDeclaration.g.h new file mode 100644 index 00000000000..2d83f3c9d3b --- /dev/null +++ b/crates/bindings-cpp/include/spacetimedb/internal/autogen/EnvironmentDeclaration.g.h @@ -0,0 +1,15 @@ +#pragma once +#include "EnvironmentConstraint.g.h" +namespace SpacetimeDB::Internal { +SPACETIMEDB_INTERNAL_PRODUCT_TYPE(EnvironmentDeclaration) { + std::string name; + EnvironmentConstraint constraint; + bool optional; + void bsatn_serialize(::SpacetimeDB::bsatn::Writer& writer) const { + ::SpacetimeDB::bsatn::serialize(writer, name); + ::SpacetimeDB::bsatn::serialize(writer, constraint); + ::SpacetimeDB::bsatn::serialize(writer, optional); + } + SPACETIMEDB_PRODUCT_TYPE_EQUALITY(name, constraint, optional) +}; +} diff --git a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDefV10Section.g.h b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDefV10Section.g.h index ea2e4b5ec85..551fb3bc636 100644 --- a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDefV10Section.g.h +++ b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDefV10Section.g.h @@ -27,8 +27,9 @@ #include "RawScheduleDefV10.g.h" #include "RawViewPrimaryKeyDefV10.g.h" #include "RawHttpHandlerDefV10.g.h" +#include "EnvironmentDeclaration.g.h" namespace SpacetimeDB::Internal { -SPACETIMEDB_INTERNAL_TAGGED_ENUM(RawModuleDefV10Section, SpacetimeDB::Internal::Typespace, std::vector, std::vector, std::vector, std::vector, std::vector, std::vector, std::vector, std::vector, SpacetimeDB::Internal::CaseConversionPolicy, SpacetimeDB::Internal::ExplicitNames, std::vector, std::vector, std::vector, std::vector) +SPACETIMEDB_INTERNAL_TAGGED_ENUM(RawModuleDefV10Section, SpacetimeDB::Internal::Typespace, std::vector, std::vector, std::vector, std::vector, std::vector, std::vector, std::vector, std::vector, SpacetimeDB::Internal::CaseConversionPolicy, SpacetimeDB::Internal::ExplicitNames, std::vector, std::vector, std::vector, std::vector, std::vector) } // namespace SpacetimeDB::Internal diff --git a/crates/bindings-cpp/src/abi/wasi_shims.cpp b/crates/bindings-cpp/src/abi/wasi_shims.cpp index 430a1f9a858..ec461c8d51d 100644 --- a/crates/bindings-cpp/src/abi/wasi_shims.cpp +++ b/crates/bindings-cpp/src/abi/wasi_shims.cpp @@ -4,13 +4,14 @@ #include #include - -// SpacetimeDB imports we need for console output -// Import from spacetime_10.0 module as required by SpacetimeDB ABI +// Keep this pure ABI translation unit independent of SDK opaque types: their +// standard-library helpers include wasi/api.h in Emscripten, which conflicts +// with these standalone shim definitions. Use a distinct C++ name for the raw +// host logging import; its WebAssembly signature is the same eight i32 values. extern "C" __attribute__((import_module("spacetime_10.0"), import_name("console_log"))) -void console_log(uint8_t log_level, const uint8_t* target, uint32_t target_len, - const uint8_t* filename, uint32_t filename_len, uint32_t line_number, - const uint8_t* message, uint32_t message_len); +void wasi_console_log(uint8_t level, const uint8_t* target_ptr, size_t target_len, + const uint8_t* filename_ptr, size_t filename_len, uint32_t line_number, + const uint8_t* message_ptr, size_t message_len); // Helper macro for string literals #define CSTR(s) (uint8_t*)s, sizeof(s) - 1 @@ -150,7 +151,7 @@ __wasi_errno_t __wasi_fd_write(__wasi_fd_t fd, const __wasi_ciovec_t* iovs, // Make a single console_log call with the complete message uint8_t log_level = (fd == STDERR_FILENO) ? 1 : 2; // 1=WARN, 2=INFO - console_log(log_level, CSTR("wasi"), CSTR(__FILE__), __LINE__, + wasi_console_log(log_level, CSTR("wasi"), CSTR(__FILE__), __LINE__, buffer, offset); // Clean up heap allocation if needed @@ -190,4 +191,4 @@ void emscripten_notify_memory_growth(int32_t) { // No-op - memory growth is handled by the runtime } -} // extern "C" \ No newline at end of file +} // extern "C" diff --git a/crates/bindings-cpp/src/internal/v10_builder.cpp b/crates/bindings-cpp/src/internal/v10_builder.cpp index a931b79a7c2..be5bbd65035 100644 --- a/crates/bindings-cpp/src/internal/v10_builder.cpp +++ b/crates/bindings-cpp/src/internal/v10_builder.cpp @@ -1,3 +1,4 @@ +#include "spacetimedb/environment.h" #include "spacetimedb/internal/v10_builder.h" #include "spacetimedb/internal/autogen/AlgebraicType.g.h" #include "spacetimedb/internal/autogen/ProductType.g.h" @@ -315,6 +316,10 @@ RawModuleDefV10 V10Builder::BuildModuleDef() const { v10_module.sections.push_back(std::move(section_rls)); } + validate_environment_declarations(); + RawModuleDefV10Section section_environment; + section_environment.set<15>(environment_declarations()); + v10_module.sections.push_back(std::move(section_environment)); return v10_module; } diff --git a/crates/bindings-cpp/tests/environment/CMakeLists.txt b/crates/bindings-cpp/tests/environment/CMakeLists.txt new file mode 100644 index 00000000000..34223081b4a --- /dev/null +++ b/crates/bindings-cpp/tests/environment/CMakeLists.txt @@ -0,0 +1,15 @@ +cmake_minimum_required(VERSION 3.16) +project(environment_declaration_tests LANGUAGES CXX) +set(SPACETIMEDB_ENV_HEADER "${CMAKE_CURRENT_SOURCE_DIR}/declarations.h") +add_subdirectory(../.. sdk) +add_executable(environment_declaration_tests main.cpp other.cpp) +target_link_libraries(environment_declaration_tests PRIVATE spacetimedb_cpp_library) +enable_testing() +add_test(NAME environment_declaration_tests COMMAND environment_declaration_tests) +# This separate target deliberately has no forced prelude, modeling an existing +# module with no declarations and the generic checked accessor only. +add_executable(environment_fallback_tests fallback.cpp) +target_include_directories(environment_fallback_tests PRIVATE ../../include) +target_compile_features(environment_fallback_tests PRIVATE cxx_std_20) +add_test(NAME environment_fallback_tests COMMAND environment_fallback_tests) +set_tests_properties(environment_declaration_tests environment_fallback_tests PROPERTIES TIMEOUT 10) diff --git a/crates/bindings-cpp/tests/environment/declarations.h b/crates/bindings-cpp/tests/environment/declarations.h new file mode 100644 index 00000000000..ab7e2bba1b5 --- /dev/null +++ b/crates/bindings-cpp/tests/environment/declarations.h @@ -0,0 +1,11 @@ +#pragma once +#include +SPACETIMEDB_ENV( + (FOOBAR, std::string), + (ENABLE_EMAIL, std::string, ("true", "false")), + (LOG_LEVEL, std::optional, ("debug", "info", "error")), + (DEPLOYMENT_KIND, std::string, ("production")), + (get, std::optional), + (class, std::string), + (NUL_LITERAL, std::string, ("a\0b")) +) diff --git a/crates/bindings-cpp/tests/environment/fallback.cpp b/crates/bindings-cpp/tests/environment/fallback.cpp new file mode 100644 index 00000000000..27d94126011 --- /dev/null +++ b/crates/bindings-cpp/tests/environment/fallback.cpp @@ -0,0 +1,8 @@ +#include +#include +#include + +int main() { + static_assert(std::is_same_v>); + assert(SpacetimeDB::Internal::environment_declarations().empty()); +} diff --git a/crates/bindings-cpp/tests/environment/main.cpp b/crates/bindings-cpp/tests/environment/main.cpp new file mode 100644 index 00000000000..7d0273a8bc5 --- /dev/null +++ b/crates/bindings-cpp/tests/environment/main.cpp @@ -0,0 +1,56 @@ +#include +#include +#include +#include +#include + +using namespace SpacetimeDB; +std::string from_another_translation_unit(); +namespace { std::string payload; size_t position; unsigned calls; } +extern "C" Status env_get(const uint8_t* key, uint32_t length, BytesSource* out) { + const std::string name(reinterpret_cast(key), length); + ++calls; + if (name == "LOG_LEVEL") { *out = BytesSource{0}; return Status{0}; } + if (name == "FOOBAR") payload = calls == 1 ? "first" : "updated"; + else if (name == "ENABLE_EMAIL") payload = "false"; + else if (name == "DEPLOYMENT_KIND") payload = "production"; + else if (name == "get") payload = "reserved"; + else if (name == "class") payload = "keyword"; + else if (name == "NUL_LITERAL") payload = std::string("a\0b", 3); + else return Status{1}; + position = 0; + *out = BytesSource{1}; + return Status{0}; +} +extern "C" int16_t bytes_source_read(BytesSource, uint8_t* out, size_t* size) { + *size = std::min(*size, payload.size() - position); + std::memcpy(out, payload.data() + position, *size); + position += *size; + return position == payload.size() ? -1 : 0; +} +extern "C" void console_log(LogLevel, const uint8_t*, size_t, const uint8_t*, size_t, uint32_t, const uint8_t*, size_t) {} + +int main() { + Environment env; + static_assert(std::is_same_v); + static_assert(std::is_same_v>); + assert(env.FOOBAR() == "first"); + assert(from_another_translation_unit() == "updated"); + assert(env.ENABLE_EMAIL() == "false"); + assert(!env.LOG_LEVEL()); + assert(env.DEPLOYMENT_KIND() == "production"); + assert(env.get("get") == "reserved"); + assert(env.get("class") == "keyword"); + assert(env.NUL_LITERAL() == std::string("a\0b", 3)); + const auto& entries = Internal::environment_declarations(); + assert(entries.size() == 7); + assert(entries[0].name == "FOOBAR" && entries[0].constraint.get_tag() == 0 && !entries[0].optional); + assert(entries[1].constraint.get_tag() == 2 && entries[1].constraint.get<2>() == std::vector({"true", "false"})); + assert(entries[2].optional); + assert(entries[3].constraint.get_tag() == 1 && entries[3].constraint.get<1>() == "production"); + assert(entries[6].constraint.get<1>() == std::string("a\0b", 3)); + Internal::RawModuleDefV10Section section; + section.set<15>(entries); + assert(section.get_tag() == 15); + assert(section.get<15>() == entries); +} diff --git a/crates/bindings-cpp/tests/environment/other.cpp b/crates/bindings-cpp/tests/environment/other.cpp new file mode 100644 index 00000000000..c9ffe40f9fd --- /dev/null +++ b/crates/bindings-cpp/tests/environment/other.cpp @@ -0,0 +1,6 @@ +#include +#include +std::string from_another_translation_unit() { + return SpacetimeDB::Environment{}.FOOBAR(); +} +static_assert(std::is_same_v().env.FOOBAR()), std::string>); diff --git a/crates/bindings-csharp/Codegen.Tests/EnvironmentTests.cs b/crates/bindings-csharp/Codegen.Tests/EnvironmentTests.cs new file mode 100644 index 00000000000..d7dd4d4382e --- /dev/null +++ b/crates/bindings-csharp/Codegen.Tests/EnvironmentTests.cs @@ -0,0 +1,137 @@ +namespace SpacetimeDB.Codegen.Tests; + +using System.Reflection; +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp; + +public static class EnvironmentTests +{ + // A controlled host seam allows the generated accessor code to execute. + // Live value access remains delegated to DatabaseEnvironment.Get. + private const string Host = """ + #nullable enable + namespace SpacetimeDB { + [System.AttributeUsage(System.AttributeTargets.Struct)] + public sealed class EnvAttribute : System.Attribute {} + [System.AttributeUsage(System.AttributeTargets.Field)] + public sealed class EnvValuesAttribute(params string[] values) : System.Attribute {} + public readonly struct DatabaseEnvironment { + public string? Get(string key) => Host.Get(key); + } + public static class Host { + public static int Reads; + public static string? Get(string key) { + Reads++; + return key switch { + "REQUIRED" => Reads.ToString(), "OPTIONAL" => null, + "MODE" => "prod", "Get" => "reserved", "class" => "keyword", + _ => throw new System.InvalidOperationException("undeclared environment key") + }; + } + } + } + namespace SpacetimeDB.Internal { + public abstract record EnvironmentConstraint { + public sealed record AnyString(System.ValueTuple Value) : EnvironmentConstraint; + public sealed record Literal(string Value) : EnvironmentConstraint; + public sealed record OneOf(System.Collections.Generic.List Value) : EnvironmentConstraint; + } + public sealed record EnvironmentDeclaration(string Name, EnvironmentConstraint Constraint, bool Optional); + public static class Module { + public static System.Collections.Generic.List Declarations = new(); + public static void RegisterEnvironment(EnvironmentDeclaration value) => Declarations.Add(value); + } + } + """; + + private static (Compilation Compilation, GeneratorDriverRunResult Result) Generate( + string declaration + ) + { + var references = ((string)AppContext.GetData("TRUSTED_PLATFORM_ASSEMBLIES")!) + .Split(Path.PathSeparator) + .Select(path => MetadataReference.CreateFromFile(path)); + var parse = new CSharpParseOptions(LanguageVersion.Preview); + var compilation = CSharpCompilation.Create( + "EnvironmentFixture" + Guid.NewGuid().ToString("N"), + [CSharpSyntaxTree.ParseText(Host + declaration, parse)], + references, + new CSharpCompilationOptions(OutputKind.DynamicallyLinkedLibrary) + ); + GeneratorDriver driver = CSharpGeneratorDriver.Create( + [new EnvironmentGenerator().AsSourceGenerator()], + parseOptions: parse + ); + driver = driver.RunGeneratorsAndUpdateCompilation(compilation, out var output, out _); + return (output, driver.GetRunResult()); + } + + [Fact] + public static void NamedAccessorsKeepCheckedReadsAndRegisterCanonicalConstraints() + { + var (compilation, result) = Generate( + """ + [SpacetimeDB.Env] public struct Declarations { + public string REQUIRED; + public string? OPTIONAL; + [SpacetimeDB.EnvValues("prod", "dev")] public string MODE; + [SpacetimeDB.EnvValues("reserved")] public string Get; + public string @class; + } + public static class Usage { + public static void Check() { + var env = new SpacetimeDB.ModuleEnvironment(); + if (env.REQUIRED == env.REQUIRED || env.OPTIONAL != null || env.MODE != "prod" || + env.Get("Get") != "reserved" || env.@class != "keyword") throw new System.Exception("bad accessor"); + try { env.Get("UNKNOWN"); throw new System.Exception("unchecked generic read"); } + catch (System.InvalidOperationException) {} + var declarations = SpacetimeDB.Internal.Module.Declarations; + if (declarations.Count != 5 || declarations[0].Optional || !declarations[1].Optional || + declarations[0].Constraint is not SpacetimeDB.Internal.EnvironmentConstraint.AnyString || + declarations[2].Constraint is not SpacetimeDB.Internal.EnvironmentConstraint.OneOf { Value.Count: 2 } || + declarations[3].Constraint is not SpacetimeDB.Internal.EnvironmentConstraint.Literal { Value: "reserved" }) + throw new System.Exception("bad metadata"); + } + } + """ + ); + Assert.Empty(result.Diagnostics); + using var stream = new MemoryStream(); + var emitted = compilation.Emit(stream); + Assert.True(emitted.Success, string.Join("\n", emitted.Diagnostics)); + var assembly = Assembly.Load(stream.ToArray()); + assembly.GetType("Usage")!.GetMethod("Check")!.Invoke(null, null); + Assert.Null(assembly.GetType("SpacetimeDB.ModuleEnvironment")!.GetProperty("Get")); + } + + [Theory] + [InlineData("public int BAD;")] + [InlineData("public static string BAD;")] + [InlineData("[SpacetimeDB.EnvValues()] public string BAD;")] + [InlineData("[SpacetimeDB.EnvValues(\"x\", \"x\")] public string BAD;")] + [InlineData("[SpacetimeDB.EnvValues(null)] public string BAD;")] + public static void InvalidDeclarationsAreCompileErrors(string field) + { + var (_, result) = Generate("[SpacetimeDB.Env] public struct Declarations {" + field + "}"); + Assert.Contains( + result.Diagnostics, + diagnostic => + diagnostic.Id == "STDBENV001" && diagnostic.Severity == DiagnosticSeverity.Error + ); + } + + [Fact] + public static void EmptySchemaRetainsOnlyGenericAccess() + { + var (compilation, result) = Generate(""); + Assert.Empty(result.Diagnostics); + Assert.DoesNotContain( + compilation.GetDiagnostics(), + diagnostic => diagnostic.Severity == DiagnosticSeverity.Error + ); + Assert.Contains( + "public string? Get(string key)", + result.GeneratedTrees.Single().ToString() + ); + } +} diff --git a/crates/bindings-csharp/Codegen.Tests/Tests.cs b/crates/bindings-csharp/Codegen.Tests/Tests.cs index 4133819ef9c..5ecccedb31b 100644 --- a/crates/bindings-csharp/Codegen.Tests/Tests.cs +++ b/crates/bindings-csharp/Codegen.Tests/Tests.cs @@ -88,9 +88,13 @@ public async Task RunAndCheckGenerators( params IIncrementalGenerator[] generators ) => SampleCompilation.AddSyntaxTrees( - (await Task.WhenAll(generators.Select(RunAndCheckGenerator))).SelectMany(output => - output - ) + (await Task.WhenAll(generators.Select(RunAndCheckGenerator))) + .SelectMany(output => output) + .Concat( + generators.Any(generator => generator is SpacetimeDB.Codegen.Module) + ? RunGeneratorAndGetResult(new EnvironmentGenerator()).GeneratedTrees + : [] + ) ); } @@ -333,6 +337,7 @@ public static void @params(ProcedureContext ctx) [ new SpacetimeDB.Codegen.Type().AsSourceGenerator(), new SpacetimeDB.Codegen.Module().AsSourceGenerator(), + new EnvironmentGenerator().AsSourceGenerator(), ], driverOptions: new( disabledOutputs: IncrementalGeneratorOutputKind.None, diff --git a/crates/bindings-csharp/Codegen.Tests/fixtures/diag/snapshots/Module#FFI.verified.cs b/crates/bindings-csharp/Codegen.Tests/fixtures/diag/snapshots/Module#FFI.verified.cs index 2eb70c4a352..e81f122020d 100644 --- a/crates/bindings-csharp/Codegen.Tests/fixtures/diag/snapshots/Module#FFI.verified.cs +++ b/crates/bindings-csharp/Codegen.Tests/fixtures/diag/snapshots/Module#FFI.verified.cs @@ -649,7 +649,7 @@ public static class Handlers { } public sealed record ReducerContext : DbContext, Internal.IReducerContext { - public global::SpacetimeDB.DatabaseEnvironment Env => default; + public global::SpacetimeDB.ModuleEnvironment Env => default; public readonly Identity Sender; public readonly ConnectionId? ConnectionId; public readonly Random Rng; @@ -732,6 +732,7 @@ public Uuid NewUuidV7() public sealed partial class ProcedureContext : global::SpacetimeDB.ProcedureContextBase { + public new global::SpacetimeDB.ModuleEnvironment Env => default; private readonly Local _db = new(); internal ProcedureContext( @@ -810,6 +811,7 @@ public Uuid NewUuidV7() public sealed partial class HandlerContext : global::SpacetimeDB.HandlerContextBase { + public new global::SpacetimeDB.ModuleEnvironment Env => default; private readonly Local _db = new(); internal HandlerContext(Random random, Timestamp time) @@ -850,6 +852,8 @@ public Uuid NewUuidV7() public sealed class ProcedureTxContext : global::SpacetimeDB.ProcedureTxContextBase { + public new global::SpacetimeDB.ModuleEnvironment Env => default; + internal ProcedureTxContext(Internal.TxContext inner) : base(inner) { } @@ -859,6 +863,8 @@ internal ProcedureTxContext(Internal.TxContext inner) [Experimental("STDB_UNSTABLE")] public sealed class HandlerTxContext : global::SpacetimeDB.HandlerTxContextBase { + public new global::SpacetimeDB.ModuleEnvironment Env => default; + internal HandlerTxContext(Internal.TxContext inner) : base(inner) { } @@ -893,7 +899,7 @@ public sealed record ViewContext : DbContext, Internal.I { public Identity Sender { get; } - public global::SpacetimeDB.DatabaseEnvironment Env => default; + public global::SpacetimeDB.ModuleEnvironment Env => default; public QueryBuilder From => default; internal ViewContext(Identity sender, Internal.LocalReadOnly db) @@ -907,7 +913,7 @@ public sealed record AnonymousViewContext : DbContext, Internal.IAnonymousViewContext { - public global::SpacetimeDB.DatabaseEnvironment Env => default; + public global::SpacetimeDB.ModuleEnvironment Env => default; public QueryBuilder From => default; internal AnonymousViewContext(Internal.LocalReadOnly db) diff --git a/crates/bindings-csharp/Codegen.Tests/fixtures/explicitnames/snapshots/Module#FFI.verified.cs b/crates/bindings-csharp/Codegen.Tests/fixtures/explicitnames/snapshots/Module#FFI.verified.cs index d05060e0bd3..ad729dadac1 100644 --- a/crates/bindings-csharp/Codegen.Tests/fixtures/explicitnames/snapshots/Module#FFI.verified.cs +++ b/crates/bindings-csharp/Codegen.Tests/fixtures/explicitnames/snapshots/Module#FFI.verified.cs @@ -51,7 +51,7 @@ public static class Handlers { } public sealed record ReducerContext : DbContext, Internal.IReducerContext { - public global::SpacetimeDB.DatabaseEnvironment Env => default; + public global::SpacetimeDB.ModuleEnvironment Env => default; public readonly Identity Sender; public readonly ConnectionId? ConnectionId; public readonly Random Rng; @@ -134,6 +134,7 @@ public Uuid NewUuidV7() public sealed partial class ProcedureContext : global::SpacetimeDB.ProcedureContextBase { + public new global::SpacetimeDB.ModuleEnvironment Env => default; private readonly Local _db = new(); internal ProcedureContext( @@ -212,6 +213,7 @@ public Uuid NewUuidV7() public sealed partial class HandlerContext : global::SpacetimeDB.HandlerContextBase { + public new global::SpacetimeDB.ModuleEnvironment Env => default; private readonly Local _db = new(); internal HandlerContext(Random random, Timestamp time) @@ -252,6 +254,8 @@ public Uuid NewUuidV7() public sealed class ProcedureTxContext : global::SpacetimeDB.ProcedureTxContextBase { + public new global::SpacetimeDB.ModuleEnvironment Env => default; + internal ProcedureTxContext(Internal.TxContext inner) : base(inner) { } @@ -261,6 +265,8 @@ internal ProcedureTxContext(Internal.TxContext inner) [Experimental("STDB_UNSTABLE")] public sealed class HandlerTxContext : global::SpacetimeDB.HandlerTxContextBase { + public new global::SpacetimeDB.ModuleEnvironment Env => default; + internal HandlerTxContext(Internal.TxContext inner) : base(inner) { } @@ -276,7 +282,7 @@ public sealed record ViewContext : DbContext, Internal.I { public Identity Sender { get; } - public global::SpacetimeDB.DatabaseEnvironment Env => default; + public global::SpacetimeDB.ModuleEnvironment Env => default; public QueryBuilder From => default; internal ViewContext(Identity sender, Internal.LocalReadOnly db) @@ -290,7 +296,7 @@ public sealed record AnonymousViewContext : DbContext, Internal.IAnonymousViewContext { - public global::SpacetimeDB.DatabaseEnvironment Env => default; + public global::SpacetimeDB.ModuleEnvironment Env => default; public QueryBuilder From => default; internal AnonymousViewContext(Internal.LocalReadOnly db) diff --git a/crates/bindings-csharp/Codegen.Tests/fixtures/server/snapshots/Module#FFI.verified.cs b/crates/bindings-csharp/Codegen.Tests/fixtures/server/snapshots/Module#FFI.verified.cs index 7f196c87a30..95bbf9c0516 100644 --- a/crates/bindings-csharp/Codegen.Tests/fixtures/server/snapshots/Module#FFI.verified.cs +++ b/crates/bindings-csharp/Codegen.Tests/fixtures/server/snapshots/Module#FFI.verified.cs @@ -493,7 +493,7 @@ public static class Handlers { } public sealed record ReducerContext : DbContext, Internal.IReducerContext { - public global::SpacetimeDB.DatabaseEnvironment Env => default; + public global::SpacetimeDB.ModuleEnvironment Env => default; public readonly Identity Sender; public readonly ConnectionId? ConnectionId; public readonly Random Rng; @@ -576,6 +576,7 @@ public Uuid NewUuidV7() public sealed partial class ProcedureContext : global::SpacetimeDB.ProcedureContextBase { + public new global::SpacetimeDB.ModuleEnvironment Env => default; private readonly Local _db = new(); internal ProcedureContext( @@ -654,6 +655,7 @@ public Uuid NewUuidV7() public sealed partial class HandlerContext : global::SpacetimeDB.HandlerContextBase { + public new global::SpacetimeDB.ModuleEnvironment Env => default; private readonly Local _db = new(); internal HandlerContext(Random random, Timestamp time) @@ -694,6 +696,8 @@ public Uuid NewUuidV7() public sealed class ProcedureTxContext : global::SpacetimeDB.ProcedureTxContextBase { + public new global::SpacetimeDB.ModuleEnvironment Env => default; + internal ProcedureTxContext(Internal.TxContext inner) : base(inner) { } @@ -703,6 +707,8 @@ internal ProcedureTxContext(Internal.TxContext inner) [Experimental("STDB_UNSTABLE")] public sealed class HandlerTxContext : global::SpacetimeDB.HandlerTxContextBase { + public new global::SpacetimeDB.ModuleEnvironment Env => default; + internal HandlerTxContext(Internal.TxContext inner) : base(inner) { } @@ -727,7 +733,7 @@ public sealed record ViewContext : DbContext, Internal.I { public Identity Sender { get; } - public global::SpacetimeDB.DatabaseEnvironment Env => default; + public global::SpacetimeDB.ModuleEnvironment Env => default; public QueryBuilder From => default; internal ViewContext(Identity sender, Internal.LocalReadOnly db) @@ -741,7 +747,7 @@ public sealed record AnonymousViewContext : DbContext, Internal.IAnonymousViewContext { - public global::SpacetimeDB.DatabaseEnvironment Env => default; + public global::SpacetimeDB.ModuleEnvironment Env => default; public QueryBuilder From => default; internal AnonymousViewContext(Internal.LocalReadOnly db) diff --git a/crates/bindings-csharp/Codegen/Environment.cs b/crates/bindings-csharp/Codegen/Environment.cs new file mode 100644 index 00000000000..e049abb5fda --- /dev/null +++ b/crates/bindings-csharp/Codegen/Environment.cs @@ -0,0 +1,158 @@ +namespace SpacetimeDB.Codegen; + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text; +using System.Text.RegularExpressions; +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp; + +/// Compile-time declarations and read-only accessors, never live values. +[Generator] +public sealed class EnvironmentGenerator : IIncrementalGenerator +{ + private static readonly DiagnosticDescriptor InvalidDeclaration = + new( + "STDBENV001", + "Invalid environment declaration", + "{0}", + "SpacetimeDB", + DiagnosticSeverity.Error, + isEnabledByDefault: true + ); + + public void Initialize(IncrementalGeneratorInitializationContext context) + { + var declarations = context + .SyntaxProvider.ForAttributeWithMetadataName( + "SpacetimeDB.EnvAttribute", + (_, _) => true, + (ctx, _) => (INamedTypeSymbol)ctx.TargetSymbol + ) + .Collect(); + context.RegisterSourceOutput(declarations, Generate); + } + + private static void Generate( + SourceProductionContext context, + ImmutableArray types + ) + { + void Report(ISymbol symbol, string message) => + context.ReportDiagnostic( + Diagnostic.Create(InvalidDeclaration, symbol.Locations.FirstOrDefault(), message) + ); + if (types.Length > 1) + { + foreach (var type in types) + Report(type, "A module may have only one [SpacetimeDB.Env] declaration struct."); + } + var fields = types + .SelectMany(type => type.GetMembers().OfType()) + .Where(field => !field.IsImplicitlyDeclared) + .ToArray(); + if (fields.Length > 256 && types.Length != 0) + Report(types[0], "An environment schema may declare at most 256 variables."); + + var keys = new HashSet(StringComparer.Ordinal); + var properties = new List(); + var registrations = new List(); + foreach (var field in fields) + { + var name = field.Name; + if (field.IsStatic || field.Type.SpecialType != SpecialType.System_String) + { + Report( + field, + "Environment fields must be instance string or nullable string declarations." + ); + continue; + } + if ( + !Regex.IsMatch(name, "^[A-Za-z_][A-Za-z0-9_]*$") + || Encoding.UTF8.GetByteCount(name) > 256 + || !keys.Add(name) + ) + { + Report( + field, + "Environment names must be unique POSIX identifiers of at most 256 UTF-8 bytes." + ); + continue; + } + var optional = field.NullableAnnotation == NullableAnnotation.Annotated; + var attr = field + .GetAttributes() + .FirstOrDefault(a => + a.AttributeClass?.ToDisplayString() == "SpacetimeDB.EnvValuesAttribute" + ); + var constraint = + "new global::SpacetimeDB.Internal.EnvironmentConstraint.AnyString(default)"; + if (attr is not null) + { + var values = attr.ConstructorArguments.FirstOrDefault(); + if ( + values.Kind != TypedConstantKind.Array + || values.IsNull + || values.Values.Length == 0 + || values.Values.Any(value => + value.Value is not string text || Encoding.UTF8.GetByteCount(text) > 8192 + ) + ) + { + Report( + field, + "EnvValues requires a nonempty list of string literals of at most 8192 UTF-8 bytes each." + ); + continue; + } + var strings = values.Values.Select(value => (string)value.Value!).ToArray(); + if (strings.Distinct(StringComparer.Ordinal).Count() != strings.Length) + { + Report(field, "EnvValues must not repeat an allowed literal."); + continue; + } + constraint = + strings.Length == 1 + ? $"new global::SpacetimeDB.Internal.EnvironmentConstraint.Literal({Literal(strings[0])})" + : $"new global::SpacetimeDB.Internal.EnvironmentConstraint.OneOf(new global::System.Collections.Generic.List {{ {string.Join(", ", strings.Select(Literal))} }})"; + } + registrations.Add( + $"global::SpacetimeDB.Internal.Module.RegisterEnvironment(new({Literal(name)}, {constraint}, {(optional ? "true" : "false")}));" + ); + // Preserve the checked generic method, including a key literally + // named Get. Keywords are escaped without renaming stored keys. + if (name is "Get" or "ModuleEnvironment" or "Equals" or "GetHashCode" or "ToString") + continue; + var read = $"Get({Literal(name)})"; + if (!optional) + read += + " ?? throw new global::System.InvalidOperationException(\"Required environment value is absent\")"; + properties.Add($"public string{(optional ? "?" : "")} @{name} => {read};"); + } + context.AddSource( + "Environment.g.cs", + $$""" + // + #nullable enable + #pragma warning disable CS0436 + namespace SpacetimeDB { + public readonly struct ModuleEnvironment { + public string? Get(string key) => default(global::SpacetimeDB.DatabaseEnvironment).Get(key); + {{string.Join("\n", properties)}} + } + internal static class EnvironmentRegistration { + [global::System.Runtime.CompilerServices.ModuleInitializer] + internal static void Register() { + {{string.Join("\n", registrations)}} + } + } + } + """ + ); + } + + private static string Literal(string value) => SymbolDisplay.FormatLiteral(value, quote: true); +} diff --git a/crates/bindings-csharp/Codegen/Module.cs b/crates/bindings-csharp/Codegen/Module.cs index e3f146ad3f2..29fc16d2cfc 100644 --- a/crates/bindings-csharp/Codegen/Module.cs +++ b/crates/bindings-csharp/Codegen/Module.cs @@ -2557,7 +2557,7 @@ public static class Handlers { ))}} } public sealed record ReducerContext : DbContext, Internal.IReducerContext { - public global::SpacetimeDB.DatabaseEnvironment Env => default; + public global::SpacetimeDB.ModuleEnvironment Env => default; public readonly Identity Sender; public readonly ConnectionId? ConnectionId; public readonly Random Rng; @@ -2629,6 +2629,7 @@ public Uuid NewUuidV7() } public sealed partial class ProcedureContext : global::SpacetimeDB.ProcedureContextBase { + public new global::SpacetimeDB.ModuleEnvironment Env => default; private readonly Local _db = new(); internal ProcedureContext(Identity identity, ConnectionId? connectionId, Random random, Timestamp time) @@ -2699,6 +2700,7 @@ public Uuid NewUuidV7() } public sealed partial class HandlerContext : global::SpacetimeDB.HandlerContextBase { + public new global::SpacetimeDB.ModuleEnvironment Env => default; private readonly Local _db = new(); internal HandlerContext(Random random, Timestamp time) @@ -2736,6 +2738,7 @@ public Uuid NewUuidV7() } public sealed class ProcedureTxContext : global::SpacetimeDB.ProcedureTxContextBase { + public new global::SpacetimeDB.ModuleEnvironment Env => default; internal ProcedureTxContext(Internal.TxContext inner) : base(inner) {} public new Local Db => (Local)base.Db; @@ -2743,6 +2746,7 @@ internal ProcedureTxContext(Internal.TxContext inner) : base(inner) {} [Experimental("STDB_UNSTABLE")] public sealed class HandlerTxContext : global::SpacetimeDB.HandlerTxContextBase { + public new global::SpacetimeDB.ModuleEnvironment Env => default; internal HandlerTxContext(Internal.TxContext inner) : base(inner) {} public new Local Db => (Local)base.Db; @@ -2756,7 +2760,7 @@ public sealed record ViewContext : DbContext, Internal.I { public Identity Sender { get; } - public global::SpacetimeDB.DatabaseEnvironment Env => default; + public global::SpacetimeDB.ModuleEnvironment Env => default; public QueryBuilder From => default; internal ViewContext(Identity sender, Internal.LocalReadOnly db) @@ -2768,7 +2772,7 @@ internal ViewContext(Identity sender, Internal.LocalReadOnly db) public sealed record AnonymousViewContext : DbContext, Internal.IAnonymousViewContext { - public global::SpacetimeDB.DatabaseEnvironment Env => default; + public global::SpacetimeDB.ModuleEnvironment Env => default; public QueryBuilder From => default; internal AnonymousViewContext(Internal.LocalReadOnly db) diff --git a/crates/bindings-csharp/README.md b/crates/bindings-csharp/README.md index 289bd570ff0..a97596d4ec9 100644 --- a/crates/bindings-csharp/README.md +++ b/crates/bindings-csharp/README.md @@ -20,3 +20,30 @@ The [`Codegen`](./Codegen/) and [`Runtime`](./Runtime/) libraries are used: They provide all of the functionality needed to write SpacetimeDB modules in C#. See their READMEs for more information. + +### Declared environment + +A module may declare one `[SpacetimeDB.Env]` struct. `string` is required and +`string?` is optional. An optional `EnvValues` attribute restricts the allowed +strings; one value is a literal constraint. Values are supplied on publish, never +in module source: + +```csharp +[SpacetimeDB.Env] +public partial struct EnvironmentSchema +{ + public string API_URL; + [SpacetimeDB.EnvValues("prod", "dev")] + public string MODE; + public string? LOG_LEVEL; +} +``` + +Context access is read-only: `ctx.Env.MODE` returns `string`, while +`ctx.Env.LOG_LEVEL` returns `string?`. `ctx.Env.Get("MODE")` uses the same checked +host read. A key named `Get` keeps generic access rather than replacing the method; +C# keywords are escaped, for example `ctx.Env.@class`. Empty or absent declarations +allow no environment keys. Undeclared reads and reads from host-dispatched +submodules fail at runtime. Values are private, durable database configuration for secrets and other settings. +Database owners and authorized collaborators can read them; module code can expose +them through its own outputs. diff --git a/crates/bindings-csharp/Runtime/Attrs.cs b/crates/bindings-csharp/Runtime/Attrs.cs index 3865f925a2a..afcfcc0688e 100644 --- a/crates/bindings-csharp/Runtime/Attrs.cs +++ b/crates/bindings-csharp/Runtime/Attrs.cs @@ -1,5 +1,16 @@ namespace SpacetimeDB { + /// Declares the complete environment schema for this module. + [AttributeUsage(AttributeTargets.Struct)] + public sealed class EnvAttribute : Attribute { } + + /// Restricts one declared string to these exact permitted values. + [AttributeUsage(AttributeTargets.Field)] + public sealed class EnvValuesAttribute(params string[] values) : Attribute + { + public string[] Values { get; } = values; + } + namespace Internal { [Flags] diff --git a/crates/bindings-csharp/Runtime/Internal/Autogen/EnvironmentConstraint.g.cs b/crates/bindings-csharp/Runtime/Internal/Autogen/EnvironmentConstraint.g.cs new file mode 100644 index 00000000000..f4226020799 --- /dev/null +++ b/crates/bindings-csharp/Runtime/Internal/Autogen/EnvironmentConstraint.g.cs @@ -0,0 +1,11 @@ +// Canonical module-definition metadata; declaration constraints contain no runtime values. +#nullable enable +namespace SpacetimeDB.Internal; + +[SpacetimeDB.Type] +public partial record EnvironmentConstraint + : SpacetimeDB.TaggedEnum<( + SpacetimeDB.Unit AnyString, + string Literal, + System.Collections.Generic.List OneOf + )>; diff --git a/crates/bindings-csharp/Runtime/Internal/Autogen/EnvironmentDeclaration.g.cs b/crates/bindings-csharp/Runtime/Internal/Autogen/EnvironmentDeclaration.g.cs new file mode 100644 index 00000000000..554d77ba55a --- /dev/null +++ b/crates/bindings-csharp/Runtime/Internal/Autogen/EnvironmentDeclaration.g.cs @@ -0,0 +1,29 @@ +#nullable enable +namespace SpacetimeDB.Internal; + +[SpacetimeDB.Type] +[System.Runtime.Serialization.DataContract] +public sealed partial class EnvironmentDeclaration +{ + [System.Runtime.Serialization.DataMember(Name = "name")] + public string Name; + + [System.Runtime.Serialization.DataMember(Name = "constraint")] + public EnvironmentConstraint Constraint; + + [System.Runtime.Serialization.DataMember(Name = "optional")] + public bool Optional; + + public EnvironmentDeclaration(string Name, EnvironmentConstraint Constraint, bool Optional) + { + this.Name = Name; + this.Constraint = Constraint; + this.Optional = Optional; + } + + public EnvironmentDeclaration() + { + Name = ""; + Constraint = new EnvironmentConstraint.AnyString(default); + } +} diff --git a/crates/bindings-csharp/Runtime/Internal/Autogen/RawModuleDefV10Section.g.cs b/crates/bindings-csharp/Runtime/Internal/Autogen/RawModuleDefV10Section.g.cs index 61212c98e89..52f750e3903 100644 --- a/crates/bindings-csharp/Runtime/Internal/Autogen/RawModuleDefV10Section.g.cs +++ b/crates/bindings-csharp/Runtime/Internal/Autogen/RawModuleDefV10Section.g.cs @@ -23,6 +23,7 @@ public partial record RawModuleDefV10Section : SpacetimeDB.TaggedEnum<( System.Collections.Generic.List HttpHandlers, System.Collections.Generic.List HttpRoutes, System.Collections.Generic.List ViewPrimaryKeys, - System.Collections.Generic.List Submodules + System.Collections.Generic.List Submodules, + System.Collections.Generic.List Environment )>; } diff --git a/crates/bindings-csharp/Runtime/Internal/FFI.cs b/crates/bindings-csharp/Runtime/Internal/FFI.cs index f498e6ed8ea..cc3e8cf624b 100644 --- a/crates/bindings-csharp/Runtime/Internal/FFI.cs +++ b/crates/bindings-csharp/Runtime/Internal/FFI.cs @@ -110,13 +110,14 @@ internal static partial class FFI ; const string StdbNamespace10_7 = -#if EXPERIMENTAL_WASM_AOT +#if EXPERIMENTAL_WASM_AOT || NET10_0_OR_GREATER "spacetime_10.7" #else "bindings" #endif ; + [WasmImportLinkage] [LibraryImport(StdbNamespace10_7)] public static unsafe partial CheckedStatus env_get( byte* key, diff --git a/crates/bindings-csharp/Runtime/Internal/Module.cs b/crates/bindings-csharp/Runtime/Internal/Module.cs index 177fddd785a..035291a2f0d 100644 --- a/crates/bindings-csharp/Runtime/Internal/Module.cs +++ b/crates/bindings-csharp/Runtime/Internal/Module.cs @@ -20,6 +20,7 @@ partial class RawModuleDefV10 private readonly List httpRouteDefs = []; private readonly List viewDefs = []; private readonly List viewPrimaryKeyDefs = []; + private readonly List environment = []; private readonly List rowLevelSecurityDefs = []; private readonly Dictionary> defaultValuesByTable = new(StringComparer.Ordinal); @@ -86,6 +87,8 @@ internal void RegisterTable(RawTableDefV10 table, RawScheduleDefV10? schedule) internal void RegisterView(RawViewDefV10 view) => viewDefs.Add(view); + internal void RegisterEnvironment(EnvironmentDeclaration declaration) => environment.Add(declaration); + internal void RegisterViewPrimaryKey(string viewSourceName, IEnumerable columns) => viewPrimaryKeyDefs.Add(new RawViewPrimaryKeyDefV10(viewSourceName, [.. columns])); @@ -162,6 +165,7 @@ internal RawModuleDefV10 BuildModuleDefinition() var sections = new List { new RawModuleDefV10Section.Typespace(typespace), + new RawModuleDefV10Section.Environment(environment), }; if (typeDefs.Count > 0) @@ -427,6 +431,8 @@ public static void RegisterAnonymousView() moduleDef.RegisterView(def); } + public static void RegisterEnvironment(EnvironmentDeclaration declaration) => moduleDef.RegisterEnvironment(declaration); + public static void RegisterViewPrimaryKey(string viewSourceName, string[] columns) => moduleDef.RegisterViewPrimaryKey(viewSourceName, columns); diff --git a/crates/bindings-csharp/Runtime/build/SpacetimeDB.Runtime.targets b/crates/bindings-csharp/Runtime/build/SpacetimeDB.Runtime.targets index e3b4bd0e942..95c2f1cfdd5 100644 --- a/crates/bindings-csharp/Runtime/build/SpacetimeDB.Runtime.targets +++ b/crates/bindings-csharp/Runtime/build/SpacetimeDB.Runtime.targets @@ -65,6 +65,9 @@ + + + diff --git a/crates/bindings-macro/src/environment.rs b/crates/bindings-macro/src/environment.rs new file mode 100644 index 00000000000..8377037082e --- /dev/null +++ b/crates/bindings-macro/src/environment.rs @@ -0,0 +1,252 @@ +use proc_macro2::TokenStream; +use quote::{format_ident, quote}; +use syn::ext::IdentExt as _; +use syn::punctuated::Punctuated; +use syn::{Fields, GenericArgument, ItemStruct, LitStr, PathArguments, Token, Type}; + +pub(crate) fn expand(args: TokenStream, mut item: ItemStruct) -> syn::Result { + if !args.is_empty() { + return Err(syn::Error::new_spanned(args, "env does not accept arguments")); + } + if !item.generics.params.is_empty() || item.generics.where_clause.is_some() { + return Err(syn::Error::new_spanned( + &item.generics, + "environment declarations cannot be generic", + )); + } + if matches!(item.fields, Fields::Unnamed(_)) { + return Err(syn::Error::new_spanned( + &item.fields, + "environment declarations require named fields", + )); + } + let mut declarations = Vec::new(); + let mut signatures = Vec::new(); + let mut methods = Vec::new(); + for field in &mut item.fields { + let ident = field.ident.as_ref().expect("named fields checked"); + let name = ident.unraw().to_string(); + if name.len() > 256 || !name.bytes().all(|b| b.is_ascii_alphanumeric() || b == b'_') { + return Err(syn::Error::new_spanned( + ident, + "environment keys must be POSIX names of at most 256 bytes", + )); + } + let optional = optional_string(&field.ty)?; + let mut values: Option> = None; + for attr in field.attrs.iter().filter(|attr| attr.path().is_ident("env")) { + attr.parse_nested_meta(|meta| { + if !meta.path.is_ident("values") || values.is_some() { + return Err(meta.error("expected one values(\"literal\", ...) constraint")); + } + let content; + syn::parenthesized!(content in meta.input); + let parsed = Punctuated::::parse_terminated(&content)?; + if parsed.is_empty() { + return Err(meta.error("environment string unions must not be empty")); + } + for value in &parsed { + if value.value().len() > 8192 { + return Err(syn::Error::new_spanned( + value, + "environment literal exceeds 8192 UTF-8 bytes", + )); + } + } + values = Some(parsed.into_iter().collect()); + Ok(()) + })?; + } + field.attrs.retain(|attr| !attr.path().is_ident("env")); + let constraint = match values.as_deref() { + None => quote!(::spacetimedb::spacetimedb_lib::environment::EnvironmentConstraint::AnyString), + Some([value]) => { + quote!(::spacetimedb::spacetimedb_lib::environment::EnvironmentConstraint::Literal(#value.into())) + } + Some(values) => quote!( + ::spacetimedb::spacetimedb_lib::environment::EnvironmentConstraint::OneOf( + ::std::vec![#(#values.into()),*] + ) + ), + }; + declarations.push( + quote!(::spacetimedb::spacetimedb_lib::environment::EnvironmentDeclaration { + name: #name.into(), constraint: #constraint, optional: #optional, + }), + ); + if name == "get" { + continue; + } + let (return_type, body) = if optional { + ( + quote!(::std::option::Option<::std::string::String>), + quote!(::spacetimedb::Environment::get(self, #name)), + ) + } else { + ( + quote!(::std::string::String), + quote!(::spacetimedb::Environment::get(self, #name).expect(concat!("required environment key is missing: ", #name))), + ) + }; + signatures.push(quote! { + #[doc = concat!("Read the declared environment key `", #name, "` through the checked host ABI.")] + fn #ident(&self) -> #return_type; + }); + methods.push(quote!(fn #ident(&self) -> #return_type { #body })); + } + let vis = &item.vis; + let access = format_ident!("{}Access", item.ident.unraw()); + let symbol = format!("__preinit__20_register_environment_{}", item.ident.unraw()); + Ok(quote! { + #[allow(non_snake_case)] + #item + + /// Named read-only accessors for this module's environment declaration. + #[allow(non_snake_case)] + #vis trait #access { + #(#signatures)* + } + #[allow(non_snake_case)] + impl #access for ::spacetimedb::Environment { + #(#methods)* + } + const _: () = { + #[unsafe(export_name = #symbol)] + extern "C" fn __register_environment() { + ::spacetimedb::rt::register_environment(|| ::std::vec![#(#declarations),*]); + } + }; + }) +} + +fn optional_string(ty: &Type) -> syn::Result { + let Type::Path(path) = ty else { + return Err(syn::Error::new_spanned(ty, "expected String or Option")); + }; + if path.qself.is_none() { + let segments: Vec<_> = path + .path + .segments + .iter() + .map(|segment| segment.ident.to_string()) + .collect(); + let names: Vec<_> = segments.iter().map(String::as_str).collect(); + if matches!(names.as_slice(), ["String"] | ["std" | "alloc", "string", "String"]) + && path + .path + .segments + .iter() + .all(|segment| matches!(segment.arguments, PathArguments::None)) + { + return Ok(false); + } + if matches!(names.as_slice(), ["Option"] | ["std" | "core", "option", "Option"]) + && path + .path + .segments + .iter() + .rev() + .skip(1) + .all(|segment| matches!(segment.arguments, PathArguments::None)) + && let PathArguments::AngleBracketed(arguments) = &path.path.segments.last().unwrap().arguments + && let [GenericArgument::Type(inner)] = arguments.args.iter().collect::>().as_slice() + && !optional_string(inner)? + { + return Ok(true); + } + } + Err(syn::Error::new_spanned( + ty, + "expected String or Option; aliases and other types are not env constraints", + )) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn rejects_non_string_nested_optional_empty_union_and_invalid_names() { + for item in [ + quote!( + struct Env { + VALUE: bool, + } + ), + quote!( + struct Env { + VALUE: Option>, + } + ), + quote!( + struct Env { + #[env(values())] + VALUE: String, + } + ), + quote!( + struct Env { + #[env(values("a"), values("b"))] + VALUE: String, + } + ), + quote!( + struct Env { + 雪: String, + } + ), + quote!( + struct Env(String); + ), + quote!( + struct Env { + VALUE: T, + } + ), + ] { + assert!(expand(TokenStream::new(), syn::parse2(item).unwrap()).is_err()); + } + } + + #[test] + fn accepts_strings_optionals_constraints_and_generic_accessor_collision() { + let output = expand( + TokenStream::new(), + syn::parse_quote! { + pub struct Env { + VALUE: String, + #[env(values("false", "true"))] ENABLED: std::string::String, + #[env(values(""))] OPTIONAL: Option, + get: Option, + r#type: String, + } + }, + ) + .unwrap(); + let parsed: syn::File = syn::parse2(output).unwrap(); + let trait_item = parsed + .items + .iter() + .find_map(|item| match item { + syn::Item::Trait(item) => Some(item), + _ => None, + }) + .unwrap(); + let names: Vec<_> = trait_item + .items + .iter() + .filter_map(|item| match item { + syn::TraitItem::Fn(method) => Some(method.sig.ident.unraw().to_string()), + _ => None, + }) + .collect(); + assert_eq!(names, ["VALUE", "ENABLED", "OPTIONAL", "type"]); + assert!(expand( + TokenStream::new(), + syn::parse_quote!( + pub struct Empty {} + ) + ) + .is_ok()); + } +} diff --git a/crates/bindings-macro/src/lib.rs b/crates/bindings-macro/src/lib.rs index a3efdd3d2f4..02aaacc4028 100644 --- a/crates/bindings-macro/src/lib.rs +++ b/crates/bindings-macro/src/lib.rs @@ -8,6 +8,13 @@ // // (private documentation for the macro authors is totally fine here and you SHOULD write that!) +mod environment; + +#[proc_macro_attribute] +pub fn env(args: StdTokenStream, item: StdTokenStream) -> StdTokenStream { + ok_or_compile_error(|| environment::expand(args.into(), syn::parse(item)?)) +} + mod http; mod procedure; diff --git a/crates/bindings-typescript/README.md b/crates/bindings-typescript/README.md index 48e7cfd1535..a006b8489fd 100644 --- a/crates/bindings-typescript/README.md +++ b/crates/bindings-typescript/README.md @@ -209,3 +209,29 @@ To run the tests, do: ```sh pnpm build && pnpm test ``` + +### Declared environment + +Pass the complete environment schema to `schema`. Values are supplied at publish +time, not embedded in the module: + +```ts +const db = schema(tables, { + env: { + API_URL: t.string(), + MODE: t.enum('Mode', ['prod', 'dev']), + LOG_LEVEL: t.enum('LogLevel', ['info', 'debug']).optional(), + }, +}); +``` + +`ctx.env.MODE` has type `'prod' | 'dev'`; `ctx.env.LOG_LEVEL` also permits +`undefined`. Simple enum cases mean allowed strings only in this declaration; +ordinary enum values elsewhere retain their tagged representation. Payload enums +are rejected. The checked `ctx.env.get('LOG_LEVEL')` returns `null` for an omitted +optional value. A declared key named `get` remains accessible through the generic +method. Omitted or empty schemas allow no keys. Undeclared reads and reads entered +by the host in a submodule fail at runtime; ordinary helpers retain their caller's +scope. Values are private, durable database configuration for secrets and other settings. +Database owners and authorized collaborators can read them; module code can expose +them through its own outputs. diff --git a/crates/bindings-typescript/src/lib/autogen/types.ts b/crates/bindings-typescript/src/lib/autogen/types.ts index 3cee51f03d5..642c0431285 100644 --- a/crates/bindings-typescript/src/lib/autogen/types.ts +++ b/crates/bindings-typescript/src/lib/autogen/types.ts @@ -393,9 +393,38 @@ export const RawModuleDefV10Section = __t.enum('RawModuleDefV10Section', { get Submodules() { return __t.array(RawSubmoduleV10); }, + get Environment() { + return __t.array(EnvironmentDeclaration); + }, }); export type RawModuleDefV10Section = __Infer; +export const EnvironmentConstraint = __t.enum('EnvironmentConstraint', { + get AnyString() { + return __t.unit(); + }, + get Literal() { + return __t.string(); + }, + get OneOf() { + return __t.array(__t.string()); + }, +}); +export type EnvironmentConstraint = __Infer; + +export const EnvironmentDeclaration = __t.object('EnvironmentDeclaration', { + get name() { + return __t.string(); + }, + get constraint() { + return EnvironmentConstraint; + }, + get optional() { + return __t.bool(); + }, +}); +export type EnvironmentDeclaration = __Infer; + export const RawModuleDefV8 = __t.object('RawModuleDefV8', { get typespace() { return Typespace; diff --git a/crates/bindings-typescript/src/lib/environment.ts b/crates/bindings-typescript/src/lib/environment.ts index 3798cfb43d4..d065d5bb0aa 100644 --- a/crates/bindings-typescript/src/lib/environment.ts +++ b/crates/bindings-typescript/src/lib/environment.ts @@ -1,5 +1,56 @@ -/** Read-only database environment access. Missing keys return null; empty values return "". */ -export interface Environment { - /** Reads the current transaction, or a short snapshot outside a procedure transaction. */ - get(key: string): string | null; +import type { + OptionBuilder, + StringBuilder, + TypeBuilder, + t, +} from './type_builders'; + +/** Only strings and simple, payload-free enums constrain environment strings. */ +export type EnvironmentString = + | StringBuilder + | (TypeBuilder & { + readonly variants: Record>; + }); +export type EnvironmentSchema = Record< + string, + EnvironmentString | OptionBuilder +>; + +export type EnvironmentValue = + T extends OptionBuilder + ? EnvironmentValue | undefined + : T extends { readonly variants: infer Variants } + ? keyof Variants & string + : string; + +/** Values are read from the host on each access. Undeclared names are errors. */ +export type Environment< + Declarations extends EnvironmentSchema | undefined = undefined, +> = Declarations extends EnvironmentSchema + ? { + readonly [Key in keyof Declarations as Key extends 'get' + ? never + : Key]: EnvironmentValue; + } & { + get( + key: Key & + (string extends Key + ? unknown + : Key extends keyof Declarations + ? unknown + : never) + ): Key extends keyof Declarations + ? + | Exclude, undefined> + | (undefined extends EnvironmentValue + ? null + : never) + : string | null; + } + : { get(key: string): string | null }; + +export type EnvironmentFor = Schema extends { + env: infer Declarations extends EnvironmentSchema; } + ? Environment + : Environment; diff --git a/crates/bindings-typescript/src/lib/reducers.ts b/crates/bindings-typescript/src/lib/reducers.ts index eaa8de94b55..f2d470152ec 100644 --- a/crates/bindings-typescript/src/lib/reducers.ts +++ b/crates/bindings-typescript/src/lib/reducers.ts @@ -1,4 +1,4 @@ -import type { Environment } from './environment'; +import type { EnvironmentFor } from './environment'; import type { DbView } from '../server/db_view'; import type { Random } from '../server/rng'; import type { ConnectionId } from './connection_id'; @@ -116,7 +116,7 @@ export type ReducerCtx = Readonly<{ timestamp: Timestamp; connectionId: ConnectionId | null; db: DbView; - env: Environment; + env: EnvironmentFor; senderAuth: AuthCtx; newUuidV4(): Uuid; newUuidV7(): Uuid; diff --git a/crates/bindings-typescript/src/lib/schema.ts b/crates/bindings-typescript/src/lib/schema.ts index eb3821f6606..67b263d8397 100644 --- a/crates/bindings-typescript/src/lib/schema.ts +++ b/crates/bindings-typescript/src/lib/schema.ts @@ -205,6 +205,7 @@ export class ModuleContext { entries: [], }, submodules: [], + environment: [], }; get moduleDef(): ModuleDef { @@ -275,6 +276,7 @@ export class ModuleContext { value: module.submodules, } ); + push({ tag: 'Environment', value: module.environment }); return { sections }; } diff --git a/crates/bindings-typescript/src/server/environment.ts b/crates/bindings-typescript/src/server/environment.ts index 349e8ffed8c..203c608e08c 100644 --- a/crates/bindings-typescript/src/server/environment.ts +++ b/crates/bindings-typescript/src/server/environment.ts @@ -1,6 +1,86 @@ import { env_get } from 'spacetime:sys@2.3'; -import type { Environment } from '../lib/environment'; +import type { Environment, EnvironmentSchema } from '../lib/environment'; +import type { + EnvironmentDeclaration, + EnvironmentConstraint, + AlgebraicType, +} from '../lib/autogen/types'; +import { OptionBuilder, StringBuilder } from '../lib/type_builders'; /** Values are not cached: transaction and procedure reads retain host semantics. */ -export const environment: Environment = Object.freeze({ get: env_get }); -export type { Environment } from '../lib/environment'; +export const environment: Environment = new Proxy( + Object.freeze( + Object.assign(Object.create(null), { get: (key: string) => env_get(key) }) + ), + { + get(target, key) { + if (key === 'get') return target.get; + if (typeof key !== 'string') return undefined; + // The host rejects undeclared keys and missing required values. Optional + // named access uses undefined; the generic ABI accessor retains null. + return env_get(key) ?? undefined; + }, + } +); +export type { + Environment, + EnvironmentFor, + EnvironmentSchema, +} from '../lib/environment'; + +/** Produce metadata only. No environment value is embedded in the artifact. */ +export function environmentDeclarations( + schema: EnvironmentSchema +): EnvironmentDeclaration[] { + const entries = Object.entries(schema); + if (entries.length > 256) + throw new TypeError('Too many environment declarations'); + const bytes = new TextEncoder(); + return entries.map(([name, definition]) => { + if ( + !/^[A-Za-z_][A-Za-z0-9_]*$/.test(name) || + bytes.encode(name).length > 256 + ) { + throw new TypeError('Invalid environment declaration name'); + } + const optional = definition instanceof OptionBuilder; + const inner = optional ? definition.value : definition; + let constraint: EnvironmentConstraint; + if (inner instanceof StringBuilder) { + constraint = { tag: 'AnyString' }; + } else { + const type: AlgebraicType = inner?.algebraicType; + if (type?.tag !== 'Sum' || !('variants' in inner)) { + throw new TypeError( + `Environment '${name}' must be a string or simple enum` + ); + } + const values = type.value.variants.map(variant => { + if ( + variant.algebraicType.tag !== 'Product' || + variant.algebraicType.value.elements.length !== 0 + ) { + throw new TypeError( + `Environment '${name}' cannot use an enum payload` + ); + } + if (typeof variant.name !== 'string') + throw new TypeError( + `Environment '${name}' enum cases must have names` + ); + if (bytes.encode(variant.name).length > 8192) + throw new TypeError(`Environment '${name}' literal is too long`); + return variant.name; + }); + if (values.length === 0 || new Set(values).size !== values.length) + throw new TypeError( + `Environment '${name}' needs a nonempty literal union` + ); + constraint = + values.length === 1 + ? { tag: 'Literal', value: values[0]! } + : { tag: 'OneOf', value: values }; + } + return { name, constraint, optional }; + }); +} diff --git a/crates/bindings-typescript/src/server/http_handlers.ts b/crates/bindings-typescript/src/server/http_handlers.ts index 0c4d9469eee..b8823577f4e 100644 --- a/crates/bindings-typescript/src/server/http_handlers.ts +++ b/crates/bindings-typescript/src/server/http_handlers.ts @@ -1,4 +1,4 @@ -import type { Environment } from '../lib/environment'; +import type { EnvironmentFor } from '../lib/environment'; import type { Identity } from '../lib/identity'; import type { HttpMethod, @@ -220,7 +220,7 @@ export type HandlerAliasViews = : {}; export interface HandlerContext { - readonly env: Environment; + readonly env: EnvironmentFor; readonly timestamp: Timestamp; readonly http: HttpClient; readonly identity: Identity; diff --git a/crates/bindings-typescript/src/server/procedures.ts b/crates/bindings-typescript/src/server/procedures.ts index f4d57416e69..f5b8e2aa1e2 100644 --- a/crates/bindings-typescript/src/server/procedures.ts +++ b/crates/bindings-typescript/src/server/procedures.ts @@ -1,4 +1,4 @@ -import { environment, type Environment } from './environment'; +import { environment, type EnvironmentFor } from './environment'; import { AlgebraicType, ProductType, @@ -109,7 +109,7 @@ export type ProcedureAliasViews = : {}; export interface ProcedureCtx { - readonly env: Environment; + readonly env: EnvironmentFor; readonly sender: Identity; readonly databaseIdentity: Identity; /** @deprecated Use `databaseIdentity` instead. */ @@ -228,7 +228,7 @@ const ProcedureCtxImpl = class ProcedureCtx #uuidCounter: { value: 0 } | undefined; #random: Random | undefined; #dbView: () => DbView; - readonly env = environment; + readonly env = environment as EnvironmentFor; #dispatches: SubmoduleDispatchInfo[]; #parentPrefix: string; #asViews: object | undefined; diff --git a/crates/bindings-typescript/src/server/runtime.ts b/crates/bindings-typescript/src/server/runtime.ts index 1fac18ac95b..70c6eb81401 100644 --- a/crates/bindings-typescript/src/server/runtime.ts +++ b/crates/bindings-typescript/src/server/runtime.ts @@ -1,4 +1,4 @@ -import { environment } from './environment'; +import { environment, type EnvironmentFor } from './environment'; import * as _syscalls2_0 from 'spacetime:sys@2.0'; import * as _syscalls2_1 from 'spacetime:sys@2.1'; @@ -247,7 +247,7 @@ export const ReducerCtxImpl = class ReducerCtx< timestamp: Timestamp; connectionId: ConnectionId | null; db: DbView; - readonly env = environment; + readonly env = environment as EnvironmentFor; as: AliasViews; constructor( @@ -777,7 +777,7 @@ const BINARY_READER = new BinaryReader(new Uint8Array()); class HandlerContextImpl implements HandlerContext { - readonly env = environment; + readonly env = environment as EnvironmentFor; #identity: Identity | undefined; #uuidCounter: { value: number } | undefined; #random: Random | undefined; diff --git a/crates/bindings-typescript/src/server/schema.ts b/crates/bindings-typescript/src/server/schema.ts index c399a66d31e..95688b7f3cd 100644 --- a/crates/bindings-typescript/src/server/schema.ts +++ b/crates/bindings-typescript/src/server/schema.ts @@ -1,3 +1,4 @@ +import { environmentDeclarations, type EnvironmentSchema } from './environment'; import { moduleHooks, type ModuleDefaultExport } from 'spacetime:sys@2.0'; import { CaseConversionPolicy, @@ -322,6 +323,9 @@ export class Schema implements ModuleDefaultExport { const rawDef = this.buildRawModuleDefV10(exports, { ignoreNonModuleExports: true, }); + if (this.#ctx.moduleDef.environment.length !== 0) { + throw new TypeError('Submodules cannot declare environment variables'); + } this.#ctx.resolveHttpRoutes(); return { rawDef, @@ -747,7 +751,11 @@ export type InferSchema> = /** * Module-level settings that can be passed to `schema()`. */ -export interface ModuleSettings { +export interface ModuleSettings< + E extends EnvironmentSchema = EnvironmentSchema, +> { + /** Declared strings installed only through publishing; omitted means empty. */ + env?: E; /** * The case conversion policy for this module. * Defaults to `SnakeCase` if not specified. @@ -825,11 +833,17 @@ function registerModuleExports( } } -export function schema>( +export function schema< + const H extends Record, + const E extends EnvironmentSchema = {}, +>( entries: H, - moduleSettings?: ModuleSettings -): Schema> { - const ctx = new SchemaInner>(ctx => { + moduleSettings?: ModuleSettings +): Schema & { env: E }> { + const ctx = new SchemaInner & { env: E }>(ctx => { + ctx.moduleDef.environment = environmentDeclarations( + moduleSettings?.env ?? {} + ); // Apply module settings. if (moduleSettings?.CASE_CONVERSION_POLICY != null) { ctx.setCaseConversionPolicy(moduleSettings.CASE_CONVERSION_POLICY); @@ -884,7 +898,10 @@ export function schema>( }); } } - return { tables: tableSchemas } as SchemaDefForEntries; + return { + tables: tableSchemas, + env: moduleSettings?.env ?? {}, + } as SchemaDefForEntries & { env: E }; }); return new Schema(ctx); diff --git a/crates/bindings-typescript/src/server/views.ts b/crates/bindings-typescript/src/server/views.ts index 528c8319063..f089de2f0c1 100644 --- a/crates/bindings-typescript/src/server/views.ts +++ b/crates/bindings-typescript/src/server/views.ts @@ -1,4 +1,4 @@ -import type { Environment } from '../lib/environment'; +import type { EnvironmentFor } from '../lib/environment'; import { AlgebraicType, ProductType, @@ -82,13 +82,13 @@ export function makeAnonViewExport< export type ViewCtx = Readonly<{ sender: Identity; db: ReadonlyDbView; - env: Environment; + env: EnvironmentFor; from: QueryBuilder; }>; export type AnonymousViewCtx = Readonly<{ db: ReadonlyDbView; - env: Environment; + env: EnvironmentFor; from: QueryBuilder; }>; diff --git a/crates/bindings-typescript/tests/__mocks__/spacetime-sys.ts b/crates/bindings-typescript/tests/__mocks__/spacetime-sys.ts index 47cbf9d7039..257321cfe8e 100644 --- a/crates/bindings-typescript/tests/__mocks__/spacetime-sys.ts +++ b/crates/bindings-typescript/tests/__mocks__/spacetime-sys.ts @@ -91,3 +91,7 @@ export const procedure_http_request = ( export const procedure_start_mut_tx = (): bigint => 0n; export const procedure_commit_mut_tx = (): void => {}; export const procedure_abort_mut_tx = (): void => {}; + +export const env_get = (_key: string): string | null => { + throw new Error('mock environment read is not configured'); +}; diff --git a/crates/bindings-typescript/tests/environment.test.ts b/crates/bindings-typescript/tests/environment.test.ts new file mode 100644 index 00000000000..b78713d49e9 --- /dev/null +++ b/crates/bindings-typescript/tests/environment.test.ts @@ -0,0 +1,149 @@ +import { describe, expect, expectTypeOf, it, vi } from 'vitest'; +import { schema } from '../src/server/schema'; +import { t } from '../src/lib/type_builders'; +import { + environment, + environmentDeclarations, +} from '../src/server/environment'; +import type { EnvironmentSchema } from '../src/lib/environment'; +import { env_get } from 'spacetime:sys@2.3'; + +vi.mock('spacetime:sys@2.3', async importOriginal => ({ + ...(await importOriginal()), + env_get: vi.fn(), +})); + +const declarations = { + FOOBAR: t.string(), + ENABLE_EMAIL: t.enum('EnableEmail', ['true', 'false']), + LOG_LEVEL: t.enum('LogLevel', ['debug', 'info', 'error']).optional(), + DEPLOYMENT_KIND: t.enum('DeploymentKind', ['production']), + get: t.string().optional(), +}; + +describe('declared database environment', () => { + it('emits canonical constraint metadata and an explicit empty section', () => { + expect(environmentDeclarations(declarations)).toEqual([ + { name: 'FOOBAR', constraint: { tag: 'AnyString' }, optional: false }, + { + name: 'ENABLE_EMAIL', + constraint: { tag: 'OneOf', value: ['true', 'false'] }, + optional: false, + }, + { + name: 'LOG_LEVEL', + constraint: { tag: 'OneOf', value: ['debug', 'info', 'error'] }, + optional: true, + }, + { + name: 'DEPLOYMENT_KIND', + constraint: { tag: 'Literal', value: 'production' }, + optional: false, + }, + { name: 'get', constraint: { tag: 'AnyString' }, optional: true }, + ]); + const defined = schema({}, { env: declarations }); + const section = defined + .buildRawModuleDefV10({}) + .sections.find(section => section.tag === 'Environment'); + expect(section).toEqual({ + tag: 'Environment', + value: environmentDeclarations(declarations), + }); + expect(schema({}).buildRawModuleDefV10({}).sections).toContainEqual({ + tag: 'Environment', + value: [], + }); + // Enum values outside env retain their existing tagged-sum interpretation. + expect(Reflect.get(declarations.ENABLE_EMAIL, 'true')).toEqual({ + tag: 'true', + }); + }); + + it('rejects unsupported constraints and submodule declarations before upload', () => { + const invalid = (value: unknown) => () => + environmentDeclarations(value as EnvironmentSchema); + expect(invalid({ BAD: t.u32() })).toThrow('string or simple enum'); + expect(invalid({ BAD: t.enum('Payload', { value: t.string() }) })).toThrow( + 'enum payload' + ); + expect(invalid({ BAD: t.enum('Empty', []) })).toThrow( + 'nonempty literal union' + ); + expect(invalid({ BAD: t.string().optional().optional() })).toThrow(); + expect(invalid({ 'bad-name': t.string() })).toThrow('name'); + expect(invalid({ BAD: t.enum('Long', ['x'.repeat(8193)]) })).toThrow( + 'too long' + ); + expect( + invalid( + Object.fromEntries( + Array.from({ length: 257 }, (_, i) => [`K${i}`, t.string()]) + ) + ) + ).toThrow('Too many'); + expect(() => + schema({ child: { default: schema({}, { env: declarations }) } }) + ).toThrow('Submodules'); + expect(() => + schema({ child: { default: schema({}, { env: {} }) } }) + ).not.toThrow(); + }); + + it('preserves generic get, optional absence, host errors and uncached named reads', () => { + const get = vi.mocked(env_get); + get.mockReset(); + get + .mockReturnValueOnce('first') + .mockReturnValueOnce('') + .mockReturnValueOnce(null) + .mockReturnValueOnce('declared get'); + const named = environment as typeof environment & { + readonly FOOBAR: string; + readonly LOG_LEVEL: string | undefined; + }; + expect(named.FOOBAR).toBe('first'); + expect(named.FOOBAR).toBe(''); + expect(named.LOG_LEVEL).toBeUndefined(); + expect(named.get('get')).toBe('declared get'); + get.mockReturnValueOnce(null); + expect(named.get('LOG_LEVEL')).toBeNull(); + get.mockImplementationOnce(() => { + throw new Error('undeclared host key'); + }); + expect(() => named.get('UNDECLARED')).toThrow('undeclared host key'); + expect(get.mock.calls.map(([name]) => name)).toEqual([ + 'FOOBAR', + 'FOOBAR', + 'LOG_LEVEL', + 'get', + 'LOG_LEVEL', + 'UNDECLARED', + ]); + }); +}); + +// These declarations are compiled by the focused typecheck. Callback bodies +// need not execute to assert their schema-specific context types. +const typed = schema({}, { env: declarations }); +typed.reducer(ctx => { + expectTypeOf(ctx.env.FOOBAR).toEqualTypeOf(); + expectTypeOf(ctx.env.ENABLE_EMAIL).toEqualTypeOf<'true' | 'false'>(); + expectTypeOf(ctx.env.LOG_LEVEL).toEqualTypeOf< + 'debug' | 'info' | 'error' | undefined + >(); + expectTypeOf(ctx.env.DEPLOYMENT_KIND).toEqualTypeOf<'production'>(); + expectTypeOf(ctx.env.get('FOOBAR')).toEqualTypeOf(); + expectTypeOf(ctx.env.get('get')).toEqualTypeOf(); + // @ts-expect-error Undeclared literal names have no checked accessor. + ctx.env.get('UNDECLARED'); + // @ts-expect-error No named access to undeclared keys. + void ctx.env.UNKNOWN; + // @ts-expect-error Environment access is read-only. + ctx.env.FOOBAR = 'changed'; +}); +function rejectPayloadType() { + // @ts-expect-error Payload enums cannot declare environment strings. + schema({}, { env: { BAD: t.enum('Payload', { value: t.string() }) } }); +} +void rejectPayloadType; diff --git a/crates/bindings-typescript/vitest.config.ts b/crates/bindings-typescript/vitest.config.ts index 7cfc858d47c..e5503037be6 100644 --- a/crates/bindings-typescript/vitest.config.ts +++ b/crates/bindings-typescript/vitest.config.ts @@ -14,6 +14,7 @@ export default defineConfig({ alias: [ { find: 'spacetime:sys@2.0', replacement: sysMock }, { find: 'spacetime:sys@2.1', replacement: sysMock }, + { find: 'spacetime:sys@2.3', replacement: sysMock }, ], }, test: { diff --git a/crates/bindings/src/lib.rs b/crates/bindings/src/lib.rs index d1548bca052..1589eb54ad5 100644 --- a/crates/bindings/src/lib.rs +++ b/crates/bindings/src/lib.rs @@ -919,6 +919,29 @@ pub use spacetimedb_bindings_macro::view; pub struct QueryBuilder {} pub use query_builder::{Query, RawQuery}; +/// Declare the complete publish-time environment schema and generate named accessors. +/// +/// Fields must be `String` or `Option`; `#[env(values("a", "b"))]` +/// constrains exact strings. Values are supplied on every publish, never in metadata. +/// The macro generates an `EnvAccess` extension trait for a struct named `Env`. +/// Import that trait when the declaration lives in a different Rust module. +/// The name `get` is reserved for generic checked access. +/// +/// ```no_run +/// #[spacetimedb::env] +/// pub struct Env { +/// pub API_KEY: String, +/// #[env(values("debug", "info"))] +/// pub LOG_LEVEL: Option, +/// } +/// fn read(ctx: &spacetimedb::ReducerContext) { +/// let _: String = ctx.env.API_KEY(); +/// let _: Option = ctx.env.LOG_LEVEL(); +/// } +/// ``` +#[doc(inline)] +pub use spacetimedb_bindings_macro::env; + /// Read-only access to this database's environment store. /// /// Reads use the current transaction. In a procedure outside a transaction, @@ -930,8 +953,12 @@ pub struct Environment { } impl Environment { - /// Return None for a missing key and Some("") for a present empty value. + /// Return `None` for an absent declared optional key and `Some("")` for a present empty value. /// Keys must be POSIX environment names of at most 256 bytes. + /// + /// # Panics + /// + /// Panics if the key is undeclared, invalid, or inaccessible in the current host call. pub fn get(&self, key: &str) -> Option { rt::env_get(key) } diff --git a/crates/bindings/src/rt.rs b/crates/bindings/src/rt.rs index b09c998f5c3..bc5b0d8fae1 100644 --- a/crates/bindings/src/rt.rs +++ b/crates/bindings/src/rt.rs @@ -917,6 +917,14 @@ pub fn register_case_conversion_policy(policy: CaseConversionPolicy) { }) } +/// Register declarative ENV metadata without reading any environment values. +#[doc(hidden)] +pub fn register_environment(declarations: fn() -> Vec) { + register_describer(move |module| { + module.inner.add_environment(declarations()); + }); +} + /// A builder for a module. #[derive(Default)] pub struct ModuleBuilder { @@ -983,6 +991,7 @@ extern "C" fn __describe_module__(description: BytesSink) { } // Serialize the module to bsatn. + module.inner.ensure_environment(); let module_def = module.inner.finish(); let module_def = RawModuleDef::V10(module_def); let bytes = bsatn::to_vec(&module_def).expect("unable to serialize typespace"); diff --git a/crates/bindings/tests/environment.rs b/crates/bindings/tests/environment.rs new file mode 100644 index 00000000000..5bffb7924ff --- /dev/null +++ b/crates/bindings/tests/environment.rs @@ -0,0 +1,4 @@ +#[test] +fn environment_declaration_accessors_compile_with_exact_types() { + trybuild::TestCases::new().pass("tests/pass/environment.rs"); +} diff --git a/crates/bindings/tests/pass/environment.rs b/crates/bindings/tests/pass/environment.rs new file mode 100644 index 00000000000..87a53d23aea --- /dev/null +++ b/crates/bindings/tests/pass/environment.rs @@ -0,0 +1,24 @@ +#![deny(warnings)] + +#[spacetimedb::env] +pub struct Env { + pub REQUIRED: String, + #[env(values("false", "true"))] + pub FLAG: String, + #[env(values(""))] + pub OPTIONAL: Option, + pub get: Option, + pub r#type: String, +} + +fn reads(env: spacetimedb::Environment) { + let _: String = env.REQUIRED(); + let _: String = env.FLAG(); + let _: Option = env.OPTIONAL(); + let _: Option = env.get("get"); + let _: String = env.r#type(); +} + +fn main() { + let _ = reads as fn(spacetimedb::Environment); +} diff --git a/crates/cli/src/lib.rs b/crates/cli/src/lib.rs index 9411e6f5e5e..96bb1034bfe 100644 --- a/crates/cli/src/lib.rs +++ b/crates/cli/src/lib.rs @@ -22,6 +22,7 @@ pub use tasks::build; pub fn get_subcommands() -> Vec { vec![ publish::cli(), + env::cli(), delete::cli(), logs::cli(), call::cli(), @@ -57,6 +58,7 @@ pub async fn exec_subcommand( "describe" => describe::exec(config, args).await, "dev" => dev::exec(config, args).await, "publish" => publish::exec(config, args).await, + "env" => env::exec(config, args).await, "delete" => delete::exec(config, args).await, "logs" => logs::exec(config, args).await, "sql" => sql::exec(config, args).await, diff --git a/crates/cli/src/spacetime_config.rs b/crates/cli/src/spacetime_config.rs index b3316f2c318..cce271510ee 100644 --- a/crates/cli/src/spacetime_config.rs +++ b/crates/cli/src/spacetime_config.rs @@ -1,3 +1,5 @@ +mod environment; + use anyhow::Context; use clap::{ArgMatches, Command}; use path_clean::PathClean; @@ -183,6 +185,14 @@ impl SpacetimeConfig { let mut fields = self.additional_fields.clone(); if let Some(parent) = parent_fields { for (key, value) in parent { + if key == "env" + && let Some(child) = fields.get_mut(key) + { + let mut combined = value.clone(); + environment::overlay(&mut combined, child); + *child = combined; + continue; + } if fields.contains_key(key) { continue; } @@ -310,6 +320,9 @@ impl CommandSchemaBuilder { // Check that all the defined keys exist in clap for key in &self.keys { + if key.config_only { + continue; + } if !clap_arg_names.contains(key.clap_arg_name()) { return Err(CommandConfigError::InvalidClapReference { config_name: key.config_name().to_string(), @@ -343,6 +356,9 @@ impl CommandSchemaBuilder { let mut config_to_alias_map = HashMap::new(); for key in &self.keys { + if key.config_only { + continue; + } let config_name = key.config_name().to_string(); let clap_name = key.clap_arg_name().to_string(); @@ -398,6 +414,13 @@ impl CommandSchema { matches: &ArgMatches, config_name: &str, ) -> Result, CommandConfigError> { + if self + .keys + .iter() + .any(|key| key.config_name() == config_name && key.config_only) + { + return Ok(None); + } // Check clap with mapped name (if from_clap was used, use that name, otherwise use config name) let clap_name = self .config_to_clap @@ -428,6 +451,13 @@ impl CommandSchema { /// Check if a value was provided via CLI (not from config). /// Only returns true if the user explicitly provided the value, not if it came from a default. pub fn is_from_cli(&self, matches: &ArgMatches, config_name: &str) -> bool { + if self + .keys + .iter() + .any(|key| key.config_name() == config_name && key.config_only) + { + return false; + } // Check clap with mapped name let clap_name = self .config_to_clap @@ -567,6 +597,8 @@ impl CommandSchema { pub struct Key { /// The key name in the config file (e.g., "module-path") config_name: String, + /// This field has no value-bearing CLI argument (even if a same-named selector exists). + config_only: bool, /// The corresponding clap argument name (e.g., "project-path"), if different clap_name: Option, /// Alias for a clap argument, useful for example if we have to deprecate a clap @@ -585,6 +617,7 @@ impl Key { pub fn new(name: impl Into) -> Self { Self { config_name: name.into(), + config_only: false, clap_name: None, clap_alias: None, module_specific: false, @@ -593,6 +626,12 @@ impl Key { } } + /// Read this key exclusively from project configuration. + pub fn config_only(mut self) -> Self { + self.config_only = true; + self + } + /// Map this config key to a different clap argument name. When fetching values /// the key that is defined should be used. /// Example: Key::new("module-path").from_clap("project-path") @@ -813,8 +852,10 @@ impl SpacetimeConfig { let content = std::fs::read_to_string(path).with_context(|| format!("Failed to read config file: {}", path.display()))?; - let config: Self = json5::from_str(&content) - .map_err(|e| anyhow::anyhow!("Failed to parse config file {}: {}", path.display(), e))?; + let value = + environment::parse(&content).with_context(|| format!("Failed to parse config file {}", path.display()))?; + let config: Self = environment::decode_config(value) + .map_err(|_| anyhow::anyhow!("Invalid configuration structure in {}", path.display()))?; Ok(config) } @@ -911,8 +952,8 @@ fn load_json_value(path: &Path) -> anyhow::Result> { // comments and formatting since json5 crate doesn't support serialization. remove_source_config_from_text(path, &content); - let value: serde_json::Value = json5::from_str(&content) - .map_err(|e| anyhow::anyhow!("Failed to parse config file {}: {}", path.display(), e))?; + let value = + environment::parse(&content).with_context(|| format!("Failed to parse config file {}", path.display()))?; Ok(Some(value)) } @@ -1000,6 +1041,12 @@ fn overlay_json(base: &mut serde_json::Value, mut overlay: serde_json::Value, so base_obj.insert(key.clone(), other); } } + } else if key == "env" { + if let Some(base_env) = base_obj.get_mut(key) { + environment::overlay(base_env, value); + } else { + base_obj.insert(key.clone(), value_owned); + } } else { base_obj.insert(key.clone(), value_owned); } @@ -1068,7 +1115,7 @@ pub fn find_and_load_with_env_from(env: Option<&str>, start_dir: PathBuf) -> any } } - let config: SpacetimeConfig = serde_json::from_value(merged).context("Failed to deserialize merged config")?; + let config = environment::decode_config(merged)?; Ok(Some(LoadedConfig { config, diff --git a/crates/cli/src/spacetime_config/environment.rs b/crates/cli/src/spacetime_config/environment.rs new file mode 100644 index 00000000000..ecaa3218684 --- /dev/null +++ b/crates/cli/src/spacetime_config/environment.rs @@ -0,0 +1,286 @@ +//! Preserve JSON numeric values before the JSON5 deserializer can round them. +//! This also permits existing comments, unquoted names and trailing commas. +use serde_json::Value; + +pub(super) fn parse(content: &str) -> anyhow::Result { + let bytes = content.as_bytes(); + let mut numbers = Vec::new(); + let mut replacements = Vec::new(); + let mut strings: Vec = Vec::new(); + let mut i = 0; + while i < bytes.len() { + let start = i; + match bytes[i] { + b'\'' | b'"' => { + let quote = bytes[i]; + i += 1; + while i < bytes.len() { + if bytes[i] == b'\\' { + i = (i + 2).min(bytes.len()); + } else if bytes[i] == quote { + i += 1; + break; + } else { + i += 1; + } + } + } + b'/' if bytes.get(i + 1) == Some(&b'/') => { + i += 2; + while i < bytes.len() { + let ch = content[i..].chars().next().unwrap(); + if matches!(ch, '\n' | '\r' | '\u{2028}' | '\u{2029}') { + break; + } + i += ch.len_utf8(); + } + } + b'/' if bytes.get(i + 1) == Some(&b'*') => { + i += 2; + while i + 1 < bytes.len() && &bytes[i..i + 2] != b"*/" { + i += 1; + } + i = (i + 2).min(bytes.len()); + } + b'{' | b'}' | b'[' | b']' | b':' | b',' => i += 1, + _ if is_space(content[i..].chars().next().unwrap()) => i += content[i..].chars().next().unwrap().len_utf8(), + _ => { + while i < bytes.len() + && !is_space(content[i..].chars().next().unwrap()) + && !matches!(bytes[i], b'{' | b'}' | b'[' | b']' | b':' | b',' | b'/' | b'\'' | b'"') + { + i += content[i..].chars().next().unwrap().len_utf8(); + } + if i == start { + i += content[i..].chars().next().unwrap().len_utf8(); + } + let token = &content[start..i]; + if (token.starts_with(|c: char| c.is_ascii_digit() || matches!(c, '-' | '+' | '.')) + || matches!(token, "Infinity" | "NaN")) + && !content[i..].trim_start_matches(is_space).starts_with(':') + { + replacements.push((start, i)); + numbers.push(token); + continue; + } + } + } + if matches!(bytes[start], b'\'' | b'"') { + strings.push(json5::from_str(&content[start..i]).map_err(|_| anyhow::anyhow!("Invalid JSON5 string"))?); + } + } + // Check decoded strings as well: Unicode escapes must not manufacture an + // internal marker and cause a string to be interpreted as a number. + let mut prefix = "__spacetime_numeric_".to_owned(); + while content.contains(&prefix) || strings.iter().any(|s| s.contains(&prefix)) { + prefix.push('_'); + } + let mut text = String::with_capacity(content.len()); + let mut previous = 0; + for (index, (start, end)) in replacements.into_iter().enumerate() { + text.push_str(&content[previous..start]); + text.push_str(&format!("\"{prefix}{index}\"")); + previous = end; + } + text.push_str(&content[previous..]); + // Parser diagnostics may quote the source line, which can contain secrets. + let mut value: Value = json5::from_str(&text).map_err(|_| anyhow::anyhow!("Invalid JSON5 configuration"))?; + restore(&mut value, &prefix, &numbers, None)?; + Ok(value) +} + +/// Deserialize through JSON text so Serde's flattened-field buffer does not +/// receive visit_u128 from Value's deserializer. That buffer cannot represent +/// u128, while the arbitrary-precision JSON parser preserves its decimal token. +/// Diagnostics deliberately discard the original error, which may quote values. +pub(super) fn decode_config(value: Value) -> anyhow::Result { + let encoded = serde_json::to_vec(&value).map_err(|_| anyhow::anyhow!("Invalid configuration structure"))?; + serde_json::from_slice(&encoded).map_err(|_| anyhow::anyhow!("Invalid configuration structure")) +} + +fn is_space(ch: char) -> bool { + ch.is_whitespace() || ch == '\u{feff}' +} + +fn restore(value: &mut Value, prefix: &str, numbers: &[&str], env_key: Option<&str>) -> anyhow::Result<()> { + match value { + Value::String(s) => { + if let Some(index) = s.strip_prefix(prefix).and_then(|s| s.parse::().ok()) { + let token = numbers[index]; + *value = if let Some(env_key) = env_key { + Value::Number(token.parse().map_err(|_| { + anyhow::anyhow!( + "Environment key {:?}: numeric config input must use JSON number syntax", + env_key + ) + })?) + } else { + // Preserve existing JSON5 conveniences outside env. JSON numbers retain + // arbitrary precision throughout layering and config serialization. + token + .parse() + .map(Value::Number) + .or_else(|_| json5::from_str(token)) + .map_err(|_| anyhow::anyhow!("Invalid numeric configuration input"))? + }; + } + } + Value::Object(object) => { + for (key, value) in object { + if key == "env" && env_key.is_none() { + if let Value::Object(env) = value { + for (name, value) in env { + restore(value, prefix, numbers, Some(name))?; + } + } else { + restore(value, prefix, numbers, Some("env"))?; + } + } else { + restore(value, prefix, numbers, env_key)?; + } + } + } + Value::Array(values) => { + for value in values { + restore(value, prefix, numbers, env_key)?; + } + } + _ => {} + } + Ok(()) +} + +/// Merge only object-valued env maps. Invalid higher precedence input remains +/// invalid instead of silently falling back to the lower layer. +pub(super) fn overlay(base: &mut Value, higher: &Value) { + if let (Some(base), Some(higher)) = (base.as_object_mut(), higher.as_object()) { + base.extend(higher.clone()); + } else { + *base = higher.clone(); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::spacetime_config::{find_and_load_with_env_from, SpacetimeConfig}; + #[test] + fn numeric_input_is_lossless_with_json5_comments_and_strings() { + let value = parse( + r#"{ // comment 111 + env: { HUGE: 9007199254740993123456789, DECIMAL: 0.1234567890123456789012345, + EXP: 1.000000000000000000001e+300, STRING: '12 // 99', BOOL: false, }, + 'server': 'http://127.0.0.1:9', /* 222 */ 'num-replicas': 3, + }"#, + ) + .unwrap(); + assert_eq!(value["env"]["HUGE"].to_string(), "9007199254740993123456789"); + assert_eq!(value["env"]["DECIMAL"].to_string(), "0.1234567890123456789012345"); + assert_eq!(value["env"]["EXP"].to_string(), "1.000000000000000000001e+300"); + assert_eq!(value["env"]["STRING"], "12 // 99"); + assert_eq!(value["num-replicas"], 3); + let unicode_lines = parse("{ // comment\u{2028} env: { NUMBER: 9007199254740993123456789\u{feff}} }").unwrap(); + assert_eq!(unicode_lines["env"]["NUMBER"].to_string(), "9007199254740993123456789"); + } + #[test] + fn numeric_extensions_and_parse_errors_do_not_quote_secret_input() { + for input in ["NaN", "Infinity", "-Infinity", "0xFF", "+2", ".5"] { + assert!(parse(&format!("{{env: {{KEY: {input}}}}}")).is_err()); + } + let error = parse("{env: {KEY: 'generated-secret-sentinel', broken }").unwrap_err(); + assert!(!format!("{error:#}").contains("generated-secret-sentinel")); + let value = parse("{env: {A: '__spacetime_numeric_0', B: 2}}").unwrap(); + assert_eq!(value["env"]["A"], "__spacetime_numeric_0"); + assert_eq!(value["env"]["B"], 2); + let escaped = parse(r#"{env: {A: '\u005f_spacetime_numeric_999', B: 2}}"#).unwrap(); + assert_eq!(escaped["env"]["A"], "__spacetime_numeric_999"); + } + #[test] + fn four_layers_and_parent_child_merge_individual_keys() { + let dir = tempfile::tempdir().unwrap(); + for (file, json) in [ + ( + "spacetime.json", + r#"{database:'parent', env:{A:'base',B:1},children:[{database:'child',env:{B:2,C:'child'}}]}"#, + ), + ( + "spacetime.local.json", + r#"{env:{A:'local'},children:[{env:{D:'local-child'}}]}"#, + ), + ( + "spacetime.prod.json", + r#"{env:{A:'prod',E:true},children:[{env:{B:3}}]}"#, + ), + ( + "spacetime.prod.local.json", + r#"{env:{A:'prod-local'},children:[{env:{}}]}"#, + ), + ] { + std::fs::write(dir.path().join(file), json).unwrap(); + } + let config = find_and_load_with_env_from(Some("prod"), dir.path().to_owned()) + .unwrap() + .unwrap(); + let targets = config.config.collect_all_targets_with_inheritance(); + assert_eq!( + targets[0].fields["env"], + serde_json::json!({"A":"prod-local","B":1,"E":true}) + ); + assert_eq!( + targets[1].fields["env"], + serde_json::json!({"A":"prod-local","B":3,"C":"child","D":"local-child","E":true}) + ); + // Direct loads use the same lossless parser. + let base = SpacetimeConfig::load(&dir.path().join("spacetime.json")).unwrap(); + assert_eq!(base.additional_fields["env"]["B"], 1); + } + #[test] + fn loaded_flattened_configuration_preserves_full_precision_and_redacts_structure_errors() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("spacetime.json"); + for number in [ + "9007199254740993123456789", + "0.1234567890123456789012345", + "1.000000000000000000001e+300", + ] { + std::fs::write( + &path, + format!(r#"{{"database":"owned","env":{{"NUMBER":{number}}},"children":[{{"database":"child"}}]}}"#), + ) + .unwrap(); + let direct = SpacetimeConfig::load(&path).unwrap(); + let layered = find_and_load_with_env_from(None, dir.path().to_owned()) + .unwrap() + .unwrap() + .config; + for config in [direct, layered] { + for target in config.collect_all_targets_with_inheritance() { + assert_eq!(target.fields["env"]["NUMBER"].to_string(), number); + } + } + } + std::fs::write( + &path, + r#"{"children":"private-structure-sentinel","env":{"NUMBER":9007199254740993123456789}}"#, + ) + .unwrap(); + for error in [ + SpacetimeConfig::load(&path).unwrap_err(), + find_and_load_with_env_from(None, dir.path().to_owned()).err().unwrap(), + ] { + let error = format!("{error:#}"); + assert!(!error.contains("private-structure-sentinel")); + assert!(!error.contains("9007199254740993123456789")); + } + } + + #[test] + fn invalid_higher_layer_is_not_an_empty_map_or_fallback() { + let mut base = serde_json::json!({"A":"base"}); + overlay(&mut base, &Value::Null); + assert!(base.is_null()); + let config: SpacetimeConfig = + serde_json::from_value(serde_json::json!({"env":{"A":1},"children":[{"env":null}]})).unwrap(); + assert!(config.collect_all_targets_with_inheritance()[1].fields["env"].is_null()); + } +} diff --git a/crates/cli/src/subcommands/env.rs b/crates/cli/src/subcommands/env.rs new file mode 100644 index 00000000000..58876664f48 --- /dev/null +++ b/crates/cli/src/subcommands/env.rs @@ -0,0 +1,284 @@ +//! Read-only conveniences over the private st_env SQL table. +use anyhow::{ensure, Context}; +use clap::{Arg, ArgAction, ArgMatches, Command}; +use spacetimedb_lib::environment::{validate_key, MAX_ENV_KEY_BYTES, MAX_ENV_VALUE_BYTES, MAX_ENV_VARS}; + +use super::{ + db_arg_resolution::{load_config_db_targets, resolve_database_arg}, + sql, +}; +use crate::{api::ClientApi, common_args, Config}; + +pub fn cli() -> Command { + let target = |command: Command| { + command + .arg( + Arg::new("database") + .index(1) + .required(true) + .help("The database name, identity, or configured target"), + ) + .arg(common_args::server()) + .arg(common_args::anonymous()) + .arg(common_args::yes()) + .arg(common_args::confirmed()) + .arg( + Arg::new("no_config") + .long("no-config") + .action(ArgAction::SetTrue) + .help("Ignore project configuration when resolving the database target"), + ) + }; + Command::new("env") + .about("Inspect published database environment variables") + .subcommand_required(true) + .subcommand(target( + Command::new("get").about("Read one published environment value").arg( + Arg::new("key") + .index(2) + .required(true) + .help("The declared environment key to read"), + ), + )) + .subcommand(target( + Command::new("list").about("List published environment keys (never values)"), + )) +} + +#[derive(Clone)] +enum Query { + List, + Get(String), +} +impl Query { + fn sql(&self) -> anyhow::Result { + match self { + Self::List => Ok("SELECT key FROM st_env".into()), + Self::Get(key) => { + // POSIX names cannot contain quotes or SQL syntax. + validate_key(key).map_err(|_| anyhow::anyhow!("Invalid environment key name"))?; + Ok(format!("SELECT value FROM st_env WHERE key = '{key}'")) + } + } + } +} + +pub async fn exec(config: Config, args: &ArgMatches) -> anyhow::Result<()> { + let (command, args) = args.subcommand().context("Expected env get or list")?; + let query = match command { + "list" => Query::List, + "get" => Query::Get( + args.get_one::("key") + .context("Expected environment key")? + .clone(), + ), + _ => anyhow::bail!("Environment values can only be changed by publishing"), + }; + query.sql()?; + let targets = load_config_db_targets(args.get_flag("no_config"))?; + let database = resolve_database_arg( + args.get_one::("database").map(String::as_str), + targets.as_deref(), + "spacetime env get/list ", + )?; + let con = sql::parse_req(config, args, &database.database, database.server.as_deref()).await?; + let mut request = ClientApi::new(con).sql(); + if let Some(confirmed) = args.get_one::("confirmed") { + request = request.query(&[("confirmed", confirmed)]); + } + print!("{}", fetch(request, query).await?); + Ok(()) +} + +async fn fetch(request: reqwest::RequestBuilder, query: Query) -> anyhow::Result { + use futures::StreamExt; + let response = request + .timeout(std::time::Duration::from_secs(30)) + .body(query.sql()?) + .send() + .await?; + ensure!( + response.status().is_success(), + "Environment read failed with HTTP {}", + response.status() + ); + let mut body = Vec::new(); + let limit = MAX_ENV_VARS * (MAX_ENV_KEY_BYTES + MAX_ENV_VALUE_BYTES) * 6 + 64 * 1024; + let mut stream = response.bytes_stream(); + while let Some(chunk) = stream.next().await { + let chunk = chunk?; + ensure!( + body.len().saturating_add(chunk.len()) <= limit, + "Environment read response exceeds limit" + ); + body.extend_from_slice(&chunk); + } + render(&body, &query) +} + +fn render(body: &[u8], query: &Query) -> anyhow::Result { + // Only project the requested single string column; do not dump an error or + // unexpected response which could contain unrequested secret values. + let results: Vec>> = + serde_json::from_slice(body).map_err(|_| anyhow::anyhow!("Invalid environment read response"))?; + ensure!(results.len() == 1, "Invalid environment read result count"); + let result = &results[0]; + let expected = match query { + Query::List => "key", + Query::Get(_) => "value", + }; + ensure!( + result.schema.elements.len() == 1 + && result.schema.elements[0].name.as_deref() == Some(expected) + && result.schema.elements[0].algebraic_type == spacetimedb_lib::AlgebraicType::String, + "Invalid environment read projection" + ); + let mut values = Vec::new(); + for row in &result.rows { + ensure!(row.len() == 1, "Invalid environment read row"); + if matches!(query, Query::List) { + validate_key(&row[0]).context("Invalid environment key in response")?; + } + ensure!( + row[0].len() <= MAX_ENV_VALUE_BYTES, + "Environment read value exceeds limit" + ); + values.push(row[0].as_str()); + } + match query { + Query::List => { + ensure!(values.len() <= MAX_ENV_VARS, "Environment key count exceeds limit"); + values.sort_unstable(); + } + Query::Get(_) => ensure!(values.len() == 1, "Environment key is absent"), + } + Ok(values.into_iter().map(|v| format!("{v}\n")).collect()) +} + +#[cfg(test)] +mod tests { + use super::*; + fn body(column: &'static str, rows: Vec>) -> Vec { + serde_json::to_vec(&[spacetimedb_client_api_messages::http::SqlStmtResult { + schema: spacetimedb_lib::sats::ProductType::from([(column, spacetimedb_lib::AlgebraicType::String)]), + rows, + total_duration_micros: 0, + stats: Default::default(), + }]) + .unwrap() + } + #[test] + fn read_only_commands_and_safe_queries() { + for command in ["set", "del", "delete", "update"] { + assert!(cli().try_get_matches_from(["env", command, "db", "KEY"]).is_err()); + } + let matches = cli() + .try_get_matches_from([ + "env", + "get", + "db", + "KEY", + "--server", + "http://127.0.0.1:9", + "--no-config", + ]) + .unwrap(); + let get = matches.subcommand_matches("get").unwrap(); + assert_eq!(get.get_one::("database").unwrap(), "db"); + assert_eq!(get.get_one::("key").unwrap(), "KEY"); + assert_eq!(Query::List.sql().unwrap(), "SELECT key FROM st_env"); + assert_eq!( + Query::Get("KEY".into()).sql().unwrap(), + "SELECT value FROM st_env WHERE key = 'KEY'" + ); + assert!(Query::Get("x';DELETE FROM st_env;--".into()).sql().is_err()); + } + #[test] + fn list_projects_keys_and_rejects_unexpected_secret_columns() { + assert_eq!( + render(&body("key", vec![vec!["Z"], vec!["A"]]), &Query::List).unwrap(), + "A\nZ\n" + ); + let err = render(&body("value", vec![vec!["generated-secret-sentinel"]]), &Query::List).unwrap_err(); + assert!(!format!("{err:#}").contains("generated-secret-sentinel")); + assert_eq!( + render(&body("value", vec![vec![""]]), &Query::Get("A".into())).unwrap(), + "\n" + ); + assert!(render(&body("value", vec![]), &Query::Get("A".into())).is_err()); + } + #[tokio::test] + async fn actual_loopback_queries_and_error_redaction() { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + for (query, status, response, expected) in [ + (Query::List, "200 OK", body("key", vec![vec!["KEY"]]), Some("KEY\n")), + ( + Query::Get("KEY".into()), + "200 OK", + body("value", vec![vec!["generated-read-sentinel"]]), + Some("generated-read-sentinel\n"), + ), + ( + Query::Get("KEY".into()), + "403 Forbidden", + b"generated-error-secret".to_vec(), + None, + ), + ] { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let expected_sql = query.sql().unwrap(); + let server = tokio::spawn(async move { + let (mut stream, _) = listener.accept().await.unwrap(); + let mut input = Vec::new(); + let (head, length) = loop { + let mut chunk = [0; 1024]; + let n = stream.read(&mut chunk).await.unwrap(); + assert!(n > 0); + input.extend_from_slice(&chunk[..n]); + assert!(input.len() <= 16384); + if let Some(end) = input.windows(4).position(|w| w == b"\r\n\r\n") { + let headers = std::str::from_utf8(&input[..end]).unwrap(); + assert!(headers.starts_with("POST /v1/database/owned/sql HTTP/1.1\r\n")); + let length: usize = headers + .lines() + .find_map(|line| { + line.to_ascii_lowercase() + .strip_prefix("content-length: ") + .map(str::to_owned) + }) + .unwrap() + .parse() + .unwrap(); + break (end + 4, length); + } + }; + while input.len() < head + length { + let mut chunk = [0; 1024]; + let n = stream.read(&mut chunk).await.unwrap(); + assert!(n > 0); + input.extend_from_slice(&chunk[..n]); + } + assert_eq!(&input[head..head + length], expected_sql.as_bytes()); + let headers = format!("HTTP/1.1 {status}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", response.len()); + stream.write_all(headers.as_bytes()).await.unwrap(); + stream.write_all(&response).await.unwrap(); + stream.shutdown().await.unwrap(); + }); + let client = reqwest::Client::builder() + .no_proxy() + .redirect(reqwest::redirect::Policy::none()) + .build() + .unwrap(); + let result = fetch(client.post(format!("http://{address}/v1/database/owned/sql")), query).await; + match expected { + Some(expected) => assert_eq!(result.unwrap(), expected), + None => assert!(!format!("{:#}", result.unwrap_err()).contains("generated-error-secret")), + } + tokio::time::timeout(std::time::Duration::from_secs(5), server) + .await + .unwrap() + .unwrap(); + } + } +} diff --git a/crates/cli/src/subcommands/mod.rs b/crates/cli/src/subcommands/mod.rs index af0d2e364c3..9c659a5880f 100644 --- a/crates/cli/src/subcommands/mod.rs +++ b/crates/cli/src/subcommands/mod.rs @@ -5,6 +5,7 @@ pub mod delete; pub mod describe; pub mod dev; pub mod dns; +pub mod env; pub mod generate; pub mod init; pub mod list; diff --git a/crates/cli/src/subcommands/publish.rs b/crates/cli/src/subcommands/publish.rs index 745664880f0..b734b37cba8 100644 --- a/crates/cli/src/subcommands/publish.rs +++ b/crates/cli/src/subcommands/publish.rs @@ -1,3 +1,5 @@ +mod environment; + use anyhow::{ensure, Context}; use clap::Arg; use clap::ArgAction::{self, Set, SetTrue}; @@ -6,8 +8,8 @@ use reqwest::{StatusCode, Url}; use spacetimedb_client_api_messages::name::{is_identity, parse_database_name, PublishResult}; use spacetimedb_client_api_messages::name::{DatabaseNameError, PrePublishResult, PrettyPrintStyle, PublishOp}; use std::collections::HashMap; +use std::env; use std::path::PathBuf; -use std::{env, fs}; use crate::common_args::parse_optional_dotnet_version; use crate::common_args::ClearMode; @@ -87,6 +89,7 @@ pub fn build_publish_schema(command: &clap::Command) -> Result( ) .await? }; - let program_bytes = fs::read(path_to_program)?; + let program_bytes = environment::read_program(&path_to_program)?; + let module_schema = environment::inspect(&program_bytes, host_type).await?; + let environment = environment::resolve( + module_schema.environment(), + command_config.get_config_value("env"), + |key| std::env::var_os(key), + )?; + print!("{}", environment.display()); let server_address = { let url = Url::parse(&database_host)?; @@ -583,7 +593,10 @@ async fn execute_publish_configs<'a>( database_host ); - let client = reqwest::Client::new(); + // The body contains secrets. Never replay it to a redirect destination. + let client = reqwest::Client::builder() + .redirect(reqwest::redirect::Policy::none()) + .build()?; // If a name was given, ensure to percent-encode it. // We also use PUT with a name or identity, and POST otherwise. let mut builder = if let Some(name_or_identity) = name_or_identity { @@ -634,8 +647,24 @@ async fn execute_publish_configs<'a>( // Set the host type. builder = builder.query(&[("host_type", host_type)]); - let res = builder.body(program_bytes).send().await?; - let response: PublishResult = res.json_or_error().await?; + let payload = spacetimedb_client_api_messages::publish::PublishRequest { + module: program_bytes, + environment: environment.values, + } + .encode()?; + let res = builder + .header( + reqwest::header::CONTENT_TYPE, + spacetimedb_client_api_messages::publish::CONTENT_TYPE, + ) + .body(payload) + .send() + .await?; + anyhow::ensure!(res.status().is_success(), "Publish failed with HTTP {}", res.status()); + let response: PublishResult = res + .json() + .await + .map_err(|_| anyhow::anyhow!("Invalid publish response"))?; match response { PublishResult::Success { domain, diff --git a/crates/cli/src/subcommands/publish/environment.rs b/crates/cli/src/subcommands/publish/environment.rs new file mode 100644 index 00000000000..17fc67efb62 --- /dev/null +++ b/crates/cli/src/subcommands/publish/environment.rs @@ -0,0 +1,180 @@ +//! Resolve a complete, declared environment without consulting stored values. +use std::collections::BTreeMap; +use std::ffi::OsString; +use std::path::PathBuf; +use std::process::Stdio; +use std::time::Duration; + +use anyhow::{ensure, Context}; +use serde_json::Value; +use spacetimedb_lib::environment::EnvironmentSchema; +use spacetimedb_lib::{sats::serde::SerdeWrapper, RawModuleDef}; +use spacetimedb_schema::def::ModuleDef; +use tokio::io::AsyncReadExt; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum Source { + Config, + Shell, +} +impl std::fmt::Display for Source { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(match self { + Self::Config => "config", + Self::Shell => "shell", + }) + } +} + +// Deliberately no Debug: values are credentials, not diagnostics. +pub(super) struct Resolved { + pub values: BTreeMap, + pub sources: BTreeMap, +} +impl Resolved { + pub fn display(&self) -> String { + use std::fmt::Write; + let mut output = String::new(); + for (name, source) in &self.sources { + let _ = writeln!(output, "Environment {name} ({source})"); + } + output + } +} + +pub(super) fn resolve( + schema: &EnvironmentSchema, + config: Option<&Value>, + mut shell: impl FnMut(&str) -> Option, +) -> anyhow::Result { + let mut resolved = Resolved { + values: BTreeMap::new(), + sources: BTreeMap::new(), + }; + if let Some(config) = config { + let config = config.as_object().context("Environment config must be an object")?; + for (name, value) in config { + ensure!( + schema.get(name).is_some(), + "Environment key {name:?}: key is not declared" + ); + let value = match value { + Value::String(value) => value.clone(), + Value::Bool(value) => value.to_string(), + Value::Number(value) => value.to_string(), + _ => anyhow::bail!("Environment key {name:?}: config input must be a string, boolean or JSON number"), + }; + resolved.values.insert(name.clone(), value); + resolved.sources.insert(name.clone(), Source::Config); + } + } + // Lookup only the new artifact's declared names, never enumerate ambient values. + for declaration in schema.declarations() { + if let Some(value) = shell(&declaration.name) { + let value = value + .into_string() + .map_err(|_| anyhow::anyhow!("Environment key {:?}: shell input must be UTF-8", declaration.name))?; + resolved.values.insert(declaration.name.clone(), value); + resolved.sources.insert(declaration.name.clone(), Source::Shell); + } + } + schema.validate_values(&resolved.values)?; + Ok(resolved) +} + +pub(super) fn read_program(path: &std::path::Path) -> anyhow::Result> { + use std::io::Read; + let mut bytes = Vec::new(); + std::fs::File::open(path)? + .take(spacetimedb_client_api_messages::publish::MAX_MODULE_BYTES as u64 + 1) + .read_to_end(&mut bytes)?; + ensure!( + bytes.len() <= spacetimedb_client_api_messages::publish::MAX_MODULE_BYTES, + "Module exceeds publish size limit" + ); + Ok(bytes) +} + +const MAX_SCHEMA_BYTES: u64 = 16 * 1024 * 1024; +const INSPECT_TIMEOUT: Duration = Duration::from_secs(60); + +/// Inspect exactly the artifact bytes that will be uploaded. The private copy +/// prevents path replacement between inspection and upload, including --bin-path. +/// This invokes only local extraction, never a server or a saved CLI context. +pub(super) async fn inspect(program: &[u8], host_type: &str) -> anyhow::Result { + let extractor = std::env::var_os("SPACETIMEDB_SCHEMA_EXTRACTOR") + .map(PathBuf::from) + .map(Ok) + .unwrap_or_else(|| crate::util::resolve_sibling_binary("spacetimedb-standalone"))?; + inspect_with(extractor, program.to_vec(), host_type.to_owned(), INSPECT_TIMEOUT).await +} + +async fn inspect_with( + extractor: PathBuf, + program: Vec, + host_type: String, + deadline: Duration, +) -> anyhow::Result { + let (mut send, mut recv) = tokio::sync::oneshot::channel(); + // This owner retains the child and private file until actual reaping, even + // when its caller drops while extraction or stdout reading is in progress. + tokio::spawn(async move { + let result = async { + let dir = tempfile::tempdir().context("Cannot create private module inspection directory")?; + let module = dir.path().join("module"); + tokio::fs::write(&module, program) + .await + .context("Cannot prepare module inspection input")?; + let mut child = tokio::process::Command::new(extractor) + .arg("extract-schema") + .arg(&module) + .arg("--host-type") + .arg(host_type.to_ascii_lowercase()) + .env_clear() + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::null()) + .kill_on_drop(true) + .spawn() + .context("Cannot start local module schema inspection")?; + let mut output = Vec::new(); + let mut stdout = child + .stdout + .take() + .context("Module inspection stdout unavailable")? + .take(MAX_SCHEMA_BYTES + 1); + let result = tokio::select! { + biased; + _ = send.closed() => Err(anyhow::anyhow!("Module inspection cancelled")), + result = tokio::time::timeout(deadline, async { + stdout.read_to_end(&mut output).await.context("Cannot read local module schema")?; + ensure!(output.len() as u64 <= MAX_SCHEMA_BYTES, "Local module schema exceeds output limit"); + let status = child.wait().await.context("Cannot reap local module inspector")?; + ensure!(status.success(), "Local module schema inspection failed"); + Ok(()) + }) => result.unwrap_or_else(|_| Err(anyhow::anyhow!("Local module schema inspection timed out"))), + }; + if result.is_err() { + // Queue termination, then retain ownership through positive reaping. + let _ = child.start_kill(); + child + .wait() + .await + .context("Cannot reap failed local module inspector")?; + } + result?; + // Neither parser nor validation diagnostics may echo schema literals. + let SerdeWrapper::(raw) = serde_json::from_slice(&output) + .map_err(|_| anyhow::anyhow!("Local module inspector returned invalid schema data"))?; + let schema = + ModuleDef::try_from(raw).map_err(|_| anyhow::anyhow!("Local module schema validation failed"))?; + Ok(schema) + } + .await; + let _ = send.send(result); + }); + (&mut recv).await.context("Local module inspection owner failed")? +} + +#[cfg(test)] +mod tests; diff --git a/crates/cli/src/subcommands/publish/environment/tests.rs b/crates/cli/src/subcommands/publish/environment/tests.rs new file mode 100644 index 00000000000..5af04aa6877 --- /dev/null +++ b/crates/cli/src/subcommands/publish/environment/tests.rs @@ -0,0 +1,307 @@ +use super::*; +use spacetimedb_lib::environment::{EnvironmentConstraint as Constraint, EnvironmentDeclaration as Declaration}; + +fn schema() -> EnvironmentSchema { + EnvironmentSchema::new(vec![ + Declaration { + name: "A".into(), + constraint: Constraint::AnyString, + optional: false, + }, + Declaration { + name: "B".into(), + constraint: Constraint::OneOf(vec!["true".into(), "false".into()]), + optional: false, + }, + Declaration { + name: "C".into(), + constraint: Constraint::AnyString, + optional: false, + }, + Declaration { + name: "OPTIONAL".into(), + constraint: Constraint::AnyString, + optional: true, + }, + ]) + .unwrap() +} + +#[test] +fn declared_shell_overrides_are_complete_and_redacted() { + let mut checked = Vec::new(); + let resolved = resolve( + &schema(), + Some(&serde_json::json!({"A":"config-sentinel","B":false})), + |name| { + checked.push(name.to_owned()); + match name { + "C" => Some("shell-sentinel".into()), + "A" => Some("".into()), + _ => None, + } + }, + ) + .unwrap(); + assert_eq!( + resolved.values, + BTreeMap::from([ + ("A".into(), "".into()), + ("B".into(), "false".into()), + ("C".into(), "shell-sentinel".into()) + ]) + ); + assert_eq!(checked, vec!["A", "B", "C", "OPTIONAL"]); + assert_eq!( + resolved.display(), + "Environment A (shell)\nEnvironment B (config)\nEnvironment C (shell)\n" + ); + assert!(!resolved.display().contains("sentinel")); + // No declaration means no ambient lookup, including PATH or credentials. + let empty = resolve(&EnvironmentSchema::default(), None, |_| panic!("ambient access")).unwrap(); + assert!(empty.values.is_empty()); +} + +#[test] +fn missing_required_never_reuses_old_values_and_optional_disappears() { + let config = serde_json::json!({"A":"first","B":true,"C":"first","OPTIONAL":"old"}); + let first = resolve(&schema(), Some(&config), |_| None).unwrap(); + assert!(first.values.contains_key("OPTIONAL")); + let second = resolve( + &schema(), + Some(&serde_json::json!({"A":"next","B":false,"C":"next"})), + |_| None, + ) + .unwrap(); + assert!(!second.values.contains_key("OPTIONAL")); + let error = resolve(&schema(), Some(&serde_json::json!({"A":"first","B":true})), |_| None) + .err() + .unwrap(); + assert!(error.to_string().contains('C')); +} + +#[test] +fn invalid_inputs_fail_without_values_or_lower_priority_fallback() { + for input in [Value::Null, serde_json::json!([]), serde_json::json!({})] { + let config = serde_json::json!({"A":input,"B":true,"C":"private-sentinel"}); + let error = resolve(&schema(), Some(&config), |_| None).err().unwrap(); + assert!(!format!("{error:#}").contains("private-sentinel")); + } + let config = serde_json::json!({"A":"private-sentinel","B":false,"C":"private-sentinel"}); + let error = resolve(&schema(), Some(&config), |name| { + (name == "B").then(|| "invalid-shell-secret".into()) + }) + .err() + .unwrap(); + let error = format!("{error:#}"); + assert!(error.contains('B')); + assert!(!error.contains("invalid-shell-secret") && !error.contains("private-sentinel")); + let error = resolve(&schema(), Some(&serde_json::json!({"UNDECLARED":"secret"})), |_| { + panic!("must fail first") + }) + .err() + .unwrap(); + assert!(error.to_string().contains("UNDECLARED")); +} + +#[test] +fn env_layer_selector_is_not_a_value_source() { + let command = super::super::cli(); + let args = command + .clone() + .try_get_matches_from(["publish", "db", "--env", "prod"]) + .unwrap(); + let schema = super::super::build_publish_schema(&command).unwrap(); + let config = crate::spacetime_config::CommandConfig::new( + &schema, + std::collections::HashMap::from([("env".into(), serde_json::json!({"A":"from-config"}))]), + &args, + ) + .unwrap(); + assert_eq!(config.get_config_value("env").unwrap()["A"], "from-config"); + assert!(!config.is_from_cli("env")); +} + +#[test] +fn number_boolean_and_empty_string_conversion_has_no_float_rounding() { + let schema = EnvironmentSchema::new( + ["NUMBER", "BOOL", "EMPTY"] + .map(|name| Declaration { + name: name.into(), + constraint: Constraint::AnyString, + optional: false, + }) + .to_vec(), + ) + .unwrap(); + let config = serde_json::from_str(r#"{"NUMBER":9007199254740993123456789,"BOOL":false,"EMPTY":""}"#).unwrap(); + let resolved = resolve(&schema, Some(&config), |_| None).unwrap(); + assert_eq!(resolved.values["NUMBER"], "9007199254740993123456789"); + assert_eq!(resolved.values["BOOL"], "false"); + assert_eq!(resolved.values["EMPTY"], ""); +} + +#[cfg(unix)] +#[test] +fn non_utf8_declared_shell_value_is_rejected_without_bytes() { + use std::os::unix::ffi::OsStringExt; + let error = resolve(&schema(), None, |name| { + (name == "A").then(|| OsString::from_vec(vec![0xff, 0xfe])) + }) + .err() + .unwrap(); + assert!(error.to_string().contains("UTF-8")); +} + +// These fixtures invoke only a locally generated executable in an owned tempdir. +// No CLI config, server credentials or user environment are imported. +#[cfg(unix)] +fn inspector(script: &str) -> (tempfile::TempDir, PathBuf) { + use std::os::unix::fs::PermissionsExt; + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("inspector"); + std::fs::write(&path, format!("#!/bin/sh\n{script}\n")).unwrap(); + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o700)).unwrap(); + (dir, path) +} + +#[cfg(unix)] +#[tokio::test] +async fn local_inspection_passes_exact_bytes_host_and_requires_success() { + use spacetimedb_lib::db::raw_def::v10::{RawModuleDefV10, RawModuleDefV10Section}; + let raw = RawModuleDef::V10(RawModuleDefV10 { + sections: vec![RawModuleDefV10Section::Environment(schema().into_declarations())], + }); + let json = serde_json::to_string(&SerdeWrapper(raw)).unwrap(); + let (dir, extractor) = inspector(&format!( + "[ \"$1\" = extract-schema ] && [ \"$3\" = --host-type ] && [ \"$4\" = js ] || exit 2\n[ \"$(/bin/cat \"$2\")\" = exact-artifact ] || exit 3\nprintf '%s' '{}'", json.replace('\'', "'\\''") + )); + let result = inspect_with( + extractor, + b"exact-artifact".to_vec(), + "Js".into(), + Duration::from_secs(5), + ) + .await + .unwrap(); + assert_eq!(result.environment(), &schema()); + drop(dir); + let (_dir, extractor) = inspector(&format!("printf '%s' '{}'; exit 9", json.replace('\'', "'\\''"))); + assert!( + inspect_with(extractor, b"anything".to_vec(), "Wasm".into(), Duration::from_secs(5)) + .await + .is_err() + ); +} + +#[cfg(unix)] +#[tokio::test] +async fn invalid_and_oversize_inspector_output_never_becomes_diagnostics() { + for script in [ + "printf 'generated-inspector-secret'", + "exec /usr/bin/head -c 16777217 /dev/zero", + ] { + let (_dir, extractor) = inspector(script); + let error = inspect_with(extractor, b"input".to_vec(), "Wasm".into(), Duration::from_secs(5)) + .await + .unwrap_err(); + assert!(!format!("{error:#}").contains("generated-inspector-secret")); + } +} + +#[cfg(unix)] +#[tokio::test] +async fn inspection_timeout_and_dropped_waiter_reap_exact_child() { + // A shell-only busy loop has no descendants and records the exact child PID. + // kill(0) via the owned process is not used for proof: wait for /bin/kill -0 + // to report ESRCH after our owner has called wait, including cancellation. + async fn wait_pid(path: &std::path::Path) -> String { + tokio::time::timeout(Duration::from_secs(5), async { + loop { + if let Ok(pid) = tokio::fs::read_to_string(path).await + && !pid.trim().is_empty() + { + break pid; + } + tokio::task::yield_now().await; + } + }) + .await + .unwrap() + } + async fn gone(pid: &str) { + tokio::time::timeout(Duration::from_secs(5), async { + loop { + let status = tokio::process::Command::new("/bin/kill") + .args(["-0", pid.trim()]) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status() + .await + .unwrap(); + if !status.success() { + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .unwrap(); + } + for cancel in [false, true] { + let (dir, extractor) = inspector("placeholder"); + let pid_file = dir.path().join("pid"); + std::fs::write( + &extractor, + format!( + "#!/bin/sh\nprintf '%s' \"$$\" > '{}'\nwhile :; do :; done\n", + pid_file.display() + ), + ) + .unwrap(); + let operation = tokio::spawn(inspect_with( + extractor, + b"input".to_vec(), + "Wasm".into(), + if cancel { + Duration::from_secs(30) + } else { + Duration::from_millis(300) + }, + )); + let pid = wait_pid(&pid_file).await; + if cancel { + operation.abort(); + let _ = operation.await; + } else { + assert!(operation.await.unwrap().is_err()); + } + gone(&pid).await; + } +} + +#[tokio::test] +#[ignore = "requires explicit locally built ENV-aware standalone and declared Wasm fixture paths"] +async fn actual_precompiled_declarations_are_inspected_without_server_or_values() { + let extractor = + PathBuf::from(std::env::var_os("SPACETIMEDB_ENV_CLI_TEST_EXTRACTOR").expect("explicit inspector path")); + let module = PathBuf::from(std::env::var_os("SPACETIMEDB_ENV_CLI_TEST_MODULE").expect("explicit module path")); + assert!(extractor.is_absolute() && module.is_absolute()); + let program = read_program(&module).unwrap(); + let inspected = inspect_with(extractor, program, "Wasm".into(), INSPECT_TIMEOUT) + .await + .unwrap(); + let schema = inspected.environment(); + assert!(inspected.environment_declared()); + assert!(!schema.get("REQUIRED").unwrap().optional); + assert_eq!( + schema.get("MODE").unwrap().constraint, + Constraint::OneOf(vec!["other".into(), "ready".into()]) + ); + let config = serde_json::json!({"REQUIRED":"generated-local-inspection-sentinel","MODE":"ready"}); + let resolved = resolve(schema, Some(&config), |_| None).unwrap(); + assert_eq!(resolved.values.len(), 2); + assert!(!resolved.display().contains("generated-local-inspection-sentinel")); + assert!(resolve(schema, None, |_| None).is_err()); +} diff --git a/crates/client-api-messages/src/lib.rs b/crates/client-api-messages/src/lib.rs index bf2f1dc9ca5..67b58de659f 100644 --- a/crates/client-api-messages/src/lib.rs +++ b/crates/client-api-messages/src/lib.rs @@ -4,3 +4,5 @@ pub mod energy; pub mod http; pub mod name; pub mod websocket; + +pub mod publish; diff --git a/crates/client-api-messages/src/publish.rs b/crates/client-api-messages/src/publish.rs new file mode 100644 index 00000000000..fea29732139 --- /dev/null +++ b/crates/client-api-messages/src/publish.rs @@ -0,0 +1,117 @@ +//! Complete publish input. Environment values travel only in the request body. +use serde::{Deserialize, Deserializer, Serialize}; +use serde_with::{base64::Base64, serde_as}; +use spacetimedb_lib::environment::{validate_key, validate_value, MAX_ENV_VARS}; +use std::collections::BTreeMap; + +pub const CONTENT_TYPE: &str = "application/vnd.spacetimedb.publish+json"; +pub const MAX_MODULE_BYTES: usize = 128 * 1024 * 1024; +/// Includes base64 module expansion and worst-case JSON escaping of configuration. +pub const MAX_REQUEST_BYTES: usize = 192 * 1024 * 1024; + +/// Values deliberately have no Debug representation. Omission is an empty map, +/// including for a publish of an unchanged module. +#[serde_as] +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct PublishRequest { + #[serde_as(as = "Base64")] + pub module: Vec, + #[serde(default, deserialize_with = "deserialize_environment")] + pub environment: BTreeMap, +} + +#[derive(Debug, Clone, Copy, thiserror::Error)] +pub enum PublishRequestError { + #[error("invalid publish request body")] + Invalid, + #[error("publish request exceeds size limit")] + TooLarge, +} + +impl PublishRequest { + pub fn decode(body: &[u8]) -> Result { + if body.len() > MAX_REQUEST_BYTES { + return Err(PublishRequestError::TooLarge); + } + // Never expose serde's error text: it can quote a supplied secret. + let request: Self = serde_json::from_slice(body).map_err(|_| PublishRequestError::Invalid)?; + request.validate()?; + Ok(request) + } + + pub fn encode(&self) -> Result, PublishRequestError> { + self.validate()?; + serde_json::to_vec(self).map_err(|_| PublishRequestError::Invalid) + } + + fn validate(&self) -> Result<(), PublishRequestError> { + if self.module.len() > MAX_MODULE_BYTES || self.environment.len() > MAX_ENV_VARS { + return Err(PublishRequestError::TooLarge); + } + for (key, value) in &self.environment { + validate_key(key).map_err(|_| PublishRequestError::Invalid)?; + validate_value(value).map_err(|_| PublishRequestError::TooLarge)?; + } + Ok(()) + } +} + +fn deserialize_environment<'de, D: Deserializer<'de>>(de: D) -> Result, D::Error> { + struct Visitor; + impl<'de> serde::de::Visitor<'de> for Visitor { + type Value = BTreeMap; + fn expecting(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("a complete map of environment strings") + } + fn visit_map>(self, mut map: A) -> Result { + use serde::de::Error; + let mut values = BTreeMap::new(); + while let Some(key) = map.next_key::()? { + if values.len() >= MAX_ENV_VARS || validate_key(&key).is_err() || values.contains_key(&key) { + return Err(A::Error::custom("invalid environment keys")); + } + let value = map.next_value::()?; + if validate_value(&value).is_err() { + return Err(A::Error::custom("environment value exceeds size limit")); + } + values.insert(key, value); + } + Ok(values) + } + } + de.deserialize_map(Visitor) +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn roundtrip_and_omission_preserve_complete_string_input() { + let request = PublishRequest { + module: vec![0, 1, 255], + environment: BTreeMap::from([("EMPTY".into(), "".into()), ("TOKEN".into(), "雪\0false".into())]), + }; + let decoded = PublishRequest::decode(&request.encode().unwrap()).unwrap(); + assert_eq!(decoded.module, request.module); + assert_eq!(decoded.environment, request.environment); + assert!(PublishRequest::decode(br#"{"module":""}"#) + .unwrap() + .environment + .is_empty()); + } + #[test] + fn malformed_inputs_and_duplicate_keys_are_rejected_without_values() { + for body in [ + r#"{"module":"","environment":{"KEY":true}}"#, + r#"{"module":"","environment":{"KEY":null}}"#, + r#"{"module":"","environment":{"KEY":"first","KEY":"secret-marker"}}"#, + r#"{"module":"","environment":{"KEY":["secret-marker"]}}"#, + r#"{"module":"secret-marker"}"#, + r#"{"module":"","unknown":"secret-marker"}"#, + ] { + let error = PublishRequest::decode(body.as_bytes()).err().expect("must reject"); + assert!(!format!("{error:?}: {error}").contains("secret-marker")); + } + } +} diff --git a/crates/client-api/src/lib.rs b/crates/client-api/src/lib.rs index 68fd0a61bb5..9423849ddf8 100644 --- a/crates/client-api/src/lib.rs +++ b/crates/client-api/src/lib.rs @@ -206,6 +206,26 @@ impl Host { .update_module_host(database, host_type, self.replica_id, program_bytes, policy) .await } + + pub async fn update_with_environment( + &self, + database: Database, + host_type: HostType, + program_bytes: Box<[u8]>, + policy: MigrationPolicy, + environment: std::collections::BTreeMap, + ) -> anyhow::Result { + self.host_controller + .update_module_host_with_environment( + database, + host_type, + self.replica_id, + program_bytes, + policy, + environment, + ) + .await + } } /// Parameters for publishing a database. /// @@ -215,6 +235,8 @@ pub struct DatabaseDef { pub database_identity: Identity, /// The compiled program of the database module. pub program_bytes: Bytes, + /// Complete publish input, never persisted in the public Database record. + pub environment: std::collections::BTreeMap, /// The desired number of replicas the database shall have. /// /// If `None`, the edition default is used. @@ -232,6 +254,7 @@ pub struct DatabaseDef { pub struct DatabaseResetDef { pub database_identity: Identity, pub program_bytes: Option, + pub environment: std::collections::BTreeMap, pub num_replicas: Option, pub host_type: Option, } diff --git a/crates/client-api/src/routes/database.rs b/crates/client-api/src/routes/database.rs index f170f9b290e..5d98a065976 100644 --- a/crates/client-api/src/routes/database.rs +++ b/crates/client-api/src/routes/database.rs @@ -1,3 +1,6 @@ +mod publish_environment; +use publish_environment::{ModuleBody, PublishBody}; + use std::borrow::Cow; use std::future::Future; use std::num::NonZeroU8; @@ -835,7 +838,10 @@ pub async fn reset( host_type, }): Query, Extension(auth): Extension, - program_bytes: Option, + PublishBody { + program_bytes, + environment, + }: PublishBody, ) -> axum::response::Result> { let database_identity = database.database_identity; @@ -856,6 +862,7 @@ pub async fn reset( DatabaseResetDef { database_identity, program_bytes, + environment, num_replicas, host_type: Some(host_type), }, @@ -933,7 +940,10 @@ pub async fn publish( update_confirmation_timeout: confirmation_timeout, }): Query, Extension(auth): Extension, - program_bytes: Bytes, + PublishBody { + program_bytes, + environment, + }: PublishBody, ) -> axum::response::Result> { // If `clear`, check that the database exists and delegate to `reset`. // If it doesn't exist, ignore the `clear` parameter. @@ -963,13 +973,17 @@ pub async fn publish( host_type, }), Extension(auth), - Some(program_bytes), + PublishBody { + program_bytes, + environment, + }, ) .await; } } } + let program_bytes = program_bytes.unwrap_or_default(); let (database_identity, db_name) = get_or_create_identity_and_name(&ctx, &auth, name_or_identity.as_ref()).await?; let maybe_parent_database_identity = match parent.as_ref() { None => None, @@ -1033,6 +1047,7 @@ pub async fn publish( DatabaseDef { database_identity, program_bytes, + environment, num_replicas, host_type, parent, @@ -1217,7 +1232,7 @@ pub async fn pre_publish Extension(ResolvedDatabase(database)): Extension, Query(PrePublishQueryParams { style, host_type }): Query, Extension(auth): Extension, - program_bytes: Bytes, + ModuleBody(program_bytes): ModuleBody, ) -> axum::response::Result> { let database_identity = database.database_identity; @@ -1236,6 +1251,7 @@ pub async fn pre_publish DatabaseDef { database_identity, program_bytes, + environment: Default::default(), num_replicas: None, host_type, parent: None, diff --git a/crates/client-api/src/routes/database/publish_environment.rs b/crates/client-api/src/routes/database/publish_environment.rs new file mode 100644 index 00000000000..0287d226688 --- /dev/null +++ b/crates/client-api/src/routes/database/publish_environment.rs @@ -0,0 +1,139 @@ +//! Bounded publish extraction. Neither errors nor Debug output retain configuration values. +use axum::body::{to_bytes, Bytes}; +use axum::extract::{FromRequest, Request}; +use axum::response::{IntoResponse, Response}; +use http::{header, StatusCode}; +use spacetimedb_client_api_messages::publish::{PublishRequest, CONTENT_TYPE, MAX_MODULE_BYTES, MAX_REQUEST_BYTES}; +use std::collections::BTreeMap; + +pub struct PublishBody { + pub program_bytes: Option, + pub environment: BTreeMap, +} + +async fn bounded_body(request: Request, limit: usize) -> Result { + if request + .headers() + .get(header::CONTENT_LENGTH) + .and_then(|value| value.to_str().ok()) + .and_then(|value| value.parse::().ok()) + .is_some_and(|len| len > limit as u64) + { + return Err((StatusCode::PAYLOAD_TOO_LARGE, "publish request exceeds size limit").into_response()); + } + to_bytes(request.into_body(), limit) + .await + .map_err(|_| (StatusCode::PAYLOAD_TOO_LARGE, "publish request exceeds size limit").into_response()) +} + +#[async_trait::async_trait] +impl FromRequest for PublishBody { + type Rejection = Response; + + async fn from_request(request: Request, _state: &S) -> Result { + let envelope = request + .headers() + .get(header::CONTENT_TYPE) + .and_then(|value| value.to_str().ok()) + .is_some_and(|value| { + value + .split(';') + .next() + .unwrap_or_default() + .trim() + .eq_ignore_ascii_case(CONTENT_TYPE) + }); + let bytes = bounded_body(request, if envelope { MAX_REQUEST_BYTES } else { MAX_MODULE_BYTES }).await?; + if envelope { + let request = PublishRequest::decode(&bytes) + .map_err(|_| (StatusCode::BAD_REQUEST, "invalid publish request body").into_response())?; + Ok(Self { + program_bytes: (!request.module.is_empty()).then_some(request.module.into()), + environment: request.environment, + }) + } else { + // An absent reset body retains the program, but never retains env values. + Ok(Self { + program_bytes: (!bytes.is_empty()).then_some(bytes), + environment: BTreeMap::new(), + }) + } + } +} + +pub struct ModuleBody(pub Bytes); + +#[async_trait::async_trait] +impl FromRequest for ModuleBody { + type Rejection = Response; + + async fn from_request(request: Request, _state: &S) -> Result { + bounded_body(request, MAX_MODULE_BYTES).await.map(Self) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use axum::body::Body; + + #[tokio::test] + async fn legacy_envelope_and_empty_reset_have_complete_input_semantics() { + let legacy = PublishBody::from_request(Request::new(Body::from("module")), &()) + .await + .unwrap(); + assert_eq!(legacy.program_bytes.unwrap(), "module"); + assert!(legacy.environment.is_empty()); + let empty = PublishBody::from_request(Request::new(Body::empty()), &()) + .await + .unwrap(); + assert!(empty.program_bytes.is_none()); + assert!(empty.environment.is_empty()); + let input = PublishRequest { + module: vec![1, 2, 3], + environment: BTreeMap::from([("TOKEN".into(), "雪\0".into())]), + }; + let request = Request::builder() + .header(header::CONTENT_TYPE, CONTENT_TYPE) + .body(Body::from(input.encode().unwrap())) + .unwrap(); + let extracted = PublishBody::from_request(request, &()).await.unwrap(); + assert_eq!(extracted.environment, input.environment); + assert_eq!(extracted.program_bytes.unwrap(), input.module); + let reset = PublishRequest { + module: vec![], + environment: input.environment, + }; + let request = Request::builder() + .header(header::CONTENT_TYPE, CONTENT_TYPE) + .body(Body::from(reset.encode().unwrap())) + .unwrap(); + let extracted = PublishBody::from_request(request, &()).await.unwrap(); + assert!(extracted.program_bytes.is_none()); + assert_eq!(extracted.environment, reset.environment); + } + + #[tokio::test] + async fn streamed_limits_apply_without_global_body_limit_and_errors_are_redacted() { + let stream = futures::stream::iter([ + Ok::<_, std::io::Error>(Bytes::from_static(b"12345")), + Ok(Bytes::from_static(b"67890")), + ]); + let error = bounded_body(Request::new(Body::from_stream(stream)), 8) + .await + .unwrap_err(); + assert_eq!(error.into_response().status(), StatusCode::PAYLOAD_TOO_LARGE); + let request = Request::builder() + .header(header::CONTENT_TYPE, CONTENT_TYPE) + .body(Body::from(r#"{"module":"","environment":{"KEY":["secret-marker"]}}"#)) + .unwrap(); + let error = PublishBody::from_request(request, &()) + .await + .err() + .unwrap() + .into_response(); + assert_eq!(error.status(), StatusCode::BAD_REQUEST); + let body = to_bytes(error.into_body(), 1024).await.unwrap(); + assert!(!String::from_utf8_lossy(&body).contains("secret-marker")); + } +} diff --git a/crates/core/src/db/environment.rs b/crates/core/src/db/environment.rs index 87b7a7231f7..10718637d13 100644 --- a/crates/core/src/db/environment.rs +++ b/crates/core/src/db/environment.rs @@ -1,15 +1,16 @@ //! Dedicated access to the private environment store. //! -//! Mutation callers must authorize owner/admin access before calling these -//! helpers and commit through the normal module transaction machinery so -//! dependent views refresh. Helpers never acquire a second transaction. +//! Publishing replaces the complete environment inside the program transaction. +//! Helpers never acquire a second transaction or expose individual mutation APIs. use super::relational_db::{MutTx, RelationalDB}; use crate::error::DBError; use spacetimedb_datastore::error::DatastoreError; use spacetimedb_datastore::locking_tx_datastore::state_view::StateView; use spacetimedb_datastore::system_tables::{StEnvFields, StEnvRow, ST_ENV_ID}; -use spacetimedb_lib::environment::{validate_key, validate_value, EnvironmentValidationError, MAX_ENV_VARS}; +use spacetimedb_lib::environment::{ + validate_key, EnvironmentSchema, EnvironmentSchemaError, EnvironmentValidationError, +}; use spacetimedb_sats::AlgebraicValue; use std::collections::BTreeMap; @@ -18,6 +19,8 @@ pub enum EnvironmentError { #[error(transparent)] Validation(#[from] EnvironmentValidationError), #[error(transparent)] + Schema(#[from] EnvironmentSchemaError), + #[error(transparent)] Datastore(#[from] DatastoreError), #[error(transparent)] Database(#[from] DBError), @@ -43,29 +46,37 @@ pub fn snapshot(state: &impl StateView) -> Result, Envi .collect() } -/// Insert or replace one key. Validation occurs before any mutation. -pub fn set(db: &RelationalDB, tx: &mut MutTx, key: &str, value: &str) -> Result<(), EnvironmentError> { - validate_key(key)?; - validate_value(value)?; - let previous = get(tx, key)?; - if previous.is_none() && tx.table_row_count(ST_ENV_ID).unwrap_or(0) >= MAX_ENV_VARS as u64 { - return Err(EnvironmentValidationError::TooManyVariables.into()); +/// Apply a complete publish configuration in the caller's program transaction. +/// Validate every input before modifying any row, even if the caller chooses to +/// recover from a validation error and commit other transaction work. +pub fn replace( + db: &RelationalDB, + tx: &mut MutTx, + schema: &EnvironmentSchema, + values: &BTreeMap, +) -> Result<(), EnvironmentError> { + schema.validate_values(values)?; + let previous = snapshot(tx)?; + for (key, value) in &previous { + if values.get(key) != Some(value) { + delete(db, tx, key)?; + } } - if previous.as_deref() == Some(value) { - return Ok(()); + for (key, value) in values { + if previous.get(key) != Some(value) { + tx.insert_via_serialize_bsatn( + ST_ENV_ID, + &StEnvRow { + key: key.clone(), + value: value.clone(), + }, + )?; + } } - delete(db, tx, key)?; - tx.insert_via_serialize_bsatn( - ST_ENV_ID, - &StEnvRow { - key: key.into(), - value: value.into(), - }, - )?; Ok(()) } -pub fn delete(db: &RelationalDB, tx: &mut MutTx, key: &str) -> Result { +fn delete(db: &RelationalDB, tx: &mut MutTx, key: &str) -> Result { validate_key(key)?; let pointer = tx .iter_by_col_eq(ST_ENV_ID, StEnvFields::Key, &AlgebraicValue::String(key.into()))? @@ -83,49 +94,73 @@ mod tests { use super::*; use crate::db::relational_db::tests_utils::TestDB; use spacetimedb_datastore::execution_context::Workload; + use spacetimedb_lib::environment::{EnvironmentConstraint, EnvironmentDeclaration}; + + fn schema() -> EnvironmentSchema { + EnvironmentSchema::new(vec![ + EnvironmentDeclaration { + name: "REQUIRED".into(), + constraint: EnvironmentConstraint::AnyString, + optional: false, + }, + EnvironmentDeclaration { + name: "OPTIONAL".into(), + constraint: EnvironmentConstraint::AnyString, + optional: true, + }, + ]) + .unwrap() + } #[test] - fn missing_empty_nul_update_and_rollback() { + fn replacement_preserves_empty_and_nul_and_removes_omitted_values() { let db = TestDB::in_memory().unwrap(); - db.with_auto_commit(Workload::ForTests, |tx| -> Result<(), EnvironmentError> { - assert_eq!(get(tx, "EMPTY")?, None); - set(&db, tx, "EMPTY", "")?; - set(&db, tx, "NUL", "a\0b")?; - assert_eq!(get(tx, "EMPTY")?, Some(String::new())); - assert_eq!(get(tx, "NUL")?, Some("a\0b".into())); - Ok(()) + let initial = BTreeMap::from([("REQUIRED".into(), "".into()), ("OPTIONAL".into(), "a\0b".into())]); + db.with_auto_commit(Workload::ForTests, |tx| replace(&db, tx, &schema(), &initial)) + .unwrap(); + db.with_read_only(Workload::ForTests, |tx| assert_eq!(snapshot(tx).unwrap(), initial)); + let next = BTreeMap::from([("REQUIRED".into(), "new".into())]); + db.with_auto_commit(Workload::ForTests, |tx| replace(&db, tx, &schema(), &next)) + .unwrap(); + db.with_read_only(Workload::ForTests, |tx| assert_eq!(snapshot(tx).unwrap(), next)); + db.with_auto_commit(Workload::ForTests, |tx| { + replace(&db, tx, &EnvironmentSchema::default(), &BTreeMap::new()) }) .unwrap(); - let result = db.with_auto_commit(Workload::ForTests, |tx| -> Result<(), EnvironmentError> { - set(&db, tx, "EMPTY", "changed")?; - delete(&db, tx, "NUL")?; - Err(EnvironmentValidationError::InvalidKey.into()) - }); - assert!(result.is_err()); - db.with_read_only(Workload::ForTests, |tx| { - assert_eq!( - snapshot(tx).unwrap(), - BTreeMap::from([("EMPTY".into(), "".into()), ("NUL".into(), "a\0b".into())]) - ); - }); + db.with_read_only(Workload::ForTests, |tx| assert!(snapshot(tx).unwrap().is_empty())); } #[test] - fn capacity_and_value_limits_precede_mutation() { + fn invalid_complete_input_does_not_reuse_stored_values_or_mutate() { let db = TestDB::in_memory().unwrap(); + let initial = BTreeMap::from([("REQUIRED".into(), "old".into()), ("OPTIONAL".into(), "keep".into())]); db.with_auto_commit(Workload::ForTests, |tx| -> Result<(), EnvironmentError> { - for i in 0..MAX_ENV_VARS { - set(&db, tx, &format!("K{i}"), "")?; + replace(&db, tx, &schema(), &initial)?; + for invalid in [ + BTreeMap::new(), + BTreeMap::from([ + ("REQUIRED".into(), "new".into()), + ("UNKNOWN".into(), "secret-marker".into()), + ]), + BTreeMap::from([("REQUIRED".into(), "x".repeat(8193))]), + ] { + assert!(replace(&db, tx, &schema(), &invalid).is_err()); + assert_eq!(snapshot(tx)?, initial); } - assert!(set(&db, tx, "EXTRA", "").is_err()); - set(&db, tx, "K0", "updated")?; - assert!(set(&db, tx, "K0", &"x".repeat(8193)).is_err()); - assert_eq!(get(tx, "K0")?.as_deref(), Some("updated")); - assert!(delete(&db, tx, "K1")?); - assert!(!delete(&db, tx, "MISSING")?); - set(&db, tx, "EXTRA", "")?; Ok(()) }) .unwrap(); + let failed_publish = db.with_auto_commit(Workload::ForTests, |tx| -> Result<(), EnvironmentError> { + replace( + &db, + tx, + &schema(), + &BTreeMap::from([("REQUIRED".into(), "updated".into())]), + )?; + // Simulate a later failure in the same publish transaction. + Err(EnvironmentValidationError::InvalidKey.into()) + }); + assert!(failed_publish.is_err()); + db.with_read_only(Workload::ForTests, |tx| assert_eq!(snapshot(tx).unwrap(), initial)); } } diff --git a/crates/core/src/host/host_controller.rs b/crates/core/src/host/host_controller.rs index ff017b733c8..38a198599b6 100644 --- a/crates/core/src/host/host_controller.rs +++ b/crates/core/src/host/host_controller.rs @@ -25,7 +25,7 @@ use crate::worker_metrics::{ record_module_host_init_attempt, record_module_host_init_failure, record_module_host_unexpected_exit, ModuleHostInitFailureCause, WORKER_METRICS, }; -use anyhow::{anyhow, bail, Context}; +use anyhow::{bail, Context}; use async_trait::async_trait; use durability::{Durability, EmptyHistory}; use log::{info, trace, warn}; @@ -92,6 +92,14 @@ where pub type ProgramStorage = Arc; +/// Private complete configuration for a not-yet-initialized database generation. +/// Implementations must verify the exact persisted database identity, program and +/// bootstrap generation. This source is never consulted during ordinary reopen. +#[async_trait] +pub trait InitialEnvironmentSource: Send + Sync { + async fn load(&self, database: &Database) -> anyhow::Result>; +} + /// A launched module host plus any pending controldb program-bootstrap completion work. pub struct ModuleHostWithBootstrap { pub module: ModuleHost, @@ -187,6 +195,7 @@ pub struct HostController { default_config: db::Config, /// The [`ProgramStorage`] to query when instantiating a module. program_storage: ProgramStorage, + initial_environment_source: Option>, /// The [`EnergyMonitor`] used by this controller. energy_monitor: Arc, /// The [`MemoryObserver`] used by this controller. @@ -357,6 +366,7 @@ impl HostController { hosts: <_>::default(), default_config, program_storage, + initial_environment_source: None, energy_monitor, memory_observer, persistence, @@ -368,6 +378,12 @@ impl HostController { } } + /// Install the private bootstrap input source before this controller is shared. + pub fn with_initial_environment_source(mut self, source: Arc) -> Self { + self.initial_environment_source = Some(source); + self + } + /// Replace the [`ProgramStorage`] used by this controller. pub fn set_program_storage(&mut self, ps: ProgramStorage) { self.program_storage = ps; @@ -514,6 +530,16 @@ impl HostController { /// This is not necessary during hotswap publishes, /// as the automigration planner and executor accomplish the same validity checks. pub async fn check_module_validity(&self, database: Database, program: Program) -> anyhow::Result> { + self.check_module_validity_with_environment(database, program, Default::default()) + .await + } + + pub async fn check_module_validity_with_environment( + &self, + database: Database, + program: Program, + environment: std::collections::BTreeMap, + ) -> anyhow::Result> { let (program, launched) = Host::try_init_in_memory_to_check( &self.runtimes, self.page_pool.clone(), @@ -529,12 +555,20 @@ impl HostController { ) .await?; - let InitDatabaseResult { reducer, .. } = launched.module_host.init_database(program).await?; + let result = launched + .module_host + .init_database_with_environment(program, environment) + .await; + let info = launched.module_host.info.clone(); + // Validation never starts scheduled work. Release its receiver before + // waiting for scheduler closure, including when initialization failed. + drop(launched.scheduler_starter); + launched.module_host.exit().await; + let InitDatabaseResult { reducer, .. } = result?; if let Some(call_result) = reducer { Result::from(call_result)?; } - - Ok(launched.module_host.info) + Ok(info) } /// Update the [`ModuleHost`] identified by `replica_id` to the given @@ -553,6 +587,27 @@ impl HostController { replica_id: u64, program_bytes: Box<[u8]>, policy: MigrationPolicy, + ) -> anyhow::Result { + self.update_module_host_with_environment( + database, + host_type, + replica_id, + program_bytes, + policy, + Default::default(), + ) + .await + } + + #[tracing::instrument(level = "trace", skip_all, err)] + pub async fn update_module_host_with_environment( + &self, + database: Database, + host_type: HostType, + replica_id: u64, + program_bytes: Box<[u8]>, + policy: MigrationPolicy, + environment: std::collections::BTreeMap, ) -> anyhow::Result { let program = Program::from_bytes(host_type.into(), program_bytes); trace!( @@ -605,12 +660,14 @@ impl HostController { this.energy_monitor.clone(), this.unregister_fn(replica_id, database_identity), this.db_cores.take(), + environment, ) - .await?; + .await; + // Rejected publication leaves the existing host usable. Restore it + // before propagating validation or migration failure to the caller. *guard = Some(host); - - Ok::<_, anyhow::Error>(update_result) + update_result }) .await??; @@ -657,6 +714,25 @@ impl HostController { /// and deregister it from the controller. #[tracing::instrument(level = "trace", skip_all)] pub async fn exit_module_host(&self, replica_id: u64, timeout: Duration) -> Result<(), anyhow::Error> { + let start = Instant::now(); + if tokio::time::timeout(timeout, self.exit_module_host_and_join(replica_id)) + .await + .is_err() + { + warn!( + "replica={replica_id} shutdown timed out after {}s", + start.elapsed().as_secs_f32() + ); + } + Ok(()) + } + + /// Wait for actual module and database closure, without treating an elapsed + /// request deadline as completion. The caller must retain this future and + /// exclude new launch admission until it returns, including if its own + /// request waiter is cancelled. + #[tracing::instrument(level = "trace", skip_all)] + pub async fn exit_module_host_and_join(&self, replica_id: u64) -> Result<(), anyhow::Error> { let Some(lock) = self.hosts.lock().remove(&replica_id) else { return Ok(()); }; @@ -677,35 +753,24 @@ impl HostController { }); defer!(warn_blocked.abort()); - let shutdown = tokio::time::timeout(timeout, async { - let mut guard = lock.write_owned().await; - let Some(host) = guard.take() else { - return; - }; - let module = host.module.borrow().clone(); - let info = module.info(); - - let database_identity = info.database_identity; - let table_names = info.module_def.tables().map(|t| t.name.deref()); + let mut guard = lock.write_owned().await; + let Some(host) = guard.take() else { + return Ok(()); + }; + let module = host.module.borrow().clone(); + let info = module.info(); - // Ensure we clear the metrics even if the future is cancelled. - defer!(remove_database_gauges(&database_identity, table_names)); + let database_identity = info.database_identity; + let table_names = info.module_def.tables().map(|t| t.name.deref()); - info!("replica={replica_id} database={database_identity} exiting module"); - module.exit().await; - info!("replica={replica_id} database={database_identity} exiting database"); - module.relational_db().shutdown().await; - info!("replica={replica_id} database={database_identity} module host exited"); - }) - .await; - - if shutdown.is_err() { - warn!( - "replica={replica_id} shutdown timed out after {}s", - start.elapsed().as_secs_f32() - ); - } + // Ensure we clear the metrics even if the future is cancelled. + defer!(remove_database_gauges(&database_identity, table_names)); + info!("replica={replica_id} database={database_identity} exiting module"); + module.exit().await; + info!("replica={replica_id} database={database_identity} exiting database"); + module.relational_db().shutdown().await; + info!("replica={replica_id} database={database_identity} module host exited"); Ok(()) } @@ -1030,32 +1095,25 @@ fn repair_stale_view_backing_tables_on_launch(launched: &LaunchedModule) -> anyh /// If the `db` is not initialized yet (i.e. its program hash is `None`), /// return an error. /// -/// Otherwise, if `db.program_hash` matches the given `program_hash`, do -/// nothing and return an empty `UpdateDatabaseResult`. -/// -/// Otherwise, invoke `module.update_database` and return the result. +/// Otherwise publish the complete environment with the module, including when +/// its program hash is unchanged. async fn update_module( db: &RelationalDB, module: &ModuleHost, program: Program, old_module_info: Arc, policy: MigrationPolicy, + environment: std::collections::BTreeMap, ) -> anyhow::Result { let addr = db.database_identity(); - match stored_program_hash(db)? { - None => Err(anyhow!("database `{addr}` not yet initialized")), - Some(stored) => { - let res = if stored == program.hash { - info!("database `{}` up to date with program `{}`", addr, program.hash); - UpdateDatabaseResult::NoUpdateNeeded - } else { - info!("updating `{}` from {} to {}", addr, stored, program.hash); - module.update_database(program, old_module_info, policy).await? - }; - - Ok(res) - } - } + let Some(stored) = stored_program_hash(db)? else { + bail!("database `{addr}` not yet initialized"); + }; + info!("publishing `{}` from {} to {}", addr, stored, program.hash); + // Even an unchanged program publishes a complete replacement environment. + module + .update_database_with_environment(program, old_module_info, policy, environment) + .await } /// Encapsulates a database, associated module, and auxiliary state. @@ -1194,6 +1252,14 @@ impl Host { } }; let bootstrap_generation = database.bootstrap_generation; + let initial_environment = if program_needs_init { + match &host_controller.initial_environment_source { + Some(source) => source.load(&database).await?, + None => Default::default(), + } + } else { + Default::default() + }; let mut bootstrap_completion = Some(BootstrapCompletion::durable(bootstrap_generation)); let relational_db = Arc::new(db); @@ -1284,7 +1350,10 @@ impl Host { }; if program_needs_init { - let InitDatabaseResult { reducer, tx_offset } = launched.module_host.init_database(program).await?; + let InitDatabaseResult { reducer, tx_offset } = launched + .module_host + .init_database_with_environment(program, initial_environment) + .await?; if let Some(call_result) = reducer { validate_init_reducer_call_result(call_result)?; } @@ -1414,6 +1483,7 @@ impl Host { energy_monitor: Arc, on_panic: impl Fn() + Send + Sync + 'static, core: AllocatedJobCore, + environment: std::collections::BTreeMap, ) -> anyhow::Result { let replica_ctx = &self.replica_ctx; let (scheduler, scheduler_starter) = Scheduler::open(self.replica_ctx.relational_db().clone()); @@ -1432,8 +1502,25 @@ impl Host { // Get the old module info to diff against when building a migration plan. let old_module_info = self.module.borrow().info.clone(); - let update_result = - update_module(replica_ctx.relational_db(), &module, program, old_module_info, policy).await?; + let update_result = match update_module( + replica_ctx.relational_db(), + &module, + program, + old_module_info, + policy, + environment, + ) + .await + { + Ok(result) => result, + Err(error) => { + // This candidate was never installed or scheduled. Close its + // receiver first so cleanup cannot wait on an unstarted actor. + drop(scheduler_starter); + module.exit().await; + return Err(error); + } + }; // Only replace the module + scheduler if the update succeeded. // Otherwise, we want the database to continue running with the old state. @@ -1470,7 +1557,10 @@ impl Host { let old_module = old_watcher.borrow().clone(); old_module.exit().await; } - _ => {} + _ => { + drop(scheduler_starter); + module.exit().await; + } } Ok(update_result) @@ -1678,6 +1768,42 @@ where mod tests { use super::*; + #[tokio::test] + async fn positive_close_stays_pending_past_a_waiter_deadline_until_host_ownership_is_released() { + use crate::db::persistence::LocalPersistenceProvider; + use spacetimedb_paths::FromPathUnchecked; + let temp = tempfile::tempdir().unwrap(); + let directory = Arc::new(ServerDataDir::from_path_unchecked(temp.path().to_owned())); + let controller = HostController::new( + directory.clone(), + db::Config { + storage: db::Storage::Memory, + page_pool_max_size: None, + }, + HostRuntimeConfig::new(WasmConfig::default(), V8Config::default(), ModuleHttpConfig::default()), + Arc::new(|_| std::future::ready(anyhow::Ok(None))), + Arc::new(NullEnergyMonitor), + Arc::new(()), + Arc::new(LocalPersistenceProvider::new(directory)), + JobCores::without_pinned_cores(), + ); + let cell = Arc::new(AsyncRwLock::new(None)); + controller.hosts.lock().insert(17, cell.clone()); + let accepted_reader = cell.clone().read_owned().await; + let mut close = tokio::spawn(async move { controller.exit_module_host_and_join(17).await }); + assert!(tokio::time::timeout(Duration::from_millis(30), &mut close) + .await + .is_err()); + // A deadline on the observer did not complete or discard the owned close. + assert!(!close.is_finished()); + drop(accepted_reader); + tokio::time::timeout(Duration::from_secs(5), close) + .await + .unwrap() + .unwrap() + .unwrap(); + } + fn reducer_call_result(outcome: ReducerOutcome) -> ReducerCallResult { ReducerCallResult { outcome, diff --git a/crates/core/src/host/instance_env.rs b/crates/core/src/host/instance_env.rs index 1bfb4f7f7c7..ac9885b83a1 100644 --- a/crates/core/src/host/instance_env.rs +++ b/crates/core/src/host/instance_env.rs @@ -50,6 +50,9 @@ pub struct InstanceEnv { pub func_type: FuncCallType, /// The name of the last, including current, function to be executed by this environment. pub func_name: Option, + /// Bound by the host after validating this instance's module metadata. + environment_module: Option<(spacetimedb_lib::Hash, Arc)>, + environment_call_active: bool, /// Are we in an anonymous tx context? in_anon_tx: bool, /// A procedure's last known transaction offset. @@ -236,6 +239,8 @@ impl InstanceEnv { // run a function func_type: FuncCallType::Reducer, func_name: None, + environment_module: None, + environment_call_active: false, in_anon_tx: false, procedure_last_tx_offset: None, } @@ -252,6 +257,7 @@ impl InstanceEnv { self.start_instant = Instant::now(); self.func_type = func_type; self.func_name = Some(name); + self.environment_call_active = true; } /// Returns the name of the most recent reducer to be run in this environment, @@ -260,9 +266,29 @@ impl InstanceEnv { self.func_name.as_deref() } - /// Swap in a temporary function type, returning the previous one. - pub fn swap_func_type(&mut self, func_type: FuncCallType) -> FuncCallType { - mem::replace(&mut self.func_type, func_type) + pub(crate) fn bind_environment_module( + &mut self, + hash: spacetimedb_lib::Hash, + def: Arc, + ) { + self.environment_module = Some((hash, def)); + } + + pub(crate) fn finish_funcall(&mut self) { + self.environment_call_active = false; + } + + /// Nested host-dispatched view refreshes must use the view's namespace, + /// then restore the enclosing procedure's namespace and dependency tracking. + pub(crate) fn swap_func_context( + &mut self, + name: Option, + func_type: FuncCallType, + ) -> (Option, FuncCallType) { + ( + mem::replace(&mut self.func_name, name), + mem::replace(&mut self.func_type, func_type), + ) } fn get_tx(&self) -> Result + '_, GetTxError> { @@ -282,23 +308,58 @@ impl InstanceEnv { self.replica_ctx.relational_db() } - /// Dedicated read-only environment access. Missing reads also register a - /// dependency so a later insert refreshes a view that observed absence. + /// Read configuration using the schema of this exact module instance. + /// Missing optional reads also register a view dependency on `st_env`. pub(crate) fn env_get(&self, key: &str) -> Result, NodesError> { - use crate::db::environment; use spacetimedb_datastore::system_tables::ST_ENV_ID; spacetimedb_lib::environment::validate_key(key).map_err(|_| NodesError::InvalidEnvironmentKey)?; - let read = |state: &_| environment::get(state, key).map_err(|err| NodesError::from(DBError::Other(err.into()))); + if !self.environment_call_active || self.func_name.as_ref().is_none_or(|name| name.is_namespaced()) { + return Err(DBError::Other(anyhow::anyhow!( + "environment access requires a host-dispatched root module function" + )) + .into()); + } if let Ok(mut tx) = self.get_tx() { tx.record_table_scan(&self.func_type, ST_ENV_ID); - return read(&*tx); + return self.read_declared_environment(&*tx, key); } if !matches!(self.func_type, FuncCallType::Procedure) { return Err(NodesError::NotInTransaction); } - self.relational_db().with_read_only(Workload::Internal, |tx| { - environment::get(tx, key).map_err(|err| NodesError::from(DBError::Other(err.into()))) - }) + self.relational_db() + .with_read_only(Workload::Internal, |tx| self.read_declared_environment(tx, key)) + } + + fn read_declared_environment(&self, state: &impl StateView, key: &str) -> Result, NodesError> { + use spacetimedb_datastore::system_tables::{StModuleFields, ST_MODULE_ID}; + let fail = |message| NodesError::from(DBError::Other(anyhow::anyhow!("{message}"))); + let (hash, module) = self + .environment_module + .as_ref() + .ok_or_else(|| fail("environment schema is not available"))?; + let declaration = module + .environment() + .get(key) + .ok_or_else(|| fail("environment key is not declared"))?; + // Check inside this same snapshot. A suspended old procedure must never + // combine its declarations with values installed for a different module. + let row = state + .iter(ST_MODULE_ID) + .map_err(DBError::from)? + .next() + .ok_or_else(|| fail("database program is not initialized"))?; + let current_hash = spacetimedb_datastore::system_tables::read_hash_from_col(row, StModuleFields::ProgramHash) + .map_err(DBError::from)?; + if current_hash != *hash { + return Err(fail("module was replaced while this function was running")); + } + let value = crate::db::environment::get(state, key).map_err(|error| DBError::Other(error.into()))?; + if value.is_none() && !declaration.optional { + return Err(fail( + "required environment value is missing from the published configuration", + )); + } + Ok(value) } pub(crate) fn get_jwt_payload(&self, connection_id: ConnectionId) -> Result, NodesError> { @@ -1487,36 +1548,148 @@ mod test { Ok(db) } + fn bind_test_environment(env: &mut InstanceEnv) -> Result { + use spacetimedb_lib::db::raw_def::v10::RawModuleDefV10Builder; + use spacetimedb_lib::environment::{EnvironmentConstraint, EnvironmentDeclaration}; + let mut builder = RawModuleDefV10Builder::new(); + builder.add_environment( + [("A", false), ("MISSING", true)] + .into_iter() + .map(|(name, optional)| EnvironmentDeclaration { + name: name.into(), + constraint: EnvironmentConstraint::AnyString, + optional, + }) + .collect(), + ); + let module: spacetimedb_schema::def::ModuleDef = builder.finish().try_into()?; + let program = spacetimedb_datastore::traits::Program::from_bytes( + spacetimedb_datastore::system_tables::ModuleKind::WASM, + b"environment-unit-test".as_slice(), + ); + env.bind_environment_module(program.hash, Arc::new(module)); + Ok(program) + } + #[test] fn environment_reads_use_active_transaction_and_track_missing_view_dependency() -> Result<()> { use crate::db::environment; use spacetimedb_datastore::locking_tx_datastore::ViewCallInfo; use spacetimedb_primitives::ViewId; + use std::collections::BTreeMap; let db = relational_db()?; let (mut env, _runtime) = instance_env(db.clone())?; + let program = bind_test_environment(&mut env)?; + let schema = env.environment_module.as_ref().unwrap().1.environment().clone(); + env.start_funcall( + NamespacedIdentifier::from(spacetimedb_schema::identifier::Identifier::new("root".into())?), + Timestamp::now(), + FuncCallType::Reducer, + ); assert!(matches!(env.env_get("A"), Err(NodesError::NotInTransaction))); - env.func_type = FuncCallType::Procedure; - assert_eq!(env.env_get("A")?, None); let mut tx = begin_mut_tx(&db); - environment::set(&db, &mut tx, "A", "uncommitted")?; + db.update_program(&mut tx, program)?; + environment::replace( + &db, + &mut tx, + &schema, + &BTreeMap::from([("A".into(), "uncommitted".into())]), + )?; env.tx.set_raw(tx); assert_eq!(env.env_get("A")?.as_deref(), Some("uncommitted")); + assert!(env.env_get("UNDECLARED").is_err()); let view = ViewCallInfo::anonymous(ViewId(88)); - env.func_type = FuncCallType::View(view.clone()); + env.start_funcall( + NamespacedIdentifier::from(spacetimedb_schema::identifier::Identifier::new("view".into())?), + Timestamp::now(), + FuncCallType::View(view.clone()), + ); assert_eq!(env.env_get("MISSING")?, None); let tx = env.tx.take()?; db.commit_tx(tx)?; let mut tx = begin_mut_tx(&db); - environment::set(&db, &mut tx, "MISSING", "")?; + environment::replace( + &db, + &mut tx, + &schema, + &BTreeMap::from([("A".into(), "uncommitted".into()), ("MISSING".into(), "".into())]), + )?; assert!(tx.views_for_refresh().any(|dependency| dependency == &view)); let (_, metrics, reducer) = db.rollback_mut_tx(tx); db.report_mut_tx_metrics(reducer, metrics, None); - env.func_type = FuncCallType::Procedure; + env.start_funcall( + NamespacedIdentifier::from(spacetimedb_schema::identifier::Identifier::new("procedure".into())?), + Timestamp::now(), + FuncCallType::Procedure, + ); assert_eq!(env.env_get("MISSING")?, None); assert!(matches!(env.env_get("A=B"), Err(NodesError::InvalidEnvironmentKey))); Ok(()) } + #[test] + fn environment_rejects_submodules_finished_calls_and_replaced_programs() -> Result<()> { + use crate::db::environment; + use spacetimedb_datastore::traits::Program; + use std::collections::BTreeMap; + let db = relational_db()?; + let (mut env, _runtime) = instance_env(db.clone())?; + let program = bind_test_environment(&mut env)?; + let schema = env.environment_module.as_ref().unwrap().1.environment().clone(); + let mut tx = begin_mut_tx(&db); + db.update_program(&mut tx, program)?; + environment::replace( + &db, + &mut tx, + &schema, + &BTreeMap::from([("A".into(), "old-value".into())]), + )?; + db.commit_tx(tx)?; + assert!(env.env_get("A").is_err()); + env.start_funcall( + NamespacedIdentifier::from(spacetimedb_schema::identifier::Identifier::new("procedure".into())?), + Timestamp::now(), + FuncCallType::Procedure, + ); + assert_eq!(env.env_get("A")?.as_deref(), Some("old-value")); + let previous = env.swap_func_context( + Some(NamespacedIdentifier::from_segments(vec![ + spacetimedb_schema::identifier::Identifier::new("child".into())?, + spacetimedb_schema::identifier::Identifier::new("view".into())?, + ])), + FuncCallType::Procedure, + ); + assert!(env.env_get("A").is_err()); + env.swap_func_context(previous.0, previous.1); + assert_eq!(env.env_get("A")?.as_deref(), Some("old-value")); + env.finish_funcall(); + assert!(env.env_get("A").is_err()); + env.start_funcall( + NamespacedIdentifier::from(spacetimedb_schema::identifier::Identifier::new("procedure".into())?), + Timestamp::now(), + FuncCallType::Procedure, + ); + let mut tx = begin_mut_tx(&db); + let newer = Program::from_bytes( + spacetimedb_datastore::system_tables::ModuleKind::WASM, + b"new-code".as_slice(), + ); + db.update_program(&mut tx, newer)?; + environment::replace( + &db, + &mut tx, + &schema, + &BTreeMap::from([("A".into(), "new-secret".into())]), + )?; + db.commit_tx(tx)?; + assert!(env.env_get("A").is_err()); + env.start_mutable_tx()?; + assert!(env.env_get("A").is_err()); + let tx = env.take_mutable_tx_for_commit()?; + env.rollback_procedure_tx(tx); + Ok(()) + } + #[test] fn module_cannot_access_environment_by_guessed_table_and_index_ids() -> Result<()> { use spacetimedb_datastore::system_tables::ST_ENV_ID; diff --git a/crates/core/src/host/mod.rs b/crates/core/src/host/mod.rs index f28a515c910..d0a1f23c560 100644 --- a/crates/core/src/host/mod.rs +++ b/crates/core/src/host/mod.rs @@ -26,8 +26,8 @@ mod wasm_common; pub use disk_storage::DiskStorage; pub use host_controller::{ extract_schema, BootstrapCompletion, CallProcedureReturn, CallResult, ExternalDurability, ExternalStorage, - HostController, HostRuntimeConfig, MigratePlanResult, ModuleHostWithBootstrap, ProcedureCallResult, ProgramStorage, - ReducerCallResult, ReducerCallResultWithTxOffset, ReducerOutcome, + HostController, HostRuntimeConfig, InitialEnvironmentSource, MigratePlanResult, ModuleHostWithBootstrap, + ProcedureCallResult, ProgramStorage, ReducerCallResult, ReducerCallResultWithTxOffset, ReducerOutcome, }; pub use module_host::{ InitDatabaseResult, ModuleHost, NoSuchModule, ProcedureCallError, ReducerCallError, UpdateDatabaseResult, diff --git a/crates/core/src/host/module_host.rs b/crates/core/src/host/module_host.rs index 25eb09e6382..685a6046e13 100644 --- a/crates/core/src/host/module_host.rs +++ b/crates/core/src/host/module_host.rs @@ -608,15 +608,23 @@ pub(crate) fn init_database( replica_ctx: &ReplicaContext, module_def: &ModuleDef, program: Program, + environment: std::collections::BTreeMap, call_reducer: impl FnOnce(Option, CallReducerParams) -> (ReducerCallResultWithTxOffset, bool), ) -> (anyhow::Result, bool) { - extract_trapped(init_database_inner(replica_ctx, module_def, program, call_reducer)) + extract_trapped(init_database_inner( + replica_ctx, + module_def, + program, + environment, + call_reducer, + )) } fn init_database_inner( replica_ctx: &ReplicaContext, module_def: &ModuleDef, program: Program, + environment: std::collections::BTreeMap, call_reducer: impl FnOnce(Option, CallReducerParams) -> (ReducerCallResultWithTxOffset, bool), ) -> anyhow::Result<(InitDatabaseResult, bool)> { log::debug!("init database"); @@ -674,6 +682,7 @@ fn init_database_inner( .with_context(|| format!("failed to create row-level security for table `{table_id}`: `{sql}`",))?; } + crate::db::environment::replace(stdb, tx, module_def.environment(), &environment)?; stdb.set_initialized(tx, program)?; anyhow::Ok(()) @@ -1621,6 +1630,18 @@ impl fmt::Debug for ViewCallResult { } impl ViewCallResult { + fn into_materialized_tx(self, db: &RelationalDB, trapped: bool) -> Result { + let error = match self.outcome { + ViewOutcome::Success if !trapped => return Ok(self.tx), + ViewOutcome::Success => "View instance trapped during materialization".to_owned(), + ViewOutcome::Failed(error) => error, + ViewOutcome::BudgetExceeded => "View terminated due to insufficient budget".to_owned(), + }; + let (_, metrics, reducer) = db.rollback_mut_tx(self.tx); + db.report_mut_tx_metrics(reducer, metrics, None); + Err(ViewCallError::InternalError(error)) + } + pub fn default(tx: MutTxId) -> Self { Self { outcome: ViewOutcome::Success, @@ -1700,6 +1721,9 @@ pub enum ClientConnectedError { pub struct RefInstance<'a, I: WasmInstance> { pub common: &'a mut InstanceCommon, pub instance: &'a mut I, + // Invocation-local disposal state survives errors propagated through SQL or + // subscription helpers, whose Result error does not carry a success tuple. + pub(crate) trapped: bool, } macro_rules! call_view_command_method { @@ -2937,17 +2961,26 @@ impl ModuleHost { /// Passing [`Workload::Sql`] will update the instance's last-used timestamp. /// Passing [`Workload::Subscribe`] will also increment the subscriber's refcount. pub fn materialize_views( - mut tx: MutTxId, + tx: MutTxId, instance: &mut RefInstance<'_, I>, view_collector: &impl CollectViews, caller: Identity, workload: Workload, ) -> Result<(MutTxId, bool), ViewCallError> { use FunctionArgs::*; + let db = instance.instance.replica_ctx().relational_db().clone(); + // Keep all earlier view materializations and subscription refcounts in + // the same rollback boundary if any later view fails. + let mut tx = scopeguard::guard(Some(tx), |tx| { + if let Some(tx) = tx { + let (_, metrics, reducer) = db.rollback_mut_tx(tx); + db.report_mut_tx_metrics(reducer, metrics, None); + } + }); let mut view_ids = HashSet::new(); view_collector.collect_views(&mut view_ids); for view_id in view_ids { - let st_view_row = tx.lookup_st_view(view_id)?; + let st_view_row = tx.as_ref().unwrap().lookup_st_view(view_id)?; let view_name: NamespacedIdentifier = st_view_row.view_name.into(); let view_id = st_view_row.view_id; let table_id = st_view_row.table_id.ok_or(ViewCallError::TableDoesNotExist(view_id))?; @@ -2959,25 +2992,30 @@ impl ModuleHost { }; let view_call = ViewCallInfo::from_args(view_id, args); let sender = args.sender(); - let is_materialized = tx.is_view_materialized(&view_call)?; + let is_materialized = tx.as_ref().unwrap().is_view_materialized(&view_call)?; if !is_materialized { - let (res, trapped) = - Self::call_view(instance, tx, &view_name, view_id, table_id, Nullary, caller, sender)?; - tx = res.tx; - if trapped { - return Ok((tx, true)); - } + let (res, trapped) = Self::call_view( + instance, + tx.take().unwrap(), + &view_name, + view_id, + table_id, + Nullary, + caller, + sender, + )?; + *tx = Some(res.into_materialized_tx(&db, trapped)?); } - // If this is a sql call, we only update this view's "last called" timestamp + let tx = tx.as_mut().unwrap(); + // These changes commit only after every requested view succeeds. if let Workload::Sql = workload { tx.update_view_timestamp(view_call.clone(), args)?; } - // If this is a subscribe call, we also increment this view's subscriber count if let Workload::Subscribe = workload { tx.subscribe_view(view_call, args, caller)?; } } - Ok((tx, false)) + Ok((ScopeGuard::into_inner(tx).unwrap(), false)) } /// Refreshes every view made stale by `tx`. @@ -3124,6 +3162,11 @@ impl ModuleHost { sender: Option, timestamp: Timestamp, ) -> Result<(ViewCallResult, bool), ViewCallError> { + let db = instance.instance.replica_ctx().relational_db().clone(); + let tx = scopeguard::guard(tx, |tx| { + let (_, metrics, reducer) = db.rollback_mut_tx(tx); + db.report_mut_tx_metrics(reducer, metrics, None); + }); let module_def = &instance.common.info().module_def; let (global_fn_ptr, view_def, owning_def) = module_def .view_by_name_with_global_fn_ptr(view_name) @@ -3135,7 +3178,7 @@ impl ModuleHost { Ok(Self::call_view_inner( instance, - tx, + ScopeGuard::into_inner(tx), view_name, view_id, table_id, @@ -3177,16 +3220,26 @@ impl ModuleHost { view_typespace, }; - instance.common.call_view_with_tx(tx, params, instance.instance) + let (result, trapped) = instance.common.call_view_with_tx(tx, params, instance.instance); + instance.trapped |= trapped; + (result, trapped) } pub async fn init_database(&self, program: Program) -> Result { + self.init_database_with_environment(program, Default::default()).await + } + + pub async fn init_database_with_environment( + &self, + program: Program, + environment: std::collections::BTreeMap, + ) -> Result { call_instance!( self, "", - program, - |p, inst| inst.init_database(p), - |p, inst| inst.init_database(p).await, + (program, environment), + |(program, environment), inst| inst.init_database(program, environment), + |(program, environment), inst| inst.init_database(program, environment).await, )? .map_err(InitDatabaseError::Other) } @@ -3196,13 +3249,24 @@ impl ModuleHost { program: Program, old_module_info: Arc, policy: MigrationPolicy, + ) -> Result { + self.update_database_with_environment(program, old_module_info, policy, Default::default()) + .await + } + + pub async fn update_database_with_environment( + &self, + program: Program, + old_module_info: Arc, + policy: MigrationPolicy, + environment: std::collections::BTreeMap, ) -> Result { call_instance!( self, "", - (program, old_module_info, policy), - |(a, b, c), inst| inst.update_database(a, b, c), - |(a, b, c), inst| inst.update_database(a, b, c).await, + (program, old_module_info, policy, environment), + |(a, b, c, d), inst| inst.update_database(a, b, c, d), + |(a, b, c, d), inst| inst.update_database(a, b, c, d).await, )? } @@ -3660,6 +3724,69 @@ mod tests { use spacetimedb_sats::product; use std::sync::Arc; + #[test] + fn failed_view_materialization_rolls_back_prior_rows_and_subscriber_counts() -> anyhow::Result<()> { + use super::{ViewCallResult, ViewOutcome}; + use crate::db::relational_db::tests_utils::begin_mut_tx; + use spacetimedb_datastore::locking_tx_datastore::{ViewCallInfo, ViewInstanceArgs}; + use spacetimedb_lib::db::raw_def::v10::RawModuleDefV10Builder; + use spacetimedb_lib::ProductType; + use spacetimedb_schema::def::ModuleDef; + + let db = TestDB::in_memory()?; + let mut builder = RawModuleDefV10Builder::new(); + let row = builder.add_algebraic_type( + [], + "Row", + AlgebraicType::Product(ProductType::from_iter([("value", AlgebraicType::U8)])), + true, + ); + builder.add_view( + "earlier", + 0, + true, + true, + ProductType::unit(), + AlgebraicType::array(row.into()), + ); + let module: ModuleDef = builder.finish().try_into()?; + let mut tx = begin_mut_tx(&db); + let (view_id, table_id) = db.create_view(&mut tx, &module, module.view("earlier").unwrap())?; + db.commit_tx(tx)?; + let call = ViewCallInfo::anonymous(view_id); + + for (outcome, trapped) in [ + (ViewOutcome::Failed("denied".into()), false), + (ViewOutcome::Failed("trap".into()), true), + (ViewOutcome::BudgetExceeded, true), + (ViewOutcome::Success, true), + ] { + let mut tx = begin_mut_tx(&db); + // A preceding view succeeded in this same multi-view request. + db.materialize_view_call(&mut tx, table_id, call.clone(), vec![product![9_u8]])?; + tx.subscribe_view(call.clone(), ViewInstanceArgs::Anonymous, Identity::ZERO)?; + assert_eq!(tx.active_subscribers_for_view(view_id), vec![(Identity::ZERO, 1)]); + let mut result = ViewCallResult::default(tx); + result.outcome = outcome; + assert!(result.into_materialized_tx(&db, trapped).is_err()); + + let tx = begin_mut_tx(&db); + assert!(tx.active_subscribers_for_view(view_id).is_empty()); + assert!(!tx.is_view_materialized(&call)?); + assert_eq!(db.iter_mut(&tx, table_id)?.count(), 0); + let _ = db.rollback_mut_tx(tx); + } + // Success retains the owned transaction for the normal commit path. + let mut tx = begin_mut_tx(&db); + db.materialize_view_call(&mut tx, table_id, call.clone(), vec![product![9_u8]])?; + tx.subscribe_view(call, ViewInstanceArgs::Anonymous, Identity::ZERO)?; + db.commit_tx(ViewCallResult::default(tx).into_materialized_tx(&db, false)?)?; + let tx = begin_mut_tx(&db); + assert_eq!(tx.active_subscribers_for_view(view_id), vec![(Identity::ZERO, 1)]); + let _ = db.rollback_mut_tx(tx); + Ok(()) + } + fn v2_client_config() -> ClientConfig { ClientConfig { protocol: Protocol::Binary, diff --git a/crates/core/src/host/v8/mod.rs b/crates/core/src/host/v8/mod.rs index 6f357962bbb..90a83691fd8 100644 --- a/crates/core/src/host/v8/mod.rs +++ b/crates/core/src/host/v8/mod.rs @@ -400,6 +400,7 @@ impl JsInstanceEnv { /// This resets all of the state associated to a single function call, /// and returns instrumentation records. fn finish_funcall(&mut self) -> ExecutionTimings { + self.instance_env.finish_funcall(); let total_duration = self.reducer_start().elapsed(); let func_name = self.log_record_function().unwrap_or("").to_owned(); @@ -424,7 +425,9 @@ impl JsInstanceEnv { } } - fn set_module_def(&mut self, module_def: Arc) { + fn set_module_def(&mut self, module_def: Arc, module_hash: spacetimedb_lib::Hash) { + self.instance_env + .bind_environment_module(module_hash, module_def.clone()); self.module_def = Some(module_def); } @@ -475,11 +478,13 @@ impl JsMainInstance { program: Program, old_module_info: Arc, policy: MigrationPolicy, + environment: std::collections::BTreeMap, ) -> anyhow::Result { self.request(UpdateDatabaseRequest { program, old_module_info, policy, + environment, }) .await } @@ -535,8 +540,12 @@ impl JsMainInstance { self.request(DisconnectClientRequest { client_id }).await } - pub async fn init_database(&self, program: Program) -> anyhow::Result { - self.request(InitDatabaseRequest { program }).await + pub async fn init_database( + &self, + program: Program, + environment: std::collections::BTreeMap, + ) -> anyhow::Result { + self.request(InitDatabaseRequest { program, environment }).await } pub async fn call_view(&self, cmd: ViewCommand) -> ViewCommandResult { @@ -620,6 +629,7 @@ js_main_request! { program: Program, old_module_info: Arc, policy: MigrationPolicy, + environment: std::collections::BTreeMap, } => "update_database", anyhow::Result, UpdateDatabase } @@ -662,6 +672,7 @@ js_main_request! { js_main_request! { InitDatabaseRequest { program: Program, + environment: std::collections::BTreeMap, } => "init_database", anyhow::Result, InitDatabase } @@ -804,6 +815,7 @@ enum JsMainWorkerRequest { program: Program, old_module_info: Arc, policy: MigrationPolicy, + environment: std::collections::BTreeMap, }, /// See [`JsMainInstance::call_reducer`]. CallReducer { @@ -864,6 +876,7 @@ enum JsMainWorkerRequest { InitDatabase { reply_tx: JsReplyTx>, program: Program, + environment: std::collections::BTreeMap, }, } @@ -1398,8 +1411,9 @@ fn handle_main_worker_request( program, old_module_info, policy, + environment, } => handle_worker_request("update_database", reply_tx, || { - let res = instance_common.update_database(program, old_module_info, policy, inst); + let res = instance_common.update_database(program, old_module_info, policy, environment, inst); (res, false) }), JsMainWorkerRequest::CallReducer { reply_tx, params } => { @@ -1496,14 +1510,16 @@ fn handle_main_worker_request( (res, trapped) }) } - JsMainWorkerRequest::InitDatabase { reply_tx, program } => { - handle_worker_request("init_database", reply_tx, || { - let call_reducer = |tx, params| instance_common.call_reducer_with_tx(tx, params, inst); - let (res, trapped): (Result, bool) = - init_database(replica_ctx, &info.module_def, program, call_reducer); - (res, trapped) - }) - } + JsMainWorkerRequest::InitDatabase { + reply_tx, + program, + environment, + } => handle_worker_request("init_database", reply_tx, || { + let call_reducer = |tx, params| instance_common.call_reducer_with_tx(tx, params, inst); + let (res, trapped): (Result, bool) = + init_database(replica_ctx, &info.module_def, program, environment, call_reducer); + (res, trapped) + }), } } @@ -1668,7 +1684,10 @@ where return; } Ok(Ok((crf, module_common))) => { - env_on_isolate_unwrap(scope).set_module_def(module_common.info().module_def.clone()); + env_on_isolate_unwrap(scope).set_module_def( + module_common.info().module_def.clone(), + module_common.info().module_hash, + ); if let Some(result_tx) = startup_result_tx.take() && result_tx.send(Ok(module_common.clone())).is_err() @@ -1872,8 +1891,8 @@ impl WasmInstance for V8Instance<'_, '_, '_> { self.scope.get_slot::().unwrap().instance_env.tx.clone() } - fn set_module_def(&mut self, module_def: Arc) { - env_on_isolate_unwrap(self.scope).set_module_def(module_def); + fn set_module_def(&mut self, module_def: Arc, module_hash: spacetimedb_lib::Hash) { + env_on_isolate_unwrap(self.scope).set_module_def(module_def, module_hash); } fn call_reducer(&mut self, op: ReducerOp<'_>, budget: FunctionBudget) -> ReducerExecuteResult { diff --git a/crates/core/src/host/v8/syscall/common.rs b/crates/core/src/host/v8/syscall/common.rs index 52b78463def..e00cf40d1f0 100644 --- a/crates/core/src/host/v8/syscall/common.rs +++ b/crates/core/src/host/v8/syscall/common.rs @@ -865,9 +865,9 @@ fn call_view( fn_ptr: ViewFnPtr, sender: Option, ) -> SysCallResult { - let prev_func_type = get_env(scope)? + let (prev_func_name, prev_func_type) = get_env(scope)? .instance_env - .swap_func_type(FuncCallType::View(view_call.clone())); + .swap_func_context(Some(view_name.clone()), FuncCallType::View(view_call.clone())); let result = { let args = crate::host::ArgsTuple::nullary(); @@ -900,7 +900,9 @@ fn call_view( } }; - get_env(scope)?.instance_env.swap_func_type(prev_func_type); + get_env(scope)? + .instance_env + .swap_func_context(prev_func_name, prev_func_type); result.map_err(|err| match err { ErrorOrException::Err(err) => TypeError(format!( diff --git a/crates/core/src/host/wasm_common/module_host_actor.rs b/crates/core/src/host/wasm_common/module_host_actor.rs index aee984fe3a5..2733d4cefe3 100644 --- a/crates/core/src/host/wasm_common/module_host_actor.rs +++ b/crates/core/src/host/wasm_common/module_host_actor.rs @@ -37,6 +37,7 @@ use spacetimedb_auth::identity::ConnectionAuthCtx; use spacetimedb_datastore::db_metrics::DB_METRICS; use spacetimedb_datastore::error::{DatastoreError, ViewError}; use spacetimedb_datastore::execution_context::{self, ReducerContext, Workload}; +use spacetimedb_datastore::locking_tx_datastore::state_view::StateView; use spacetimedb_datastore::locking_tx_datastore::{FuncCallType, MutTxId, ViewCallInfo, ViewInstanceArgs}; use spacetimedb_datastore::traits::{IsolationLevel, Program}; use spacetimedb_execution::ExecutionParams; @@ -85,7 +86,7 @@ pub trait WasmInstance { fn tx_slot(&self) -> TxSlot; - fn set_module_def(&mut self, module_def: Arc); + fn set_module_def(&mut self, module_def: Arc, module_hash: Hash); fn call_reducer(&mut self, op: ReducerOp<'_>, budget: FunctionBudget) -> ReducerExecuteResult; @@ -190,6 +191,15 @@ pub(crate) fn run_query_for_view( // Validate shape and disallow views-on-views. for plan in &plans { + // This SQL originates in module code, not an authenticated external + // query. Check every source, including non-returned join inputs, before + // any plan executes. The checked env accessor is the only module path. + ensure!( + !plan + .table_ids() + .any(spacetimedb_datastore::system_tables::is_module_restricted_table), + "module SQL views cannot read a module-restricted table" + ); let Some(source_schema) = plan.return_table() else { bail!("query does not return plain table rows"); }; @@ -431,7 +441,7 @@ impl WasmModuleHostActor { impl WasmModuleHostActor { fn make_from_instance(&self, mut instance: T::Instance) -> WasmModuleInstance { let common = InstanceCommon::new(&self.common); - instance.set_module_def(common.info().module_def.clone()); + instance.set_module_def(common.info().module_def.clone(), common.info().module_hash); WasmModuleInstance { instance, common, @@ -489,9 +499,10 @@ impl WasmModuleInstance { program: Program, old_module_info: Arc, policy: MigrationPolicy, + environment: std::collections::BTreeMap, ) -> anyhow::Result { self.common - .update_database(program, old_module_info, policy, &mut self.instance) + .update_database(program, old_module_info, policy, environment, &mut self.instance) } pub fn call_reducer(&mut self, params: CallReducerParams) -> ReducerCallResult { @@ -545,11 +556,15 @@ impl WasmModuleInstance { res } - pub fn init_database(&mut self, program: Program) -> anyhow::Result { + pub fn init_database( + &mut self, + program: Program, + environment: std::collections::BTreeMap, + ) -> anyhow::Result { let module_def = &self.common.info.clone().module_def; let replica_ctx = &self.instance.replica_ctx().clone(); let call_reducer = |tx, params| self.call_reducer_with_tx_offset(tx, params); - let (res, trapped) = init_database(replica_ctx, module_def, program, call_reducer); + let (res, trapped) = init_database(replica_ctx, module_def, program, environment, call_reducer); self.trapped = trapped; res } @@ -621,6 +636,42 @@ impl WasmModuleInstance { } } +/// Client disconnection and materialized-view refresh are independent effects. +/// A breaking migration does not remove surviving cached view instances. +struct UpdateEffects { + refresh_views: bool, + disconnect_clients: bool, +} + +impl UpdateEffects { + fn after_migration(result: crate::db::update::UpdateResult, tx: &MutTxId) -> Self { + use crate::db::update::UpdateResult; + Self { + refresh_views: matches!(result, UpdateResult::EvaluateSubscribedViews) + || tx.views_for_refresh().next().is_some(), + disconnect_clients: matches!(result, UpdateResult::RequiresClientDisconnect), + } + } + + fn committed( + &self, + tx_offset: TransactionOffset, + durable_offset: Option, + ) -> UpdateDatabaseResult { + if self.disconnect_clients { + UpdateDatabaseResult::UpdatePerformedWithClientDisconnect { + tx_offset, + durable_offset, + } + } else { + UpdateDatabaseResult::UpdatePerformed { + tx_offset, + durable_offset, + } + } + } +} + pub struct InstanceCommon { info: Arc, energy_monitor: Arc, @@ -649,6 +700,7 @@ impl InstanceCommon { program: Program, old_module_info: Arc, policy: MigrationPolicy, + environment: std::collections::BTreeMap, inst: &mut I, ) -> Result { let replica_ctx = inst.replica_ctx().clone(); @@ -674,7 +726,22 @@ impl InstanceCommon { let program_hash = program.hash; let host_type = HostType::from(program.kind); let tx = stdb.begin_mut_tx(IsolationLevel::Serializable, Workload::Internal); - let (mut tx, _) = stdb.with_auto_rollback(tx, |tx| stdb.update_program(tx, program))?; + let (mut tx, _) = stdb.with_auto_rollback(tx, |tx| -> anyhow::Result<()> { + use spacetimedb_datastore::system_tables::{StModuleFields, ST_MODULE_ID}; + let row = tx + .iter(ST_MODULE_ID)? + .next() + .context("database program is not initialized")?; + let current_hash = + spacetimedb_datastore::system_tables::read_hash_from_col(row, StModuleFields::ProgramHash)?; + anyhow::ensure!( + current_hash == old_module_info.module_hash, + "database program changed before publication" + ); + crate::db::environment::replace(stdb, tx, self.info.module_def.environment(), &environment)?; + stdb.update_program(tx, program)?; + Ok(()) + })?; system_logger.info(&format!("Updated program to {program_hash}")); let auth_ctx = AuthCtx::for_current(replica_ctx.database.owner_identity); @@ -717,45 +784,31 @@ impl InstanceCommon { }; let durable_offset = stdb.durable_tx_offset(); - let res: UpdateDatabaseResult = match res { - crate::db::update::UpdateResult::Success => { - let tx_offset = succeed(self.info.clone(), FunctionBudget::ZERO, Duration::ZERO, tx); - UpdateDatabaseResult::UpdatePerformed { - tx_offset, - durable_offset, - } - } - crate::db::update::UpdateResult::EvaluateSubscribedViews => { - let (out, _, trapped) = self.evaluate_subscribed_views(tx, inst)?; - tx = out.tx; - if trapped || out.outcome != ViewOutcome::Success { - let msg = match trapped { - true => "Trapped while evaluating views during database update".to_string(), - false => format!( - "Views evaluation did not complete successfully during database update: {:?}", - out.outcome - ), - }; - - let (_, tx_metrics, reducer) = stdb.rollback_mut_tx(tx); - stdb.report_mut_tx_metrics(reducer, tx_metrics, None); - UpdateDatabaseResult::ErrorExecutingMigration(anyhow::anyhow!(msg)) - } else { - let tx_offset = - succeed(self.info.clone(), out.execution_budget_used, out.total_duration, tx); - UpdateDatabaseResult::UpdatePerformed { - tx_offset, - durable_offset, - } - } - } - crate::db::update::UpdateResult::RequiresClientDisconnect => { - let tx_offset = succeed(self.info.clone(), FunctionBudget::ZERO, Duration::ZERO, tx); - UpdateDatabaseResult::UpdatePerformedWithClientDisconnect { - tx_offset, - durable_offset, - } + let effects = UpdateEffects::after_migration(res, &tx); + let res = if effects.refresh_views { + // Resolve surviving materializations through the new module, + // even when this migration also requires client disconnection. + let (out, _, trapped) = self.evaluate_subscribed_views(tx, inst)?; + tx = out.tx; + if trapped || out.outcome != ViewOutcome::Success { + let msg = match trapped { + true => "Trapped while evaluating views during database update".to_string(), + false => format!( + "Views evaluation did not complete successfully during database update: {:?}", + out.outcome + ), + }; + + let (_, tx_metrics, reducer) = stdb.rollback_mut_tx(tx); + stdb.report_mut_tx_metrics(reducer, tx_metrics, None); + UpdateDatabaseResult::ErrorExecutingMigration(anyhow::anyhow!(msg)) + } else { + let tx_offset = succeed(self.info.clone(), out.execution_budget_used, out.total_duration, tx); + effects.committed(tx_offset, durable_offset) } + } else { + let tx_offset = succeed(self.info.clone(), FunctionBudget::ZERO, Duration::ZERO, tx); + effects.committed(tx_offset, durable_offset) }; Ok(res) @@ -778,32 +831,22 @@ impl InstanceCommon { params: CallProcedureParams, inst: &mut I, ) -> (CallProcedureReturn, bool) { + // Resolve authority and type refs from the same canonical flattened ID. + // A ProcedureDef's local name alone does not identify its host scope. + let (op, procedure_def, owning_def) = + ProcedureOp::for_module(&self.info.module_def, ¶ms).expect("validated procedure id should resolve"); + let procedure_name = op.name.clone(); let CallProcedureParams { timestamp, caller_identity, - caller_connection_id, timer, - procedure_id, - args, + .. } = params; - // We've already validated by this point that the procedure exists, - // so it's fine to use the panicking `procedure_by_id`. - let procedure_def = self.info.module_def.procedure_by_id(procedure_id); - let procedure_name = &procedure_def.name; - // TODO(observability): Add tracing spans, energy, metrics? // These will require further thinking once we implement procedure suspend/resume, // and so are not worth doing yet. - let op = ProcedureOp { - id: procedure_id, - name: procedure_name.clone().into(), - caller_identity, - caller_connection_id, - timestamp, - arg_bytes: args.get_bsatn().clone(), - }; let energy_fingerprint = FunctionFingerprint { module_hash: self.info.module_hash, module_identity: self.info.owner_identity, @@ -833,7 +876,7 @@ impl InstanceCommon { WORKER_METRICS .wasm_instance_errors - .with_label_values(&self.info.database_identity, &self.info.module_hash, procedure_name) + .with_label_values(&self.info.database_identity, &self.info.module_hash, &procedure_name) .inc(); // TODO(procedure-energy): @@ -846,7 +889,7 @@ impl InstanceCommon { } Ok(return_val) => { let return_type = &procedure_def.return_type; - let seed = spacetimedb_sats::WithTypespace::new(self.info.module_def.typespace(), return_type); + let seed = spacetimedb_sats::WithTypespace::new(owning_def.typespace(), return_type); seed.deserialize(bsatn::Deserializer::new(&mut &return_val[..])) .map_err(|err| ProcedureCallError::InternalError(format!("{err}"))) .map(|return_val| ProcedureCallResult { @@ -1167,6 +1210,7 @@ impl InstanceCommon { let mut inst = RefInstance { instance: inst, common: self, + trapped: false, }; let (res, trapped) = match cmds { ViewCommand::AddSingleSubscription { @@ -1253,7 +1297,7 @@ impl InstanceCommon { if let Err(err) = &res { error_target.send(&info.subscriptions, err); } - (res, trapped) + (res, trapped || inst.trapped) } pub(in crate::host) fn handle_sql_cmd( @@ -1264,6 +1308,7 @@ impl InstanceCommon { let mut inst = RefInstance { instance: inst, common: self, + trapped: false, }; let SqlCommand { db, @@ -1280,9 +1325,9 @@ impl InstanceCommon { result: Ok(result), head, }, - trapped, + trapped || inst.trapped, ), - Err(err) => (SqlCommandResult { result: Err(err), head }, false), + Err(err) => (SqlCommandResult { result: Err(err), head }, inst.trapped), } } @@ -1449,6 +1494,7 @@ impl InstanceCommon { let mut instance = RefInstance { common: self, instance: inst, + trapped: false, }; ModuleHost::call_views_with_tx_at(tx, &mut instance, caller, timestamp) } @@ -1962,6 +2008,27 @@ pub struct ProcedureOp { pub arg_bytes: Bytes, } +impl ProcedureOp { + fn for_module<'a>( + module: &'a ModuleDef, + params: &CallProcedureParams, + ) -> Option<(Self, &'a spacetimedb_schema::def::ProcedureDef, &'a ModuleDef)> { + let (name, def, owning) = module.get_procedure_by_id_with_module(params.procedure_id)?; + Some(( + Self { + id: params.procedure_id, + name, + caller_identity: params.caller_identity, + caller_connection_id: params.caller_connection_id, + timestamp: params.timestamp, + arg_bytes: params.args.get_bsatn().clone(), + }, + def, + owning, + )) + } +} + impl InstanceOp for ProcedureOp { fn name(&self) -> &NamespacedIdentifier { &self.name @@ -2006,6 +2073,242 @@ mod tests { use spacetimedb_sats::raw_identifier::RawIdentifier; use spacetimedb_schema::def::ModuleDef; + #[test] + fn breaking_migration_preserves_disconnect_and_refreshes_surviving_environment_view() -> anyhow::Result<()> { + use super::UpdateEffects; + use crate::db::{environment, update}; + use crate::host::UpdateDatabaseResult; + use spacetimedb_datastore::execution_context::Workload; + use spacetimedb_datastore::locking_tx_datastore::FuncCallType; + use spacetimedb_datastore::system_tables::{ModuleKind, ST_ENV_ID}; + use spacetimedb_datastore::traits::Program; + use spacetimedb_lib::db::raw_def::{ + v10::{RawModuleDefV10Builder, RawModuleDefV10Section}, + v9::TableAccess, + }; + use spacetimedb_lib::environment::{EnvironmentConstraint, EnvironmentDeclaration}; + use spacetimedb_lib::identity::AuthCtx; + use spacetimedb_schema::auto_migrate::ponder_migrate; + use std::collections::BTreeMap; + + struct TestLogger; + impl update::UpdateLogger for TestLogger { + fn info(&self, _: &str) {} + } + fn module(include_obsolete_table: bool) -> ModuleDef { + let mut builder = RawModuleDefV10Builder::new(); + let row = builder.add_algebraic_type( + [], + "EnvironmentViewRow", + AlgebraicType::Product(ProductType::from_iter([("value", AlgebraicType::String)])), + true, + ); + builder.add_view( + "environment_view", + 0, + true, + true, + ProductType::unit(), + AlgebraicType::array(AlgebraicType::Ref(row)), + ); + if include_obsolete_table { + builder + .build_table_with_new_type("obsolete", ProductType::from_iter([("id", AlgebraicType::U64)]), true) + .with_access(TableAccess::Public) + .finish(); + } + let mut raw = builder.finish(); + raw.sections + .push(RawModuleDefV10Section::Environment(vec![EnvironmentDeclaration { + name: "TOKEN".into(), + constraint: EnvironmentConstraint::AnyString, + optional: false, + }])); + raw.try_into().expect("valid ENV view module") + } + + let db = TestDB::in_memory()?; + let old = module(true); + let new = module(false); + let before = BTreeMap::from([("TOKEN".into(), "before".into())]); + let after = BTreeMap::from([("TOKEN".into(), "after".into())]); + let mut tx = begin_mut_tx(&db); + db.update_program( + &mut tx, + Program::from_bytes(ModuleKind::WASM, b"old-program".as_slice()), + )?; + for table in old.tables() { + update::create_table_from_def(&db, &mut tx, &old, table)?; + } + let (view_id, _) = db.create_view(&mut tx, &old, old.view("environment_view").unwrap())?; + let call = ViewCallInfo::anonymous(view_id); + // Ordinary SQL materialization has no live subscriber to disconnect. + tx.update_view_timestamp(call.clone(), ViewInstanceArgs::Anonymous)?; + tx.record_table_scan(&FuncCallType::View(call.clone()), ST_ENV_ID); + environment::replace(&db, &mut tx, old.environment(), &before)?; + db.commit_tx(tx)?; + + let mut tx = begin_mut_tx(&db); + environment::replace(&db, &mut tx, new.environment(), &after)?; + db.update_program( + &mut tx, + Program::from_bytes(ModuleKind::WASM, b"new-program".as_slice()), + )?; + let plan = ponder_migrate(&old, &new)?; + assert!( + plan.breaks_client(), + "removing the unrelated table must require disconnection" + ); + let result = update::update_database(&db, &mut tx, AuthCtx::for_testing(), plan, &TestLogger)?; + assert!(matches!(result, update::UpdateResult::RequiresClientDisconnect)); + assert!(tx.views_for_refresh().any(|dirty| *dirty == call)); + let effects = UpdateEffects::after_migration(result, &tx); + assert!(effects.refresh_views); + assert!(effects.disconnect_clients); + let calls = collect_subscribed_view_calls(&tx, &new, Identity::ZERO)?; + assert_eq!(calls.len(), 1); + assert_eq!(calls[0].view_id, view_id); + assert_eq!(&*calls[0].view_name, "environment_view"); + assert!(tx.active_subscribers_for_view(view_id).is_empty()); + let (_send, receive) = tokio::sync::oneshot::channel(); + assert!(matches!( + effects.committed(receive, None), + UpdateDatabaseResult::UpdatePerformedWithClientDisconnect { .. } + )); + + // The failed-view branch can roll back the same complete migration tx. + let _ = db.rollback_mut_tx(tx); + db.with_read_only(Workload::ForTests, |tx| { + assert_eq!(environment::snapshot(tx).unwrap(), before); + }); + let tx = begin_mut_tx(&db); + assert!(db.table_id_from_name_mut(&tx, "obsolete")?.is_some()); + let _ = db.rollback_mut_tx(tx); + Ok(()) + } + + #[test] + fn module_sql_views_cannot_read_environment_directly_or_through_a_join() -> anyhow::Result<()> { + use super::run_query_for_view; + use crate::db::environment; + use spacetimedb_lib::environment::{EnvironmentConstraint, EnvironmentDeclaration, EnvironmentSchema}; + use spacetimedb_primitives::ViewId; + use spacetimedb_sats::product; + use std::collections::BTreeMap; + + let db = TestDB::in_memory()?; + let visible = db.create_table_for_test( + "visible", + &[("key", AlgebraicType::String), ("value", AlgebraicType::String)], + &[0.into()], + )?; + let mut tx = begin_mut_tx(&db); + tx.insert_via_serialize_bsatn(visible, &product!("TOKEN", "ordinary-value"))?; + let schema = EnvironmentSchema::new(vec![EnvironmentDeclaration { + name: "TOKEN".into(), + constraint: EnvironmentConstraint::AnyString, + optional: false, + }])?; + environment::replace( + &db, + &mut tx, + &schema, + &BTreeMap::from([("TOKEN".into(), "private-value".into())]), + )?; + let row_type = ProductType::from_iter([("key", AlgebraicType::String), ("value", AlgebraicType::String)]); + let call = ViewCallInfo::anonymous(ViewId(99)); + let run = |tx: &mut _, query| run_query_for_view(tx, query, &row_type, &call, db.database_identity()); + assert_eq!( + run(&mut tx, "SELECT * FROM visible")?, + vec![product!("TOKEN", "ordinary-value")] + ); + for query in [ + "SELECT * FROM st_env", + "SELECT e.* FROM st_env AS e", + "SELECT v.* FROM visible AS v JOIN st_env AS e ON v.key = e.key", + "SELECT e.* FROM visible AS v JOIN st_env AS e ON v.key = e.key", + ] { + let error = run(&mut tx, query).expect_err("module SQL must use the checked environment accessor"); + assert!( + error.to_string().contains("module-restricted table"), + "unexpected query failure for {query}: {error:#}" + ); + assert!(!error.to_string().contains("private-value")); + } + let _ = db.rollback_mut_tx(tx); + Ok(()) + } + + #[test] + fn procedure_operations_resolve_root_and_nested_host_scope_and_typespace() { + use super::{CallProcedureParams, InstanceOp, ProcedureOp}; + use crate::host::ArgsTuple; + use spacetimedb_lib::db::raw_def::v10::{ + RawModuleDefV10, RawModuleDefV10Builder, RawModuleDefV10Section, RawSubmoduleV10, + }; + use spacetimedb_lib::de::DeserializeSeed; + use spacetimedb_lib::Timestamp; + use spacetimedb_primitives::ProcedureId; + use spacetimedb_sats::{bsatn, AlgebraicValue, ProductValue, WithTypespace}; + + fn module(value_type: AlgebraicType) -> RawModuleDefV10 { + let mut builder = RawModuleDefV10Builder::new(); + let result_type = builder.add_algebraic_type( + [], + "ResultRow", + AlgebraicType::Product(ProductType::from_iter([("value", value_type)])), + true, + ); + builder.add_procedure("read_env", ProductType::unit(), AlgebraicType::Ref(result_type)); + builder.finish() + } + + let mut child = module(AlgebraicType::String); + child + .sections + .push(RawModuleDefV10Section::Submodules(vec![RawSubmoduleV10 { + namespace: "nested".into(), + module: module(AlgebraicType::Bool), + }])); + let mut root = module(AlgebraicType::U64); + root.sections + .push(RawModuleDefV10Section::Submodules(vec![RawSubmoduleV10 { + namespace: "lib".into(), + module: child, + }])); + let module: ModuleDef = root.try_into().expect("valid nested module"); + for (index, expected_name) in ["read_env", "lib.read_env", "lib.nested.read_env"] + .into_iter() + .enumerate() + { + let params = CallProcedureParams::from_system( + Timestamp::UNIX_EPOCH, + Identity::ZERO, + ProcedureId::from(index), + ArgsTuple::nullary(), + ); + let (op, def, owning) = ProcedureOp::for_module(&module, ¶ms).unwrap(); + assert_eq!(&**op.name(), expected_name); + assert_eq!(op.name().is_namespaced(), index != 0); + assert_eq!(op.id, params.procedure_id); + assert_eq!(&*def.name, "read_env", "declaration names remain local"); + if index == 2 { + let expected = AlgebraicValue::Product(ProductValue::from_iter([AlgebraicValue::Bool(true)])); + let bytes = bsatn::to_vec(&expected).unwrap(); + let decoded = WithTypespace::new(owning.typespace(), &def.return_type) + .deserialize(bsatn::Deserializer::new(&mut &bytes[..])) + .unwrap(); + assert_eq!( + decoded, expected, + "nested type refs must not use the root U64 typespace" + ); + } + } + assert!(module + .get_procedure_by_id_with_module(ProcedureId::from(3usize)) + .is_none()); + } + fn module_def_for_view(name: &str, is_anonymous: bool) -> ModuleDef { let mut builder = RawModuleDefV9Builder::new(); let name = RawIdentifier::new(name); diff --git a/crates/core/src/host/wasmtime/wasm_instance_env.rs b/crates/core/src/host/wasmtime/wasm_instance_env.rs index 23d33983440..0c22ee54694 100644 --- a/crates/core/src/host/wasmtime/wasm_instance_env.rs +++ b/crates/core/src/host/wasmtime/wasm_instance_env.rs @@ -298,7 +298,9 @@ impl WasmInstanceEnv { self.mem = Some(mem); } - pub fn set_module_def(&mut self, module_def: Arc) { + pub fn set_module_def(&mut self, module_def: Arc, module_hash: spacetimedb_lib::Hash) { + self.instance_env + .bind_environment_module(module_hash, module_def.clone()); self.module_def = Some(module_def) } @@ -379,6 +381,7 @@ impl WasmInstanceEnv { /// /// This resets the call times and clears the arguments source and error sink. pub fn finish_funcall(&mut self, result_sink: u32) -> (ExecutionTimings, Vec) { + self.instance_env.finish_funcall(); // For the moment, // we only explicitly clear the source/sink buffers and the "syscall" times. // TODO: should we be clearing `iters` and/or `timing_spans`? @@ -1889,10 +1892,10 @@ impl WasmInstanceEnv { fn_ptr: ViewFnPtr, sender: Option, ) -> anyhow::Result { - let prev_func_type = caller + let (prev_func_name, prev_func_type) = caller .data_mut() .instance_env - .swap_func_type(FuncCallType::View(view_call.clone())); + .swap_func_context(Some(view_name.clone()), FuncCallType::View(view_call.clone())); let mut nested_result_sink = None; let call_result = (|| -> anyhow::Result { @@ -1924,7 +1927,10 @@ impl WasmInstanceEnv { Ok(code) })(); - caller.data_mut().instance_env.swap_func_type(prev_func_type); + caller + .data_mut() + .instance_env + .swap_func_context(prev_func_name, prev_func_type); let result_bytes = { let env = caller.data_mut(); diff --git a/crates/core/src/host/wasmtime/wasmtime_module.rs b/crates/core/src/host/wasmtime/wasmtime_module.rs index a5316ceb95c..aa589df3b5a 100644 --- a/crates/core/src/host/wasmtime/wasmtime_module.rs +++ b/crates/core/src/host/wasmtime/wasmtime_module.rs @@ -625,8 +625,8 @@ impl module_host_actor::WasmInstance for WasmtimeInstance { self.store.data().instance_env().tx.clone() } - fn set_module_def(&mut self, module_def: Arc) { - self.store.data_mut().set_module_def(module_def); + fn set_module_def(&mut self, module_def: Arc, module_hash: spacetimedb_lib::Hash) { + self.store.data_mut().set_module_def(module_def, module_hash); } #[tracing::instrument(level = "trace", skip_all)] diff --git a/crates/core/src/sql/execute.rs b/crates/core/src/sql/execute.rs index e927921c886..1f2c587b762 100644 --- a/crates/core/src/sql/execute.rs +++ b/crates/core/src/sql/execute.rs @@ -91,7 +91,7 @@ fn run_inner( let stmt = compile_sql_stmt(&sql_text, &SchemaViewer::new(tx, &auth), &auth)?; // Check mutation authority while the automatic rollback guard owns // the transaction, including rejected administrative statements. - if matches!(&stmt, Statement::DML(_) | Statement::Environment(_)) && !auth.has_write_access() { + if matches!(&stmt, Statement::DML(_)) && !auth.has_write_access() { return Err(anyhow!( "Caller {} is not authorized to run SQL mutations", auth.caller() @@ -101,7 +101,7 @@ fn run_inner( && dml.table_id() == spacetimedb_datastore::system_tables::ST_ENV_ID { return Err(anyhow!( - "Use SET env.KEY or DELETE env.KEY to modify database environment variables" + "Database environment variables can only be changed by publishing" )); } Ok(stmt) @@ -157,19 +157,9 @@ fn run_inner( trapped, )) } - stmt @ (Statement::DML(_) | Statement::Environment(_)) => { - // Evaluate the mutation + Statement::DML(stmt) => { let (mut tx, _) = db.with_auto_rollback(tx, |tx| -> anyhow::Result<()> { - match stmt { - Statement::DML(stmt) => execute_dml_stmt(&auth, stmt, tx, &mut metrics)?, - Statement::Environment(environment) => match environment.value { - Some(value) => crate::db::environment::set(&db, tx, &environment.key, &value)?, - None => { - crate::db::environment::delete(&db, tx, &environment.key)?; - } - }, - Statement::Select(_) => unreachable!(), - } + execute_dml_stmt(&auth, stmt, tx, &mut metrics)?; Ok(()) })?; @@ -267,8 +257,10 @@ pub(crate) mod tests { use spacetimedb_schema::table_name::TableName; #[test] - fn environment_sql_enforces_permissions_escaping_limits_and_rollback() { + fn environment_sql_is_read_only_including_for_owner() { + use spacetimedb_lib::environment::{EnvironmentConstraint, EnvironmentDeclaration, EnvironmentSchema}; use spacetimedb_lib::identity::SqlPermission; + use std::collections::BTreeMap; let db = TestDB::in_memory().unwrap(); let runtime = tokio::runtime::Runtime::new().unwrap(); let owner = AuthCtx::for_current(Identity::ZERO); @@ -277,63 +269,48 @@ pub(crate) mod tests { Arc::new(|permission| matches!(permission, SqlPermission::Read(_))), ); let outsider = AuthCtx::new(Identity::ZERO, Identity::ONE); + let value = "secret-marker"; + let schema = EnvironmentSchema::new(vec![EnvironmentDeclaration { + name: "TOKEN".into(), + constraint: EnvironmentConstraint::AnyString, + optional: false, + }]) + .unwrap(); + db.with_auto_commit(Workload::ForTests, |tx| { + crate::db::environment::replace(&db, tx, &schema, &BTreeMap::from([("TOKEN".into(), value.into())])) + }) + .unwrap(); let execute = |statement: &str, auth: AuthCtx| { runtime.block_on(run(db.clone(), statement.to_string(), auth, None, None, &mut vec![])) }; - let value = "it's \\quoted;\nUTF-8 é\0tail"; - execute( - &format!( - "/* prefix */ SET env.Mixed_Key = '{}'; -- suffix", - value.replace('\'', "''") - ), - owner.clone(), - ) - .unwrap(); - execute("SET env.EMPTY TO ''", owner.clone()).unwrap(); - let rows = execute("SELECT value FROM st_env WHERE key = 'Mixed_Key'", viewer.clone()) - .unwrap() - .rows; - assert_eq!(rows, vec![product![value]]); - assert!(execute("SELECT * FROM st_env", outsider.clone()).is_err()); - for auth in [viewer, outsider] { - assert!(execute("SET env.Mixed_Key = 'forbidden'", auth.clone()).is_err()); - assert!(execute("DELETE env.Mixed_Key", auth).is_err()); - } - for statement in [ - "SET env.EMPTY = 5".to_string(), - "SET env.\"BAD-KEY\" = 'value'".to_string(), - format!("SET env.EMPTY = '{}'", "x".repeat(8193)), - "SET env.EMPTY = 'changed'; DELETE env.Mixed_Key".to_string(), - "DELETE env.Mixed_Key WHERE true".to_string(), - "INSERT INTO st_env (key, value) VALUES ('BYPASS', 'value')".to_string(), - "UPDATE st_env SET value = 'bypass'".to_string(), - "DELETE FROM st_env".to_string(), - ] { - assert!( - execute(&statement, owner.clone()).is_err(), - "unexpectedly accepted {statement}" - ); - } - // Every rejected path released its transaction and preserved old data. assert_eq!( - execute("SELECT value FROM st_env WHERE key = 'EMPTY'", owner.clone()) + execute("SELECT value FROM st_env WHERE key = 'TOKEN'", viewer.clone()) .unwrap() .rows, - vec![product![""]] + vec![product![value]] ); + assert!(execute("SELECT * FROM st_env", outsider.clone()).is_err()); + for auth in [owner.clone(), viewer, outsider] { + for statement in [ + "SET env.TOKEN = 'forbidden'", + "DELETE env.TOKEN", + "INSERT INTO st_env (key, value) VALUES ('BYPASS', 'value')", + "UPDATE st_env SET value = 'bypass'", + "DELETE FROM st_env", + ] { + assert!( + execute(statement, auth.clone()).is_err(), + "unexpectedly accepted {statement}" + ); + } + } + // Rejected writes must release their transactions and preserve data. assert_eq!( - execute("SELECT value FROM st_env WHERE key = 'Mixed_Key'", owner.clone()) + execute("SELECT value FROM st_env WHERE key = 'TOKEN'", owner) .unwrap() .rows, vec![product![value]] ); - execute("SET env.EMPTY = 'updated'", owner.clone()).unwrap(); - execute("DELETE env.EMPTY", owner.clone()).unwrap(); - execute("DELETE env.EMPTY", owner.clone()).unwrap(); - assert!(execute("SELECT value FROM st_env WHERE key = 'EMPTY'", owner) - .unwrap() - .rows - .is_empty()); } /// Short-cut for simplify test execution diff --git a/crates/expr/src/errors.rs b/crates/expr/src/errors.rs index d61ade64f54..1e7315cb4eb 100644 --- a/crates/expr/src/errors.rs +++ b/crates/expr/src/errors.rs @@ -132,10 +132,6 @@ pub struct DmlOnView { #[derive(Error, Debug)] pub enum TypingError { - #[error(transparent)] - Environment(#[from] spacetimedb_lib::environment::EnvironmentValidationError), - #[error("environment values must be SQL string literals")] - EnvironmentValueType, #[error(transparent)] Unsupported(#[from] Unsupported), #[error(transparent)] diff --git a/crates/expr/src/statement.rs b/crates/expr/src/statement.rs index 3293553f1b6..9fbdb869a20 100644 --- a/crates/expr/src/statement.rs +++ b/crates/expr/src/statement.rs @@ -31,12 +31,6 @@ use super::{ pub enum Statement { Select(ProjectList), DML(DML), - Environment(EnvironmentWrite), -} - -pub struct EnvironmentWrite { - pub key: Box, - pub value: Option>, } pub enum DML { @@ -460,21 +454,6 @@ pub fn parse_and_type_sql(sql: &str, tx: &impl SchemaView, _auth: &AuthCtx) -> T SqlAst::Update(update) => Ok(Statement::DML(DML::Update(type_update(update, tx)?))), SqlAst::Set(set) => Ok(Statement::DML(DML::Insert(type_and_rewrite_set(set, tx)?))), SqlAst::Show(show) => Ok(Statement::Select(type_and_rewrite_show(show, tx)?)), - SqlAst::Environment(environment) => { - // Resolve through the normal private-table visibility check. - tx.schema("st_env").ok_or_else(|| Unresolved::table("st_env"))?; - let key = &*environment.key.0; - spacetimedb_lib::environment::validate_key(key)?; - let value = match environment.value { - None => None, - Some(SqlLiteral::Str(value)) => { - spacetimedb_lib::environment::validate_value(&value)?; - Some(value) - } - Some(_) => return Err(TypingError::EnvironmentValueType), - }; - Ok(Statement::Environment(EnvironmentWrite { key: key.into(), value })) - } } } diff --git a/crates/lib/src/db/raw_def/v10.rs b/crates/lib/src/db/raw_def/v10.rs index b21d8470b84..9aec59ffaa8 100644 --- a/crates/lib/src/db/raw_def/v10.rs +++ b/crates/lib/src/db/raw_def/v10.rs @@ -101,6 +101,9 @@ pub enum RawModuleDefV10Section { /// Submodules, keyed by the namespace they are registered under. Submodules(Vec), + + /// Declared publish-only configuration. Even an empty section requires ENV support. + Environment(Vec), } #[derive(Debug, Clone, SpacetimeType)] @@ -715,6 +718,27 @@ impl RawModuleDefV10Builder { Default::default() } + /// Declare a complete environment schema, including an explicit empty schema. + /// Repeated calls remain repeated sections so host validation rejects ambiguity. + pub fn add_environment(&mut self, declarations: Vec) -> &mut Self { + self.module + .sections + .push(RawModuleDefV10Section::Environment(declarations)); + self + } + + /// New ENV-aware bindings declare an empty schema when no declaration is registered. + pub fn ensure_environment(&mut self) { + if !self + .module + .sections + .iter() + .any(|section| matches!(section, RawModuleDefV10Section::Environment(_))) + { + self.add_environment(Vec::new()); + } + } + /// Get mutable access to the typespace section, creating it if missing. fn typespace_mut(&mut self) -> &mut Typespace { let idx = self diff --git a/crates/lib/src/environment.rs b/crates/lib/src/environment.rs index 17b5c3f3293..8b6f6db38d8 100644 --- a/crates/lib/src/environment.rs +++ b/crates/lib/src/environment.rs @@ -3,6 +3,9 @@ pub const MAX_ENV_KEY_BYTES: usize = 256; pub const MAX_ENV_VALUE_BYTES: usize = 8 * 1024; pub const MAX_ENV_VARS: usize = 256; +/// Total key and literal bytes in a declaration schema, independent of runtime values. +pub const MAX_ENV_SCHEMA_BYTES: usize = 2 * 1024 * 1024; +pub const MAX_ENV_UNION_ENTRIES: usize = 256; #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum EnvironmentValidationError { @@ -60,3 +63,321 @@ mod tests { assert!(validate_value(&"é".repeat(4097)).is_err()); } } + +/// Host-validated string constraints. Values remain strings in every module SDK. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, crate::SpacetimeType)] +#[sats(crate = crate)] +pub enum EnvironmentConstraint { + AnyString, + Literal(String), + OneOf(Vec), +} + +/// Declaration metadata, never an environment value supplied during publishing. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, crate::SpacetimeType)] +#[sats(crate = crate)] +pub struct EnvironmentDeclaration { + pub name: String, + pub constraint: EnvironmentConstraint, + pub optional: bool, +} + +/// An environment schema whose declarations have passed host validation. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct EnvironmentSchema { + declarations: std::collections::BTreeMap, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub enum EnvironmentSchemaErrorKind { + InvalidName, + TooManyDeclarations, + DuplicateDeclaration, + EmptyUnion, + TooManyUnionEntries, + SchemaTooLarge, + LiteralTooLarge, + Undeclared, + MissingRequired, + ValueTooLarge, + ConstraintMismatch, +} + +/// Errors identify a key and rule, and never contain a supplied or allowed value. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub struct EnvironmentSchemaError { + pub key: Option, + pub kind: EnvironmentSchemaErrorKind, +} + +impl std::fmt::Display for EnvironmentSchemaError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + if let Some(key) = &self.key { + write!(f, "environment key {key:?}: ")?; + } + f.write_str(match self.kind { + EnvironmentSchemaErrorKind::InvalidName => "invalid name", + EnvironmentSchemaErrorKind::TooManyDeclarations => "too many declarations", + EnvironmentSchemaErrorKind::DuplicateDeclaration => "duplicate declaration", + EnvironmentSchemaErrorKind::EmptyUnion => "string union must not be empty", + EnvironmentSchemaErrorKind::TooManyUnionEntries => "string union has too many entries", + EnvironmentSchemaErrorKind::SchemaTooLarge => "declaration schema exceeds size limit", + EnvironmentSchemaErrorKind::LiteralTooLarge => "declared literal exceeds value size limit", + EnvironmentSchemaErrorKind::Undeclared => "key is not declared", + EnvironmentSchemaErrorKind::MissingRequired => "required value is missing", + EnvironmentSchemaErrorKind::ValueTooLarge => "value exceeds size limit", + EnvironmentSchemaErrorKind::ConstraintMismatch => "value does not satisfy its declared string constraint", + }) + } +} + +impl std::error::Error for EnvironmentSchemaError {} + +impl EnvironmentSchema { + fn validate_metadata(declarations: &[EnvironmentDeclaration]) -> Result<(), EnvironmentSchemaError> { + use EnvironmentSchemaErrorKind as Kind; + if declarations.len() > MAX_ENV_VARS { + return Err(EnvironmentSchemaError { + key: None, + kind: Kind::TooManyDeclarations, + }); + } + let mut bytes = 0usize; + for declaration in declarations { + // Never retain or format unvalidated key bytes in diagnostics. + validate_key(&declaration.name).map_err(|_| EnvironmentSchemaError { + key: None, + kind: Kind::InvalidName, + })?; + let error = |kind| EnvironmentSchemaError { + key: Some(declaration.name.clone()), + kind, + }; + bytes += declaration.name.len(); + if bytes > MAX_ENV_SCHEMA_BYTES { + return Err(error(Kind::SchemaTooLarge)); + } + let literals = match &declaration.constraint { + EnvironmentConstraint::AnyString => &[][..], + EnvironmentConstraint::Literal(value) => std::slice::from_ref(value), + EnvironmentConstraint::OneOf(values) => { + if values.is_empty() { + return Err(error(Kind::EmptyUnion)); + } + if values.len() > MAX_ENV_UNION_ENTRIES { + return Err(error(Kind::TooManyUnionEntries)); + } + values.as_slice() + } + }; + for value in literals { + validate_value(value).map_err(|_| error(Kind::LiteralTooLarge))?; + bytes += value.len(); + if bytes > MAX_ENV_SCHEMA_BYTES { + return Err(error(Kind::SchemaTooLarge)); + } + } + } + Ok(()) + } + + pub fn new(declarations: Vec) -> Result { + Self::validate_metadata(&declarations)?; + let mut schema = Self::default(); + for mut declaration in declarations { + if let EnvironmentConstraint::OneOf(values) = &mut declaration.constraint { + values.sort_unstable(); + values.dedup(); + } + if schema.declarations.contains_key(&declaration.name) { + return Err(EnvironmentSchemaError { + key: Some(declaration.name), + kind: EnvironmentSchemaErrorKind::DuplicateDeclaration, + }); + } + schema.declarations.insert(declaration.name.clone(), declaration); + } + Ok(schema) + } + + /// Check bounds before cloning raw untrusted metadata into the validated schema. + pub fn from_declarations(declarations: &[EnvironmentDeclaration]) -> Result { + Self::validate_metadata(declarations)?; + Self::new(declarations.to_vec()) + } + + pub fn get(&self, name: &str) -> Option<&EnvironmentDeclaration> { + self.declarations.get(name) + } + + pub fn declarations(&self) -> impl ExactSizeIterator { + self.declarations.values() + } + + pub fn into_declarations(self) -> Vec { + self.declarations.into_values().collect() + } + + pub fn is_empty(&self) -> bool { + self.declarations.is_empty() + } + + /// Validate a complete publish input. Existing stored values are not inputs. + pub fn validate_values( + &self, + values: &std::collections::BTreeMap, + ) -> Result<(), EnvironmentSchemaError> { + use EnvironmentSchemaErrorKind as Kind; + for (name, value) in values { + validate_key(name).map_err(|_| EnvironmentSchemaError { + key: None, + kind: Kind::InvalidName, + })?; + let error = |kind| EnvironmentSchemaError { + key: Some(name.clone()), + kind, + }; + let declaration = self.get(name).ok_or_else(|| error(Kind::Undeclared))?; + validate_value(value).map_err(|_| error(Kind::ValueTooLarge))?; + let matches = match &declaration.constraint { + EnvironmentConstraint::AnyString => true, + EnvironmentConstraint::Literal(expected) => value == expected, + EnvironmentConstraint::OneOf(allowed) => allowed.binary_search(value).is_ok(), + }; + if !matches { + return Err(error(Kind::ConstraintMismatch)); + } + } + for declaration in self.declarations() { + if !declaration.optional && !values.contains_key(&declaration.name) { + return Err(EnvironmentSchemaError { + key: Some(declaration.name.clone()), + kind: Kind::MissingRequired, + }); + } + } + Ok(()) + } +} + +#[cfg(test)] +mod schema_tests { + use super::*; + use std::collections::BTreeMap; + + fn declaration(name: &str, constraint: EnvironmentConstraint, optional: bool) -> EnvironmentDeclaration { + EnvironmentDeclaration { + name: name.into(), + constraint, + optional, + } + } + + #[test] + fn complete_inputs_preserve_optional_empty_and_exact_string_constraints() { + let schema = EnvironmentSchema::new(vec![ + declaration("REQUIRED", EnvironmentConstraint::AnyString, false), + declaration( + "MODE", + EnvironmentConstraint::OneOf(vec!["false".into(), "true".into(), "false".into()]), + false, + ), + declaration("OPTIONAL", EnvironmentConstraint::Literal("".into()), true), + ]) + .unwrap(); + let mut values = BTreeMap::from([("REQUIRED".into(), "\0雪".into()), ("MODE".into(), "false".into())]); + schema.validate_values(&values).unwrap(); + values.insert("OPTIONAL".into(), "".into()); + schema.validate_values(&values).unwrap(); + values.insert("MODE".into(), "False".into()); + assert_eq!( + schema.validate_values(&values).unwrap_err().kind, + EnvironmentSchemaErrorKind::ConstraintMismatch + ); + values.remove("MODE"); + assert_eq!( + schema.validate_values(&values).unwrap_err().kind, + EnvironmentSchemaErrorKind::MissingRequired + ); + values.insert("UNDECLARED".into(), "secret-marker".into()); + let error = schema.validate_values(&values).unwrap_err(); + assert_eq!(error.kind, EnvironmentSchemaErrorKind::Undeclared); + assert!(!format!("{error:?}: {error}").contains("secret-marker")); + } + + #[test] + fn declaration_limits_count_absent_optionals_and_reject_invalid_metadata() { + let optional = declaration("A", EnvironmentConstraint::AnyString, true); + assert_eq!( + EnvironmentSchema::new(vec![optional.clone(), optional]) + .unwrap_err() + .kind, + EnvironmentSchemaErrorKind::DuplicateDeclaration + ); + assert_eq!( + EnvironmentSchema::new( + (0..=MAX_ENV_VARS) + .map(|i| declaration(&format!("K{i}"), EnvironmentConstraint::AnyString, true)) + .collect() + ) + .unwrap_err() + .kind, + EnvironmentSchemaErrorKind::TooManyDeclarations + ); + for (name, constraint, expected) in [ + ( + "A-B", + EnvironmentConstraint::AnyString, + EnvironmentSchemaErrorKind::InvalidName, + ), + ( + "A", + EnvironmentConstraint::OneOf(vec![]), + EnvironmentSchemaErrorKind::EmptyUnion, + ), + ( + "A", + EnvironmentConstraint::Literal("x".repeat(MAX_ENV_VALUE_BYTES + 1)), + EnvironmentSchemaErrorKind::LiteralTooLarge, + ), + ] { + assert_eq!( + EnvironmentSchema::new(vec![declaration(name, constraint, true)]) + .unwrap_err() + .kind, + expected + ); + } + assert!(EnvironmentSchema::default().validate_values(&BTreeMap::new()).is_ok()); + assert_eq!( + EnvironmentSchema::default() + .validate_values(&BTreeMap::from([("A".into(), "".into())])) + .unwrap_err() + .kind, + EnvironmentSchemaErrorKind::Undeclared + ); + } + + #[test] + fn raw_metadata_is_bounded_before_copying_or_formatting_untrusted_keys() { + let invalid = format!("private-marker\n{}", "x".repeat(100_000)); + let error = + EnvironmentSchema::new(vec![declaration(&invalid, EnvironmentConstraint::AnyString, true)]).unwrap_err(); + assert_eq!(error.key, None); + assert!(!format!("{error:?}: {error}").contains("private-marker")); + let error = EnvironmentSchema::new(vec![declaration( + "A", + EnvironmentConstraint::OneOf(vec!["".into(); MAX_ENV_UNION_ENTRIES + 1]), + true, + )]) + .unwrap_err(); + assert_eq!(error.kind, EnvironmentSchemaErrorKind::TooManyUnionEntries); + let error = EnvironmentSchema::new(vec![declaration( + "A", + EnvironmentConstraint::OneOf(vec!["x".repeat(MAX_ENV_VALUE_BYTES); MAX_ENV_UNION_ENTRIES]), + true, + )]) + .unwrap_err(); + assert_eq!(error.kind, EnvironmentSchemaErrorKind::SchemaTooLarge); + } +} diff --git a/crates/query/src/lib.rs b/crates/query/src/lib.rs index 69168032a0a..717f9d38092 100644 --- a/crates/query/src/lib.rs +++ b/crates/query/src/lib.rs @@ -63,7 +63,7 @@ pub fn compile_sql_stmt(sql: &str, tx: &impl SchemaView, auth: &AuthCtx) -> Resu } match parse_and_type_sql(sql, tx, auth)? { - stmt @ (Statement::DML(_) | Statement::Environment(_)) => Ok(stmt), + stmt @ Statement::DML(_) => Ok(stmt), Statement::Select(expr) => Ok(Statement::Select(resolve_views_for_sql(tx, expr, auth)?)), } } diff --git a/crates/schema/src/def.rs b/crates/schema/src/def.rs index 521bcb35aa9..2516894ceea 100644 --- a/crates/schema/src/def.rs +++ b/crates/schema/src/def.rs @@ -179,6 +179,9 @@ pub struct ModuleDef { /// Submodules, keyed by the namespace they are registered under. submodules: IndexMap, + + environment: spacetimedb_lib::environment::EnvironmentSchema, + environment_declared: bool, } #[derive(Debug, Clone, Copy, Eq, PartialEq)] @@ -190,6 +193,16 @@ pub enum RawModuleDefVersion { } impl ModuleDef { + /// The validated root environment schema. Legacy modules have an empty schema. + pub fn environment(&self) -> &spacetimedb_lib::environment::EnvironmentSchema { + &self.environment + } + + /// Whether the raw module explicitly required environment support. + pub fn environment_declared(&self) -> bool { + self.environment_declared + } + /// The raw module definition version this module was authored under. pub fn raw_module_def_version(&self) -> RawModuleDefVersion { self.raw_module_def_version @@ -823,15 +836,27 @@ impl ModuleDef { /// Look up a procuedure by its id, returning `None` if it doesn't exist. pub fn get_procedure_by_id(&self, id: ProcedureId) -> Option<&ProcedureDef> { + self.get_procedure_by_id_with_module(id).map(|(_, def, _)| def) + } + + /// Resolve a flattened wire ID to its host-qualified name, definition and + /// owning module. Procedure definitions store local names and type refs. + pub fn get_procedure_by_id_with_module( + &self, + id: ProcedureId, + ) -> Option<(NamespacedIdentifier, &ProcedureDef, &ModuleDef)> { let idx = id.idx(); if idx < self.procedures.len() { - return self.procedures.get_index(idx).map(|(_, def)| def); + return self + .procedures + .get_index(idx) + .map(|(_, def)| (self.path.join(def.name.clone()), def, self)); } let mut offset = self.procedures.len(); for submodule in self.submodules.values() { let count = submodule.procedure_count(); if idx < offset + count { - return submodule.get_procedure_by_id(ProcedureId::from(idx - offset)); + return submodule.get_procedure_by_id_with_module(ProcedureId::from(idx - offset)); } offset += count; } @@ -986,6 +1011,8 @@ impl From for RawModuleDefV9 { http_routes: _, raw_module_def_version: _, submodules: _, + environment: _, + environment_declared: _, } = val; // Extract column defaults from tables before consuming tables @@ -1046,9 +1073,14 @@ impl From for RawModuleDefV10 { http_routes, raw_module_def_version: _, submodules, + environment, + environment_declared, } = val; let mut sections = Vec::new(); + if environment_declared { + sections.push(RawModuleDefV10Section::Environment(environment.into_declarations())); + } let mut explicit_names = ExplicitNames::default(); sections.push(RawModuleDefV10Section::Typespace(typespace)); diff --git a/crates/schema/src/def/validate/v10.rs b/crates/schema/src/def/validate/v10.rs index d6303f3a81c..f164642a615 100644 --- a/crates/schema/src/def/validate/v10.rs +++ b/crates/schema/src/def/validate/v10.rs @@ -78,6 +78,7 @@ impl From for ValidationCase { /// Validate a `RawModuleDefV10` and convert it into a `ModuleDef`, /// or return a stream of errors if the definition is invalid. pub fn validate(def: RawModuleDefV10) -> Result { + let environment = validate_environment(&def); let mut typespace = def.typespace().cloned().unwrap_or_else(|| Typespace::EMPTY.clone()); let known_type_definitions = def.types().into_iter().flatten().map(|def| def.ty); let case_policy = def.case_conversion_policy().into(); @@ -297,10 +298,13 @@ pub fn validate(def: RawModuleDefV10) -> Result { .map(|rls| (rls.sql.clone(), rls.to_owned())) .collect(); - let ((tables, types, reducers, procedures, views, (http_handlers, http_routes)), submodules) = - (tables_types_reducers_procedures_views, submodules) - .combine_errors() - .map_err(|errors: ValidationErrors| errors.sort_deduplicate())?; + let ( + (tables, types, reducers, procedures, views, (http_handlers, http_routes)), + submodules, + (environment, environment_declared), + ) = (tables_types_reducers_procedures_views, submodules, environment) + .combine_errors() + .map_err(|errors: ValidationErrors| errors.sort_deduplicate())?; let typespace_for_generate = typespace_for_generate.finish(); @@ -322,6 +326,8 @@ pub fn validate(def: RawModuleDefV10) -> Result { http_routes, raw_module_def_version: RawModuleDefVersion::V10, submodules, + environment, + environment_declared, }; // Submodules were validated in isolation, so their defs carry root-relative names. @@ -334,6 +340,22 @@ pub fn validate(def: RawModuleDefV10) -> Result { Ok(module_def) } +fn validate_environment(def: &RawModuleDefV10) -> Result<(spacetimedb_lib::environment::EnvironmentSchema, bool)> { + let mut sections = def.sections.iter().filter_map(|section| match section { + RawModuleDefV10Section::Environment(declarations) => Some(declarations), + _ => None, + }); + let Some(declarations) = sections.next() else { + return Ok((Default::default(), false)); + }; + if sections.next().is_some() { + return Err(ValidationError::RepeatedEnvironmentSection.into()); + } + let schema = spacetimedb_lib::environment::EnvironmentSchema::from_declarations(declarations) + .map_err(|error| ValidationError::Environment { error })?; + Ok((schema, true)) +} + /// Validate that each submodule's namespace is a valid identifier of at most 63 characters, /// that no two submodules share the same namespace, and that no submodule declares lifecycle /// reducers (lifecycle reducers are only permitted in the root module). @@ -365,6 +387,11 @@ fn validate_submodules(submodules: Vec) -> Result { + if !def.environment().is_empty() { + errors.push(ValidationError::EnvironmentInSubmodule { + namespace: submodule.namespace.clone(), + }); + } for (lifecycle, opt_id) in def.lifecycle_reducers_map() { if opt_id.is_some() { errors.push(ValidationError::LifecycleInSubmodule { @@ -2796,3 +2823,78 @@ mod tests { }); } } + +#[cfg(test)] +mod environment_tests { + use super::*; + use spacetimedb_lib::environment::{EnvironmentConstraint, EnvironmentDeclaration}; + + fn declared(name: &str) -> RawModuleDefV10 { + RawModuleDefV10 { + sections: vec![RawModuleDefV10Section::Environment(vec![EnvironmentDeclaration { + name: name.into(), + constraint: EnvironmentConstraint::AnyString, + optional: true, + }])], + } + } + + #[test] + fn environment_schema_round_trip_preserves_explicit_empty_and_exact_keys() { + let legacy = validate(RawModuleDefV10::default()).unwrap(); + assert!(legacy.environment().is_empty()); + assert!(!legacy.environment_declared()); + let explicit = validate(RawModuleDefV10 { + sections: vec![RawModuleDefV10Section::Environment(vec![])], + }) + .unwrap(); + assert!(explicit.environment().is_empty()); + assert!(explicit.environment_declared()); + let raw: RawModuleDefV10 = explicit.into(); + assert!(validate(raw).unwrap().environment_declared()); + let module = validate(declared("Mixed_CASE")).unwrap(); + assert!(module.environment().get("Mixed_CASE").is_some()); + assert!(module.environment().get("mixed_case").is_none()); + let raw: RawModuleDefV10 = module.into(); + assert!(validate(raw).unwrap().environment().get("Mixed_CASE").is_some()); + assert_eq!( + spacetimedb_lib::bsatn::to_vec(&RawModuleDefV10Section::Environment(vec![])).unwrap(), + vec![15, 0, 0, 0, 0] + ); + } + + #[test] + fn environment_rejects_ambiguous_sections_and_nested_declarations() { + let mut duplicate = declared("A"); + duplicate.sections.push(RawModuleDefV10Section::Environment(vec![])); + assert!(validate(duplicate) + .unwrap_err() + .into_iter() + .any(|error| matches!(error, ValidationError::RepeatedEnvironmentSection))); + let nested = RawModuleDefV10 { + sections: vec![RawModuleDefV10Section::Submodules(vec![RawSubmoduleV10 { + namespace: "outer".into(), + module: RawModuleDefV10 { + sections: vec![RawModuleDefV10Section::Submodules(vec![RawSubmoduleV10 { + namespace: "inner".into(), + module: declared("SECRET"), + }])], + }, + }])], + }; + assert!(validate(nested) + .unwrap_err() + .into_iter() + .any(|error| matches!(error, ValidationError::EnvironmentInSubmodule { .. }))); + let empty = RawModuleDefV10 { + sections: vec![RawModuleDefV10Section::Submodules(vec![RawSubmoduleV10 { + namespace: "allowed".into(), + module: RawModuleDefV10 { + sections: vec![RawModuleDefV10Section::Environment(vec![])], + }, + }])], + }; + assert!(validate(empty).is_ok()); + assert!(validate(declared("INVALID-NAME")).is_err()); + } +} diff --git a/crates/schema/src/def/validate/v9.rs b/crates/schema/src/def/validate/v9.rs index 8de4927afa2..1ba7c03c7b9 100644 --- a/crates/schema/src/def/validate/v9.rs +++ b/crates/schema/src/def/validate/v9.rs @@ -171,6 +171,8 @@ pub fn validate(def: RawModuleDefV9) -> Result { http_routes: Vec::new(), raw_module_def_version: RawModuleDefVersion::V9OrEarlier, submodules: IndexMap::new(), + environment: Default::default(), + environment_declared: false, }; // Records each def's namespace. V9 has no submodules, so this just resolves everything at diff --git a/crates/schema/src/error.rs b/crates/schema/src/error.rs index e9408a35482..770c44c7a3e 100644 --- a/crates/schema/src/error.rs +++ b/crates/schema/src/error.rs @@ -22,6 +22,14 @@ pub type ValidationErrors = ErrorStream; #[derive(thiserror::Error, Debug, PartialOrd, Ord, PartialEq, Eq)] #[non_exhaustive] pub enum ValidationError { + #[error("module has repeated environment sections")] + RepeatedEnvironmentSection, + #[error("invalid environment declaration: {error}")] + Environment { + error: spacetimedb_lib::environment::EnvironmentSchemaError, + }, + #[error("submodule {namespace:?} cannot declare environment variables")] + EnvironmentInSubmodule { namespace: String }, #[error("name `{name}` is used for multiple entities")] DuplicateName { name: RawIdentifier }, #[error("name `{name}` is used for multiple types")] diff --git a/crates/smoketests/modules/Cargo.lock b/crates/smoketests/modules/Cargo.lock index 69d26e1f356..3bfe68ba8c2 100644 --- a/crates/smoketests/modules/Cargo.lock +++ b/crates/smoketests/modules/Cargo.lock @@ -756,6 +756,13 @@ dependencies = [ "spacetimedb", ] +[[package]] +name = "smoketest-module-environment-publish" +version = "0.1.0" +dependencies = [ + "spacetimedb", +] + [[package]] name = "smoketest-module-fail-initial-publish-broken" version = "0.1.0" diff --git a/crates/smoketests/modules/Cargo.toml b/crates/smoketests/modules/Cargo.toml index 63dc67687eb..5184afb1947 100644 --- a/crates/smoketests/modules/Cargo.toml +++ b/crates/smoketests/modules/Cargo.toml @@ -119,6 +119,7 @@ members = [ "new-user-flow", "module-nested-op", "noop", + "environment-publish", "fail-initial-publish-broken", "fail-initial-publish-fixed", diff --git a/crates/smoketests/modules/environment-publish/Cargo.toml b/crates/smoketests/modules/environment-publish/Cargo.toml new file mode 100644 index 00000000000..8ed5366b336 --- /dev/null +++ b/crates/smoketests/modules/environment-publish/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "smoketest-module-environment-publish" +version = "0.1.0" +edition = "2021" +publish = false + +[lib] +crate-type = ["cdylib"] + +[dependencies] +spacetimedb.workspace = true diff --git a/crates/smoketests/modules/environment-publish/src/lib.rs b/crates/smoketests/modules/environment-publish/src/lib.rs new file mode 100644 index 00000000000..57d82e8b7d2 --- /dev/null +++ b/crates/smoketests/modules/environment-publish/src/lib.rs @@ -0,0 +1,46 @@ +use spacetimedb::{ReducerContext, Table}; + +#[spacetimedb::env] +pub struct Env { + pub SMOKE_REQUIRED: String, + #[env(values("ready", "other"))] + pub SMOKE_MODE: String, + pub SMOKE_OPTIONAL: Option, + pub SMOKE_EMPTY: Option, + pub SMOKE_NUMBER: Option, + pub SMOKE_FLAG: Option, +} + +#[spacetimedb::table(accessor = initial_environment, public)] +pub struct InitialEnvironment { + required: String, + mode: String, + optional: Option, +} + +#[spacetimedb::reducer(init)] +pub fn init(ctx: &ReducerContext) { + ctx.db.initial_environment().insert(InitialEnvironment { + required: ctx.env.SMOKE_REQUIRED(), + mode: ctx.env.SMOKE_MODE(), + optional: ctx.env.SMOKE_OPTIONAL(), + }); +} + +#[spacetimedb::reducer] +pub fn check_environment( + ctx: &ReducerContext, + required: String, + mode: String, + optional: Option, + empty: Option, + number: Option, + flag: Option, +) { + assert_eq!(ctx.env.SMOKE_REQUIRED(), required); + assert_eq!(ctx.env.SMOKE_MODE(), mode); + assert_eq!(ctx.env.SMOKE_OPTIONAL(), optional); + assert_eq!(ctx.env.SMOKE_EMPTY(), empty); + assert_eq!(ctx.env.SMOKE_NUMBER(), number); + assert_eq!(ctx.env.SMOKE_FLAG(), flag); +} diff --git a/crates/smoketests/tests/standalone/cli/environment.rs b/crates/smoketests/tests/standalone/cli/environment.rs new file mode 100644 index 00000000000..aed684811ce --- /dev/null +++ b/crates/smoketests/tests/standalone/cli/environment.rs @@ -0,0 +1,439 @@ +//! Publish-only environment configuration through the real CLI and local server. +use serde_json::{json, Value}; +use spacetimedb_guard::ensure_binaries_built; +use spacetimedb_smoketests::{modules, random_string, Smoketest}; +use std::{ + fs, + io::{Read as _, Seek as _}, + path::PathBuf, + process::{Child, Command, Output, Stdio}, + time::{Duration, Instant}, +}; + +const KEYS: &[&str] = &[ + "SMOKE_REQUIRED", + "SMOKE_MODE", + "SMOKE_OPTIONAL", + "SMOKE_EMPTY", + "SMOKE_NUMBER", + "SMOKE_FLAG", +]; + +struct Fixture { + test: Smoketest, + database: String, + wasm: PathBuf, +} + +impl Fixture { + fn new() -> Self { + // Check before the harness can connect or copy a prior login. These are + // standalone-only tests, including when accidentally run in a remote job. + for key in [ + "SPACETIME_REMOTE_SERVER", + "SPACETIME_USE_AUTH_HOST", + "SPACETIME_SMOKETEST_BASE_CONFIG_PATH", + ] { + assert!( + std::env::var_os(key).is_none(), + "ENV smoke test requires isolated local settings ({key})" + ); + } + let test = Smoketest::builder() + .precompiled_module("environment-publish") + .autopublish(false) + .build(); + assert!(test.guard.is_some()); + let address = test + .server_url + .strip_prefix("http://") + .unwrap() + .parse::() + .unwrap(); + assert_eq!(address.ip(), std::net::IpAddr::V4(std::net::Ipv4Addr::LOCALHOST)); + assert_ne!(address.port(), 0); + let wasm = test.project_dir.path().join("published.wasm"); + fs::copy(modules::precompiled_module("environment-publish"), &wasm).unwrap(); + // A misleading local source proves --bin-path reads declarations from + // these exact bytes, rather than trusting nearby source/config metadata. + fs::create_dir(test.project_dir.path().join("src")).unwrap(); + fs::write( + test.project_dir.path().join("src/lib.rs"), + "#[spacetimedb::env] pub struct Env { pub WRONG_SOURCE_DECLARATION: String }", + ) + .unwrap(); + let fixture = Self { + test, + database: format!("environment-{}", random_string()), + wasm, + }; + fixture.success(&["login", "--server-issued-login", &fixture.test.server_url], &[]); + fixture + } + + fn command(&self, args: &[&str], shell: &[(&str, &str)]) -> Output { + let mut command = Command::new(ensure_binaries_built()); + command.env_clear(); + // Runtime executables are already built. No user credentials, remote + // settings, or ambient module variables enter these child processes. + for key in ["PATH", "SystemRoot", "WINDIR", "TMP", "TEMP", "TMPDIR"] { + if let Some(value) = std::env::var_os(key) { + command.env(key, value); + } + } + command + .env("HOME", self.test.project_dir.path()) + .env("USERPROFILE", self.test.project_dir.path()) + .env("XDG_CONFIG_HOME", self.test.project_dir.path()) + .env("NO_PROXY", "*") + .env("no_proxy", "*") + .envs(shell.iter().copied()) + .arg("--config-path") + .arg(&self.test.config_path) + .args(args) + .current_dir(self.test.project_dir.path()) + .stdin(Stdio::null()); + bounded_output(command) + } + + fn success(&self, args: &[&str], shell: &[(&str, &str)]) -> String { + let output = self.command(args, shell); + assert!( + output.status.success(), + "local ENV CLI command failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + String::from_utf8(output.stdout).unwrap() + } + + fn config(&self, environment: Option) { + for file in [ + "spacetime.local.json", + "spacetime.prod.json", + "spacetime.prod.local.json", + ] { + let path = self.test.project_dir.path().join(file); + if path.exists() { + fs::remove_file(path).unwrap(); + } + } + let mut config = json!({"database": self.database}); + if let Some(environment) = environment { + config["env"] = environment; + } + self.write("spacetime.json", config); + } + + fn write(&self, file: &str, value: Value) { + fs::write( + self.test.project_dir.path().join(file), + serde_json::to_vec(&value).unwrap(), + ) + .unwrap(); + } + + fn publish(&self, shell: &[(&str, &str)], extra: &[&str]) -> Output { + let mut args = vec![ + "publish", + &self.database, + "--bin-path", + self.wasm.to_str().unwrap(), + "--server", + &self.test.server_url, + "--yes", + ]; + args.extend_from_slice(extra); + self.command(&args, shell) + } + + fn published(&self, shell: &[(&str, &str)], extra: &[&str]) -> String { + let before = fs::read(&self.wasm).unwrap(); + let output = self.publish(shell, extra); + assert!( + output.status.success(), + "publish failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!(fs::read(&self.wasm).unwrap(), before); + String::from_utf8(output.stdout).unwrap() + } + + fn get(&self, key: &str) -> String { + self.success( + &[ + "env", + "get", + &self.database, + key, + "--server", + &self.test.server_url, + "--no-config", + ], + &[], + ) + } + + fn list(&self) -> String { + self.success( + &[ + "env", + "list", + &self.database, + "--server", + &self.test.server_url, + "--no-config", + ], + &[], + ) + } + + fn typed(&self, required: &str, mode: &str, rest: [Option<&str>; 4]) { + let option = |value: Option<&str>| match value { + Some(value) => json!({"some": value}), + None => json!({"none": []}), + }; + let arguments = [ + json!(required), + json!(mode), + option(rest[0]), + option(rest[1]), + option(rest[2]), + option(rest[3]), + ] + .map(|value| value.to_string()); + let mut args = vec![ + "call", + &self.database, + "check_environment", + "--no-config", + "--server", + &self.test.server_url, + ]; + args.extend(arguments.iter().map(String::as_str)); + self.success(&args, &[]); + } + + fn sql(&self, statement: &str) -> Output { + self.command( + &[ + "sql", + &self.database, + statement, + "--server", + &self.test.server_url, + "--no-config", + ], + &[], + ) + } +} + +// Keep ownership through failure/timeout and avoid pipe backpressure. Output is +// generated fixture data; the cap also prevents accidental unbounded diagnostics. +fn bounded_output(mut command: Command) -> Output { + struct OwnedChild(Option); + impl Drop for OwnedChild { + fn drop(&mut self) { + if let Some(child) = self.0.as_mut() { + let _ = child.kill(); + let _ = child.wait(); + } + } + } + let mut stdout = tempfile::tempfile().unwrap(); + let mut stderr = tempfile::tempfile().unwrap(); + let mut child = OwnedChild(Some( + command + .stdout(Stdio::from(stdout.try_clone().unwrap())) + .stderr(Stdio::from(stderr.try_clone().unwrap())) + .spawn() + .unwrap(), + )); + let deadline = Instant::now() + Duration::from_secs(90); + let status = loop { + assert!( + stdout.metadata().unwrap().len() <= 1024 * 1024 && stderr.metadata().unwrap().len() <= 1024 * 1024, + "local ENV CLI output exceeds bound" + ); + if let Some(status) = child.0.as_mut().unwrap().try_wait().unwrap() { + child.0.take(); // Already reaped: never signal this process identifier again. + break status; + } + assert!(Instant::now() < deadline, "local ENV CLI command timed out"); + std::thread::sleep(Duration::from_millis(10)); + }; + stdout.rewind().unwrap(); + stderr.rewind().unwrap(); + let mut out = Vec::new(); + let mut err = Vec::new(); + stdout.read_to_end(&mut out).unwrap(); + stderr.read_to_end(&mut err).unwrap(); + Output { + status, + stdout: out, + stderr: err, + } +} + +#[test] +fn cli_environment_layers_shell_and_exact_precompiled_declarations() { + let f = Fixture::new(); + f.write( + "spacetime.json", + json!({"database":"unused-parent", "env":{ + "SMOKE_REQUIRED":"base-required", "SMOKE_MODE":"ready", "SMOKE_OPTIONAL":"base-optional", + "SMOKE_EMPTY":"base-empty", "SMOKE_FLAG": false + }, "children":[{"database":f.database,"env":{"SMOKE_OPTIONAL":"child-optional"}}]}), + ); + // Use a raw JSON number to verify there is no f64 round trip. + fs::write(f.test.project_dir.path().join("spacetime.local.json"), + r#"{"env":{"SMOKE_REQUIRED":"local-required","SMOKE_NUMBER":9007199254740993123456789},"children":[{"env":{"SMOKE_OPTIONAL":"child-local"}}]}"#).unwrap(); + f.write( + "spacetime.prod.json", + json!({"env":{"SMOKE_REQUIRED":"prod-required","SMOKE_MODE":"other"}, + "children":[{"env":{"SMOKE_EMPTY":"child-prod"}}]}), + ); + f.write( + "spacetime.prod.local.json", + json!({"env":{"SMOKE_REQUIRED":"prod-local-required"}, + "children":[{"env":{"SMOKE_OPTIONAL":"child-final"}}]}), + ); + let output = f.published( + &[ + ("SMOKE_REQUIRED", "shell-required"), + ("SMOKE_EMPTY", ""), + ("SMOKE_UNDECLARED", "ambient-not-published"), + ], + &["--env", "prod"], + ); + for key in KEYS { + assert!(output.contains(key)); + } + for value in [ + "shell-required", + "child-final", + "9007199254740993123456789", + "ambient-not-published", + ] { + assert!(!output.contains(value), "publish display leaked a fixture value"); + } + assert!(output.contains("SMOKE_REQUIRED (shell)")); + assert!(output.contains("SMOKE_NUMBER (config)")); + assert_eq!(f.get("SMOKE_EMPTY"), "\n"); + assert_eq!(f.get("SMOKE_NUMBER"), "9007199254740993123456789\n"); + assert_eq!(f.get("SMOKE_FLAG"), "false\n"); + f.typed( + "shell-required", + "other", + [ + Some("child-final"), + Some(""), + Some("9007199254740993123456789"), + Some("false"), + ], + ); + let mut keys = KEYS.to_vec(); + keys.sort_unstable(); + assert_eq!(f.list(), format!("{}\n", keys.join("\n"))); + let initial = f.sql("SELECT required, mode FROM initial_environment"); + assert!(initial.status.success()); + let initial = String::from_utf8(initial.stdout).unwrap(); + assert!(initial.contains("shell-required") && initial.contains("other")); +} + +#[test] +fn cli_environment_replacement_rejection_and_read_only_commands() { + let f = Fixture::new(); + f.config(Some( + json!({"SMOKE_REQUIRED":"initial-sentinel","SMOKE_MODE":"ready","SMOKE_OPTIONAL":"remove-me"}), + )); + f.published(&[], &[]); + f.config(Some( + json!({"SMOKE_REQUIRED":"replacement-sentinel","SMOKE_MODE":"other"}), + )); + f.published(&[], &[]); + f.typed("replacement-sentinel", "other", [None; 4]); + assert_eq!(f.list(), "SMOKE_MODE\nSMOKE_REQUIRED\n"); + assert!(!f + .command( + &[ + "env", + "get", + &f.database, + "SMOKE_OPTIONAL", + "--server", + &f.test.server_url, + "--no-config" + ], + &[] + ) + .status + .success()); + for input in [ + json!({"SMOKE_MODE":"ready"}), + json!({"SMOKE_REQUIRED":"rejected-sentinel","SMOKE_MODE":"invalid-sentinel"}), + json!({"SMOKE_REQUIRED":"rejected-sentinel","SMOKE_MODE":"ready","UNKNOWN":"unknown-sentinel"}), + json!({"SMOKE_REQUIRED":"rejected-sentinel","SMOKE_MODE":"ready","SMOKE_OPTIONAL":{}}), + ] { + f.config(Some(input)); + let output = f.publish(&[], &[]); + assert!(!output.status.success()); + for value in ["rejected-sentinel", "invalid-sentinel", "unknown-sentinel"] { + assert!(!String::from_utf8_lossy(&output.stdout).contains(value)); + assert!(!String::from_utf8_lossy(&output.stderr).contains(value)); + } + assert_eq!(f.get("SMOKE_REQUIRED"), "replacement-sentinel\n"); + f.typed("replacement-sentinel", "other", [None; 4]); + } + // Invalid local configuration must not prevent an explicit read-only target. + assert_eq!(f.list(), "SMOKE_MODE\nSMOKE_REQUIRED\n"); + for statement in [ + "SET env.SMOKE_REQUIRED = 'bypass'", + "DELETE env.SMOKE_REQUIRED", + "INSERT INTO st_env (key, value) VALUES ('BYPASS', 'value')", + "UPDATE st_env SET value = 'bypass'", + "DELETE FROM st_env", + ] { + assert!(!f.sql(statement).status.success()); + assert_eq!(f.get("SMOKE_REQUIRED"), "replacement-sentinel\n"); + } + for operation in ["set", "delete", "unset"] { + assert!(!f + .command( + &[ + "env", + operation, + &f.database, + "SMOKE_REQUIRED", + "--server", + &f.test.server_url + ], + &[] + ) + .status + .success()); + } +} + +#[test] +fn cli_environment_initial_rejection_clear_and_omitted_payload() { + let mut f = Fixture::new(); + f.config(None); + assert!(!f.publish(&[], &[]).status.success()); + f.config(Some(json!({"SMOKE_REQUIRED":"clear-initial","SMOKE_MODE":"ready"}))); + f.published(&[], &[]); + f.config(Some( + json!({"SMOKE_REQUIRED":"clear-replaced","SMOKE_MODE":"other","SMOKE_EMPTY":""}), + )); + f.published(&[], &["--delete-data"]); + f.typed("clear-replaced", "other", [None, Some(""), None, None]); + let initial = f.sql("SELECT required FROM initial_environment"); + assert!(initial.status.success()); + let initial = String::from_utf8(initial.stdout).unwrap(); + assert!(initial.contains("clear-replaced") && !initial.contains("clear-initial")); + // A legacy module with no ENV declaration receives an empty complete input. + f.config(None); + f.wasm = modules::precompiled_module("noop"); + f.published(&[("SMOKE_REQUIRED", "must-not-be-ambient")], &["--delete-data"]); + assert_eq!(f.list(), ""); +} diff --git a/crates/smoketests/tests/standalone/cli/mod.rs b/crates/smoketests/tests/standalone/cli/mod.rs index 8c2b6481318..ee04d64547e 100644 --- a/crates/smoketests/tests/standalone/cli/mod.rs +++ b/crates/smoketests/tests/standalone/cli/mod.rs @@ -1,5 +1,6 @@ mod auth; mod dev; +mod environment; mod generate; mod list; mod server; diff --git a/crates/sql-parser/src/ast/sql.rs b/crates/sql-parser/src/ast/sql.rs index a7593cc4582..be7b753f395 100644 --- a/crates/sql-parser/src/ast/sql.rs +++ b/crates/sql-parser/src/ast/sql.rs @@ -19,15 +19,6 @@ pub enum SqlAst { Set(SqlSet), /// SHOW var Show(SqlShow), - /// Administrative environment mutation, distinct from generic table DML. - Environment(SqlEnvironment), -} - -#[derive(Debug)] -pub struct SqlEnvironment { - pub key: SqlIdent, - /// None is DELETE; a string literal, including empty, is SET. - pub value: Option, } impl SqlAst { diff --git a/crates/sql-parser/src/parser/sql.rs b/crates/sql-parser/src/parser/sql.rs index a70ea4d5b39..0508baae5b2 100644 --- a/crates/sql-parser/src/parser/sql.rs +++ b/crates/sql-parser/src/parser/sql.rs @@ -133,13 +133,11 @@ use sqlparser::{ Value, Values, }, dialect::PostgreSqlDialect, - keywords::Keyword, parser::Parser, - tokenizer::Token, }; use crate::ast::{ - sql::{SqlAst, SqlDelete, SqlEnvironment, SqlInsert, SqlSelect, SqlSet, SqlShow, SqlUpdate, SqlValues}, + sql::{SqlAst, SqlDelete, SqlInsert, SqlSelect, SqlSet, SqlShow, SqlUpdate, SqlValues}, SqlIdent, }; @@ -150,36 +148,7 @@ use super::{ /// Parse a SQL string pub fn parse_sql(sql: &str) -> SqlParseResult { - // DELETE env.KEY is a SpacetimeDB administrative statement, not the - // PostgreSQL DELETE FROM grammar. Use the same tokenizer and expression - // parser, including comments and SQL string escaping, for this extension. - let mut parser = Parser::new(&PostgreSqlDialect {}).try_with_sql(sql)?; - let verb = parser.peek_token().token; - let environment_prefix = matches!(parser.peek_nth_token(1).token, - Token::Word(word) if word.quote_style.is_none() && word.value.eq_ignore_ascii_case("env")) - && parser.peek_nth_token(2).token == Token::Period; - if environment_prefix - && matches!(&verb, Token::Word(word) if matches!(word.keyword, Keyword::SET | Keyword::DELETE)) - { - parser.next_token(); - parser.next_token(); - parser.next_token(); - let key = SqlIdent(parser.parse_identifier()?.value.into()); - let value = if matches!(verb, Token::Word(word) if word.keyword == Keyword::SET) { - if !parser.parse_keyword(Keyword::TO) { - parser.expect_token(&Token::Eq)?; - } - Some(parse_literal_expr(parser.parse_expr()?, SqlUnsupported::Assignment)?) - } else { - None - }; - let _ = parser.consume_token(&Token::SemiColon); - if parser.peek_token().token != Token::EOF { - return Err(SqlUnsupported::MultiStatement.into()); - } - return Ok(SqlAst::Environment(SqlEnvironment { key, value })); - } - let mut stmts = parser.parse_statements()?; + let mut stmts = Parser::parse_sql(&PostgreSqlDialect {}, sql)?; if stmts.len() > 1 { return Err(SqlUnsupported::MultiStatement.into()); } diff --git a/crates/standalone/src/control_db.rs b/crates/standalone/src/control_db.rs index a637d38afcc..9432b6ee097 100644 --- a/crates/standalone/src/control_db.rs +++ b/crates/standalone/src/control_db.rs @@ -1,3 +1,5 @@ +mod environment; + use anyhow::Context; use sled::transaction::{ self, ConflictableTransactionError, ConflictableTransactionResult, TransactionError, TransactionResult, @@ -339,19 +341,18 @@ impl ControlDb { let scan_key: &[u8] = b""; for result in tree.range(scan_key..) { let (_key, value) = result?; - let database = compat::Database::from_slice(&value)?.into(); + let database = self.decode_database(&value)?; databases.push(database); } Ok(databases) } pub fn get_database_by_id(&self, id: u64) -> Result> { - for database in self.get_databases()? { - if database.id == id { - return Ok(Some(database)); - } - } - Ok(None) + self.db + .open_tree("database")? + .get(id.to_be_bytes())? + .map(|bytes| self.decode_database(&bytes)) + .transpose() } pub fn get_database_by_identity(&self, identity: &Identity) -> Result> { @@ -359,12 +360,13 @@ impl ControlDb { let key = identity.to_be_byte_array(); let value = tree.get(&key[..])?; if let Some(value) = value { - let database = compat::Database::from_slice(&value[..])?.into(); + let database = self.decode_database(&value)?; return Ok(Some(database)); } Ok(None) } + #[cfg(test)] pub fn insert_database(&self, mut database: Database) -> Result { let id = self.db.generate_id()?; let tree = self.db.open_tree("database_by_identity")?; @@ -388,23 +390,6 @@ impl ControlDb { Ok(id) } - pub(crate) fn update_database(&self, database: Database) -> Result<()> { - let Some(stored_database) = self.get_database_by_identity(&database.database_identity)? else { - return Err(Error::DatabaseNotFound(database.database_identity)); - }; - - let tree = self.db.open_tree("database_by_identity")?; - let buf = sled::IVec::from(compat::Database::from(database).to_vec()?); - tree.insert(stored_database.database_identity.to_be_byte_array(), buf.clone())?; - tree.flush()?; - - let tree = self.db.open_tree("database")?; - tree.insert(stored_database.id.to_be_bytes(), buf)?; - tree.flush()?; - - Ok(()) - } - pub fn is_database_locked(&self, database_identity: &Identity) -> Result { let tree = self.db.open_tree("database_locks")?; let key = database_identity.to_be_byte_array(); @@ -419,20 +404,7 @@ impl ControlDb { } pub fn delete_database(&self, id: u64) -> Result> { - let tree = self.db.open_tree("database")?; - let tree_by_identity = self.db.open_tree("database_by_identity")?; - - if let Some(old_value) = tree.get(id.to_be_bytes())? { - let database = compat::Database::from_slice(&old_value[..])?; - let key = database.database_identity().to_be_byte_array(); - - tree_by_identity.remove(&key[..])?; - tree.remove(id.to_be_bytes())?; - tree.flush()?; - return Ok(Some(id)); - } - - Ok(None) + self.delete_database_and_environment(id) } pub fn get_replicas(&self) -> Result> { diff --git a/crates/standalone/src/control_db/environment.rs b/crates/standalone/src/control_db/environment.rs new file mode 100644 index 00000000000..ede423c3935 --- /dev/null +++ b/crates/standalone/src/control_db/environment.rs @@ -0,0 +1,316 @@ +//! Private bootstrap inputs, separate from the historical public Database encoding. +use super::*; +use spacetimedb_client_api_messages::publish::PublishRequest; +use spacetimedb_lib::Hash; +use std::collections::BTreeMap; + +const METADATA_TREE: &str = "database_bootstrap"; +const VALUES_TREE: &str = "initial_environment"; + +#[derive(serde::Serialize, serde::Deserialize)] +#[serde(deny_unknown_fields)] +struct Bootstrap { + version: u8, + database_id: u64, + identity: Identity, + program: Hash, + generation: u64, +} + +fn invalid() -> Error { + Error::Other(anyhow::anyhow!("invalid private bootstrap record")) +} + +impl Bootstrap { + fn decode(bytes: &[u8], database: &Database) -> Result { + if bytes.len() > 1024 { + return Err(invalid()); + } + let value: Self = serde_json::from_slice(bytes).map_err(|_| invalid())?; + if value.version != 1 + || value.database_id != database.id + || value.identity != database.database_identity + || value.program != database.initial_program + || value.generation == 0 + { + return Err(invalid()); + } + Ok(value) + } +} + +impl ControlDb { + pub(super) fn decode_database(&self, bytes: &[u8]) -> Result { + let mut database: Database = compat::Database::from_slice(bytes)?.into(); + if let Some(bytes) = self.db.open_tree(METADATA_TREE)?.get(database.id.to_be_bytes())? { + database.bootstrap_generation = Bootstrap::decode(&bytes, &database)?.generation; + } + Ok(database) + } + + /// Recheck the exact persisted generation and program before releasing any values. + pub(crate) fn initial_environment(&self, database: &Database) -> Result> { + let databases = self.db.open_tree("database_by_identity")?; + let metadata = self.db.open_tree(METADATA_TREE)?; + let values = self.db.open_tree(VALUES_TREE)?; + let result: TransactionResult, Error> = + (&databases, &metadata, &values).transaction(|(databases, metadata, values)| { + let persisted = databases + .get(database.database_identity.to_be_byte_array())? + .ok_or_else(|| { + ConflictableTransactionError::Abort(Error::DatabaseNotFound(database.database_identity)) + })?; + let stored: Database = compat::Database::from_slice(&persisted) + .map_err(|_| ConflictableTransactionError::Abort(invalid()))? + .into(); + if stored.id != database.id + || stored.initial_program != database.initial_program + || stored.owner_identity != database.owner_identity + { + return transaction::abort(invalid()); + } + match metadata.get(database.id.to_be_bytes())? { + Some(bytes) => { + let binding = + Bootstrap::decode(&bytes, &stored).map_err(ConflictableTransactionError::Abort)?; + if binding.generation != database.bootstrap_generation { + return transaction::abort(invalid()); + } + let bytes = values + .get(database.id.to_be_bytes())? + .ok_or_else(|| ConflictableTransactionError::Abort(invalid()))?; + Ok(Some(bytes)) + } + None if database.bootstrap_generation == 0 => { + if values.get(database.id.to_be_bytes())?.is_some() { + return transaction::abort(invalid()); + } + Ok(None) + } + None => transaction::abort(invalid()), + } + }); + let bytes = result.map_err(transaction_error)?; + match bytes { + None => Ok(BTreeMap::new()), + Some(bytes) => { + let request = PublishRequest::decode(&bytes).map_err(|_| invalid())?; + if !request.module.is_empty() { + return Err(invalid()); + } + Ok(request.environment) + } + } + } + + /// Install the desired initial database, its complete private input and a + /// durable leader nomination together. Cancellation before host launch can + /// therefore recover through the ordinary leader lookup. + pub(crate) fn install_database_with_environment( + &self, + mut database: Database, + expected: Option<&Database>, + environment: BTreeMap, + previous_replicas: &[Replica], + ) -> Result<(Database, Replica)> { + if expected.is_none() { + database.id = self.db.generate_id()?; + } + database.bootstrap_generation = expected + .map_or(Some(1), |old| old.bootstrap_generation.checked_add(1)) + .ok_or_else(invalid)?; + let replica = Replica { + id: self.db.generate_id()?, + database_id: database.id, + node_id: 0, + leader: true, + }; + let input = PublishRequest { + module: Vec::new(), + environment, + } + .encode() + .map_err(|_| invalid())?; + let binding = Bootstrap { + version: 1, + database_id: database.id, + identity: database.database_identity, + program: database.initial_program, + generation: database.bootstrap_generation, + }; + let binding = serde_json::to_vec(&binding).map_err(|_| invalid())?; + let encoded = compat::Database::from(database.clone()).to_vec()?; + let encoded_replica = bsatn::to_vec(&replica)?; + let databases = self.db.open_tree("database")?; + let identities = self.db.open_tree("database_by_identity")?; + let metadata = self.db.open_tree(METADATA_TREE)?; + let values = self.db.open_tree(VALUES_TREE)?; + let replicas = self.db.open_tree("replica")?; + let result: TransactionResult<(), Error> = (&databases, &identities, &metadata, &values, &replicas) + .transaction(|(databases, identities, metadata, values, replicas)| { + let identity_key = database.database_identity.to_be_byte_array(); + match (expected, identities.get(identity_key)?) { + (None, None) => {} + (Some(expected), Some(bytes)) => { + let stored: Database = compat::Database::from_slice(&bytes) + .map_err(|_| ConflictableTransactionError::Abort(invalid()))? + .into(); + if stored.id != expected.id + || stored.initial_program != expected.initial_program + || stored.owner_identity != expected.owner_identity + { + return transaction::abort(invalid()); + } + let generation = match metadata.get(stored.id.to_be_bytes())? { + Some(bytes) => { + Bootstrap::decode(&bytes, &stored) + .map_err(ConflictableTransactionError::Abort)? + .generation + } + None => 0, + }; + if generation != expected.bootstrap_generation { + return transaction::abort(invalid()); + } + } + (None, Some(_)) => { + return transaction::abort(Error::DatabaseAlreadyExists(database.database_identity)) + } + (Some(_), None) => return transaction::abort(Error::DatabaseNotFound(database.database_identity)), + } + databases.insert(&database.id.to_be_bytes(), encoded.clone())?; + identities.insert(&identity_key, encoded.clone())?; + metadata.insert(&database.id.to_be_bytes(), binding.clone())?; + values.insert(&database.id.to_be_bytes(), input.clone())?; + for previous in previous_replicas { + if previous.database_id != database.id { + return transaction::abort(invalid()); + } + replicas.remove(&previous.id.to_be_bytes())?; + } + replicas.insert(&replica.id.to_be_bytes(), encoded_replica.clone())?; + databases.flush(); + Ok(()) + }); + result.map_err(transaction_error)?; + self.db.flush()?; + Ok((database, replica)) + } + + pub(super) fn delete_database_and_environment(&self, id: u64) -> Result> { + let databases = self.db.open_tree("database")?; + let identities = self.db.open_tree("database_by_identity")?; + let metadata = self.db.open_tree(METADATA_TREE)?; + let values = self.db.open_tree(VALUES_TREE)?; + let result: TransactionResult, Error> = + (&databases, &identities, &metadata, &values).transaction(|(databases, identities, metadata, values)| { + let Some(bytes) = databases.get(id.to_be_bytes())? else { + return Ok(None); + }; + let database = + compat::Database::from_slice(&bytes).map_err(|_| ConflictableTransactionError::Abort(invalid()))?; + identities.remove(&database.database_identity().to_be_byte_array())?; + databases.remove(&id.to_be_bytes())?; + metadata.remove(&id.to_be_bytes())?; + values.remove(&id.to_be_bytes())?; + databases.flush(); + Ok(Some(id)) + }); + let result = result.map_err(transaction_error)?; + self.db.flush()?; + Ok(result) + } +} + +fn transaction_error(error: TransactionError) -> Error { + match error { + TransactionError::Abort(error) => error, + TransactionError::Storage(error) => error.into(), + } +} + +#[async_trait::async_trait] +impl spacetimedb::host::InitialEnvironmentSource for ControlDb { + async fn load(&self, database: &Database) -> anyhow::Result> { + let source = self.clone(); + let database = database.clone(); + spacetimedb::util::asyncify(move || source.initial_environment(&database)) + .await + .map_err(Into::into) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use spacetimedb::messages::control_db::HostType; + + fn database() -> Database { + Database { + id: 0, + database_identity: Identity::ZERO, + owner_identity: Identity::ZERO, + host_type: HostType::Wasm, + initial_program: spacetimedb_lib::hash_bytes(b"module"), + bootstrap_generation: 0, + } + } + + #[test] + fn bootstrap_is_durable_atomic_generation_bound_and_deleted_with_database() -> anyhow::Result<()> { + let temp = tempfile::tempdir()?; + let (original, original_replica) = { + let control = ControlDb::at(temp.path())?; + control.install_database_with_environment( + database(), + None, + BTreeMap::from([("VALUE".into(), "first".into())]), + &[], + )? + }; + let control = ControlDb::at(temp.path())?; + let loaded = control.get_database_by_id(original.id)?.unwrap(); + assert_eq!(loaded.bootstrap_generation, 1); + assert_eq!(control.initial_environment(&loaded)?["VALUE"], "first"); + assert_eq!( + control.get_leader_replica_by_database(loaded.id).unwrap().id, + original_replica.id + ); + let (replaced, new_replica) = control.install_database_with_environment( + loaded.clone(), + Some(&loaded), + BTreeMap::new(), + &[original_replica], + )?; + assert_eq!(replaced.bootstrap_generation, 2); + assert!(control.initial_environment(&original).is_err()); + assert!(control.initial_environment(&replaced)?.is_empty()); + assert_eq!(control.get_replicas_by_database(loaded.id)?.len(), 1); + assert_eq!( + control.get_leader_replica_by_database(loaded.id).unwrap().id, + new_replica.id + ); + assert!(control + .install_database_with_environment(loaded.clone(), Some(&loaded), BTreeMap::new(), &[]) + .is_err()); + assert_eq!(control.get_database_by_id(loaded.id)?.unwrap().bootstrap_generation, 2); + control.delete_database(loaded.id)?; + assert!(control.initial_environment(&replaced).is_err()); + assert!(control.db.open_tree(VALUES_TREE)?.is_empty()); + Ok(()) + } + + #[test] + fn legacy_absence_is_empty_but_missing_new_values_fails_closed() -> anyhow::Result<()> { + let temp = tempfile::tempdir()?; + let control = ControlDb::at(temp.path())?; + let id = control.insert_database(database())?; + let legacy = control.get_database_by_id(id)?.unwrap(); + assert!(control.initial_environment(&legacy)?.is_empty()); + let (new, _) = + control.install_database_with_environment(legacy.clone(), Some(&legacy), BTreeMap::new(), &[])?; + control.db.open_tree(VALUES_TREE)?.remove(new.id.to_be_bytes())?; + assert!(control.initial_environment(&new).is_err()); + Ok(()) + } +} diff --git a/crates/standalone/src/environment_tests.rs b/crates/standalone/src/environment_tests.rs new file mode 100644 index 00000000000..abeb5a84193 --- /dev/null +++ b/crates/standalone/src/environment_tests.rs @@ -0,0 +1,190 @@ +//! Actual module persistence with explicit local input, without a server or CLI configuration. +use super::*; +use spacetimedb::host::FunctionArgs; +use spacetimedb_client_api::{ControlStateWriteAccess as _, DatabaseDef}; +use spacetimedb_lib::{bsatn, sats::product, AlgebraicValue}; +use spacetimedb_paths::cli::{PrivKeyPath, PubKeyPath}; +use spacetimedb_paths::FromPathUnchecked; +use std::collections::BTreeMap; + +type Values = BTreeMap; + +async fn read(env: &StandaloneEnv, database: u64, key: &str) -> anyhow::Result { + let module = env.leader(database).await?.module().await?; + Ok(module + .call_procedure( + Identity::ZERO, + None, + None, + "read_environment", + FunctionArgs::Bsatn(bsatn::to_vec(&product![key])?.into()), + ) + .await + .result? + .return_val) +} + +#[tokio::test] +#[ignore = "requires an explicitly configured local environment-test Wasm artifact"] +async fn real_module_reopen_and_no_artifact_reset_preserve_complete_environment_semantics() -> anyhow::Result<()> { + let module_path = std::path::PathBuf::from( + std::env::var_os("SPACETIMEDB_ENV_STANDALONE_TEST_MODULE") + .context("SPACETIMEDB_ENV_STANDALONE_TEST_MODULE must name the owned local fixture")?, + ); + anyhow::ensure!(module_path.is_absolute(), "fixture path must be absolute"); + let bytes = axum::body::Bytes::from(std::fs::read(module_path)?); + anyhow::ensure!(bytes.starts_with(b"\0asm"), "fixture must be a Wasm module"); + let temp = tempfile::tempdir()?; + let keys = temp.path().join("keys"); + std::fs::create_dir(&keys)?; + let ca = CertificateAuthority { + jwt_pub_key_path: PubKeyPath(keys.join("public")), + jwt_priv_key_path: PrivKeyPath(keys.join("private")), + }; + let data_dir = Arc::new(ServerDataDir::from_path_unchecked(temp.path().join("data"))); + data_dir.create()?; + let config = StandaloneOptions { + db_config: db::Config { + storage: db::Storage::Disk, + page_pool_max_size: None, + }, + durability: DurabilityConfig::default(), + websocket: WebSocketOptions::default(), + module_http: ModuleHttpConfig::default(), + wasm: WasmConfig::default(), + v8: V8Config::default(), + }; + let env = StandaloneEnv::init(config, &ca, data_dir, JobCores::without_pinned_cores()).await?; + let test_env = env.clone(); + let run = tokio::spawn(async move { + let env = test_env; + let initial = Values::from([ + ("REQUIRED".into(), "initial-required".into()), + ("MODE".into(), "ready".into()), + ]); + let spec = |environment| DatabaseDef { + database_identity: Identity::ZERO, + program_bytes: bytes.clone(), + environment, + num_replicas: None, + host_type: HostType::Wasm, + parent: None, + organization: None, + }; + log::info!("ENV standalone fixture: initial publication"); + assert!(env + .publish_database(&Identity::ZERO, spec(initial.clone()), MigrationPolicy::Compatible) + .await? + .is_none()); + let database = env.control_db.get_database_by_identity(&Identity::ZERO)?.unwrap(); + let replica = env.control_db.get_leader_replica_by_database(database.id).unwrap(); + log::info!("ENV standalone fixture: rejected publication preserves live host"); + let rejected = env + .publish_database(&Identity::ZERO, spec(Values::new()), MigrationPolicy::Compatible) + .await; + assert!( + rejected.as_ref().is_err() + || rejected + .as_ref() + .unwrap() + .as_ref() + .is_some_and(|result| !result.was_successful()) + ); + assert_eq!( + read(&env, database.id, "REQUIRED").await?, + AlgebraicValue::from(Some("initial-required".to_owned())) + ); + log::info!("ENV standalone fixture: same-program update"); + let mut updated = initial.clone(); + updated.insert("REQUIRED".into(), "republished".into()); + assert!(env + .publish_database(&Identity::ZERO, spec(updated), MigrationPolicy::Compatible) + .await? + .unwrap() + .was_successful()); + assert_eq!( + read(&env, database.id, "REQUIRED").await?, + AlgebraicValue::from(Some("republished".to_owned())) + ); + // The old private bootstrap input intentionally remains distinct. Ordinary + // reopen must consult persisted st_module/st_env and skip that input. + assert_eq!( + env.control_db.initial_environment(&database)?["REQUIRED"], + "initial-required" + ); + log::info!("ENV standalone fixture: positive close and normal reopen"); + let replica_id = replica.id; + env.own_publication( + move |owner| async move { owner.host_controller.exit_module_host_and_join(replica_id).await }, + ) + .await?; + assert_eq!( + read(&env, database.id, "REQUIRED").await?, + AlgebraicValue::from(Some("republished".to_owned())) + ); + // Init asserts initial-required. A successful reopen with republished proves + // that init was not run again and old bootstrap input was not restored. + assert!(env + .reset_database( + &Identity::ZERO, + DatabaseResetDef { + database_identity: Identity::ZERO, + program_bytes: None, + environment: Values::new(), + num_replicas: None, + host_type: None, + } + ) + .await + .is_err()); + assert_eq!( + env.control_db + .get_database_by_id(database.id)? + .unwrap() + .bootstrap_generation, + 1 + ); + assert_eq!( + read(&env, database.id, "REQUIRED").await?, + AlgebraicValue::from(Some("republished".to_owned())) + ); + log::info!("ENV standalone fixture: complete no-artifact reset"); + let mut reset = initial; + reset.insert("EMPTY".into(), "reset-input".into()); + env.reset_database( + &Identity::ZERO, + DatabaseResetDef { + database_identity: Identity::ZERO, + program_bytes: None, + environment: reset, + num_replicas: None, + host_type: None, + }, + ) + .await?; + assert_eq!( + env.control_db + .get_database_by_id(database.id)? + .unwrap() + .bootstrap_generation, + 2 + ); + assert_ne!( + env.control_db.get_leader_replica_by_database(database.id).unwrap().id, + replica.id + ); + assert_eq!( + read(&env, database.id, "EMPTY").await?, + AlgebraicValue::from(Some("reset-input".to_owned())) + ); + anyhow::Ok(()) + }) + .await; + // Join physical cleanup even if an assertion in the accepted fixture task + // panicked. Never let a failed test detach its database host. + log::info!("ENV standalone fixture: positive cleanup"); + env.delete_database(&Identity::ZERO, &Identity::ZERO).await?; + run??; + assert!(env.control_db.get_database_by_identity(&Identity::ZERO)?.is_none()); + Ok(()) +} diff --git a/crates/standalone/src/lib.rs b/crates/standalone/src/lib.rs index a79f9814f5d..efe11d860fa 100644 --- a/crates/standalone/src/lib.rs +++ b/crates/standalone/src/lib.rs @@ -1,4 +1,6 @@ mod control_db; +#[cfg(test)] +mod environment_tests; pub mod subcommands; pub mod util; pub mod version; @@ -16,7 +18,7 @@ use spacetimedb::db::persistence::{DurabilityConfig, LocalPersistenceProvider}; use spacetimedb::energy::{EnergyBalance, EnergyQuanta, NullEnergyMonitor}; use spacetimedb::host::{DiskStorage, HostController, HostRuntimeConfig, MigratePlanResult, UpdateDatabaseResult}; use spacetimedb::identity::{AuthCtx, Identity}; -use spacetimedb::messages::control_db::{Database, Node, Replica}; +use spacetimedb::messages::control_db::{Database, HostType, Node, Replica}; use spacetimedb::metrics::ENGINE_METRICS; use spacetimedb::subscription::row_list_builder_pool::BsatnRowListBuilderPool; use spacetimedb::util::jobs::JobCores; @@ -34,7 +36,8 @@ use spacetimedb_paths::server::{ModuleLogsDir, PidFile, ServerDataDir}; use spacetimedb_paths::standalone::StandaloneDataDirExt; use spacetimedb_schema::auto_migrate::{MigrationPolicy, PrettyPrintStyle}; use spacetimedb_table::page_pool::PagePool; -use std::sync::Arc; +use std::sync::{Arc, Weak}; +#[cfg(test)] use std::time::Duration; pub use spacetimedb_client_api::routes::subscribe::{BIN_PROTOCOL, TEXT_PROTOCOL}; @@ -51,6 +54,8 @@ pub struct StandaloneOptions { pub struct StandaloneEnv { control_db: ControlDb, + publication_lock: Arc>, + weak_self: Weak, program_store: Arc, host_controller: HostController, client_actor_index: ClientActorIndex, @@ -90,7 +95,8 @@ impl StandaloneEnv { Arc::new(()), persistence_provider, db_cores, - ); + ) + .with_initial_environment_source(Arc::new(control_db.clone())); let client_actor_index = ClientActorIndex::new(); let jwt_keys = certs.get_or_create_keys()?; @@ -102,8 +108,10 @@ impl StandaloneEnv { metrics_registry.register(Box::new(&*DB_METRICS)).unwrap(); metrics_registry.register(Box::new(&*DATA_SIZE_METRICS)).unwrap(); - Ok(Arc::new(Self { + Ok(Arc::new_cyclic(|weak_self| Self { control_db, + publication_lock: Arc::new(tokio::sync::RwLock::new(())), + weak_self: weak_self.clone(), program_store, host_controller, client_actor_index, @@ -176,20 +184,14 @@ impl NodeDelegate for StandaloneEnv { } async fn leader(&self, database_id: u64) -> Result { - let Some(leader) = self.control_db.get_leader_replica_by_database(database_id) else { - return Err(GetLeaderHostError::NoSuchReplica); - }; - - let Some(database) = self.control_db.get_database_by_id(database_id)? else { - return Err(GetLeaderHostError::NoSuchDatabase); - }; - - self.host_controller - .get_or_launch_module_host(database, leader.id) - .await - .map_err(|source| GetLeaderHostError::LaunchError { source })?; - - Ok(Host::new(leader.id, self.host_controller.clone())) + let guard = self.publication_lock.clone().read_owned().await; + let owner = self.weak_self.upgrade().expect("standalone owner exists during lookup"); + tokio::spawn(async move { + let _guard = guard; + owner.leader_under_publication(database_id).await + }) + .await + .map_err(|error| GetLeaderHostError::LaunchError { source: error.into() })? } fn module_logs_dir(&self, replica_id: u64) -> ModuleLogsDir { @@ -276,6 +278,153 @@ impl spacetimedb_client_api::ControlStateWriteAccess for StandaloneEnv { publisher: &Identity, spec: spacetimedb_client_api::DatabaseDef, policy: MigrationPolicy, + ) -> anyhow::Result> { + let publisher = *publisher; + self.own_publication(move |owner| async move { owner.publish_database_owned(&publisher, spec, policy).await }) + .await + } + + async fn migrate_plan( + &self, + spec: spacetimedb_client_api::DatabaseDef, + style: PrettyPrintStyle, + ) -> anyhow::Result { + let existing_db = self.control_db.get_database_by_identity(&spec.database_identity)?; + + match existing_db { + Some(db) => { + let host = self.leader(db.id).await?; + self.host_controller + .migrate_plan( + db, + spec.host_type, + host.replica_id, + spec.program_bytes.to_vec().into(), + style, + ) + .await + } + None => anyhow::bail!( + "Database `{}` does not exist", + spec.database_identity.to_abbreviated_hex() + ), + } + } + + async fn delete_database(&self, _caller_identity: &Identity, database_identity: &Identity) -> anyhow::Result<()> { + let caller_identity = *_caller_identity; + let database_identity = *database_identity; + self.own_publication(move |owner| async move { + owner.delete_database_owned(&caller_identity, &database_identity).await + }) + .await + } + + async fn reset_database(&self, _caller_identity: &Identity, spec: DatabaseResetDef) -> anyhow::Result<()> { + let caller_identity = *_caller_identity; + self.own_publication(move |owner| async move { owner.reset_database_owned(&caller_identity, spec).await }) + .await + } + + async fn add_energy(&self, identity: &Identity, amount: EnergyQuanta) -> anyhow::Result<()> { + let balance = self + .control_db + .get_energy_balance(identity)? + .unwrap_or(EnergyBalance::ZERO); + + let balance = balance.saturating_add_energy(amount); + + self.control_db.set_energy_balance(*identity, balance)?; + Ok(()) + } + async fn withdraw_energy(&self, _identity: &Identity, _amount: EnergyQuanta) -> anyhow::Result<()> { + // The energy balance code is obsolete. + Ok(()) + } + + async fn register_tld(&self, identity: &Identity, tld: Tld) -> anyhow::Result { + Ok(self.control_db.spacetime_register_tld(tld, *identity)?) + } + + async fn create_dns_record( + &self, + owner_identity: &Identity, + domain: &DomainName, + database_identity: &Identity, + ) -> anyhow::Result { + Ok(self + .control_db + .spacetime_insert_domain(database_identity, domain.clone(), *owner_identity, true)?) + } + + async fn replace_dns_records( + &self, + database_identity: &Identity, + owner_identity: &Identity, + domain_names: &[DomainName], + ) -> anyhow::Result { + Ok(self + .control_db + .spacetime_replace_domains(database_identity, owner_identity, domain_names)?) + } + + async fn set_database_lock( + &self, + _caller_identity: &Identity, + database_identity: &Identity, + locked: bool, + ) -> anyhow::Result<()> { + let Some(_database) = self.control_db.get_database_by_identity(database_identity)? else { + anyhow::bail!("Database not found: {}", database_identity.to_abbreviated_hex()); + }; + self.control_db.set_database_lock(database_identity, locked)?; + Ok(()) + } +} + +impl StandaloneEnv { + /// Admission and completion are owned together. Losing an HTTP waiter cannot + /// release the reset fence while HostController still owns accepted work. + async fn own_publication(&self, operation: F) -> anyhow::Result + where + T: Send + 'static, + F: FnOnce(Arc) -> Fut + Send + 'static, + Fut: std::future::Future> + Send + 'static, + { + let guard = self.publication_lock.clone().write_owned().await; + let owner = self + .weak_self + .upgrade() + .expect("standalone owner exists during publication"); + tokio::spawn(async move { + let _guard = guard; + operation(owner).await + }) + .await? + } + + async fn leader_under_publication(&self, database_id: u64) -> Result { + let Some(leader) = self.control_db.get_leader_replica_by_database(database_id) else { + return Err(GetLeaderHostError::NoSuchReplica); + }; + + let Some(database) = self.control_db.get_database_by_id(database_id)? else { + return Err(GetLeaderHostError::NoSuchDatabase); + }; + + self.host_controller + .get_or_launch_module_host(database, leader.id) + .await + .map_err(|source| GetLeaderHostError::LaunchError { source })?; + + Ok(Host::new(leader.id, self.host_controller.clone())) + } + + async fn publish_database_owned( + &self, + publisher: &Identity, + spec: spacetimedb_client_api::DatabaseDef, + policy: MigrationPolicy, ) -> anyhow::Result> { let existing_db = self.control_db.get_database_by_identity(&spec.database_identity)?; @@ -301,28 +450,42 @@ impl spacetimedb_client_api::ControlStateWriteAccess for StandaloneEnv { // Instantiate a temporary database in order to check that the module is valid. // This will e.g. typecheck RLS filters. self.host_controller - .check_module_validity(database.clone(), program) + .check_module_validity_with_environment(database.clone(), program, spec.environment.clone()) .await?; let program_hash = self.program_store.put(&spec.program_bytes).await?; debug_assert_eq!(_hash_for_assert, program_hash); - let database_id = self.control_db.insert_database(database)?; - - self.schedule_replicas(database_id, num_replicas).await?; + let (database, replica) = + self.control_db + .install_database_with_environment(database, None, spec.environment, &[])?; + // The leader nomination and input are durable already. If this + // waiter is cancelled, ordinary lookup resumes the same input. + self.on_insert_replica(&replica).await?; + debug_assert_eq!(database.id, replica.database_id); Ok(None) } // The database already exists, so we'll try to update it. // If that fails, we'll keep the old one. Some(database) => { + anyhow::ensure!( + database.owner_identity == *publisher, + "database ownership changed before publication" + ); let database_id = database.id; let database_identity = database.database_identity; - let leader = self.leader(database_id).await?; + let leader = self.leader_under_publication(database_id).await?; let update_result = leader - .update(database, spec.host_type, spec.program_bytes.to_vec().into(), policy) + .update_with_environment( + database, + spec.host_type, + spec.program_bytes.to_vec().into(), + policy, + spec.environment, + ) .await?; if update_result.was_successful() { let replicas = self.control_db.get_replicas_by_database(database_id)?; @@ -372,37 +535,18 @@ impl spacetimedb_client_api::ControlStateWriteAccess for StandaloneEnv { } } - async fn migrate_plan( + async fn delete_database_owned( &self, - spec: spacetimedb_client_api::DatabaseDef, - style: PrettyPrintStyle, - ) -> anyhow::Result { - let existing_db = self.control_db.get_database_by_identity(&spec.database_identity)?; - - match existing_db { - Some(db) => { - let host = self.leader(db.id).await?; - self.host_controller - .migrate_plan( - db, - spec.host_type, - host.replica_id, - spec.program_bytes.to_vec().into(), - style, - ) - .await - } - None => anyhow::bail!( - "Database `{}` does not exist", - spec.database_identity.to_abbreviated_hex() - ), - } - } - - async fn delete_database(&self, _caller_identity: &Identity, database_identity: &Identity) -> anyhow::Result<()> { + caller_identity: &Identity, + database_identity: &Identity, + ) -> anyhow::Result<()> { let Some(database) = self.control_db.get_database_by_identity(database_identity)? else { return Ok(()); }; + anyhow::ensure!( + database.owner_identity == *caller_identity, + "database ownership changed before deletion" + ); self.control_db.delete_database(database.id)?; for instance in self.control_db.get_replicas_by_database(database.id)? { @@ -412,93 +556,51 @@ impl spacetimedb_client_api::ControlStateWriteAccess for StandaloneEnv { Ok(()) } - async fn reset_database(&self, _caller_identity: &Identity, spec: DatabaseResetDef) -> anyhow::Result<()> { - let mut database = self + async fn reset_database_owned(&self, caller_identity: &Identity, spec: DatabaseResetDef) -> anyhow::Result<()> { + let previous = self .control_db .get_database_by_identity(&spec.database_identity)? .with_context(|| format!("Database `{}` does not exist", spec.database_identity))?; - let database_id = database.id; - - if let Some(program) = spec.program_bytes { - if let Some(host_type) = spec.host_type { - database.host_type = host_type; + anyhow::ensure!( + previous.owner_identity == *caller_identity, + "database ownership changed before reset" + ); + let mut database = previous.clone(); + let program = match spec.program_bytes { + Some(bytes) => { + let host_type = spec.host_type.unwrap_or(database.host_type); + Program::from_bytes(host_type.into(), &bytes[..]) + } + None => { + // A reset without an artifact retains the currently committed + // module, not the original bootstrap program or its old values. + let module = self.leader_under_publication(database.id).await?.module().await?; + module + .relational_db() + .program()? + .context("database is not initialized")? } - let program_bytes = &program[..]; - let program = Program::from_bytes(database.host_type.into(), program_bytes); - let _hash_for_assert = program.hash; - - database.initial_program = program.hash; - - self.host_controller - .check_module_validity(database.clone(), program) - .await?; - let _stored_hash_for_assert = self.program_store.put(program_bytes).await?; - debug_assert_eq!(_hash_for_assert, _stored_hash_for_assert); - } - self.control_db.update_database(database)?; - - for instance in self.control_db.get_replicas_by_database(database_id)? { - self.delete_replica(instance.id).await?; - } - // Standalone only support a single replica. - let num_replicas = 1; - self.schedule_replicas(database_id, num_replicas).await?; - - Ok(()) - } - - async fn add_energy(&self, identity: &Identity, amount: EnergyQuanta) -> anyhow::Result<()> { - let balance = self - .control_db - .get_energy_balance(identity)? - .unwrap_or(EnergyBalance::ZERO); - - let balance = balance.saturating_add_energy(amount); - - self.control_db.set_energy_balance(*identity, balance)?; - Ok(()) - } - async fn withdraw_energy(&self, _identity: &Identity, _amount: EnergyQuanta) -> anyhow::Result<()> { - // The energy balance code is obsolete. - Ok(()) - } - - async fn register_tld(&self, identity: &Identity, tld: Tld) -> anyhow::Result { - Ok(self.control_db.spacetime_register_tld(tld, *identity)?) - } - - async fn create_dns_record( - &self, - owner_identity: &Identity, - domain: &DomainName, - database_identity: &Identity, - ) -> anyhow::Result { - Ok(self - .control_db - .spacetime_insert_domain(database_identity, domain.clone(), *owner_identity, true)?) - } - - async fn replace_dns_records( - &self, - database_identity: &Identity, - owner_identity: &Identity, - domain_names: &[DomainName], - ) -> anyhow::Result { - Ok(self - .control_db - .spacetime_replace_domains(database_identity, owner_identity, domain_names)?) - } - - async fn set_database_lock( - &self, - _caller_identity: &Identity, - database_identity: &Identity, - locked: bool, - ) -> anyhow::Result<()> { - let Some(_database) = self.control_db.get_database_by_identity(database_identity)? else { - anyhow::bail!("Database not found: {}", database_identity.to_abbreviated_hex()); }; - self.control_db.set_database_lock(database_identity, locked)?; + database.host_type = HostType::from(program.kind); + database.initial_program = program.hash; + self.host_controller + .check_module_validity_with_environment(database.clone(), program.clone(), spec.environment.clone()) + .await?; + let stored = self.program_store.put(&program.bytes).await?; + anyhow::ensure!(stored == program.hash, "stored reset program changed"); + let previous_replicas = self.control_db.get_replicas_by_database(database.id)?; + // Keep old nominations until all requested closes succeed. The owned + // publication guard excludes leader admission across close and commit. + for replica in &previous_replicas { + self.on_delete_replica(replica.id).await?; + } + let (_, replica) = self.control_db.install_database_with_environment( + database, + Some(&previous), + spec.environment, + &previous_replicas, + )?; + self.on_insert_replica(&replica).await?; Ok(()) } } @@ -567,21 +669,6 @@ impl StandaloneEnv { Ok(()) } - async fn schedule_replicas(&self, database_id: u64, num_replicas: u8) -> Result<(), anyhow::Error> { - // Just scheduling a bunch of replicas to the only machine - for i in 0..num_replicas { - let replica = Replica { - id: 0, - database_id, - node_id: 0, - leader: i == 0, - }; - self.insert_replica(replica).await?; - } - - Ok(()) - } - async fn on_insert_replica(&self, instance: &Replica) -> Result<(), anyhow::Error> { if instance.leader { let database = self @@ -593,7 +680,7 @@ impl StandaloneEnv { instance.database_id, instance.id ) })?; - self.leader(database.id).await?; + self.leader_under_publication(database.id).await?; } Ok(()) @@ -604,9 +691,7 @@ impl StandaloneEnv { // replicas which have been deleted. This will just drop // them from memory, but will not remove them from disk. We need // some kind of database lifecycle manager long term. - self.host_controller - .exit_module_host(replica_id, Duration::from_secs(30)) - .await?; + self.host_controller.exit_module_host_and_join(replica_id).await?; Ok(()) } @@ -690,4 +775,74 @@ mod tests { Ok(()) } + #[tokio::test] + async fn cancelled_publication_waiter_keeps_mutations_and_leader_admission_fenced() -> Result<()> { + let tempdir = TempDir::new()?; + // Use one subdir for keys and another for the data dir. + let keys = tempdir.path().join("keys"); + let root = tempdir.path().join("data"); + let data_dir = Arc::new(ServerDataDir::from_path_unchecked(root)); + + fs::create_dir(&keys)?; + data_dir.create()?; + + let pub_key = PubKeyPath(keys.join("public")); + let priv_key = PrivKeyPath(keys.join("private")); + let ca = CertificateAuthority { + jwt_pub_key_path: pub_key, + jwt_priv_key_path: priv_key, + }; + + // Create the keys. + ca.get_or_create_keys()?; + let config = StandaloneOptions { + db_config: db::Config { + storage: Storage::Memory, + page_pool_max_size: None, + }, + durability: DurabilityConfig::default(), + websocket: WebSocketOptions::default(), + module_http: ModuleHttpConfig::default(), + wasm: WasmConfig::default(), + v8: V8Config::default(), + }; + + let env = StandaloneEnv::init(config, &ca, data_dir.clone(), JobCores::without_pinned_cores()).await?; + + let (started, started_rx) = tokio::sync::oneshot::channel(); + let (release, release_rx) = tokio::sync::oneshot::channel(); + let owner = env.clone(); + let waiter = tokio::spawn(async move { + owner + .own_publication(move |_owner| async move { + let _ = started.send(()); + release_rx.await?; + Ok(()) + }) + .await + }); + started_rx.await?; + waiter.abort(); + assert!(waiter.await.unwrap_err().is_cancelled()); + + // Both a later mutation and ordinary leader lookup must stay behind the + // accepted operation even after its request future has disappeared. + let next_owner = env.clone(); + let mut next = tokio::spawn(async move { next_owner.own_publication(|_| async { Ok(()) }).await }); + assert!(tokio::time::timeout(Duration::from_millis(30), &mut next) + .await + .is_err()); + let read_owner = env.clone(); + let mut reader = tokio::spawn(async move { read_owner.leader(u64::MAX).await }); + assert!(tokio::time::timeout(Duration::from_millis(30), &mut reader) + .await + .is_err()); + release.send(()).unwrap(); + tokio::time::timeout(Duration::from_secs(5), next).await???; + assert!(matches!( + tokio::time::timeout(Duration::from_secs(5), reader).await??, + Err(GetLeaderHostError::NoSuchReplica) + )); + Ok(()) + } } diff --git a/crates/testing/src/modules.rs b/crates/testing/src/modules.rs index 465e22c3d6e..09d35280562 100644 --- a/crates/testing/src/modules.rs +++ b/crates/testing/src/modules.rs @@ -1,3 +1,4 @@ +use std::collections::BTreeMap; use std::env; use std::future::Future; use std::panic::AssertUnwindSafe; @@ -67,6 +68,46 @@ pub struct ModuleHandle { } impl ModuleHandle { + /// Publish a complete configuration through the standalone control API. + pub async fn republish_environment( + &self, + environment: BTreeMap, + ) -> anyhow::Result { + let program = self + .client + .module() + .relational_db() + .program()? + .expect("published module"); + self.republish_program(program.bytes.into(), program.kind.into(), environment) + .await + } + + /// Publish exact replacement artifact bytes and their complete configuration. + pub async fn republish_program( + &self, + program_bytes: Bytes, + host_type: HostType, + environment: BTreeMap, + ) -> anyhow::Result { + self.env + .publish_database( + &Identity::ZERO, + DatabaseDef { + database_identity: self.db_identity, + program_bytes, + environment, + num_replicas: None, + host_type, + parent: None, + organization: None, + }, + MigrationPolicy::Compatible, + ) + .await? + .ok_or_else(|| anyhow::anyhow!("expected an update to the existing database")) + } + async fn call_reducer_result(&self, reducer: &str, args: FunctionArgs) -> anyhow::Result { let result = self .client @@ -235,6 +276,18 @@ pub enum CompilationMode { } impl CompiledModule { + /// Use an artifact built by an external toolchain, such as Linux NativeAOT. + /// This changes only compilation; publication and execution still use the + /// same in-process standalone server as locally compiled fixtures. + pub fn from_artifact(name: &str, host_type: HostType, path: PathBuf) -> Self { + Self { + name: name.to_owned(), + path, + host_type, + program_bytes: OnceLock::new(), + } + } + pub fn compile(name: &str, mode: CompilationMode) -> Self { let (path, host_type) = spacetimedb_cli::build( &module_path(name), @@ -245,12 +298,7 @@ impl CompiledModule { None, ) .expect("Module compilation failed"); - Self { - name: name.to_owned(), - path, - host_type: host_type.parse().unwrap(), - program_bytes: OnceLock::new(), - } + Self::from_artifact(name, host_type.parse().unwrap(), path) } pub fn path(&self) -> &Path { @@ -279,10 +327,22 @@ impl CompiledModule { where R: FnOnce(ModuleHandle) -> F, F: Future, + { + self.with_module_async_with_environment(config, BTreeMap::new(), routine) + } + + pub fn with_module_async_with_environment( + &self, + config: Config, + environment: BTreeMap, + routine: R, + ) where + R: FnOnce(ModuleHandle) -> F, + F: Future, { with_runtime(move |runtime| { runtime.block_on(async { - let module = self.load_module(config, None).await; + let module = self.load_module_with_environment(config, None, environment).await; let env = module.env.clone(); let db_identity = module.db_identity; let routine_result = AssertUnwindSafe(routine(module)).catch_unwind().await.map(drop); @@ -311,6 +371,16 @@ impl CompiledModule { /// without resetting the database. /// This is used to speed up benchmarks running under callgrind (it allows them to reuse native-compiled wasm modules). pub async fn load_module(&self, config: Config, reuse_db_path: Option<&RootDir>) -> ModuleHandle { + self.load_module_with_environment(config, reuse_db_path, BTreeMap::new()) + .await + } + + pub async fn load_module_with_environment( + &self, + config: Config, + reuse_db_path: Option<&RootDir>, + environment: BTreeMap, + ) -> ModuleHandle { let paths = match reuse_db_path { Some(path) => SpacetimePaths::from_root_dir(path), None => { @@ -350,6 +420,7 @@ impl CompiledModule { DatabaseDef { database_identity: db_identity, program_bytes: self.program_bytes(), + environment, num_replicas: None, host_type: self.host_type, parent: None, diff --git a/crates/testing/tests/environment.rs b/crates/testing/tests/environment.rs index 7cdf4fecf0e..96a29b29bf9 100644 --- a/crates/testing/tests/environment.rs +++ b/crates/testing/tests/environment.rs @@ -1,14 +1,20 @@ -//! Actual module calls exercise environment ABI, bindings, and snapshot semantics. +//! Actual publication and module calls exercise configuration atomicity and ABI enforcement. use serial_test::serial; +use spacetimedb::client::{messages::SerializableMessage, OutboundMessage}; use spacetimedb::host::{FunctionArgs, ModuleHost}; +use spacetimedb_client_api_messages::websocket::v1 as ws_v1; use spacetimedb_lib::identity::AuthCtx; use spacetimedb_lib::{bsatn, sats::product, AlgebraicValue, Identity}; -use spacetimedb_testing::modules::{CompilationMode, CompiledModule, DEFAULT_CONFIG}; +use spacetimedb_testing::modules::{CompilationMode, CompiledModule, ModuleHandle, DEFAULT_CONFIG}; +use std::collections::BTreeMap; +use std::time::Duration; -async fn sql(module: &ModuleHost, statement: String) -> Vec { +type Values = BTreeMap; + +async fn sql(module: &ModuleHost, statement: &str) -> Vec { spacetimedb::sql::execute::run( module.relational_db().clone(), - statement, + statement.to_string(), AuthCtx::for_current(Identity::ZERO), Some(module.info.subscriptions.clone()), Some(module.clone()), @@ -19,13 +25,190 @@ async fn sql(module: &ModuleHost, statement: String) -> Vec ModuleHost { + let result = handle.republish_environment(values.clone()).await.unwrap(); + assert!(result.was_successful(), "configuration publication failed"); + handle.client.module() +} + +async fn read(module: &ModuleHost, key: &str) -> AlgebraicValue { + module + .call_procedure( + Identity::ZERO, + None, + None, + "read_environment", + FunctionArgs::Bsatn(bsatn::to_vec(&product![key]).unwrap().into()), + ) + .await + .result + .unwrap() + .return_val +} + +async fn next_message(handle: &mut ModuleHandle) -> OutboundMessage { + tokio::time::timeout(Duration::from_secs(10), handle.recv_message()) + .await + .expect("timed out waiting for environment subscription update") + .expect("environment subscription disconnected") +} + +async fn expect_view_update(handle: &mut ModuleHandle) { + let message = next_message(handle).await; + assert!(matches!(message, OutboundMessage::V1(SerializableMessage::TxUpdate(_)))); + // Replacing the one-row view must send both the old row's deletion and + // the new row's insertion to an already connected subscriber. + assert_eq!(message.num_rows(), Some(2)); +} + +async fn check_submodule_scope(handle: &mut ModuleHandle, values: &mut Values) { + values.insert("EMPTY".into(), "root-visible".into()); + let module = publish(handle, values).await; + assert!(module.info.module_def.reducer_by_name("lib.env_read_reducer").is_some()); + let child = module + .call_reducer( + Identity::ZERO, + None, + None, + None, + None, + "lib.env_read_reducer", + FunctionArgs::Nullary, + ) + .await; + assert!(child.is_err() || child.unwrap().outcome.into_result().is_err()); + for procedure in ["lib.env_read_procedure", "lib.env_read_in_tx"] { + assert!(module.info.module_def.procedure_by_name(procedure).is_some()); + let result = module + .call_procedure(Identity::ZERO, None, None, procedure, FunctionArgs::Nullary) + .await; + assert!(result.result.is_err(), "submodule procedure read the root environment"); + } + for view in ["lib.env_read_view", "lib.env_read_sql_view", "env_read_root_sql_view"] { + assert!(module.info.module_def.view_by_name_with_module(view).is_some()); + let result = spacetimedb::sql::execute::run( + module.relational_db().clone(), + format!("SELECT * FROM {view}"), + AuthCtx::for_current(Identity::ZERO), + Some(module.info.subscriptions.clone()), + Some(module.clone()), + &mut vec![], + ) + .await; + let error = format!( + "{:#}", + result.expect_err("view bypassed the environment read interface") + ); + assert!(!error.contains("not found"), "view failed before dispatch: {error}"); + // A valid view and a forbidden view share one actual subscription + // request. It must fail as a whole, without an initial-success message. + let request_id = 880; + let subscribe = ws_v1::ClientMessage::::Subscribe(ws_v1::Subscribe { + query_strings: ["SELECT * FROM my_player".into(), format!("SELECT * FROM {view}").into()].into(), + request_id, + }); + let _ = handle.send(bsatn::to_vec(&subscribe).unwrap()).await; + let message = next_message(handle).await; + let OutboundMessage::V1(SerializableMessage::Subscription(message)) = message else { + panic!("failed view subscription returned a success or unexpected message: {message:?}"); + }; + assert_eq!(message.request_id, Some(request_id)); + let spacetimedb::client::messages::SubscriptionResult::Error(error) = message.result else { + panic!("forbidden view subscription returned rows"); + }; + assert!(!error.message.is_empty()); + assert!(!error.message.contains("not found")); + } + // HTTP routes are root entries today. Calling an exported child callback + // as an ordinary helper retains that root entry's authority. + assert_eq!( + &handle.call_http_route_get("/env-child").await.unwrap()[..], + b"root-visible" + ); +} + +// SQL, subscription materialization, and ordinary reducers all use the same +// main Wasmtime instance. The fixture view sets a guest-global marker and traps; +// the next reducer checks the marker is absent in the replacement instance. +async fn check_wasm_trap_disposal(handle: &mut ModuleHandle) { + let module = handle.client.module(); + let view = "SELECT * FROM environment_trap"; + let result = spacetimedb::sql::execute::run( + module.relational_db().clone(), + view.to_string(), + AuthCtx::for_current(Identity::ZERO), + Some(module.info.subscriptions.clone()), + Some(module.clone()), + &mut vec![], + ) + .await; + let error = format!("{:#}", result.expect_err("trapped view returned SQL success")); + assert!(!error.contains("not found"), "view failed before dispatch: {error}"); + expect_clean_wasm_instance(&module).await; + + // Keep the existing successful ENV-view subscription while adding this + // separate query. The failing request must report an error, not rows. + let request_id = 890; + let subscribe = ws_v1::ClientMessage::::SubscribeSingle(ws_v1::SubscribeSingle { + query: view.into(), + request_id, + query_id: ws_v1::QueryId::new(890), + }); + let _ = handle.send(bsatn::to_vec(&subscribe).unwrap()).await; + let message = next_message(handle).await; + let OutboundMessage::V1(SerializableMessage::Subscription(message)) = message else { + panic!("trapped view subscription returned success or an unexpected message: {message:?}"); + }; + assert_eq!(message.request_id, Some(request_id)); + let spacetimedb::client::messages::SubscriptionResult::Error(error) = message.result else { + panic!("trapped view subscription returned rows"); + }; + assert!(!error.message.is_empty()); + assert!(!error.message.contains("not found")); + expect_clean_wasm_instance(&module).await; +} + +async fn expect_clean_wasm_instance(module: &ModuleHost) { + module + .call_reducer( + Identity::ZERO, + None, + None, + None, + None, + "expect_environment", + FunctionArgs::Bsatn(bsatn::to_vec(&product!["MISSING", None::]).unwrap().into()), + ) + .await + .unwrap() + .outcome + .into_result() + .unwrap(); } fn exercise_fixture(name: &str) { - CompiledModule::compile(name, CompilationMode::Debug).with_module_async(DEFAULT_CONFIG, |handle| async move { - let module = handle.client.module(); + let initial = if name == "environment-test" { + Values::from([ + ("REQUIRED".into(), "initial-required".into()), + ("MODE".into(), "ready".into()), + ]) + } else { + Values::new() + }; + // NativeAOT's WebAssembly compiler requires a supported compiler host. + // Allow this one fixture to consume the exact artifact built there while + // exercising all normal publication and runtime paths below. + let artifact = (name == "module-test-cs") + .then(|| std::env::var_os("SPACETIMEDB_ENV_CSHARP_MODULE")) + .flatten(); + let compiled = match artifact { + Some(path) => { + CompiledModule::from_artifact(name, spacetimedb::messages::control_db::HostType::Wasm, path.into()) + } + None => CompiledModule::compile(name, CompilationMode::Debug), + }; + compiled.with_module_async_with_environment(DEFAULT_CONFIG, initial.clone(), |mut handle| async move { + let mut values = initial; for (key, expected) in [ ("MISSING", None), ("EMPTY", Some("".to_string())), @@ -34,9 +217,9 @@ fn exercise_fixture(name: &str) { ("MAXIMUM", Some("é".repeat(4096))), ] { if let Some(value) = &expected { - set_environment(&module, key, value).await; + values.insert(key.into(), value.clone()); } - let args = product![key, expected.clone()]; + let module = publish(&handle, &values).await; let result = module .call_reducer( Identity::ZERO, @@ -45,46 +228,46 @@ fn exercise_fixture(name: &str) { None, None, "expect_environment", - FunctionArgs::Bsatn(bsatn::to_vec(&args).unwrap().into()), + FunctionArgs::Bsatn(bsatn::to_vec(&product![key, expected.clone()]).unwrap().into()), ) - .await; - let result = result - .map_err(anyhow::Error::from) - .and_then(|r| r.outcome.into_result()); - assert!( - result.is_ok(), - "{name} {key}: {result:?}; module log: {}", - handle.read_log(None).await - ); - let read = || FunctionArgs::Bsatn(bsatn::to_vec(&product![key]).unwrap().into()); - let result = module - .call_procedure(Identity::ZERO, None, None, "read_environment", read()) .await - .result - .unwrap() - .return_val; - assert_eq!(result, AlgebraicValue::from(expected.clone())); + .unwrap(); + result.outcome.into_result().unwrap(); + assert_eq!(read(&module, key).await, AlgebraicValue::from(expected.clone())); if expected.is_some() { - set_environment(&module, key, "updated").await; - let result = module - .call_procedure(Identity::ZERO, None, None, "read_environment", read()) - .await - .result - .unwrap() - .return_val; - assert_eq!(result, AlgebraicValue::from(Some("updated".to_string()))); - sql(&module, format!("DELETE env.{key}")).await; - let result = module - .call_procedure(Identity::ZERO, None, None, "read_environment", read()) - .await - .result - .unwrap() - .return_val; - assert_eq!(result, AlgebraicValue::from(None::)); + values.insert(key.into(), "updated".into()); + let module = publish(&handle, &values).await; + assert_eq!( + read(&module, key).await, + AlgebraicValue::from(Some("updated".to_string())) + ); + values.remove(key); + let module = publish(&handle, &values).await; + assert_eq!(read(&module, key).await, AlgebraicValue::from(None::)); } } if name == "environment-test" { - set_environment(&module, "HANDLER", "handler snapshot").await; + // Required values cannot be inherited from the previous publish, + // and an invalid literal cannot replace the previous configuration. + for invalid in [ + Values::new(), + Values::from([ + ("REQUIRED".into(), "initial-required".into()), + ("MODE".into(), "invalid-secret-marker".into()), + ]), + ] { + let result = handle.republish_environment(invalid).await; + assert!(result.as_ref().is_err() || !result.as_ref().unwrap().was_successful()); + assert_eq!( + read(&handle.client.module(), "REQUIRED").await, + AlgebraicValue::from(Some("initial-required".to_string())) + ); + } + // A same-module publish does not run init again. Its new required + // value deliberately differs from the value init asserted. + values.insert("REQUIRED".into(), "republished".into()); + values.insert("HANDLER".into(), "handler snapshot".into()); + let module = publish(&handle, &values).await; let (_, body) = module .call_http_handler( module.info.module_def.http_handler_ids_and_defs().next().unwrap().0, @@ -100,23 +283,39 @@ fn exercise_fixture(name: &str) { .await .unwrap(); assert_eq!(&body[..], b"handler snapshot"); - // This view first reads a missing key. Its dependency must survive - // absence, and normal SQL mutations must invalidate its cached row. - let read_view = || "SELECT * FROM environment_value".to_string(); - assert_eq!(sql(&module, read_view()).await, vec![product![None::]]); - set_environment(&module, "WATCHED", "first").await; - assert_eq!( - sql(&module, read_view()).await, - vec![product![Some("first".to_string())]] - ); - set_environment(&module, "WATCHED", "second").await; + let view = "SELECT * FROM environment_value"; + assert_eq!(sql(&module, view).await, vec![product![None::]]); + let subscribe = ws_v1::ClientMessage::::Subscribe(ws_v1::Subscribe { + query_strings: [view.into()].into(), + request_id: 71, + }); + handle.send(bsatn::to_vec(&subscribe).unwrap()).await.unwrap(); + let initial_update = next_message(&mut handle).await; + assert!(matches!( + initial_update, + OutboundMessage::V1(SerializableMessage::Subscribe(_)) + )); + assert_eq!(initial_update.num_rows(), Some(1)); + for value in ["first", "second"] { + values.insert("WATCHED".into(), value.into()); + let module = publish(&handle, &values).await; + assert_eq!(sql(&module, view).await, vec![product![Some(value.to_string())]]); + expect_view_update(&mut handle).await; + } + let mut invalid = values.clone(); + invalid.insert("WATCHED".into(), "fail-view".into()); + let failed = handle.republish_environment(invalid).await; + assert!(failed.as_ref().is_err() || !failed.as_ref().unwrap().was_successful()); assert_eq!( - sql(&module, read_view()).await, + sql(&handle.client.module(), view).await, vec![product![Some("second".to_string())]] ); - sql(&module, "DELETE env.WATCHED".into()).await; - assert_eq!(sql(&module, read_view()).await, vec![product![None::]]); - set_environment(&module, "LIMIT", &"x".repeat(8192)).await; + values.remove("WATCHED"); + let module = publish(&handle, &values).await; + assert_eq!(sql(&module, view).await, vec![product![None::]]); + expect_view_update(&mut handle).await; + values.insert("LIMIT".into(), "x".repeat(8192)); + let module = publish(&handle, &values).await; for _ in 0..2 { module .call_reducer( @@ -134,44 +333,153 @@ fn exercise_fixture(name: &str) { .into_result() .unwrap(); } + check_wasm_trap_disposal(&mut handle).await; + } + if name == "module-test-ts" { + check_submodule_scope(&mut handle, &mut values).await; + } + for key in ["UNDECLARED", "A=B"] { + let result = handle + .client + .module() + .call_reducer( + Identity::ZERO, + None, + None, + None, + None, + "expect_environment", + FunctionArgs::Bsatn(bsatn::to_vec(&product![key, None::]).unwrap().into()), + ) + .await; + assert!(result.is_err() || result.unwrap().outcome.into_result().is_err()); } - let args = product!["A=B", None::]; - let result = module - .call_reducer( - Identity::ZERO, - None, - None, - None, - None, - "expect_environment", - FunctionArgs::Bsatn(bsatn::to_vec(&args).unwrap().into()), - ) - .await; - module.exit().await; - assert!(result.is_err() || result.unwrap().outcome.into_result().is_err()); }); } #[test] #[serial] -fn rust_environment_reads_are_not_cached_and_preserve_missing_empty_utf8_and_nul() { +fn rust_environment_publish_is_atomic_and_reads_follow_declared_configuration() { exercise_fixture("environment-test"); } #[test] #[serial] -fn typescript_environment_reads_are_not_cached_and_preserve_missing_empty_utf8_and_nul() { +fn typescript_environment_publish_and_checked_reads() { exercise_fixture("module-test-ts"); } #[test] #[serial] -fn cpp_environment_reads_are_not_cached_and_preserve_missing_empty_utf8_and_nul() { +fn cpp_environment_publish_and_checked_reads() { exercise_fixture("module-test-cpp"); } #[test] #[serial] -fn csharp_environment_reads_are_not_cached_and_preserve_missing_empty_utf8_and_nul() { +fn csharp_environment_publish_and_checked_reads() { exercise_fixture("module-test-cs"); } + +#[cfg(feature = "allow_loopback_http_for_tests")] +#[test] +#[serial] +fn suspended_procedure_cannot_read_environment_from_a_replacement_program() { + use anyhow::Context as _; + use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; + + let initial = Values::from([ + ("REQUIRED".into(), "initial-required".into()), + ("MODE".into(), "ready".into()), + ]); + CompiledModule::compile("environment-test", CompilationMode::Debug).with_module_async_with_environment( + DEFAULT_CONFIG, + initial.clone(), + |handle| async move { + for explicit_tx in [false, true] { + let old = handle.client.module(); + let program = old.relational_db().program().unwrap().unwrap(); + let old_hash = program.hash; + let mut replacement = program.bytes.to_vec(); + // A valid custom section changes the exact program hash without + // changing the schema or behavior of this real Wasm module. + replacement.extend_from_slice(&[0, 3, 1, b'e', u8::from(explicit_tx)]); + let values = Values::from([ + ("REQUIRED".into(), "new-program-value".into()), + ("MODE".into(), "ready".into()), + ]); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let url = format!("http://{}/hold", listener.local_addr().unwrap()); + let (entered_tx, entered_rx) = tokio::sync::oneshot::channel(); + let (release_tx, release_rx) = tokio::sync::oneshot::channel(); + let mut server = tokio::spawn(async move { + let (mut stream, _) = tokio::time::timeout(Duration::from_secs(10), listener.accept()).await??; + let mut request = Vec::new(); + while !request.ends_with(b"\r\n\r\n") { + anyhow::ensure!(request.len() < 4096, "test request exceeded its header bound"); + request.push(tokio::time::timeout(Duration::from_secs(10), stream.read_u8()).await??); + } + entered_tx + .send(()) + .map_err(|_| anyhow::anyhow!("test coordinator closed"))?; + tokio::time::timeout(Duration::from_secs(20), release_rx).await??; + stream + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\nConnection: close\r\n\r\n") + .await?; + stream.shutdown().await?; + anyhow::Ok(()) + }); + let call = old.call_procedure( + Identity::ZERO, + None, + None, + "read_environment_after_http", + FunctionArgs::Bsatn(bsatn::to_vec(&product![url, explicit_tx]).unwrap().into()), + ); + let publish_while_suspended = async { + tokio::time::timeout(Duration::from_secs(10), entered_rx) + .await + .context("procedure never reached owned HTTP barrier")??; + let publish = handle.republish_program(replacement.into(), program.kind.into(), values); + let release_after_commit = async { + tokio::time::timeout(Duration::from_secs(20), async { + loop { + if old.relational_db().program()?.unwrap().hash != old_hash { + return anyhow::Ok(()); + } + tokio::time::sleep(Duration::from_millis(5)).await; + } + }) + .await + .context("replacement program did not commit while procedure was suspended")??; + release_tx.send(()).map_err(|_| anyhow::anyhow!("HTTP barrier closed")) + }; + // Drive publication even when it waits for the old procedure + // to finish; release only after the new program is committed. + let (published, released) = tokio::join!(publish, release_after_commit); + released?; + anyhow::ensure!(published?.was_successful(), "replacement publication failed"); + anyhow::Ok(()) + }; + let (result, coordinated) = tokio::join!(call, publish_while_suspended); + let server_result = match tokio::time::timeout(Duration::from_secs(35), &mut server).await { + Ok(result) => result.unwrap(), + Err(_) => { + server.abort(); + let _ = server.await; + Err(anyhow::anyhow!("owned HTTP test server did not finish")) + } + }; + // Join the owned listener and publication before reporting any + // assertion failure, so failure cannot leave a live test host. + coordinated.unwrap(); + server_result.unwrap(); + assert!(result.result.is_err(), "old procedure read the replacement environment"); + assert_eq!( + read(&handle.client.module(), "REQUIRED").await, + AlgebraicValue::from(Some("new-program-value".to_string())) + ); + } + }, + ); +} diff --git a/docs/docs/00200-core-concepts/00100-databases/00300-spacetime-publish.md b/docs/docs/00200-core-concepts/00100-databases/00300-spacetime-publish.md index f8aaf847553..2f91291133c 100644 --- a/docs/docs/00200-core-concepts/00100-databases/00300-spacetime-publish.md +++ b/docs/docs/00200-core-concepts/00100-databases/00300-spacetime-publish.md @@ -104,6 +104,12 @@ spacetime publish --delete-data For all available publishing options and flags, see the [`spacetime publish` CLI reference](../../00300-resources/00200-reference/00100-cli-reference/00100-cli-reference.md#spacetime-publish). +### Environment Variables + +Modules can declare environment variables for configuration and secrets. Each publish supplies the complete set of values from the target's `env` configuration and declared shell variables. Required values must be supplied on every publish; omitted optional values are removed. The module and its environment update atomically. + +See [Environment Variables](./00700-environment-variables.md) for declarations, SDK accessors, publishing examples, and private tables for secrets that need to change without republishing. + ## Next Steps After publishing: diff --git a/docs/docs/00200-core-concepts/00100-databases/00700-environment-variables.md b/docs/docs/00200-core-concepts/00100-databases/00700-environment-variables.md new file mode 100644 index 00000000000..25eb6069208 --- /dev/null +++ b/docs/docs/00200-core-concepts/00100-databases/00700-environment-variables.md @@ -0,0 +1,235 @@ +--- +title: Environment Variables +slug: /databases/environment-variables +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Environment Variables + +Environment variables store configuration and secrets for a database, such as API keys and deployment settings. A module declares the names it accepts and any allowed values. Each publish supplies the complete set of values for that module. Module code reads them through `ctx.env`, or `ctx.Env` in C#. + +Use environment variables for configuration that changes when a module is published. For secrets or configuration that must change without publishing, use a [private table](#dynamically-editable-or-untyped-secrets). + +This guide assumes a module set up using a quickstart, such as the [Rust quickstart](../../00100-intro/00200-quickstarts/00500-rust.md), and familiarity with [publishing](./00300-spacetime-publish.md). + +## Declare and read variables + +Declare every environment key in the module. All values are strings. A declaration can accept any string, one exact string, or a set of allowed strings. Optional declarations permit an absent value. + +These examples declare a required `API_KEY`, a required `MODE` restricted to `development` or `production`, and an optional `LOG_LEVEL` restricted to `info` or `debug`. + + + + +Pass the declaration as the `env` option to `schema`. Include the module's existing tables in the first argument if it has any. + +```typescript +import { schema, t } from 'spacetimedb/server'; + +const spacetimedb = schema( + {}, + { + env: { + API_KEY: t.string(), + MODE: t.enum('Mode', ['development', 'production']), + LOG_LEVEL: t.enum('LogLevel', ['info', 'debug']).optional(), + }, + } +); + +export default spacetimedb; +``` + +Inside a reducer, procedure, or view, read values from its context: + +```typescript +const apiKey: string = ctx.env.API_KEY; +const mode: 'development' | 'production' = ctx.env.MODE; +const logLevel: 'info' | 'debug' | undefined = ctx.env.LOG_LEVEL; +const checked: string | null = ctx.env.get('LOG_LEVEL'); +``` + +Within an environment declaration, a simple enum specifies allowed strings. Enums used elsewhere in the module retain their usual tagged representation. An enum with one case restricts the value to that string. Enums with payloads cannot be used as environment constraints. + + + + +Add one environment declaration to the module: + +```rust +#[spacetimedb::env] +pub struct Env { + pub API_KEY: String, + #[env(values("development", "production"))] + pub MODE: String, + #[env(values("info", "debug"))] + pub LOG_LEVEL: Option, +} +``` + +Inside a reducer, procedure, or view, read values from its context: + +```rust +let api_key: String = ctx.env.API_KEY(); +let mode: String = ctx.env.MODE(); +let log_level: Option = ctx.env.LOG_LEVEL(); +let checked: Option = ctx.env.get("LOG_LEVEL"); +``` + +`String` requires a value, and `Option` permits absence. `#[env(values(...))]` restricts the allowed strings. Supplying one string makes it an exact-value constraint. + + + + +Add one environment declaration to the module: + +```csharp +#nullable enable + +[SpacetimeDB.Env] +public partial struct EnvironmentSchema +{ + public string API_KEY; + [SpacetimeDB.EnvValues("development", "production")] + public string MODE; + [SpacetimeDB.EnvValues("info", "debug")] + public string? LOG_LEVEL; +} +``` + +Inside a reducer, procedure, or view, read values from its context: + +```csharp +string apiKey = ctx.Env.API_KEY; +string mode = ctx.Env.MODE; +string? logLevel = ctx.Env.LOG_LEVEL; +string? checkedValue = ctx.Env.Get("LOG_LEVEL"); +``` + +`string` requires a value, and `string?` permits absence. `[SpacetimeDB.EnvValues(...)]` restricts the allowed strings. Supplying one string makes it an exact-value constraint. + + + + +Declare the environment in a dedicated header named `environment.h`: + +```cpp +#pragma once +#include + +SPACETIMEDB_ENV( + (API_KEY, std::string), + (MODE, std::string, ("development", "production")), + (LOG_LEVEL, std::optional, ("info", "debug")) +) +``` + +In `CMakeLists.txt`, set the header path **before** adding the SpacetimeDB SDK directory: + +```cmake +set(SPACETIMEDB_ENV_HEADER "${CMAKE_CURRENT_SOURCE_DIR}/environment.h") +``` + +The SDK's CMake target includes this declaration consistently in the SDK and module source files that use the context type. Including it manually in just one source file is insufficient. + +Inside a reducer or procedure, read values from its context: + +```cpp +std::string api_key = ctx.env.API_KEY(); +std::string mode = ctx.env.MODE(); +std::optional log_level = ctx.env.LOG_LEVEL(); +std::optional checked = ctx.env.get("LOG_LEVEL"); +``` + +`std::string` requires a value, and `std::optional` permits absence. The optional third element restricts the allowed strings. Supplying one string makes it an exact-value constraint. + + + + +The string-key getter checks the same declaration and permissions as named accessors. Reading an undeclared key fails; it does not return an absent value. Named optional accessors return `None`, `undefined`, `null`, or `std::nullopt`, depending on the language. The TypeScript string-key getter uses `null` for absence. + +Key names are exact and case-sensitive. The getter name `get`, or `Get` in C#, is reserved: a key with that name remains accessible through the string-key getter. A module with no environment declarations accepts no keys. + +Reads inside a transaction use that transaction's snapshot. In a procedure outside a transaction, each read uses a separate snapshot. To read several keys consistently, group the reads in `with_tx` in Rust or C++, `withTx` in TypeScript, or `WithTx` in C#. + +## Supply values when publishing + +Add non-secret defaults to the selected database target in `spacetime.json`: + +```json +{ + "database": "env-example", + "server": "http://127.0.0.1:3000", + "module-path": "./spacetimedb", + "env": { + "MODE": "development", + "LOG_LEVEL": "info" + } +} +``` + +With a local server running on port 3000, publish from the directory containing that configuration. This example uses a disposable development value: + +```bash +API_KEY='development-only-key' spacetime publish +``` + +For real credentials, supply the value through the publishing process's environment or an appropriately ignored local configuration file. Keep secrets out of checked-in configuration and module source. + +The CLI resolves each declared key in this order: + +1. A value in the publishing process's environment. +2. A value in the resolved configuration's `env` map. +3. Absence, which is accepted only for an optional declaration. + +A shell value overrides JSON even when it is an empty string or the key is absent from JSON. Already-exported variables behave the same as inline assignments. Unrelated shell variables are ignored unless the module declares their names. The CLI displays supplied key names and their sources, without printing their values. + +The configuration files `spacetime.json`, `spacetime.local.json`, `spacetime.{environment}.json`, and `spacetime.{environment}.local.json` apply in increasing precedence, where _environment_ is the environment selected with `--env`. Their `env` maps merge by key, as do maps inherited by child database targets. A higher-precedence value replaces that key while preserving unrelated keys. An empty map does not erase inherited keys. + +JSON strings pass through unchanged. Booleans and numbers are converted to strings, so `false` supplies `"false"`; declarations still validate strings. Use JSON strings when exact numeric spelling matters. Arrays, objects, and `null` are rejected, as are JSON keys the module has not declared. An invalid effective value rejects the publish rather than falling back to a lower-precedence value. + +### Every publish replaces the complete environment + +Publishing validates the supplied values and installs them atomically with the module, before initialization or migration. Invalid environment configuration leaves the previous module and values unchanged. Changing only values still requires publishing, and does not rerun `init` on an existing database. + +Previously stored values are **not** defaults for the next publish. Every publish must supply each required value again. An optional value omitted from all effective inputs is removed. To remove `LOG_LEVEL` in the example, remove it from every applicable configuration layer and unset any exported `LOG_LEVEL` before publishing. An empty string is a value, not a deletion instruction. + +The same rules apply to precompiled modules published with `--bin-path`. The CLI reads declarations from the artifact being published. + +## Inspect published values + +The database owner and collaborators with private-table read access can inspect the environment. For the local database above: + +```bash +spacetime env list env-example --server http://127.0.0.1:3000 +spacetime env get env-example MODE --server http://127.0.0.1:3000 +``` + +`env list` prints keys only. `env get` prints the requested value and fails if it is absent. Reading a secret with `env get` therefore exposes that secret in the command's output. + +Values are stored in the private system table `st_env`. Authorized SQL reads are also supported: + +```sql +SELECT key FROM st_env; +SELECT value FROM st_env WHERE key = 'MODE'; +``` + +SQL writes to `st_env`, module-side writes, and separate CLI setters are not supported. Changes go through a publish with the complete desired environment. + +## Access and limits + +Reducers, procedures, views, and HTTP handlers entered by the host in the root module can read its declared environment. Host-dispatched submodule entry points cannot read it, and submodules cannot declare a nonempty environment. Ordinary helper calls retain their calling entry point's access, including helpers defined in libraries or submodules. Root code can also pass a value to a helper explicitly. + +A procedure suspended across a publish cannot read values belonging to a replacement program. Environment reads in views participate in dependency tracking, so publishing changed values refreshes affected views. Module code remains responsible for what it returns or logs: returning a secret from a public view exposes that value to clients. + +Keys must match `[A-Za-z_][A-Za-z0-9_]*`. The limits are 256 bytes per key, 8 KiB per value, and 256 declarations per database. Length limits count UTF-8 bytes. Values may be empty if their declaration accepts an empty string. + +## Dynamically editable or untyped secrets + +Use an ordinary [private table](../00300-tables/00400-access-permissions.md) when a secret must change without republishing, or when its keys and allowed values should not be declared in the environment schema. For example, a table with a string primary-key column and a string value column can store arbitrary secret names and values. The table itself still has typed columns; its individual keys need no environment declarations or constraints. + +Update that table through reducers that explicitly authorize the caller. Keeping a table private controls direct client reads; it does not authorize calls to a reducer that modifies or returns its contents. Apply the same care to views, procedure results, and logs. Private tables follow the database's normal private-table permissions, including administrative reads. + +Unlike environment variables, these values follow the table's ordinary update and migration behavior. They do not receive environment schema validation or complete replacement on every publish. diff --git a/docs/docs/00300-resources/00200-reference/00100-cli-reference/00100-cli-reference.md b/docs/docs/00300-resources/00200-reference/00100-cli-reference/00100-cli-reference.md index 6a065748f69..7c9ee00f3f7 100644 --- a/docs/docs/00300-resources/00200-reference/00100-cli-reference/00100-cli-reference.md +++ b/docs/docs/00300-resources/00200-reference/00100-cli-reference/00100-cli-reference.md @@ -11,6 +11,9 @@ This document contains the help content for the `spacetime` command-line program * [`spacetime`↴](#spacetime) * [`spacetime publish`↴](#spacetime-publish) +* [`spacetime env`↴](#spacetime-env) +* [`spacetime env get`↴](#spacetime-env-get) +* [`spacetime env list`↴](#spacetime-env-list) * [`spacetime delete`↴](#spacetime-delete) * [`spacetime logs`↴](#spacetime-logs) * [`spacetime call`↴](#spacetime-call) @@ -48,6 +51,7 @@ This document contains the help content for the `spacetime` command-line program ###### **Subcommands:** * `publish` — Create and update a SpacetimeDB database +* `env` — Inspect published database environment variables * `delete` — Deletes a SpacetimeDB database * `logs` — Prints logs from a SpacetimeDB database * `call` — Invokes a function (reducer or procedure) in a database. WARNING: This command is UNSTABLE and subject to breaking changes. @@ -82,7 +86,7 @@ Create and update a SpacetimeDB database **Usage:** `spacetime publish [OPTIONS] [name|identity]` -Run `spacetime help publish` for more detailed information. +Every publish replaces the complete declared environment. Put an env map in spacetime.json; declared shell variables override config values (including empty strings). The CLI displays supplied keys and sources, never values. Optional values omitted from every input are removed. --env selects config file layers. Run `spacetime help publish` for more detailed information. ###### **Arguments:** @@ -137,6 +141,66 @@ Run `spacetime help publish` for more detailed information. +## `spacetime env` + +Inspect published database environment variables + +**Usage:** `spacetime env ` + +###### **Subcommands:** + +* `get` — Read one published environment value +* `list` — List published environment keys (never values) + + + +## `spacetime env get` + +Read one published environment value + +**Usage:** `spacetime env get [OPTIONS] ` + +###### **Arguments:** + +* `` — The declared environment key to read +* `` — The database name, identity, or configured target + +###### **Options:** + +* `-s`, `--server ` — The nickname, host name or URL of the server +* `--anonymous` — Perform this action with an anonymous identity +* `-y`, `--yes` — Run non-interactively wherever possible. This will answer "yes" to almost all prompts, but will sometimes answer "no" to preserve non-interactivity (e.g. when prompting whether to log in with spacetimedb.com). +* `--confirmed ` — Instruct the server to deliver only updates of confirmed transactions + + Possible values: `true`, `false` + +* `--no-config` — Ignore project configuration when resolving the database target + + + +## `spacetime env list` + +List published environment keys (never values) + +**Usage:** `spacetime env list [OPTIONS] ` + +###### **Arguments:** + +* `` — The database name, identity, or configured target + +###### **Options:** + +* `-s`, `--server ` — The nickname, host name or URL of the server +* `--anonymous` — Perform this action with an anonymous identity +* `-y`, `--yes` — Run non-interactively wherever possible. This will answer "yes" to almost all prompts, but will sometimes answer "no" to preserve non-interactivity (e.g. when prompting whether to log in with spacetimedb.com). +* `--confirmed ` — Instruct the server to deliver only updates of confirmed transactions + + Possible values: `true`, `false` + +* `--no-config` — Ignore project configuration when resolving the database target + + + ## `spacetime delete` Deletes a SpacetimeDB database diff --git a/docs/docusaurus.config.ts b/docs/docusaurus.config.ts index 3ff913ba0ff..accf916617a 100644 --- a/docs/docusaurus.config.ts +++ b/docs/docusaurus.config.ts @@ -3,6 +3,7 @@ import type * as Preset from '@docusaurus/preset-classic'; import rehypeShiki, { RehypeShikiOptions } from '@shikijs/rehype'; import bash from 'shiki/langs/bash.mjs'; import c from 'shiki/langs/c.mjs'; +import cmake from 'shiki/langs/cmake.mjs'; import csharp from 'shiki/langs/csharp.mjs'; import fsharp from 'shiki/langs/fsharp.mjs'; import json from 'shiki/langs/json.mjs'; @@ -158,6 +159,7 @@ const config: Config = { toml, python, c, + cmake, cpp, protobuf, fsharp, diff --git a/modules/environment-test/src/lib.rs b/modules/environment-test/src/lib.rs index 07c8c6aedd8..8803534eee7 100644 --- a/modules/environment-test/src/lib.rs +++ b/modules/environment-test/src/lib.rs @@ -1,7 +1,35 @@ use spacetimedb::{AnonymousViewContext, ProcedureContext, ReducerContext, SpacetimeType}; +use std::sync::atomic::{AtomicBool, Ordering}; + +static VIEW_TRAP_ENTERED: AtomicBool = AtomicBool::new(false); + +#[spacetimedb::env] +pub struct Env { + pub REQUIRED: String, + #[env(values("ready", "other"))] + pub MODE: String, + pub MISSING: Option, + pub EMPTY: Option, + pub UTF8: Option, + pub NUL: Option, + pub MAXIMUM: Option, + pub HANDLER: Option, + pub WATCHED: Option, + pub LIMIT: Option, +} + +#[spacetimedb::reducer(init)] +pub fn init(ctx: &ReducerContext) { + assert_eq!(ctx.env.REQUIRED(), "initial-required"); + assert_eq!(ctx.env.MODE(), "ready"); +} #[spacetimedb::reducer] pub fn expect_environment(ctx: &ReducerContext, key: String, expected: Option) { + assert!( + !VIEW_TRAP_ENTERED.load(Ordering::Relaxed), + "trapped Wasm view instance was reused" + ); assert_eq!(ctx.env.get(&key), expected); assert_eq!(ctx.as_read_only().env.get(&key), expected); assert_eq!(ctx.as_anonymous_read_only().env.get(&key), expected); @@ -14,6 +42,24 @@ pub fn read_environment(ctx: &mut ProcedureContext, key: String) -> Option Option { + let request = spacetimedb::http::Request::builder() + .uri(url) + .extension(spacetimedb::http::Timeout::from(spacetimedb::TimeDuration::from( + std::time::Duration::from_secs(30), + ))) + .body(()) + .unwrap(); + assert!(ctx.http.send(request).unwrap().status().is_success()); + if explicit_tx { + ctx.with_tx(|tx| tx.env.REQUIRED().into()) + } else { + ctx.env.REQUIRED().into() + } +} + #[derive(SpacetimeType)] pub struct EnvironmentValue { pub value: Option, @@ -21,9 +67,17 @@ pub struct EnvironmentValue { #[spacetimedb::view(accessor = environment_value, public)] pub fn environment_value(ctx: &AnonymousViewContext) -> Option { - Some(EnvironmentValue { - value: ctx.env.get("WATCHED"), - }) + let value = ctx.env.WATCHED(); + assert_ne!(value.as_deref(), Some("fail-view")); + Some(EnvironmentValue { value }) +} + +/// A Rust panic is a Wasm trap. The next main-instance reducer verifies that +/// the guest-global marker did not survive the failed SQL/subscription call. +#[spacetimedb::view(accessor = environment_trap, public)] +pub fn environment_trap(_ctx: &AnonymousViewContext) -> Option { + VIEW_TRAP_ENTERED.store(true, Ordering::Relaxed); + panic!("intentional environment view trap"); } /// Hand-written ABI callers cannot retain unbounded host allocations. diff --git a/modules/module-test-cpp/CMakeLists.txt b/modules/module-test-cpp/CMakeLists.txt index ae5c37ca153..eccf682be83 100644 --- a/modules/module-test-cpp/CMakeLists.txt +++ b/modules/module-test-cpp/CMakeLists.txt @@ -34,6 +34,9 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Emscripten") endif() +# Declare the same context type in module and SDK compilation units. +set(SPACETIMEDB_ENV_HEADER "${CMAKE_CURRENT_SOURCE_DIR}/environment.h") + # Link the SpacetimeDB library add_subdirectory(${SPACETIMEDB_CPP_LIBRARY_PATH} ${CMAKE_CURRENT_BINARY_DIR}/spacetimedb_cpp_library) target_link_libraries(${OUTPUT_NAME} PRIVATE spacetimedb_cpp_library) diff --git a/modules/module-test-cpp/environment.h b/modules/module-test-cpp/environment.h new file mode 100644 index 00000000000..e0e7e5d34f7 --- /dev/null +++ b/modules/module-test-cpp/environment.h @@ -0,0 +1,9 @@ +#pragma once +#include +SPACETIMEDB_ENV( + (MISSING, std::optional), + (EMPTY, std::optional), + (UTF8, std::optional), + (NUL, std::optional), + (MAXIMUM, std::optional) +) diff --git a/modules/module-test-cpp/src/lib.cpp b/modules/module-test-cpp/src/lib.cpp index 1102ef8be5f..fe47c4ce2a1 100644 --- a/modules/module-test-cpp/src/lib.cpp +++ b/modules/module-test-cpp/src/lib.cpp @@ -722,6 +722,8 @@ SPACETIMEDB_HTTP_ROUTER(router) { } SPACETIMEDB_REDUCER(expect_environment, ReducerContext ctx, std::string key, std::optional expected) { + // Parentheses avoid the existing enum helper macro named EMPTY(). + if ((ctx.env.EMPTY)() != ctx.env.get("EMPTY")) LOG_PANIC("named environment mismatch"); if (ctx.env.get(key) != expected) LOG_PANIC("environment value mismatch"); return Ok(); } diff --git a/modules/module-test-cs/EnvironmentTests.cs b/modules/module-test-cs/EnvironmentTests.cs index 1798d7bda44..b0630c30ec3 100644 --- a/modules/module-test-cs/EnvironmentTests.cs +++ b/modules/module-test-cs/EnvironmentTests.cs @@ -3,11 +3,22 @@ namespace SpacetimeDB.Modules.ModuleTestCs; using SpacetimeDB; +[SpacetimeDB.Env] +public partial struct ModuleEnvironmentSchema +{ + public string? MISSING; + public string? EMPTY; + public string? UTF8; + public string? NUL; + public string? MAXIMUM; +} + public static partial class EnvironmentTests { [Reducer] public static void expect_environment(ReducerContext ctx, string key, string? expected) { + if (ctx.Env.EMPTY != ctx.Env.Get("EMPTY")) throw new Exception("named environment mismatch"); if (ctx.Env.Get(key) != expected) throw new Exception("environment value mismatch"); } diff --git a/modules/module-test-ts/src/environment_sys.d.ts b/modules/module-test-ts/src/environment_sys.d.ts new file mode 100644 index 00000000000..8bf1660c106 --- /dev/null +++ b/modules/module-test-ts/src/environment_sys.d.ts @@ -0,0 +1,4 @@ +// Raw host ABI used to verify that SDK context changes cannot grant authority. +declare module 'spacetime:sys@2.3' { + export function env_get(key: string): string | null; +} diff --git a/modules/module-test-ts/src/index.ts b/modules/module-test-ts/src/index.ts index 10520d2b72c..2d903356b00 100644 --- a/modules/module-test-ts/src/index.ts +++ b/modules/module-test-ts/src/index.ts @@ -249,7 +249,13 @@ const spacetimedb = schema({ ), tableToRemove: table({ name: 'table_to_remove' }, { id: t.u32() }), lib: libSubmodule, -}); +}, { env: { + MISSING: t.string().optional(), + EMPTY: t.string().optional(), + UTF8: t.string().optional(), + NUL: t.string().optional(), + MAXIMUM: t.string().optional(), +} }); export default spacetimedb; // ───────────────────────────────────────────────────────────────────────────── @@ -549,14 +555,30 @@ export const libHello = spacetimedb.httpHandler((ctx, req) => { return libSubmodule.libHello(ctx.as.lib, req); }); +// Ordinary JS delegation retains this root host entry, even with ctx.as.lib. +// Direct host dispatch to lib.envReadHandler is the separate denied case. +export const envReadChildHandler = spacetimedb.httpHandler((ctx, req) => + libSubmodule.envReadHandler(ctx.as.lib, req) +); + +// Root entries must use the checked accessor too; returning raw st_env SQL is +// forbidden even when the same entry could legitimately call ctx.env.get. +export const envReadRootSqlView = spacetimedb.view( + { public: true }, + t.array(t.object('EnvSqlRow', { key: t.string(), value: t.string() })), + ctx => libSubmodule.uncheckedEnvironmentQuery(ctx.from.player) +); + export const router = spacetimedb.httpRouter( - new Router().get('/get', getSimple).get('/lib-hello', libHello) + new Router().get('/get', getSimple).get('/lib-hello', libHello).get('/env-child', envReadChildHandler) ); // Dedicated environment ABI integration exercised by crates/testing. export const expect_environment = spacetimedb.reducer( { key: t.string(), expected: t.option(t.string()) }, (ctx, { key, expected }) => { + if (libSubmodule.readRootEnvironmentHelper() !== ctx.env.get('EMPTY')) throw new Error('helper environment scope mismatch'); + if (ctx.env.EMPTY !== (ctx.env.get('EMPTY') ?? undefined)) throw new Error('named environment mismatch'); if (ctx.env.get(key) !== (expected ?? null)) { throw new Error('environment value mismatch'); } diff --git a/modules/module-test-ts/src/lib_submodule.ts b/modules/module-test-ts/src/lib_submodule.ts index 3c2832d602c..ff4c56b3be8 100644 --- a/modules/module-test-ts/src/lib_submodule.ts +++ b/modules/module-test-ts/src/lib_submodule.ts @@ -1,4 +1,6 @@ +/// import { schema, table, t, SyncResponse } from 'spacetimedb/server'; +import { env_get } from 'spacetime:sys@2.3'; const libData = table( { name: 'libData', public: true }, @@ -26,3 +28,37 @@ export const libCount = libSubmoduleSchema.procedure(t.u64(), ctx => export const libHello = libSubmoduleSchema.httpHandler((_ctx, _req) => { return new SyncResponse('Hello from lib submodule!'); }); + +// Ordinary helpers retain their caller's host scope. Exported module callbacks +// below are entered through the lib namespace and must be rejected by the host. +export function readRootEnvironmentHelper(): string | null { + return env_get('EMPTY'); +} +export const envReadReducer = libSubmoduleSchema.reducer(() => { + env_get('EMPTY'); +}); +export const envReadProcedure = libSubmoduleSchema.procedure(t.string(), () => + env_get('EMPTY') ?? '' +); +export const envReadInTx = libSubmoduleSchema.procedure(t.string(), ctx => + ctx.withTx(() => env_get('EMPTY') ?? '') +); +export const envReadView = libSubmoduleSchema.view( + { public: true }, t.array(t.object('EnvReadRow', { value: t.string() })), () => [{ value: env_get('EMPTY') ?? '' }] +); +export const envReadHandler = libSubmoduleSchema.httpHandler(() => + new SyncResponse(env_get('EMPTY') ?? '') +); + +// Deliberately forge module-returned SQL through an ordinary query object's +// runtime brand. SDK types are not a security boundary for this host feature. +export function uncheckedEnvironmentQuery(source: object) { + return Object.assign(Object.create(source), { + toSql: () => 'SELECT * FROM st_env', + }) as { key: string; value: string }[]; +} +export const envReadSqlView = libSubmoduleSchema.view( + { public: true }, + t.array(t.object('EnvSqlRow', { key: t.string(), value: t.string() })), + ctx => uncheckedEnvironmentQuery(ctx.from.libData) +); From 05a805392953bd44b8674b4e37ca6ea0f89251f4 Mon Sep 17 00:00:00 2001 From: Tyler Cloutier Date: Tue, 8 Sep 2026 19:25:01 -0400 Subject: [PATCH 07/34] Clarify module library terminology in environment docs --- .../00100-databases/00300-spacetime-publish.md | 2 +- .../00100-databases/00700-environment-variables.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/docs/00200-core-concepts/00100-databases/00300-spacetime-publish.md b/docs/docs/00200-core-concepts/00100-databases/00300-spacetime-publish.md index 2f91291133c..dd805b69ecf 100644 --- a/docs/docs/00200-core-concepts/00100-databases/00300-spacetime-publish.md +++ b/docs/docs/00200-core-concepts/00100-databases/00300-spacetime-publish.md @@ -108,7 +108,7 @@ For all available publishing options and flags, see the [`spacetime publish` CLI Modules can declare environment variables for configuration and secrets. Each publish supplies the complete set of values from the target's `env` configuration and declared shell variables. Required values must be supplied on every publish; omitted optional values are removed. The module and its environment update atomically. -See [Environment Variables](./00700-environment-variables.md) for declarations, SDK accessors, publishing examples, and private tables for secrets that need to change without republishing. +See [Environment Variables](./00700-environment-variables.md) for declarations, reading values in module code, publishing examples, and private tables for secrets that need to change without republishing. ## Next Steps diff --git a/docs/docs/00200-core-concepts/00100-databases/00700-environment-variables.md b/docs/docs/00200-core-concepts/00100-databases/00700-environment-variables.md index 25eb6069208..34161f9095a 100644 --- a/docs/docs/00200-core-concepts/00100-databases/00700-environment-variables.md +++ b/docs/docs/00200-core-concepts/00100-databases/00700-environment-variables.md @@ -126,13 +126,13 @@ SPACETIMEDB_ENV( ) ``` -In `CMakeLists.txt`, set the header path **before** adding the SpacetimeDB SDK directory: +In `CMakeLists.txt`, set the header path **before** adding the SpacetimeDB module library directory: ```cmake set(SPACETIMEDB_ENV_HEADER "${CMAKE_CURRENT_SOURCE_DIR}/environment.h") ``` -The SDK's CMake target includes this declaration consistently in the SDK and module source files that use the context type. Including it manually in just one source file is insufficient. +The library's CMake target includes this declaration consistently in the library and module source files that use the context type. Including it manually in just one source file is insufficient. Inside a reducer or procedure, read values from its context: From 648ae253462403b342469bcddcddcd640d33d6ed Mon Sep 17 00:00:00 2001 From: Tyler Cloutier Date: Tue, 8 Sep 2026 21:52:58 -0400 Subject: [PATCH 08/34] Address environment API and CLI review feedback --- .../include/spacetimedb/abi/abi.h | 7 +- crates/bindings-cpp/src/abi/wasi_shims.cpp | 6 +- .../bindings-csharp/Runtime/Internal/FFI.cs | 6 +- crates/bindings-csharp/Runtime/bindings.c | 2 +- .../Runtime/build/SpacetimeDB.Runtime.targets | 3 +- crates/bindings-macro/src/environment.rs | 81 ++------- crates/bindings-sys/src/lib.rs | 3 +- .../src/server/environment.ts | 13 +- .../bindings-typescript/src/server/sys.d.ts | 4 +- .../tests/environment.test.ts | 4 +- crates/bindings-typescript/vitest.config.ts | 2 +- crates/bindings/src/lib.rs | 3 +- crates/bindings/src/rt.rs | 36 ++++ crates/bindings/tests/environment.rs | 4 +- crates/bindings/tests/pass/environment.rs | 17 +- crates/bindings/tests/ui/environment_types.rs | 38 ++++ .../tests/ui/environment_types.stderr | 129 +++++++++++++ crates/cli/src/lib.rs | 1 + crates/cli/src/schema_extract.rs | 172 ++++++++++++++++++ crates/cli/src/schema_extract/tests.rs | 143 +++++++++++++++ crates/cli/src/subcommands/env.rs | 30 ++- crates/cli/src/subcommands/generate.rs | 18 +- crates/cli/src/subcommands/publish.rs | 36 +++- .../src/subcommands/publish/environment.rs | 101 +--------- .../subcommands/publish/environment/tests.rs | 131 +------------ .../cli/src/subcommands/publish/wire_tests.rs | 56 ++++++ crates/cli/src/subcommands/sql.rs | 2 +- crates/core/src/host/v8/syscall/mod.rs | 3 +- crates/core/src/host/v8/syscall/v2.rs | 4 +- crates/core/src/host/wasm_common.rs | 3 +- .../tests/standalone/cli/environment.rs | 20 +- modules/environment-test/src/lib.rs | 7 +- .../module-test-ts/src/environment_sys.d.ts | 2 +- modules/module-test-ts/src/lib_submodule.ts | 2 +- 34 files changed, 712 insertions(+), 377 deletions(-) create mode 100644 crates/bindings/tests/ui/environment_types.rs create mode 100644 crates/bindings/tests/ui/environment_types.stderr create mode 100644 crates/cli/src/schema_extract.rs create mode 100644 crates/cli/src/schema_extract/tests.rs create mode 100644 crates/cli/src/subcommands/publish/wire_tests.rs diff --git a/crates/bindings-cpp/include/spacetimedb/abi/abi.h b/crates/bindings-cpp/include/spacetimedb/abi/abi.h index 285974cff3c..502cd408be3 100644 --- a/crates/bindings-cpp/include/spacetimedb/abi/abi.h +++ b/crates/bindings-cpp/include/spacetimedb/abi/abi.h @@ -39,9 +39,8 @@ #define STDB_IMPORT_10_5(name) \ __attribute__((import_module("spacetime_10.5"), import_name(#name))) extern -// ABI10.6 is reserved for the separate invocation-authority extension. -#define STDB_IMPORT_10_7(name) \ - __attribute__((import_module("spacetime_10.7"), import_name(#name))) extern +#define STDB_IMPORT_10_6(name) \ + __attribute__((import_module("spacetime_10.6"), import_name(#name))) extern // Import opaque types into global namespace for C compatibility using SpacetimeDB::Status; @@ -63,7 +62,7 @@ using SpacetimeDB::ConsoleTimerId; extern "C" { -STDB_IMPORT_10_7(env_get) +STDB_IMPORT_10_6(env_get) Status env_get(const uint8_t* key, uint32_t key_len, BytesSource* out); // ===== Table and Index Management ===== diff --git a/crates/bindings-cpp/src/abi/wasi_shims.cpp b/crates/bindings-cpp/src/abi/wasi_shims.cpp index ec461c8d51d..a698b4177f7 100644 --- a/crates/bindings-cpp/src/abi/wasi_shims.cpp +++ b/crates/bindings-cpp/src/abi/wasi_shims.cpp @@ -9,9 +9,9 @@ // with these standalone shim definitions. Use a distinct C++ name for the raw // host logging import; its WebAssembly signature is the same eight i32 values. extern "C" __attribute__((import_module("spacetime_10.0"), import_name("console_log"))) -void wasi_console_log(uint8_t level, const uint8_t* target_ptr, size_t target_len, - const uint8_t* filename_ptr, size_t filename_len, uint32_t line_number, - const uint8_t* message_ptr, size_t message_len); +void wasi_console_log(uint8_t level, const uint8_t* target_ptr, uint32_t target_len, + const uint8_t* filename_ptr, uint32_t filename_len, uint32_t line_number, + const uint8_t* message_ptr, uint32_t message_len); // Helper macro for string literals #define CSTR(s) (uint8_t*)s, sizeof(s) - 1 diff --git a/crates/bindings-csharp/Runtime/Internal/FFI.cs b/crates/bindings-csharp/Runtime/Internal/FFI.cs index cc3e8cf624b..8957759d8eb 100644 --- a/crates/bindings-csharp/Runtime/Internal/FFI.cs +++ b/crates/bindings-csharp/Runtime/Internal/FFI.cs @@ -109,16 +109,16 @@ internal static partial class FFI #endif ; - const string StdbNamespace10_7 = + const string StdbNamespace10_6 = #if EXPERIMENTAL_WASM_AOT || NET10_0_OR_GREATER - "spacetime_10.7" + "spacetime_10.6" #else "bindings" #endif ; [WasmImportLinkage] - [LibraryImport(StdbNamespace10_7)] + [LibraryImport(StdbNamespace10_6)] public static unsafe partial CheckedStatus env_get( byte* key, uint keyLen, diff --git a/crates/bindings-csharp/Runtime/bindings.c b/crates/bindings-csharp/Runtime/bindings.c index f4d3635a5f2..b877876b37f 100644 --- a/crates/bindings-csharp/Runtime/bindings.c +++ b/crates/bindings-csharp/Runtime/bindings.c @@ -135,7 +135,7 @@ IMPORT(Status, datastore_clear, (table_id, count)); #undef SPACETIME_MODULE_VERSION -#define SPACETIME_MODULE_VERSION "spacetime_10.7" +#define SPACETIME_MODULE_VERSION "spacetime_10.6" IMPORT(Status, env_get, (const uint8_t* key, uint32_t key_len, BytesSource* source), (key, key_len, source)); #undef SPACETIME_MODULE_VERSION diff --git a/crates/bindings-csharp/Runtime/build/SpacetimeDB.Runtime.targets b/crates/bindings-csharp/Runtime/build/SpacetimeDB.Runtime.targets index 95c2f1cfdd5..45d4b570d58 100644 --- a/crates/bindings-csharp/Runtime/build/SpacetimeDB.Runtime.targets +++ b/crates/bindings-csharp/Runtime/build/SpacetimeDB.Runtime.targets @@ -65,8 +65,7 @@ - - + diff --git a/crates/bindings-macro/src/environment.rs b/crates/bindings-macro/src/environment.rs index 8377037082e..e58899d96db 100644 --- a/crates/bindings-macro/src/environment.rs +++ b/crates/bindings-macro/src/environment.rs @@ -2,7 +2,7 @@ use proc_macro2::TokenStream; use quote::{format_ident, quote}; use syn::ext::IdentExt as _; use syn::punctuated::Punctuated; -use syn::{Fields, GenericArgument, ItemStruct, LitStr, PathArguments, Token, Type}; +use syn::{Fields, ItemStruct, LitStr, Token}; pub(crate) fn expand(args: TokenStream, mut item: ItemStruct) -> syn::Result { if !args.is_empty() { @@ -32,7 +32,7 @@ pub(crate) fn expand(args: TokenStream, mut item: ItemStruct) -> syn::Result> = None; for attr in field.attrs.iter().filter(|attr| attr.path().is_ident("env")) { attr.parse_nested_meta(|meta| { @@ -71,28 +71,23 @@ pub(crate) fn expand(args: TokenStream, mut item: ItemStruct) -> syn::Result::OPTIONAL, }), ); if name == "get" { continue; } - let (return_type, body) = if optional { - ( - quote!(::std::option::Option<::std::string::String>), - quote!(::spacetimedb::Environment::get(self, #name)), - ) - } else { - ( - quote!(::std::string::String), - quote!(::spacetimedb::Environment::get(self, #name).expect(concat!("required environment key is missing: ", #name))), - ) - }; signatures.push(quote! { #[doc = concat!("Read the declared environment key `", #name, "` through the checked host ABI.")] - fn #ident(&self) -> #return_type; + fn #ident(&self) -> #ty; + }); + methods.push(quote! { + fn #ident(&self) -> #ty { + <#ty as ::spacetimedb::rt::EnvironmentValue>::get(self, #name) + } }); - methods.push(quote!(fn #ident(&self) -> #return_type { #body })); } let vis = &item.vis; let access = format_ident!("{}Access", item.ident.unraw()); @@ -119,65 +114,13 @@ pub(crate) fn expand(args: TokenStream, mut item: ItemStruct) -> syn::Result syn::Result { - let Type::Path(path) = ty else { - return Err(syn::Error::new_spanned(ty, "expected String or Option")); - }; - if path.qself.is_none() { - let segments: Vec<_> = path - .path - .segments - .iter() - .map(|segment| segment.ident.to_string()) - .collect(); - let names: Vec<_> = segments.iter().map(String::as_str).collect(); - if matches!(names.as_slice(), ["String"] | ["std" | "alloc", "string", "String"]) - && path - .path - .segments - .iter() - .all(|segment| matches!(segment.arguments, PathArguments::None)) - { - return Ok(false); - } - if matches!(names.as_slice(), ["Option"] | ["std" | "core", "option", "Option"]) - && path - .path - .segments - .iter() - .rev() - .skip(1) - .all(|segment| matches!(segment.arguments, PathArguments::None)) - && let PathArguments::AngleBracketed(arguments) = &path.path.segments.last().unwrap().arguments - && let [GenericArgument::Type(inner)] = arguments.args.iter().collect::>().as_slice() - && !optional_string(inner)? - { - return Ok(true); - } - } - Err(syn::Error::new_spanned( - ty, - "expected String or Option; aliases and other types are not env constraints", - )) -} - #[cfg(test)] mod tests { use super::*; #[test] - fn rejects_non_string_nested_optional_empty_union_and_invalid_names() { + fn rejects_empty_union_invalid_names_and_unsupported_struct_shapes() { for item in [ - quote!( - struct Env { - VALUE: bool, - } - ), - quote!( - struct Env { - VALUE: Option>, - } - ), quote!( struct Env { #[env(values())] diff --git a/crates/bindings-sys/src/lib.rs b/crates/bindings-sys/src/lib.rs index 2a3bd77b454..c683d4c8a59 100644 --- a/crates/bindings-sys/src/lib.rs +++ b/crates/bindings-sys/src/lib.rs @@ -883,8 +883,7 @@ pub mod raw { pub fn datastore_clear(table_id: TableId, out: *mut u64) -> u16; } - // ABI10.6 is reserved for the separate invocation-authority extension. - #[link(wasm_import_module = "spacetime_10.7")] + #[link(wasm_import_module = "spacetime_10.6")] unsafe extern "C" { /// Read a UTF-8 environment value. Writes INVALID for a missing key; /// present empty strings have a valid BytesSource. Returns ordinary errno. diff --git a/crates/bindings-typescript/src/server/environment.ts b/crates/bindings-typescript/src/server/environment.ts index 203c608e08c..05458efae8a 100644 --- a/crates/bindings-typescript/src/server/environment.ts +++ b/crates/bindings-typescript/src/server/environment.ts @@ -1,4 +1,4 @@ -import { env_get } from 'spacetime:sys@2.3'; +import { env_get } from 'spacetime:sys@2.2'; import type { Environment, EnvironmentSchema } from '../lib/environment'; import type { EnvironmentDeclaration, @@ -7,6 +7,11 @@ import type { } from '../lib/autogen/types'; import { OptionBuilder, StringBuilder } from '../lib/type_builders'; +// These UTF-8 byte/count limits match spacetimedb_lib::environment. +const MAX_ENV_KEY_BYTES = 256; +const MAX_ENV_VALUE_BYTES = 8 * 1024; +const MAX_ENV_VARS = 256; + /** Values are not cached: transaction and procedure reads retain host semantics. */ export const environment: Environment = new Proxy( Object.freeze( @@ -33,13 +38,13 @@ export function environmentDeclarations( schema: EnvironmentSchema ): EnvironmentDeclaration[] { const entries = Object.entries(schema); - if (entries.length > 256) + if (entries.length > MAX_ENV_VARS) throw new TypeError('Too many environment declarations'); const bytes = new TextEncoder(); return entries.map(([name, definition]) => { if ( !/^[A-Za-z_][A-Za-z0-9_]*$/.test(name) || - bytes.encode(name).length > 256 + bytes.encode(name).length > MAX_ENV_KEY_BYTES ) { throw new TypeError('Invalid environment declaration name'); } @@ -68,7 +73,7 @@ export function environmentDeclarations( throw new TypeError( `Environment '${name}' enum cases must have names` ); - if (bytes.encode(variant.name).length > 8192) + if (bytes.encode(variant.name).length > MAX_ENV_VALUE_BYTES) throw new TypeError(`Environment '${name}' literal is too long`); return variant.name; }); diff --git a/crates/bindings-typescript/src/server/sys.d.ts b/crates/bindings-typescript/src/server/sys.d.ts index 1f74debd2fc..77a9dacaf4e 100644 --- a/crates/bindings-typescript/src/server/sys.d.ts +++ b/crates/bindings-typescript/src/server/sys.d.ts @@ -124,9 +124,7 @@ declare module 'spacetime:sys@2.1' { export function datastore_clear(table_id: u32): u64; } -// sys2.2 is reserved for the separate invocation-authority extension. - -declare module 'spacetime:sys@2.3' { +declare module 'spacetime:sys@2.2' { /** Null means missing; an empty string is a present value. */ export function env_get(key: string): string | null; } diff --git a/crates/bindings-typescript/tests/environment.test.ts b/crates/bindings-typescript/tests/environment.test.ts index b78713d49e9..34fc8ae65c9 100644 --- a/crates/bindings-typescript/tests/environment.test.ts +++ b/crates/bindings-typescript/tests/environment.test.ts @@ -6,9 +6,9 @@ import { environmentDeclarations, } from '../src/server/environment'; import type { EnvironmentSchema } from '../src/lib/environment'; -import { env_get } from 'spacetime:sys@2.3'; +import { env_get } from 'spacetime:sys@2.2'; -vi.mock('spacetime:sys@2.3', async importOriginal => ({ +vi.mock('spacetime:sys@2.2', async importOriginal => ({ ...(await importOriginal()), env_get: vi.fn(), })); diff --git a/crates/bindings-typescript/vitest.config.ts b/crates/bindings-typescript/vitest.config.ts index e5503037be6..f532d437b74 100644 --- a/crates/bindings-typescript/vitest.config.ts +++ b/crates/bindings-typescript/vitest.config.ts @@ -14,7 +14,7 @@ export default defineConfig({ alias: [ { find: 'spacetime:sys@2.0', replacement: sysMock }, { find: 'spacetime:sys@2.1', replacement: sysMock }, - { find: 'spacetime:sys@2.3', replacement: sysMock }, + { find: 'spacetime:sys@2.2', replacement: sysMock }, ], }, test: { diff --git a/crates/bindings/src/lib.rs b/crates/bindings/src/lib.rs index 1589eb54ad5..04be557fdf2 100644 --- a/crates/bindings/src/lib.rs +++ b/crates/bindings/src/lib.rs @@ -921,7 +921,8 @@ pub use query_builder::{Query, RawQuery}; /// Declare the complete publish-time environment schema and generate named accessors. /// -/// Fields must be `String` or `Option`; `#[env(values("a", "b"))]` +/// Fields must resolve to `String` or `Option`, including type aliases; +/// `#[env(values("a", "b"))]` /// constrains exact strings. Values are supplied on every publish, never in metadata. /// The macro generates an `EnvAccess` extension trait for a struct named `Env`. /// Import that trait when the declaration lives in a different Rust module. diff --git a/crates/bindings/src/rt.rs b/crates/bindings/src/rt.rs index bc5b0d8fae1..486ba90ba62 100644 --- a/crates/bindings/src/rt.rs +++ b/crates/bindings/src/rt.rs @@ -917,6 +917,42 @@ pub fn register_case_conversion_policy(policy: CaseConversionPolicy) { }) } +mod environment_value_sealed { + pub trait Sealed {} + + impl Sealed for String {} + impl Sealed for Option {} +} + +/// The compiler resolves declaration types, including aliases, before selecting +/// their metadata and accessor. Sealing keeps the accepted types identical to +/// the host's string and optional-string environment model. +#[doc(hidden)] +#[diagnostic::on_unimplemented(message = "environment fields must be `String` or `Option`")] +pub trait EnvironmentValue: environment_value_sealed::Sealed + Sized { + const OPTIONAL: bool; + + fn get(environment: &crate::Environment, key: &str) -> Self; +} + +impl EnvironmentValue for String { + const OPTIONAL: bool = false; + + fn get(environment: &crate::Environment, key: &str) -> Self { + environment + .get(key) + .unwrap_or_else(|| panic!("required environment key is missing: {key}")) + } +} + +impl EnvironmentValue for Option { + const OPTIONAL: bool = true; + + fn get(environment: &crate::Environment, key: &str) -> Self { + environment.get(key) + } +} + /// Register declarative ENV metadata without reading any environment values. #[doc(hidden)] pub fn register_environment(declarations: fn() -> Vec) { diff --git a/crates/bindings/tests/environment.rs b/crates/bindings/tests/environment.rs index 5bffb7924ff..6636b9160ef 100644 --- a/crates/bindings/tests/environment.rs +++ b/crates/bindings/tests/environment.rs @@ -1,4 +1,6 @@ #[test] fn environment_declaration_accessors_compile_with_exact_types() { - trybuild::TestCases::new().pass("tests/pass/environment.rs"); + let tests = trybuild::TestCases::new(); + tests.pass("tests/pass/environment.rs"); + tests.compile_fail("tests/ui/environment_types.rs"); } diff --git a/crates/bindings/tests/pass/environment.rs b/crates/bindings/tests/pass/environment.rs index 87a53d23aea..c0f80526ea4 100644 --- a/crates/bindings/tests/pass/environment.rs +++ b/crates/bindings/tests/pass/environment.rs @@ -1,14 +1,23 @@ #![deny(warnings)] +use std::option::Option as Maybe; +use std::string::String as RenamedString; + +type RequiredAlias = RenamedString; +type OptionalAlias = Maybe; + +const _: [(); 0] = [(); ::OPTIONAL as usize]; +const _: [(); 1] = [(); ::OPTIONAL as usize]; + #[spacetimedb::env] pub struct Env { - pub REQUIRED: String, + pub REQUIRED: RequiredAlias, #[env(values("false", "true"))] pub FLAG: String, #[env(values(""))] - pub OPTIONAL: Option, - pub get: Option, - pub r#type: String, + pub OPTIONAL: OptionalAlias, + pub get: Maybe, + pub r#type: RenamedString, } fn reads(env: spacetimedb::Environment) { diff --git a/crates/bindings/tests/ui/environment_types.rs b/crates/bindings/tests/ui/environment_types.rs new file mode 100644 index 00000000000..4fa3e3aae0b --- /dev/null +++ b/crates/bindings/tests/ui/environment_types.rs @@ -0,0 +1,38 @@ +mod shadowed_string { + pub struct String; + + #[spacetimedb::env] + pub struct ShadowedString { + pub VALUE: String, + } +} + +mod shadowed_option { + pub struct Option(T); + + #[spacetimedb::env] + pub struct ShadowedOption { + pub VALUE: Option, + } +} + +#[spacetimedb::env] +pub struct Unsupported { + pub BOOL: bool, + pub NESTED: Option>, + // Even without a generated named accessor, metadata must check the type. + pub get: u32, +} + +struct Custom; + +// External code cannot extend the set of supported environment types. +impl spacetimedb::rt::EnvironmentValue for Custom { + const OPTIONAL: bool = false; + + fn get(_: &spacetimedb::Environment, _: &str) -> Self { + Self + } +} + +fn main() {} diff --git a/crates/bindings/tests/ui/environment_types.stderr b/crates/bindings/tests/ui/environment_types.stderr new file mode 100644 index 00000000000..3c7346dcb4e --- /dev/null +++ b/crates/bindings/tests/ui/environment_types.stderr @@ -0,0 +1,129 @@ +error[E0277]: the trait bound `Custom: spacetimedb::rt::environment_value_sealed::Sealed` is not satisfied + --> tests/ui/environment_types.rs:30:44 + | +30 | impl spacetimedb::rt::EnvironmentValue for Custom { + | ^^^^^^ unsatisfied trait bound + | +help: the trait `spacetimedb::rt::environment_value_sealed::Sealed` is not implemented for `Custom` + --> tests/ui/environment_types.rs:27:1 + | +27 | struct Custom; + | ^^^^^^^^^^^^^ +help: the following other types implement trait `spacetimedb::rt::environment_value_sealed::Sealed` + --> src/rt.rs + | + | impl Sealed for String {} + | ^^^^^^^^^^^^^^^^^^^^^^ `std::string::String` + | impl Sealed for Option {} + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ `std::option::Option` +note: required by a bound in `EnvironmentValue` + --> src/rt.rs + | + | pub trait EnvironmentValue: environment_value_sealed::Sealed + Sized { + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ required by this bound in `EnvironmentValue` + = note: `EnvironmentValue` is a "sealed trait", because to implement it you also need to implement `spacetimedb::rt::environment_value_sealed::Sealed`, which is not accessible; this is usually done to force you to use one of the provided types that already implement it + = help: the following types implement the trait: + std::string::String + std::option::Option + +error[E0277]: environment fields must be `String` or `Option` + --> tests/ui/environment_types.rs:6:20 + | + 6 | pub VALUE: String, + | ^^^^^^ unsatisfied trait bound + | +help: the trait `EnvironmentValue` is not implemented for `shadowed_string::String` + --> tests/ui/environment_types.rs:2:5 + | + 2 | pub struct String; + | ^^^^^^^^^^^^^^^^^ +help: the following other types implement trait `EnvironmentValue` + --> tests/ui/environment_types.rs:30:1 + | +30 | impl spacetimedb::rt::EnvironmentValue for Custom { + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ `Custom` + | + ::: src/rt.rs + | + | impl EnvironmentValue for String { + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ `std::string::String` +... + | impl EnvironmentValue for Option { + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ `std::option::Option` + +error[E0277]: environment fields must be `String` or `Option` + --> tests/ui/environment_types.rs:15:20 + | +15 | pub VALUE: Option, + | ^^^^^^^^^^^^^^ unsatisfied trait bound + | +help: the trait `EnvironmentValue` is not implemented for `shadowed_option::Option` + --> tests/ui/environment_types.rs:11:5 + | +11 | pub struct Option(T); + | ^^^^^^^^^^^^^^^^^^^^ +help: the following other types implement trait `EnvironmentValue` + --> tests/ui/environment_types.rs:30:1 + | +30 | impl spacetimedb::rt::EnvironmentValue for Custom { + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ `Custom` + | + ::: src/rt.rs + | + | impl EnvironmentValue for String { + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ `std::string::String` +... + | impl EnvironmentValue for Option { + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ `std::option::Option` + +error[E0277]: environment fields must be `String` or `Option` + --> tests/ui/environment_types.rs:21:15 + | +21 | pub BOOL: bool, + | ^^^^ the trait `EnvironmentValue` is not implemented for `bool` + | +help: the following other types implement trait `EnvironmentValue` + --> tests/ui/environment_types.rs:30:1 + | +30 | impl spacetimedb::rt::EnvironmentValue for Custom { + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ `Custom` + | + ::: src/rt.rs + | + | impl EnvironmentValue for String { + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ `std::string::String` +... + | impl EnvironmentValue for Option { + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ `std::option::Option` + +error[E0277]: environment fields must be `String` or `Option` + --> tests/ui/environment_types.rs:22:17 + | +22 | pub NESTED: Option>, + | ^^^^^^^^^^^^^^^^^^^^^^ the trait `EnvironmentValue` is not implemented for `std::option::Option>` + | +help: the trait `EnvironmentValue` is implemented for `std::option::Option` + --> src/rt.rs + | + | impl EnvironmentValue for Option { + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +error[E0277]: environment fields must be `String` or `Option` + --> tests/ui/environment_types.rs:24:14 + | +24 | pub get: u32, + | ^^^ the trait `EnvironmentValue` is not implemented for `u32` + | +help: the following other types implement trait `EnvironmentValue` + --> tests/ui/environment_types.rs:30:1 + | +30 | impl spacetimedb::rt::EnvironmentValue for Custom { + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ `Custom` + | + ::: src/rt.rs + | + | impl EnvironmentValue for String { + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ `std::string::String` +... + | impl EnvironmentValue for Option { + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ `std::option::Option` diff --git a/crates/cli/src/lib.rs b/crates/cli/src/lib.rs index 96bb1034bfe..5357e8d067b 100644 --- a/crates/cli/src/lib.rs +++ b/crates/cli/src/lib.rs @@ -4,6 +4,7 @@ mod config; pub(crate) mod detect; mod edit_distance; mod errors; +mod schema_extract; pub mod spacetime_config; mod subcommands; mod tasks; diff --git a/crates/cli/src/schema_extract.rs b/crates/cli/src/schema_extract.rs new file mode 100644 index 00000000000..c00481ae40f --- /dev/null +++ b/crates/cli/src/schema_extract.rs @@ -0,0 +1,172 @@ +//! Local schema extraction shared by publish and generate. Each invocation +//! inspects a private copy of exact bounded input bytes, bounds its output and +//! lifetime, and owns the child through kill/wait on failure or cancellation. +use anyhow::{ensure, Context}; +use spacetimedb_lib::{sats::serde::SerdeWrapper, RawModuleDef}; +use spacetimedb_schema::def::ModuleDef; +use std::{ + path::{Path, PathBuf}, + process::Stdio, + time::Duration, +}; +use tokio::io::AsyncReadExt; + +fn extractor_path() -> anyhow::Result { + std::env::var_os("SPACETIMEDB_SCHEMA_EXTRACTOR") + .map(PathBuf::from) + .map(Ok) + .unwrap_or_else(|| crate::util::resolve_sibling_binary("spacetimedb-standalone")) +} + +/// Keep generate's synchronous injectable function API. Its small dedicated +/// runtime works both inside and outside a caller's Tokio runtime; process +/// ownership and validation are identical to publish's async path. +pub(crate) fn from_path(path: &Path) -> anyhow::Result { + let bytes = read_program(path)?; + let host_type = match path.extension().and_then(|ext| ext.to_str()) { + Some("wasm") => "Wasm", + Some("js") => "Js", + _ => anyhow::bail!("Cannot determine module type from file extension"), + }; + inspect_blocking(extractor_path()?, bytes, host_type.into()) +} + +fn inspect_blocking(extractor: PathBuf, bytes: Vec, host_type: String) -> anyhow::Result { + std::thread::spawn(move || { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build()? + .block_on(inspect_with(extractor, bytes, host_type, INSPECT_TIMEOUT)) + }) + .join() + .map_err(|_| anyhow::anyhow!("Local module inspection thread failed"))? +} + +pub(crate) fn read_program(path: &std::path::Path) -> anyhow::Result> { + use std::io::Read; + let mut bytes = Vec::new(); + std::fs::File::open(path)? + .take(spacetimedb_client_api_messages::publish::MAX_MODULE_BYTES as u64 + 1) + .read_to_end(&mut bytes)?; + ensure!( + bytes.len() <= spacetimedb_client_api_messages::publish::MAX_MODULE_BYTES, + "Module exceeds publish size limit" + ); + Ok(bytes) +} + +const MAX_SCHEMA_BYTES: u64 = 16 * 1024 * 1024; +const INSPECT_TIMEOUT: Duration = Duration::from_secs(60); + +/// Inspect exactly the artifact bytes that will be uploaded. The private copy +/// prevents path replacement between inspection and upload, including --bin-path. +/// This invokes only local extraction, never a server or a saved CLI context. +pub(crate) async fn inspect(program: &[u8], host_type: &str) -> anyhow::Result { + let extractor = extractor_path()?; + inspect_with(extractor, program.to_vec(), host_type.to_owned(), INSPECT_TIMEOUT).await +} + +pub(crate) async fn inspect_with( + extractor: PathBuf, + program: Vec, + host_type: String, + deadline: Duration, +) -> anyhow::Result { + inspect_observed(extractor, program, host_type, deadline, Observation::default()).await +} + +#[derive(Default)] +struct Observation { + #[cfg(test)] + started: Option>, + #[cfg(test)] + reaped: Option>, +} +impl Observation { + fn started(&mut self, _pid: Option) { + #[cfg(test)] + if let Some(send) = self.started.take() { + let _ = send.send(_pid.expect("new child has a PID")); + } + } + fn reaped(&mut self, _status: std::process::ExitStatus) { + #[cfg(test)] + if let Some(send) = self.reaped.take() { + let _ = send.send(_status); + } + } +} + +async fn inspect_observed( + extractor: PathBuf, + program: Vec, + host_type: String, + deadline: Duration, + mut observation: Observation, +) -> anyhow::Result { + let (mut send, mut recv) = tokio::sync::oneshot::channel(); + // This owner retains the child and private file until actual reaping, even + // when its caller drops while extraction or stdout reading is in progress. + tokio::spawn(async move { + let result = async { + let dir = tempfile::tempdir().context("Cannot create private module inspection directory")?; + let module = dir.path().join("module"); + tokio::fs::write(&module, program) + .await + .context("Cannot prepare module inspection input")?; + let mut child = tokio::process::Command::new(extractor) + .arg("extract-schema") + .arg(&module) + .arg("--host-type") + .arg(host_type.to_ascii_lowercase()) + .env_clear() + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::null()) + .kill_on_drop(true) + .spawn() + .context("Cannot start local module schema inspection")?; + observation.started(child.id()); + let mut output = Vec::new(); + let mut stdout = child + .stdout + .take() + .context("Module inspection stdout unavailable")? + .take(MAX_SCHEMA_BYTES + 1); + let result = tokio::select! { + biased; + _ = send.closed() => Err(anyhow::anyhow!("Module inspection cancelled")), + result = tokio::time::timeout(deadline, async { + stdout.read_to_end(&mut output).await.context("Cannot read local module schema")?; + ensure!(output.len() as u64 <= MAX_SCHEMA_BYTES, "Local module schema exceeds output limit"); + let status = child.wait().await.context("Cannot reap local module inspector")?; + observation.reaped(status); + ensure!(status.success(), "Local module schema inspection failed"); + Ok(()) + }) => result.unwrap_or_else(|_| Err(anyhow::anyhow!("Local module schema inspection timed out"))), + }; + if result.is_err() { + // Queue termination, then retain ownership through positive reaping. + let _ = child.start_kill(); + let status = child + .wait() + .await + .context("Cannot reap failed local module inspector")?; + observation.reaped(status); + } + result?; + // Neither parser nor validation diagnostics may echo schema literals. + let SerdeWrapper::(raw) = serde_json::from_slice(&output) + .map_err(|_| anyhow::anyhow!("Local module inspector returned invalid schema data"))?; + let schema = + ModuleDef::try_from(raw).map_err(|_| anyhow::anyhow!("Local module schema validation failed"))?; + Ok(schema) + } + .await; + let _ = send.send(result); + }); + (&mut recv).await.context("Local module inspection owner failed")? +} + +#[cfg(test)] +mod tests; diff --git a/crates/cli/src/schema_extract/tests.rs b/crates/cli/src/schema_extract/tests.rs new file mode 100644 index 00000000000..d8c6daea04a --- /dev/null +++ b/crates/cli/src/schema_extract/tests.rs @@ -0,0 +1,143 @@ +use super::*; +use spacetimedb_lib::environment::{ + EnvironmentConstraint as Constraint, EnvironmentDeclaration as Declaration, EnvironmentSchema, +}; + +fn schema() -> EnvironmentSchema { + EnvironmentSchema::new(vec![ + Declaration { + name: "A".into(), + constraint: Constraint::AnyString, + optional: false, + }, + Declaration { + name: "B".into(), + constraint: Constraint::OneOf(vec!["true".into(), "false".into()]), + optional: false, + }, + Declaration { + name: "C".into(), + constraint: Constraint::AnyString, + optional: false, + }, + Declaration { + name: "OPTIONAL".into(), + constraint: Constraint::AnyString, + optional: true, + }, + ]) + .unwrap() +} + +// These fixtures invoke only a locally generated executable in an owned tempdir. +// No CLI config, server credentials or user environment are imported. +#[cfg(unix)] +fn inspector(script: &str) -> (tempfile::TempDir, PathBuf) { + use std::os::unix::fs::PermissionsExt; + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("inspector"); + std::fs::write(&path, format!("#!/bin/sh\n{script}\n")).unwrap(); + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o700)).unwrap(); + (dir, path) +} + +#[cfg(unix)] +#[tokio::test] +async fn local_inspection_passes_exact_bytes_host_and_requires_success() { + use spacetimedb_lib::db::raw_def::v10::{RawModuleDefV10, RawModuleDefV10Section}; + let raw = RawModuleDef::V10(RawModuleDefV10 { + sections: vec![RawModuleDefV10Section::Environment(schema().into_declarations())], + }); + let json = serde_json::to_string(&SerdeWrapper(raw)).unwrap(); + let (dir, extractor) = inspector(&format!( + "[ \"$1\" = extract-schema ] && [ \"$3\" = --host-type ] && [ \"$4\" = js ] || exit 2\n[ \"$(/bin/cat \"$2\")\" = exact-artifact ] || exit 3\nprintf '%s' '{}'", json.replace('\'', "'\\''") + )); + let result = inspect_with( + extractor, + b"exact-artifact".to_vec(), + "Js".into(), + Duration::from_secs(5), + ) + .await + .unwrap(); + assert_eq!(result.environment(), &schema()); + drop(dir); + let (_dir, extractor) = inspector(&format!("printf '%s' '{}'; exit 9", json.replace('\'', "'\\''"))); + assert!( + inspect_with(extractor, b"anything".to_vec(), "Wasm".into(), Duration::from_secs(5)) + .await + .is_err() + ); +} + +#[cfg(unix)] +#[tokio::test(flavor = "current_thread")] +async fn synchronous_generate_adapter_uses_same_exact_byte_protocol_inside_a_runtime() { + use spacetimedb_lib::db::raw_def::v10::{RawModuleDefV10, RawModuleDefV10Section}; + let raw = RawModuleDef::V10(RawModuleDefV10 { + sections: vec![RawModuleDefV10Section::Environment(schema().into_declarations())], + }); + let json = serde_json::to_string(&SerdeWrapper(raw)).unwrap(); + let (_dir, extractor) = inspector(&format!( + "[ \"$1\" = extract-schema ] && [ \"$3\" = --host-type ] && [ \"$4\" = wasm ] || exit 2\n[ \"$(/bin/cat \"$2\")\" = generate-exact ] || exit 3\nprintf '%s' '{}'", json.replace('\'', "'\\''") + )); + let module = inspect_blocking(extractor, b"generate-exact".to_vec(), "Wasm".into()).unwrap(); + assert_eq!(module.environment(), &schema()); +} + +#[cfg(unix)] +#[tokio::test] +async fn invalid_and_oversize_inspector_output_never_becomes_diagnostics() { + for script in [ + "printf 'generated-inspector-secret'", + "exec /usr/bin/head -c 16777217 /dev/zero", + ] { + let (_dir, extractor) = inspector(script); + let error = inspect_with(extractor, b"input".to_vec(), "Wasm".into(), Duration::from_secs(5)) + .await + .unwrap_err(); + assert!(!format!("{error:#}").contains("generated-inspector-secret")); + } +} + +#[cfg(unix)] +#[tokio::test] +async fn timeout_and_caller_drop_produce_positive_wait_receipts() { + for cancel in [false, true] { + let (_dir, extractor) = inspector("while :; do :; done"); + let (started, started_receipt) = tokio::sync::oneshot::channel(); + let (reaped, reaped_receipt) = tokio::sync::oneshot::channel(); + let operation = tokio::spawn(inspect_observed( + extractor, + b"input".to_vec(), + "Wasm".into(), + if cancel { + Duration::from_secs(30) + } else { + Duration::from_millis(300) + }, + Observation { + started: Some(started), + reaped: Some(reaped), + }, + )); + assert!( + tokio::time::timeout(Duration::from_secs(5), started_receipt) + .await + .unwrap() + .unwrap() + > 0 + ); + if cancel { + operation.abort(); + let _ = operation.await; + } else { + assert!(operation.await.unwrap().is_err()); + } + let status = tokio::time::timeout(Duration::from_secs(5), reaped_receipt) + .await + .unwrap() + .unwrap(); + assert!(!status.success()); + } +} diff --git a/crates/cli/src/subcommands/env.rs b/crates/cli/src/subcommands/env.rs index 58876664f48..20bc11a5b9b 100644 --- a/crates/cli/src/subcommands/env.rs +++ b/crates/cli/src/subcommands/env.rs @@ -149,10 +149,21 @@ fn render(body: &[u8], query: &Query) -> anyhow::Result { Query::List => { ensure!(values.len() <= MAX_ENV_VARS, "Environment key count exceeds limit"); values.sort_unstable(); + let rows = values + .into_iter() + .map(|key| Ok::<_, std::convert::Infallible>(spacetimedb_lib::sats::product![key])); + let table = sql::build_table( + spacetimedb_lib::sats::satn::PsqlClient::SpacetimeDB, + &result.schema, + rows, + )?; + Ok(format!("{table}\n")) + } + Query::Get(_) => { + ensure!(values.len() == 1, "Environment key is absent"); + Ok(format!("{}\n", values[0])) } - Query::Get(_) => ensure!(values.len() == 1, "Environment key is absent"), } - Ok(values.into_iter().map(|v| format!("{v}\n")).collect()) } #[cfg(test)] @@ -196,8 +207,12 @@ mod tests { #[test] fn list_projects_keys_and_rejects_unexpected_secret_columns() { assert_eq!( - render(&body("key", vec![vec!["Z"], vec!["A"]]), &Query::List).unwrap(), - "A\nZ\n" + render(&body("key", vec![vec!["Z"], vec!["A"]]), &Query::List) + .unwrap() + .lines() + .map(str::trim) + .collect::>(), + ["key", "-----", "\"A\"", "\"Z\""] ); let err = render(&body("value", vec![vec!["generated-secret-sentinel"]]), &Query::List).unwrap_err(); assert!(!format!("{err:#}").contains("generated-secret-sentinel")); @@ -211,7 +226,12 @@ mod tests { async fn actual_loopback_queries_and_error_redaction() { use tokio::io::{AsyncReadExt, AsyncWriteExt}; for (query, status, response, expected) in [ - (Query::List, "200 OK", body("key", vec![vec!["KEY"]]), Some("KEY\n")), + ( + Query::List, + "200 OK", + body("key", vec![vec!["KEY"]]), + Some(" key \n-------\n \"KEY\" \n"), + ), ( Query::Get("KEY".into()), "200 OK", diff --git a/crates/cli/src/subcommands/generate.rs b/crates/cli/src/subcommands/generate.rs index e312bf65ea9..14b39212458 100644 --- a/crates/cli/src/subcommands/generate.rs +++ b/crates/cli/src/subcommands/generate.rs @@ -10,11 +10,10 @@ use spacetimedb_codegen::{ UnrealCpp, AUTO_GENERATED_PREFIX, }; use spacetimedb_lib::de::serde::DeserializeWrapper; -use spacetimedb_lib::{sats, RawModuleDef}; +use spacetimedb_lib::RawModuleDef; use spacetimedb_schema; use spacetimedb_schema::def::ModuleDef; use std::path::{Path, PathBuf}; -use std::process::{Command, Stdio}; use crate::common_args::parse_optional_dotnet_version; use crate::spacetime_config::{ @@ -22,7 +21,7 @@ use crate::spacetime_config::{ }; use crate::tasks::csharp::dotnet_format; use crate::tasks::rust::rustfmt; -use crate::util::{resolve_sibling_binary, y_or_n}; +use crate::util::y_or_n; use crate::Config; use crate::{build, common_args}; use clap::builder::PossibleValue; @@ -750,18 +749,7 @@ impl Language { pub type ExtractDescriptions = fn(&Path) -> anyhow::Result; pub fn extract_descriptions(wasm_file: &Path) -> anyhow::Result { - let bin_path = std::env::var_os("SPACETIMEDB_SCHEMA_EXTRACTOR") - .map(PathBuf::from) - .map(Ok) - .unwrap_or_else(|| resolve_sibling_binary("spacetimedb-standalone"))?; - let child = Command::new(&bin_path) - .arg("extract-schema") - .arg(wasm_file) - .stdout(Stdio::piped()) - .spawn() - .with_context(|| format!("failed to spawn {}", bin_path.display()))?; - let sats::serde::SerdeWrapper::(module) = serde_json::from_reader(child.stdout.unwrap())?; - Ok(module.try_into()?) + crate::schema_extract::from_path(wasm_file) } #[cfg(test)] diff --git a/crates/cli/src/subcommands/publish.rs b/crates/cli/src/subcommands/publish.rs index b734b37cba8..5162dbeca2c 100644 --- a/crates/cli/src/subcommands/publish.rs +++ b/crates/cli/src/subcommands/publish.rs @@ -1,4 +1,6 @@ mod environment; +#[cfg(test)] +mod wire_tests; use anyhow::{ensure, Context}; use clap::Arg; @@ -322,7 +324,28 @@ i.e. only lowercase ASCII letters and numbers, separated by dashes."), .help("Use NativeAOT-LLVM compilation for C# modules (experimental, Windows only)") ) .arg(common_args::dotnet_version()) - .after_help("Every publish replaces the complete declared environment. Put an env map in spacetime.json; declared shell variables override config values (including empty strings). The CLI displays supplied keys and sources, never values. Optional values omitted from every input are removed. --env selects config file layers. Run `spacetime help publish` for more detailed information.") + .after_help("Run `spacetime help publish` for more detailed information.") + .after_long_help("Every publish replaces the complete declared environment. Put an env map in spacetime.json; declared shell variables override config values (including empty strings). The CLI displays supplied keys and sources, never values. Optional values omitted from every input are removed. --env selects config file layers. Run `spacetime help publish` for more detailed information.") +} + +fn publication_body( + module: &spacetimedb_schema::def::ModuleDef, + bytes: Vec, + environment: std::collections::BTreeMap, +) -> anyhow::Result<(&'static str, Vec)> { + if module.environment_declared() { + let body = spacetimedb_client_api_messages::publish::PublishRequest { + module: bytes, + environment, + } + .encode()?; + Ok((spacetimedb_client_api_messages::publish::CONTENT_TYPE, body)) + } else { + anyhow::ensure!(environment.is_empty(), "Module does not declare environment keys"); + // Preserve older servers for ordinary modules. Explicit empty ENV + // declarations still use the envelope, expressing replacement intent. + Ok(("application/octet-stream", bytes)) + } } fn confirm_and_clear( @@ -647,16 +670,9 @@ async fn execute_publish_configs<'a>( // Set the host type. builder = builder.query(&[("host_type", host_type)]); - let payload = spacetimedb_client_api_messages::publish::PublishRequest { - module: program_bytes, - environment: environment.values, - } - .encode()?; + let (content_type, payload) = publication_body(&module_schema, program_bytes, environment.values)?; let res = builder - .header( - reqwest::header::CONTENT_TYPE, - spacetimedb_client_api_messages::publish::CONTENT_TYPE, - ) + .header(reqwest::header::CONTENT_TYPE, content_type) .body(payload) .send() .await?; diff --git a/crates/cli/src/subcommands/publish/environment.rs b/crates/cli/src/subcommands/publish/environment.rs index 17fc67efb62..4427ae52a65 100644 --- a/crates/cli/src/subcommands/publish/environment.rs +++ b/crates/cli/src/subcommands/publish/environment.rs @@ -1,16 +1,11 @@ //! Resolve a complete, declared environment without consulting stored values. use std::collections::BTreeMap; use std::ffi::OsString; -use std::path::PathBuf; -use std::process::Stdio; -use std::time::Duration; +pub(super) use crate::schema_extract::{inspect, read_program}; use anyhow::{ensure, Context}; use serde_json::Value; use spacetimedb_lib::environment::EnvironmentSchema; -use spacetimedb_lib::{sats::serde::SerdeWrapper, RawModuleDef}; -use spacetimedb_schema::def::ModuleDef; -use tokio::io::AsyncReadExt; #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(super) enum Source { @@ -82,99 +77,5 @@ pub(super) fn resolve( Ok(resolved) } -pub(super) fn read_program(path: &std::path::Path) -> anyhow::Result> { - use std::io::Read; - let mut bytes = Vec::new(); - std::fs::File::open(path)? - .take(spacetimedb_client_api_messages::publish::MAX_MODULE_BYTES as u64 + 1) - .read_to_end(&mut bytes)?; - ensure!( - bytes.len() <= spacetimedb_client_api_messages::publish::MAX_MODULE_BYTES, - "Module exceeds publish size limit" - ); - Ok(bytes) -} - -const MAX_SCHEMA_BYTES: u64 = 16 * 1024 * 1024; -const INSPECT_TIMEOUT: Duration = Duration::from_secs(60); - -/// Inspect exactly the artifact bytes that will be uploaded. The private copy -/// prevents path replacement between inspection and upload, including --bin-path. -/// This invokes only local extraction, never a server or a saved CLI context. -pub(super) async fn inspect(program: &[u8], host_type: &str) -> anyhow::Result { - let extractor = std::env::var_os("SPACETIMEDB_SCHEMA_EXTRACTOR") - .map(PathBuf::from) - .map(Ok) - .unwrap_or_else(|| crate::util::resolve_sibling_binary("spacetimedb-standalone"))?; - inspect_with(extractor, program.to_vec(), host_type.to_owned(), INSPECT_TIMEOUT).await -} - -async fn inspect_with( - extractor: PathBuf, - program: Vec, - host_type: String, - deadline: Duration, -) -> anyhow::Result { - let (mut send, mut recv) = tokio::sync::oneshot::channel(); - // This owner retains the child and private file until actual reaping, even - // when its caller drops while extraction or stdout reading is in progress. - tokio::spawn(async move { - let result = async { - let dir = tempfile::tempdir().context("Cannot create private module inspection directory")?; - let module = dir.path().join("module"); - tokio::fs::write(&module, program) - .await - .context("Cannot prepare module inspection input")?; - let mut child = tokio::process::Command::new(extractor) - .arg("extract-schema") - .arg(&module) - .arg("--host-type") - .arg(host_type.to_ascii_lowercase()) - .env_clear() - .stdin(Stdio::null()) - .stdout(Stdio::piped()) - .stderr(Stdio::null()) - .kill_on_drop(true) - .spawn() - .context("Cannot start local module schema inspection")?; - let mut output = Vec::new(); - let mut stdout = child - .stdout - .take() - .context("Module inspection stdout unavailable")? - .take(MAX_SCHEMA_BYTES + 1); - let result = tokio::select! { - biased; - _ = send.closed() => Err(anyhow::anyhow!("Module inspection cancelled")), - result = tokio::time::timeout(deadline, async { - stdout.read_to_end(&mut output).await.context("Cannot read local module schema")?; - ensure!(output.len() as u64 <= MAX_SCHEMA_BYTES, "Local module schema exceeds output limit"); - let status = child.wait().await.context("Cannot reap local module inspector")?; - ensure!(status.success(), "Local module schema inspection failed"); - Ok(()) - }) => result.unwrap_or_else(|_| Err(anyhow::anyhow!("Local module schema inspection timed out"))), - }; - if result.is_err() { - // Queue termination, then retain ownership through positive reaping. - let _ = child.start_kill(); - child - .wait() - .await - .context("Cannot reap failed local module inspector")?; - } - result?; - // Neither parser nor validation diagnostics may echo schema literals. - let SerdeWrapper::(raw) = serde_json::from_slice(&output) - .map_err(|_| anyhow::anyhow!("Local module inspector returned invalid schema data"))?; - let schema = - ModuleDef::try_from(raw).map_err(|_| anyhow::anyhow!("Local module schema validation failed"))?; - Ok(schema) - } - .await; - let _ = send.send(result); - }); - (&mut recv).await.context("Local module inspection owner failed")? -} - #[cfg(test)] mod tests; diff --git a/crates/cli/src/subcommands/publish/environment/tests.rs b/crates/cli/src/subcommands/publish/environment/tests.rs index 5af04aa6877..7ed2bca6f2e 100644 --- a/crates/cli/src/subcommands/publish/environment/tests.rs +++ b/crates/cli/src/subcommands/publish/environment/tests.rs @@ -1,5 +1,6 @@ use super::*; use spacetimedb_lib::environment::{EnvironmentConstraint as Constraint, EnvironmentDeclaration as Declaration}; +use std::{path::PathBuf, time::Duration}; fn schema() -> EnvironmentSchema { EnvironmentSchema::new(vec![ @@ -153,134 +154,6 @@ fn non_utf8_declared_shell_value_is_rejected_without_bytes() { assert!(error.to_string().contains("UTF-8")); } -// These fixtures invoke only a locally generated executable in an owned tempdir. -// No CLI config, server credentials or user environment are imported. -#[cfg(unix)] -fn inspector(script: &str) -> (tempfile::TempDir, PathBuf) { - use std::os::unix::fs::PermissionsExt; - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("inspector"); - std::fs::write(&path, format!("#!/bin/sh\n{script}\n")).unwrap(); - std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o700)).unwrap(); - (dir, path) -} - -#[cfg(unix)] -#[tokio::test] -async fn local_inspection_passes_exact_bytes_host_and_requires_success() { - use spacetimedb_lib::db::raw_def::v10::{RawModuleDefV10, RawModuleDefV10Section}; - let raw = RawModuleDef::V10(RawModuleDefV10 { - sections: vec![RawModuleDefV10Section::Environment(schema().into_declarations())], - }); - let json = serde_json::to_string(&SerdeWrapper(raw)).unwrap(); - let (dir, extractor) = inspector(&format!( - "[ \"$1\" = extract-schema ] && [ \"$3\" = --host-type ] && [ \"$4\" = js ] || exit 2\n[ \"$(/bin/cat \"$2\")\" = exact-artifact ] || exit 3\nprintf '%s' '{}'", json.replace('\'', "'\\''") - )); - let result = inspect_with( - extractor, - b"exact-artifact".to_vec(), - "Js".into(), - Duration::from_secs(5), - ) - .await - .unwrap(); - assert_eq!(result.environment(), &schema()); - drop(dir); - let (_dir, extractor) = inspector(&format!("printf '%s' '{}'; exit 9", json.replace('\'', "'\\''"))); - assert!( - inspect_with(extractor, b"anything".to_vec(), "Wasm".into(), Duration::from_secs(5)) - .await - .is_err() - ); -} - -#[cfg(unix)] -#[tokio::test] -async fn invalid_and_oversize_inspector_output_never_becomes_diagnostics() { - for script in [ - "printf 'generated-inspector-secret'", - "exec /usr/bin/head -c 16777217 /dev/zero", - ] { - let (_dir, extractor) = inspector(script); - let error = inspect_with(extractor, b"input".to_vec(), "Wasm".into(), Duration::from_secs(5)) - .await - .unwrap_err(); - assert!(!format!("{error:#}").contains("generated-inspector-secret")); - } -} - -#[cfg(unix)] -#[tokio::test] -async fn inspection_timeout_and_dropped_waiter_reap_exact_child() { - // A shell-only busy loop has no descendants and records the exact child PID. - // kill(0) via the owned process is not used for proof: wait for /bin/kill -0 - // to report ESRCH after our owner has called wait, including cancellation. - async fn wait_pid(path: &std::path::Path) -> String { - tokio::time::timeout(Duration::from_secs(5), async { - loop { - if let Ok(pid) = tokio::fs::read_to_string(path).await - && !pid.trim().is_empty() - { - break pid; - } - tokio::task::yield_now().await; - } - }) - .await - .unwrap() - } - async fn gone(pid: &str) { - tokio::time::timeout(Duration::from_secs(5), async { - loop { - let status = tokio::process::Command::new("/bin/kill") - .args(["-0", pid.trim()]) - .stdin(Stdio::null()) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .status() - .await - .unwrap(); - if !status.success() { - break; - } - tokio::time::sleep(Duration::from_millis(10)).await; - } - }) - .await - .unwrap(); - } - for cancel in [false, true] { - let (dir, extractor) = inspector("placeholder"); - let pid_file = dir.path().join("pid"); - std::fs::write( - &extractor, - format!( - "#!/bin/sh\nprintf '%s' \"$$\" > '{}'\nwhile :; do :; done\n", - pid_file.display() - ), - ) - .unwrap(); - let operation = tokio::spawn(inspect_with( - extractor, - b"input".to_vec(), - "Wasm".into(), - if cancel { - Duration::from_secs(30) - } else { - Duration::from_millis(300) - }, - )); - let pid = wait_pid(&pid_file).await; - if cancel { - operation.abort(); - let _ = operation.await; - } else { - assert!(operation.await.unwrap().is_err()); - } - gone(&pid).await; - } -} - #[tokio::test] #[ignore = "requires explicit locally built ENV-aware standalone and declared Wasm fixture paths"] async fn actual_precompiled_declarations_are_inspected_without_server_or_values() { @@ -289,7 +162,7 @@ async fn actual_precompiled_declarations_are_inspected_without_server_or_values( let module = PathBuf::from(std::env::var_os("SPACETIMEDB_ENV_CLI_TEST_MODULE").expect("explicit module path")); assert!(extractor.is_absolute() && module.is_absolute()); let program = read_program(&module).unwrap(); - let inspected = inspect_with(extractor, program, "Wasm".into(), INSPECT_TIMEOUT) + let inspected = crate::schema_extract::inspect_with(extractor, program, "Wasm".into(), Duration::from_secs(60)) .await .unwrap(); let schema = inspected.environment(); diff --git a/crates/cli/src/subcommands/publish/wire_tests.rs b/crates/cli/src/subcommands/publish/wire_tests.rs new file mode 100644 index 00000000000..70781417800 --- /dev/null +++ b/crates/cli/src/subcommands/publish/wire_tests.rs @@ -0,0 +1,56 @@ +use super::*; +use spacetimedb_lib::{ + db::raw_def::v10::{RawModuleDefV10, RawModuleDefV10Section}, + RawModuleDef, +}; +use spacetimedb_schema::def::ModuleDef; +use std::collections::BTreeMap; + +fn schema(declared: bool) -> ModuleDef { + let mut sections = vec![RawModuleDefV10Section::Typespace(Default::default())]; + if declared { + sections.push(RawModuleDefV10Section::Environment(vec![])); + } + ModuleDef::try_from(RawModuleDef::V10(RawModuleDefV10 { sections })).unwrap() +} + +#[test] +fn ordinary_and_explicit_empty_declarations_choose_distinct_wire_formats() { + let bytes = b"exact selected module bytes\0\xff".to_vec(); + let (kind, body) = publication_body(&schema(false), bytes.clone(), BTreeMap::new()).unwrap(); + assert_eq!(kind, "application/octet-stream"); + assert_eq!(body, bytes); + let (kind, body) = publication_body(&schema(true), bytes.clone(), BTreeMap::new()).unwrap(); + assert_eq!(kind, spacetimedb_client_api_messages::publish::CONTENT_TYPE); + let envelope = spacetimedb_client_api_messages::publish::PublishRequest::decode(&body).unwrap(); + assert_eq!(envelope.module, bytes); + assert!(envelope.environment.is_empty()); + let error = publication_body( + &schema(false), + bytes, + BTreeMap::from([("KEY".into(), "secret-sentinel".into())]), + ) + .unwrap_err(); + assert!(!format!("{error:#}").contains("secret-sentinel")); +} + +#[test] +fn short_help_is_concise_and_long_help_explains_environment_replacement() { + let short = cli().render_help().to_string(); + let long = cli() + .render_long_help() + .to_string() + .split_whitespace() + .collect::>() + .join(" "); + assert!(!short.contains("Every publish replaces")); + assert!(short.contains("spacetime help publish")); + for text in [ + "Every publish replaces the complete declared environment", + "including empty strings", + "Optional values omitted", + "--env selects config file layers", + ] { + assert!(long.contains(text), "missing long-help guidance: {text}"); + } +} diff --git a/crates/cli/src/subcommands/sql.rs b/crates/cli/src/subcommands/sql.rs index d0a75e17d76..5b782d7106b 100644 --- a/crates/cli/src/subcommands/sql.rs +++ b/crates/cli/src/subcommands/sql.rs @@ -287,7 +287,7 @@ pub async fn exec(config: Config, args: &ArgMatches) -> Result<(), anyhow::Error } /// Generates a [`tabled::Table`] from a schema and rows, using the style of a psql table. -fn build_table( +pub(super) fn build_table( client: PsqlClient, schema: &ProductType, rows: impl Iterator>, diff --git a/crates/core/src/host/v8/syscall/mod.rs b/crates/core/src/host/v8/syscall/mod.rs index 029d5836282..467e7f26562 100644 --- a/crates/core/src/host/v8/syscall/mod.rs +++ b/crates/core/src/host/v8/syscall/mod.rs @@ -62,8 +62,7 @@ fn resolve_sys_module_inner<'scope>( (1, 3) => Ok(v1::sys_v1_3(scope)), (2, 0) => Ok(v2::sys_v2_0(scope)), (2, 1) => Ok(v2::sys_v2_1(scope)), - // sys2.2 is reserved for invocation authority. - (2, 3) => Ok(v2::sys_v2_3(scope)), + (2, 2) => Ok(v2::sys_v2_2(scope)), _ => Err(TypeError(format!( "Could not import {spec:?}, likely because this module was built for a newer version of SpacetimeDB.\n\ It requires sys module v{major}.{minor}, but that version is not supported by the database." diff --git a/crates/core/src/host/v8/syscall/v2.rs b/crates/core/src/host/v8/syscall/v2.rs index 8fb376ec7af..3e25145fc74 100644 --- a/crates/core/src/host/v8/syscall/v2.rs +++ b/crates/core/src/host/v8/syscall/v2.rs @@ -169,8 +169,8 @@ pub(super) fn sys_v2_1<'scope>(scope: &mut PinScope<'scope, '_>) -> Local<'scope ) } -pub(super) fn sys_v2_3<'scope>(scope: &mut PinScope<'scope, '_>) -> Local<'scope, Module> { - create_synthetic_module!(scope, "spacetime:sys@2.3", (with_sys_result, AbiCall::EnvGet, env_get),) +pub(super) fn sys_v2_2<'scope>(scope: &mut PinScope<'scope, '_>) -> Local<'scope, Module> { + create_synthetic_module!(scope, "spacetime:sys@2.2", (with_sys_result, AbiCall::EnvGet, env_get),) } fn env_get<'s>( diff --git a/crates/core/src/host/wasm_common.rs b/crates/core/src/host/wasm_common.rs index dc8baa44227..f5ab623fec3 100644 --- a/crates/core/src/host/wasm_common.rs +++ b/crates/core/src/host/wasm_common.rs @@ -444,8 +444,7 @@ macro_rules! abi_funcs { "spacetime_10.4"::datastore_delete_by_index_scan_point_bsatn, "spacetime_10.5"::datastore_clear, - // ABI10.6 is reserved for invocation authority. - "spacetime_10.7"::env_get, + "spacetime_10.6"::env_get, } $link_async! { diff --git a/crates/smoketests/tests/standalone/cli/environment.rs b/crates/smoketests/tests/standalone/cli/environment.rs index aed684811ce..1f080bd1da4 100644 --- a/crates/smoketests/tests/standalone/cli/environment.rs +++ b/crates/smoketests/tests/standalone/cli/environment.rs @@ -173,8 +173,8 @@ impl Fixture { ) } - fn list(&self) -> String { - self.success( + fn list(&self) -> Vec { + let output = self.success( &[ "env", "list", @@ -184,7 +184,13 @@ impl Fixture { "--no-config", ], &[], - ) + ); + let mut lines = output.lines().map(str::trim); + assert_eq!(lines.next(), Some("key")); + lines + .filter(|line| !line.is_empty() && !line.chars().all(|c| c == '-')) + .map(|line| serde_json::from_str::(line).unwrap()) + .collect() } fn typed(&self, required: &str, mode: &str, rest: [Option<&str>; 4]) { @@ -334,7 +340,7 @@ fn cli_environment_layers_shell_and_exact_precompiled_declarations() { ); let mut keys = KEYS.to_vec(); keys.sort_unstable(); - assert_eq!(f.list(), format!("{}\n", keys.join("\n"))); + assert_eq!(f.list(), keys); let initial = f.sql("SELECT required, mode FROM initial_environment"); assert!(initial.status.success()); let initial = String::from_utf8(initial.stdout).unwrap(); @@ -353,7 +359,7 @@ fn cli_environment_replacement_rejection_and_read_only_commands() { )); f.published(&[], &[]); f.typed("replacement-sentinel", "other", [None; 4]); - assert_eq!(f.list(), "SMOKE_MODE\nSMOKE_REQUIRED\n"); + assert_eq!(f.list(), ["SMOKE_MODE", "SMOKE_REQUIRED"]); assert!(!f .command( &[ @@ -386,7 +392,7 @@ fn cli_environment_replacement_rejection_and_read_only_commands() { f.typed("replacement-sentinel", "other", [None; 4]); } // Invalid local configuration must not prevent an explicit read-only target. - assert_eq!(f.list(), "SMOKE_MODE\nSMOKE_REQUIRED\n"); + assert_eq!(f.list(), ["SMOKE_MODE", "SMOKE_REQUIRED"]); for statement in [ "SET env.SMOKE_REQUIRED = 'bypass'", "DELETE env.SMOKE_REQUIRED", @@ -435,5 +441,5 @@ fn cli_environment_initial_rejection_clear_and_omitted_payload() { f.config(None); f.wasm = modules::precompiled_module("noop"); f.published(&[("SMOKE_REQUIRED", "must-not-be-ambient")], &["--delete-data"]); - assert_eq!(f.list(), ""); + assert!(f.list().is_empty()); } diff --git a/modules/environment-test/src/lib.rs b/modules/environment-test/src/lib.rs index 8803534eee7..8d6efa37a11 100644 --- a/modules/environment-test/src/lib.rs +++ b/modules/environment-test/src/lib.rs @@ -3,12 +3,15 @@ use std::sync::atomic::{AtomicBool, Ordering}; static VIEW_TRAP_ENTERED: AtomicBool = AtomicBool::new(false); +type RequiredString = String; +type OptionalString = Option; + #[spacetimedb::env] pub struct Env { - pub REQUIRED: String, + pub REQUIRED: RequiredString, #[env(values("ready", "other"))] pub MODE: String, - pub MISSING: Option, + pub MISSING: OptionalString, pub EMPTY: Option, pub UTF8: Option, pub NUL: Option, diff --git a/modules/module-test-ts/src/environment_sys.d.ts b/modules/module-test-ts/src/environment_sys.d.ts index 8bf1660c106..d2abea72f80 100644 --- a/modules/module-test-ts/src/environment_sys.d.ts +++ b/modules/module-test-ts/src/environment_sys.d.ts @@ -1,4 +1,4 @@ // Raw host ABI used to verify that SDK context changes cannot grant authority. -declare module 'spacetime:sys@2.3' { +declare module 'spacetime:sys@2.2' { export function env_get(key: string): string | null; } diff --git a/modules/module-test-ts/src/lib_submodule.ts b/modules/module-test-ts/src/lib_submodule.ts index ff4c56b3be8..466770be2ff 100644 --- a/modules/module-test-ts/src/lib_submodule.ts +++ b/modules/module-test-ts/src/lib_submodule.ts @@ -1,6 +1,6 @@ /// import { schema, table, t, SyncResponse } from 'spacetimedb/server'; -import { env_get } from 'spacetime:sys@2.3'; +import { env_get } from 'spacetime:sys@2.2'; const libData = table( { name: 'libData', public: true }, From 7d26b8d8a6866bcc7c4454b885eecd5914fffeeb Mon Sep 17 00:00:00 2001 From: Tyler Cloutier Date: Tue, 8 Sep 2026 23:34:12 -0400 Subject: [PATCH 09/34] Fix ENV CI coordination and restore C++ logging names --- crates/bindings-cpp/src/abi/wasi_shims.cpp | 15 ++- .../Runtime/Internal/Module.cs | 6 +- .../core/src/host/wasmtime/wasmtime_module.rs | 2 +- .../commands/workflow-coordinator/src/main.rs | 95 ++++++++++++++++++- 4 files changed, 104 insertions(+), 14 deletions(-) diff --git a/crates/bindings-cpp/src/abi/wasi_shims.cpp b/crates/bindings-cpp/src/abi/wasi_shims.cpp index a698b4177f7..37548c40d99 100644 --- a/crates/bindings-cpp/src/abi/wasi_shims.cpp +++ b/crates/bindings-cpp/src/abi/wasi_shims.cpp @@ -4,14 +4,13 @@ #include #include -// Keep this pure ABI translation unit independent of SDK opaque types: their -// standard-library helpers include wasi/api.h in Emscripten, which conflicts -// with these standalone shim definitions. Use a distinct C++ name for the raw -// host logging import; its WebAssembly signature is the same eight i32 values. + +// SpacetimeDB imports we need for console output +// Import from spacetime_10.0 module as required by SpacetimeDB ABI extern "C" __attribute__((import_module("spacetime_10.0"), import_name("console_log"))) -void wasi_console_log(uint8_t level, const uint8_t* target_ptr, uint32_t target_len, - const uint8_t* filename_ptr, uint32_t filename_len, uint32_t line_number, - const uint8_t* message_ptr, uint32_t message_len); +void console_log(uint8_t log_level, const uint8_t* target, uint32_t target_len, + const uint8_t* filename, uint32_t filename_len, uint32_t line_number, + const uint8_t* message, uint32_t message_len); // Helper macro for string literals #define CSTR(s) (uint8_t*)s, sizeof(s) - 1 @@ -151,7 +150,7 @@ __wasi_errno_t __wasi_fd_write(__wasi_fd_t fd, const __wasi_ciovec_t* iovs, // Make a single console_log call with the complete message uint8_t log_level = (fd == STDERR_FILENO) ? 1 : 2; // 1=WARN, 2=INFO - wasi_console_log(log_level, CSTR("wasi"), CSTR(__FILE__), __LINE__, + console_log(log_level, CSTR("wasi"), CSTR(__FILE__), __LINE__, buffer, offset); // Clean up heap allocation if needed diff --git a/crates/bindings-csharp/Runtime/Internal/Module.cs b/crates/bindings-csharp/Runtime/Internal/Module.cs index 035291a2f0d..438c3439146 100644 --- a/crates/bindings-csharp/Runtime/Internal/Module.cs +++ b/crates/bindings-csharp/Runtime/Internal/Module.cs @@ -87,7 +87,8 @@ internal void RegisterTable(RawTableDefV10 table, RawScheduleDefV10? schedule) internal void RegisterView(RawViewDefV10 view) => viewDefs.Add(view); - internal void RegisterEnvironment(EnvironmentDeclaration declaration) => environment.Add(declaration); + internal void RegisterEnvironment(EnvironmentDeclaration declaration) => + environment.Add(declaration); internal void RegisterViewPrimaryKey(string viewSourceName, IEnumerable columns) => viewPrimaryKeyDefs.Add(new RawViewPrimaryKeyDefV10(viewSourceName, [.. columns])); @@ -431,7 +432,8 @@ public static void RegisterAnonymousView() moduleDef.RegisterView(def); } - public static void RegisterEnvironment(EnvironmentDeclaration declaration) => moduleDef.RegisterEnvironment(declaration); + public static void RegisterEnvironment(EnvironmentDeclaration declaration) => + moduleDef.RegisterEnvironment(declaration); public static void RegisterViewPrimaryKey(string viewSourceName, string[] columns) => moduleDef.RegisterViewPrimaryKey(viewSourceName, columns); diff --git a/crates/core/src/host/wasmtime/wasmtime_module.rs b/crates/core/src/host/wasmtime/wasmtime_module.rs index aa589df3b5a..be8dd719582 100644 --- a/crates/core/src/host/wasmtime/wasmtime_module.rs +++ b/crates/core/src/host/wasmtime/wasmtime_module.rs @@ -55,7 +55,7 @@ impl WasmtimeModule { WasmtimeModule { module } } - pub const IMPLEMENTED_ABI: abi::VersionTuple = abi::VersionTuple::new(10, 7); + pub const IMPLEMENTED_ABI: abi::VersionTuple = abi::VersionTuple::new(10, 6); pub(super) fn link_imports(linker: &mut Linker) -> anyhow::Result<()> { link_imports(linker, AsyncImportMode::SyncStub) diff --git a/tools/ci/commands/workflow-coordinator/src/main.rs b/tools/ci/commands/workflow-coordinator/src/main.rs index 2ff8101d999..521b19245d0 100644 --- a/tools/ci/commands/workflow-coordinator/src/main.rs +++ b/tools/ci/commands/workflow-coordinator/src/main.rs @@ -197,6 +197,8 @@ struct Repository { #[derive(Clone, Debug, Deserialize, PartialEq, Eq)] struct PullRequestRef { sha: String, + #[serde(rename = "ref")] + branch_name: String, repo: Option, } @@ -340,10 +342,29 @@ fn related_private_pr(public_pr_number: Option) -> Result 1 { - bail!("found multiple open linked private PRs"); + let public_branch = if pulls.len() > 1 { + Some(pull_request(PUBLIC_REPO, public_pr_number)?.head.branch_name) + } else { + None + }; + select_related_private_pr(pulls, public_branch.as_deref()) +} + +fn select_related_private_pr(mut pulls: Vec, public_branch: Option<&str>) -> Result> { + if pulls.len() <= 1 { + return Ok(pulls.pop()); + } + + // Timeline references include historical links and links to other layers of + // a PR stack. A unique shared branch name identifies the companion PR; the + // exact public-submodule SHA is still checked before selecting its CI run. + if let Some(public_branch) = public_branch.filter(|branch| !branch.is_empty()) { + pulls.retain(|pull| pull.head.branch_name == public_branch); + if pulls.len() == 1 { + return Ok(pulls.pop()); + } } - Ok(pulls.pop()) + bail!("found multiple open linked private PRs without a unique matching head branch") } fn resolve_private_source(public_pr_number: Option) -> Result { @@ -574,6 +595,7 @@ mod tests { state: "open".to_owned(), head: PullRequestRef { sha: "private-sha".to_owned(), + branch_name: "tyler/environment-variables".to_owned(), repo: Some(Repository { full_name: PRIVATE_REPO.to_owned(), }), @@ -581,6 +603,73 @@ mod tests { } } + #[test] + fn related_private_pr_prefers_the_unique_exact_public_head_branch() { + let matching = pull(); + let mut downstream = pull(); + downstream.number = 43; + downstream.head.branch_name = "tyler/environment-variables-followup".to_owned(); + for candidates in [ + vec![matching.clone(), downstream.clone()], + vec![downstream, matching.clone()], + ] { + assert_eq!( + select_related_private_pr(candidates, Some("tyler/environment-variables")).unwrap(), + Some(matching.clone()) + ); + } + } + + #[test] + fn related_private_pr_preserves_absent_and_single_candidate_behavior() { + assert_eq!(select_related_private_pr(Vec::new(), None).unwrap(), None); + for public_branch in [None, Some("unrelated-branch")] { + assert_eq!( + select_related_private_pr(vec![pull()], public_branch).unwrap(), + Some(pull()) + ); + } + } + + #[test] + fn related_private_pr_rejects_multiple_matching_branches() { + let mut duplicate = pull(); + duplicate.number = 43; + assert!(select_related_private_pr(vec![pull(), duplicate], Some("tyler/environment-variables")).is_err()); + } + + #[test] + fn related_private_pr_rejects_missing_or_unmatched_public_branch() { + let mut downstream = pull(); + downstream.number = 43; + downstream.head.branch_name = "tyler/v10-abi-extensions".to_owned(); + for public_branch in [None, Some(""), Some("tyler/unrelated")] { + assert!(select_related_private_pr(vec![pull(), downstream.clone()], public_branch).is_err()); + } + } + + #[test] + fn selected_companion_still_requires_the_exact_public_submodule() { + let selected = select_related_private_pr(vec![pull()], None).unwrap().unwrap(); + assert!(ensure_public_submodule_matches(selected.number, "old-public-sha", "requested-public-sha").is_err()); + ensure_public_submodule_matches(selected.number, "requested-public-sha", "requested-public-sha").unwrap(); + } + + #[test] + fn pull_request_head_branch_uses_the_github_ref_field() { + let parsed: PullRequest = serde_json::from_value(serde_json::json!({ + "number": 42, + "state": "open", + "head": { + "sha": "private-sha", + "ref": "tyler/environment-variables", + "repo": { "full_name": PRIVATE_REPO } + } + })) + .unwrap(); + assert_eq!(parsed, pull()); + } + fn run(id: u64, title: &str, created_at: &str) -> WorkflowRun { WorkflowRun { id, From 60c17d018b5c66cb1ecac17c8d1154974d0a96cf Mon Sep 17 00:00:00 2001 From: Tyler Cloutier Date: Wed, 9 Sep 2026 00:00:45 -0400 Subject: [PATCH 10/34] Fix environment CI isolation and update handler diagnostics --- crates/bindings-typescript/package.json | 4 +- crates/bindings/tests/ui/http_handlers.stderr | 2 +- crates/cli/src/spacetime_config.rs | 2 +- .../cli/src/spacetime_config/environment.rs | 48 ++++++++++++++++++- crates/smoketests/src/lib.rs | 27 +++++++++-- .../tests/standalone/cli/environment.rs | 18 +++---- 6 files changed, 83 insertions(+), 18 deletions(-) diff --git a/crates/bindings-typescript/package.json b/crates/bindings-typescript/package.json index 8e6f94787de..db598440673 100644 --- a/crates/bindings-typescript/package.json +++ b/crates/bindings-typescript/package.json @@ -25,14 +25,14 @@ "scripts": { "build:js": "tsup", "build:types": "tsc -p tsconfig.build.json", - "build": "pnpm -s build:js && pnpm -s build:types", + "build": "pnpm run build:js && pnpm run build:types", "format": "prettier . --write --ignore-path ../../.prettierignore", "lint": "eslint . && prettier . --check --ignore-path ../../.prettierignore", "test": "vitest run", "test:typecheck": "vitest typecheck --run", "coverage": "vitest run --coverage", "brotli-size": "brotli-size dist/index.js", - "size": "pnpm -s build && size-limit", + "size": "pnpm run build && size-limit", "generate:moduledef": "cargo run -p spacetimedb-codegen --example regen-typescript-moduledef && prettier --write src/lib/autogen", "generate:client-api": "cargo run -p generate-client-api && prettier --write src/sdk/client_api", "generate:test-app": "pnpm --filter @clockworklabs/test-app generate", diff --git a/crates/bindings/tests/ui/http_handlers.stderr b/crates/bindings/tests/ui/http_handlers.stderr index 960b3cdeae9..c28936ac12e 100644 --- a/crates/bindings/tests/ui/http_handlers.stderr +++ b/crates/bindings/tests/ui/http_handlers.stderr @@ -175,7 +175,7 @@ error[E0609]: no field `db` on type `&mut HandlerContext` 53 | let _rows = ctx.db.test_table().iter(); | ^^ unknown field | - = note: available fields are: `timestamp`, `http` + = note: available fields are: `env`, `timestamp`, `http` error[E0308]: mismatched types --> tests/ui/http_handlers.rs:66:4 diff --git a/crates/cli/src/spacetime_config.rs b/crates/cli/src/spacetime_config.rs index cce271510ee..32d3030082a 100644 --- a/crates/cli/src/spacetime_config.rs +++ b/crates/cli/src/spacetime_config.rs @@ -855,7 +855,7 @@ impl SpacetimeConfig { let value = environment::parse(&content).with_context(|| format!("Failed to parse config file {}", path.display()))?; let config: Self = environment::decode_config(value) - .map_err(|_| anyhow::anyhow!("Invalid configuration structure in {}", path.display()))?; + .with_context(|| format!("Invalid configuration structure in {}", path.display()))?; Ok(config) } diff --git a/crates/cli/src/spacetime_config/environment.rs b/crates/cli/src/spacetime_config/environment.rs index ecaa3218684..c4493835376 100644 --- a/crates/cli/src/spacetime_config/environment.rs +++ b/crates/cli/src/spacetime_config/environment.rs @@ -92,10 +92,26 @@ pub(super) fn parse(content: &str) -> anyhow::Result { /// Deserialize through JSON text so Serde's flattened-field buffer does not /// receive visit_u128 from Value's deserializer. That buffer cannot represent /// u128, while the arbitrary-precision JSON parser preserves its decimal token. -/// Diagnostics deliberately discard the original error, which may quote values. +/// Diagnostics discard values while retaining a bounded, ordinary unknown field +/// name, which is useful for correcting misspelled configuration options. pub(super) fn decode_config(value: Value) -> anyhow::Result { let encoded = serde_json::to_vec(&value).map_err(|_| anyhow::anyhow!("Invalid configuration structure"))?; - serde_json::from_slice(&encoded).map_err(|_| anyhow::anyhow!("Invalid configuration structure")) + serde_json::from_slice(&encoded).map_err(|error| { + let diagnostic = error.to_string(); + if let Some((field, suffix)) = diagnostic + .strip_prefix("unknown field `") + .and_then(|message| message.split_once('`')) + && suffix.starts_with(", expected ") + && !field.is_empty() + && field.len() <= 64 + && field + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'-')) + { + return anyhow::anyhow!("unknown field `{field}`"); + } + anyhow::anyhow!("Invalid configuration structure") + }) } fn is_space(ch: char) -> bool { @@ -283,4 +299,32 @@ mod tests { serde_json::from_value(serde_json::json!({"env":{"A":1},"children":[{"env":null}]})).unwrap(); assert!(config.collect_all_targets_with_inheritance()[1].fields["env"].is_null()); } + + #[test] + fn unknown_configuration_fields_are_actionable_without_exposing_values() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("spacetime.json"); + let secret = "private-configuration-value-sentinel"; + let value = serde_json::json!({"dev": {"run_command": secret}, "env": {"TOKEN": secret}}); + std::fs::write(&path, serde_json::to_vec(&value).unwrap()).unwrap(); + for error in [ + SpacetimeConfig::load(&path).unwrap_err(), + find_and_load_with_env_from(None, dir.path().to_owned()).err().unwrap(), + ] { + let diagnostic = format!("{error:#}"); + assert!(diagnostic.contains("unknown field `run_command`")); + assert!(!diagnostic.contains(secret)); + } + + for field in [ + "control\ncharacters".to_owned(), + "x".repeat(65), + "quote`injection".to_owned(), + ] { + let error = decode_config(serde_json::json!({"dev": {field: secret}})).unwrap_err(); + assert_eq!(error.to_string(), "Invalid configuration structure"); + } + let error = decode_config(serde_json::json!({"dev": {"run": [secret]}})).unwrap_err(); + assert_eq!(error.to_string(), "Invalid configuration structure"); + } } diff --git a/crates/smoketests/src/lib.rs b/crates/smoketests/src/lib.rs index 3ec8f5b7141..f9ff97e7fe7 100644 --- a/crates/smoketests/src/lib.rs +++ b/crates/smoketests/src/lib.rs @@ -802,6 +802,7 @@ pub struct SmoketestBuilder { autopublish: bool, pg_port: Option, server_url_override: Option, + isolated_local_server: bool, cli_path: Option, } @@ -828,6 +829,7 @@ impl SmoketestBuilder { autopublish: true, pg_port: None, server_url_override: None, + isolated_local_server: false, cli_path: None, } } @@ -837,6 +839,14 @@ impl SmoketestBuilder { self } + /// Start an owned local server with a fresh CLI configuration, even in a + /// remote test job. Do not copy the remote job's login or base configuration. + /// This cannot be combined with an explicit server URL. + pub fn isolated_local_server(mut self) -> Self { + self.isolated_local_server = true; + self + } + /// Uses a specific CLI binary instead of the pre-built CLI for this test. pub fn cli_path(mut self, path: impl AsRef) -> Self { self.cli_path = Some(path.as_ref().to_path_buf()); @@ -923,6 +933,15 @@ impl SmoketestBuilder { /// Panics if the CLI/standalone binaries haven't been built or are stale. /// Run `cargo smoketest prepare` to build binaries before running tests. pub fn build(self) -> Smoketest { + assert!( + !self.isolated_local_server || self.server_url_override.is_none(), + "isolated_local_server cannot use an explicit remote server URL" + ); + let inherited_remote = if self.isolated_local_server { + None + } else { + remote_server_url() + }; // Check binaries first - this will panic with a helpful message if missing/stale if self.cli_path.is_none() { let _ = ensure_binaries_built(); @@ -936,7 +955,7 @@ impl SmoketestBuilder { // Check if we're running against a remote server let (guard, server_url, data_dir_fixture) = if let Some(fixture) = self.data_dir_fixture.as_ref() { - if self.server_url_override.is_some() || remote_server_url().is_some() { + if self.server_url_override.is_some() || inherited_remote.is_some() { panic!("data_dir_fixture requires a local server managed by the smoketest harness"); } @@ -963,7 +982,7 @@ impl SmoketestBuilder { } else if let Some(url) = self.server_url_override { eprintln!("[REMOTE] Using explicit server URL: {}", url); (None, url, None) - } else if let Some(remote_url) = remote_server_url() { + } else if let Some(remote_url) = inherited_remote { eprintln!("[REMOTE] Using remote server: {}", remote_url); (None, remote_url, None) } else { @@ -998,7 +1017,9 @@ impl SmoketestBuilder { let module_name = format!("smoketest_module_{}", random_string()); let config_path = project_dir.path().join("config.toml"); - if let Ok(base_config_path) = std::env::var("SPACETIME_SMOKETEST_BASE_CONFIG_PATH") { + if !self.isolated_local_server + && let Ok(base_config_path) = std::env::var("SPACETIME_SMOKETEST_BASE_CONFIG_PATH") + { fs::copy(&base_config_path, &config_path) .unwrap_or_else(|err| panic!("failed to copy base smoketest config from {base_config_path}: {err:#}")); } diff --git a/crates/smoketests/tests/standalone/cli/environment.rs b/crates/smoketests/tests/standalone/cli/environment.rs index 1f080bd1da4..2ba96d8adb9 100644 --- a/crates/smoketests/tests/standalone/cli/environment.rs +++ b/crates/smoketests/tests/standalone/cli/environment.rs @@ -27,23 +27,23 @@ struct Fixture { impl Fixture { fn new() -> Self { - // Check before the harness can connect or copy a prior login. These are - // standalone-only tests, including when accidentally run in a remote job. - for key in [ + // Private CI supplies remote cluster settings to the same test binary. + // This fixture must still create its own server and fresh credentials, + // without changing those settings for other tests in the process. + let remote_settings = [ "SPACETIME_REMOTE_SERVER", "SPACETIME_USE_AUTH_HOST", "SPACETIME_SMOKETEST_BASE_CONFIG_PATH", - ] { - assert!( - std::env::var_os(key).is_none(), - "ENV smoke test requires isolated local settings ({key})" - ); - } + ]; + let inherited = remote_settings.map(std::env::var_os); let test = Smoketest::builder() + .isolated_local_server() .precompiled_module("environment-publish") .autopublish(false) .build(); + assert_eq!(remote_settings.map(std::env::var_os), inherited); assert!(test.guard.is_some()); + assert!(!test.config_path.exists(), "local fixture copied inherited credentials"); let address = test .server_url .strip_prefix("http://") From 0293b627a361e2b5e182c469edee1a14dda588d6 Mon Sep 17 00:00:00 2001 From: Tyler Cloutier Date: Wed, 9 Sep 2026 00:08:51 -0400 Subject: [PATCH 11/34] Document direct HTTP environment publishing and retain query diagnostics --- crates/query/src/lib.rs | 4 +- .../00200-http-api/00300-database.md | 54 ++++++++++++++++++- 2 files changed, 54 insertions(+), 4 deletions(-) diff --git a/crates/query/src/lib.rs b/crates/query/src/lib.rs index 717f9d38092..9225393eb34 100644 --- a/crates/query/src/lib.rs +++ b/crates/query/src/lib.rs @@ -29,7 +29,7 @@ pub fn compile_subscription( auth: &AuthCtx, ) -> Result<(Vec, TableId, TableName, bool)> { if sql.len() > MAX_SQL_LENGTH { - bail!("SQL query exceeds maximum allowed length") + bail!("SQL query exceeds maximum allowed length: \"{sql:.120}...\"") } let (plan, mut has_param) = parse_and_type_sub(sql, tx, auth)?; @@ -59,7 +59,7 @@ pub fn compile_subscription( /// A utility for parsing and type checking a sql statement pub fn compile_sql_stmt(sql: &str, tx: &impl SchemaView, auth: &AuthCtx) -> Result { if sql.len() > MAX_SQL_LENGTH { - bail!("SQL query exceeds maximum allowed length") + bail!("SQL query exceeds maximum allowed length: \"{sql:.120}...\"") } match parse_and_type_sql(sql, tx, auth)? { diff --git a/docs/docs/00300-resources/00200-reference/00200-http-api/00300-database.md b/docs/docs/00300-resources/00200-reference/00200-http-api/00300-database.md index e6137bea0df..2066a7682c2 100644 --- a/docs/docs/00300-resources/00200-reference/00200-http-api/00300-database.md +++ b/docs/docs/00300-resources/00200-reference/00200-http-api/00300-database.md @@ -41,7 +41,7 @@ If no `Authorization` header is provided, a new anonymous identity will be creat #### Data -A WebAssembly module in the [binary format](https://webassembly.github.io/spec/core/binary/index.html). +A WebAssembly module in the [binary format](https://webassembly.github.io/spec/core/binary/index.html), or a [publish request with environment values](#publishing-with-environment-values). #### Returns @@ -76,7 +76,7 @@ If no `Authorization` header is provided, a new anonymous identity will be creat #### Data -A WebAssembly module in the [binary format](https://webassembly.github.io/spec/core/binary/index.html). +A WebAssembly module in the [binary format](https://webassembly.github.io/spec/core/binary/index.html), or a [publish request with environment values](#publishing-with-environment-values). #### Returns @@ -98,6 +98,56 @@ If a database with the given name exists, but the identity provided in the `Auth } } ``` +### Publishing with environment values + +Both publish endpoints accept `Content-Type: application/vnd.spacetimedb.publish+json` with this JSON body: + +```json +{ + "module": "", + "environment": { + "API_KEY": "development-only-key", + "MODE": "development" + } +} +``` + +`module` uses standard padded Base64. `environment` maps declared names to strings. The server validates the complete map against the module's declarations and installs both in one transaction. Missing required values reject the publish; omitted optional values are removed. Omitting `environment` is equivalent to `{}`, including when publishing unchanged module bytes. + +For example, this Python script publishes a Wasm module to a local server. Set `SPACETIME_TOKEN` to a token authorized to publish and `API_KEY` to the complete configuration's required value. Change the module path to the artifact you built. + +```python +import base64 +import json +import os +from pathlib import Path +from urllib.request import Request, urlopen + +body = json.dumps({ + "module": base64.b64encode(Path("module.wasm").read_bytes()).decode("ascii"), + "environment": { + "API_KEY": os.environ["API_KEY"], + "MODE": "development", + }, +}).encode("utf-8") + +request = Request( + "http://127.0.0.1:3000/v1/database/env-example?host_type=wasm", + data=body, + method="PUT", + headers={ + "Authorization": "Bearer " + os.environ["SPACETIME_TOKEN"], + "Content-Type": "application/vnd.spacetimedb.publish+json", + }, +) +with urlopen(request, timeout=60) as response: + print(response.read().decode("utf-8")) +``` + +Direct HTTP callers, including module procedures, use this same format; the server does not load project configuration or shell values for them. See [Environment Variables](../../../00200-core-concepts/00100-databases/00700-environment-variables.md) for declaration syntax and value limits. The decoded module is limited to 128 MiB and the complete encoded request to 192 MiB. + +Raw module bodies continue to work and supply an empty environment. Use `application/octet-stream` for that format. This preserves compatibility with older servers for modules that do not require ENV support; older servers do not support the JSON publish format. + ## `GET /v1/database/:name_or_identity` Get a database's identity, owner identity, host type, number of replicas and a hash of its WASM module. From 03cb28c18db7a8e93471819093f1fbd70b9b1be5 Mon Sep 17 00:00:00 2001 From: Tyler Cloutier Date: Wed, 9 Sep 2026 00:30:03 -0400 Subject: [PATCH 12/34] Add typed Rust environment enums and show values in env list --- crates/bindings-macro/src/environment.rs | 9 +- .../bindings-macro/src/environment/value.rs | 193 ++++++++++++++++++ crates/bindings-macro/src/lib.rs | 5 + crates/bindings/src/lib.rs | 40 +++- crates/bindings/src/rt.rs | 80 ++++++-- crates/bindings/tests/environment.rs | 1 + .../bindings/tests/environment_enum_values.rs | 78 +++++++ crates/bindings/tests/pass/environment.rs | 17 ++ crates/bindings/tests/ui/environment_enum.rs | 54 +++++ .../bindings/tests/ui/environment_enum.stderr | 85 ++++++++ crates/bindings/tests/ui/environment_types.rs | 11 - .../tests/ui/environment_types.stderr | 113 +++------- crates/cli/src/subcommands/env.rs | 93 +++++---- .../tests/standalone/cli/environment.rs | 15 +- crates/testing/tests/environment.rs | 76 +++++++ .../00700-environment-variables.md | 36 +++- .../00100-cli-reference.md | 4 +- modules/environment-test/src/lib.rs | 42 +++- 18 files changed, 777 insertions(+), 175 deletions(-) create mode 100644 crates/bindings-macro/src/environment/value.rs create mode 100644 crates/bindings/tests/environment_enum_values.rs create mode 100644 crates/bindings/tests/ui/environment_enum.rs create mode 100644 crates/bindings/tests/ui/environment_enum.stderr diff --git a/crates/bindings-macro/src/environment.rs b/crates/bindings-macro/src/environment.rs index e58899d96db..4640bcb40dc 100644 --- a/crates/bindings-macro/src/environment.rs +++ b/crates/bindings-macro/src/environment.rs @@ -1,3 +1,5 @@ +pub(crate) mod value; + use proc_macro2::TokenStream; use quote::{format_ident, quote}; use syn::ext::IdentExt as _; @@ -59,7 +61,7 @@ pub(crate) fn expand(args: TokenStream, mut item: ItemStruct) -> syn::Result quote!(::spacetimedb::spacetimedb_lib::environment::EnvironmentConstraint::AnyString), + None => quote!(<#ty as ::spacetimedb::rt::EnvironmentValue>::constraint()), Some([value]) => { quote!(::spacetimedb::spacetimedb_lib::environment::EnvironmentConstraint::Literal(#value.into())) } @@ -69,6 +71,11 @@ pub(crate) fn expand(args: TokenStream, mut item: ItemStruct) -> syn::Result::with_constraint(#constraint)) + } else { + constraint + }; declarations.push( quote!(::spacetimedb::spacetimedb_lib::environment::EnvironmentDeclaration { name: #name.into(), diff --git a/crates/bindings-macro/src/environment/value.rs b/crates/bindings-macro/src/environment/value.rs new file mode 100644 index 00000000000..2608cc10f69 --- /dev/null +++ b/crates/bindings-macro/src/environment/value.rs @@ -0,0 +1,193 @@ +use proc_macro2::TokenStream; +use quote::quote; +use std::collections::BTreeSet; +use syn::ext::IdentExt as _; +use syn::{Data, DeriveInput, Fields, LitStr}; + +pub(crate) fn derive(item: DeriveInput) -> syn::Result { + if !item.generics.params.is_empty() || item.generics.where_clause.is_some() { + return Err(syn::Error::new_spanned( + &item.generics, + "environment value enums cannot be generic", + )); + } + if let Some(attr) = item.attrs.iter().find(|attr| attr.path().is_ident("env")) { + return Err(syn::Error::new_spanned( + attr, + "place `#[env(value = \"...\")]` on enum variants", + )); + } + let Data::Enum(data) = &item.data else { + return Err(syn::Error::new_spanned( + &item.ident, + "EnvironmentValue requires an enum with unit variants", + )); + }; + if data.variants.is_empty() || data.variants.len() > 256 { + return Err(syn::Error::new_spanned( + &item.ident, + "environment value enums require 1 to 256 variants", + )); + } + let mut variants = Vec::new(); + let mut values = Vec::new(); + let mut unique = BTreeSet::new(); + for variant in &data.variants { + if !matches!(variant.fields, Fields::Unit) { + return Err(syn::Error::new_spanned( + &variant.fields, + "environment value variants cannot have payloads", + )); + } + let mut value: Option = None; + for attr in variant.attrs.iter().filter(|attr| attr.path().is_ident("env")) { + attr.parse_nested_meta(|meta| { + if !meta.path.is_ident("value") || value.is_some() { + return Err(meta.error("expected one value = \"literal\" mapping")); + } + value = Some(meta.value()?.parse()?); + Ok(()) + })?; + if value.is_none() { + return Err(syn::Error::new_spanned(attr, "expected value = \"literal\" mapping")); + } + } + let value = value.unwrap_or_else(|| LitStr::new(&variant.ident.unraw().to_string(), variant.ident.span())); + if value.value().len() > 8192 { + return Err(syn::Error::new_spanned( + value, + "environment literal exceeds 8192 UTF-8 bytes", + )); + } + if !unique.insert(value.value()) { + return Err(syn::Error::new_spanned( + value, + "environment variants must map to distinct strings", + )); + } + variants.push(&variant.ident); + values.push(value); + } + let constraint = match values.as_slice() { + [value] => quote!(::spacetimedb::spacetimedb_lib::environment::EnvironmentConstraint::Literal(#value.into())), + values => quote!( + ::spacetimedb::spacetimedb_lib::environment::EnvironmentConstraint::OneOf(::std::vec![#(#values.into()),*]) + ), + }; + let ident = &item.ident; + Ok(quote! { + impl ::spacetimedb::rt::EnvironmentValue for #ident { + const OPTIONAL: bool = false; + + fn constraint() -> ::spacetimedb::spacetimedb_lib::environment::EnvironmentConstraint { + #constraint + } + + fn from_environment(value: ::std::option::Option<::std::string::String>, key: &str) -> Self { + match value.as_deref() { + #(::std::option::Option::Some(#values) => Self::#variants,)* + ::std::option::Option::Some(_) => ::core::panic!("environment value does not match its declared enum: {}", key), + ::std::option::Option::None => ::core::panic!("required environment key is missing: {}", key), + } + } + } + impl ::spacetimedb::rt::RequiredEnvironmentValue for #ident {} + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn rejects_invalid_enum_shapes_mappings_and_limits() { + for input in [ + quote!( + struct Value; + ), + quote!( + enum Value {} + ), + quote!( + enum Value { + Item(T), + } + ), + quote!( + enum Value { + Item(String), + } + ), + quote!( + enum Value { + Item { field: String }, + } + ), + quote!( + enum Value { + #[env()] + Item, + } + ), + quote!( + enum Value { + #[env(values("x"))] + Item, + } + ), + quote!( + enum Value { + #[env(value = "x", value = "y")] + Item, + } + ), + quote!( + enum Value { + #[env(value = "x")] + #[env(value = "y")] + Item, + } + ), + quote!( + enum Value { + #[env(value = "Same")] + First, + Same, + } + ), + quote!( + #[env(value = "x")] + enum Value { + Item, + } + ), + ] { + assert!(derive(syn::parse2(input).unwrap()).is_err()); + } + let oversized = LitStr::new(&"é".repeat(4097), proc_macro2::Span::call_site()); + assert!(derive(syn::parse_quote!( + enum Value { + #[env(value = #oversized)] + Item, + } + )) + .is_err()); + let variants = (0..257).map(|n| quote::format_ident!("V{n}")); + assert!(derive(syn::parse_quote!(enum Value { #(#variants),* })).is_err()); + } + + #[test] + fn accepts_exact_strings_and_ordinary_enum_attributes() { + let output = derive(syn::parse_quote! { + #[derive(Debug, PartialEq)] + enum Value { + #[env(value = "in progress")] InProgress, + #[env(value = "")] Empty, + #[env(value = "héllo\0世界")] Unicode, + r#type, + } + }) + .unwrap(); + syn::parse2::(output).unwrap(); + } +} diff --git a/crates/bindings-macro/src/lib.rs b/crates/bindings-macro/src/lib.rs index 02aaacc4028..37069ccbf2c 100644 --- a/crates/bindings-macro/src/lib.rs +++ b/crates/bindings-macro/src/lib.rs @@ -15,6 +15,11 @@ pub fn env(args: StdTokenStream, item: StdTokenStream) -> StdTokenStream { ok_or_compile_error(|| environment::expand(args.into(), syn::parse(item)?)) } +#[proc_macro_derive(EnvironmentValue, attributes(env))] +pub fn derive_environment_value(item: StdTokenStream) -> StdTokenStream { + ok_or_compile_error(|| environment::value::derive(syn::parse(item)?)) +} + mod http; mod procedure; diff --git a/crates/bindings/src/lib.rs b/crates/bindings/src/lib.rs index 04be557fdf2..190065c5e1f 100644 --- a/crates/bindings/src/lib.rs +++ b/crates/bindings/src/lib.rs @@ -921,28 +921,52 @@ pub use query_builder::{Query, RawQuery}; /// Declare the complete publish-time environment schema and generate named accessors. /// -/// Fields must resolve to `String` or `Option`, including type aliases; -/// `#[env(values("a", "b"))]` -/// constrains exact strings. Values are supplied on every publish, never in metadata. -/// The macro generates an `EnvAccess` extension trait for a struct named `Env`. -/// Import that trait when the declaration lives in a different Rust module. +/// Fields may be `String`, enums deriving [`EnvironmentValue`], or `Option` of +/// either, including type aliases. Values are supplied on every publish, never +/// in metadata. The macro generates an `EnvAccess` extension trait for a struct +/// named `Env`; import that trait if the declaration lives in another module. /// The name `get` is reserved for generic checked access. /// /// ```no_run +/// #[derive(spacetimedb::EnvironmentValue)] +/// pub enum LogLevel { +/// #[env(value = "debug")] +/// Debug, +/// #[env(value = "info")] +/// Info, +/// } /// #[spacetimedb::env] /// pub struct Env { /// pub API_KEY: String, -/// #[env(values("debug", "info"))] -/// pub LOG_LEVEL: Option, +/// pub LOG_LEVEL: Option, /// } /// fn read(ctx: &spacetimedb::ReducerContext) { /// let _: String = ctx.env.API_KEY(); -/// let _: Option = ctx.env.LOG_LEVEL(); +/// let _: Option = ctx.env.LOG_LEVEL(); /// } /// ``` +/// +/// Existing `#[env(values("a", "b"))]` field constraints remain supported for +/// `String` and `Option`; enum constraints come from their variants. #[doc(inline)] pub use spacetimedb_bindings_macro::env; +/// Derive a typed environment value from an enum with unit variants. +/// +/// Each variant accepts its exact Rust name by default. Use +/// `#[env(value = "in progress")]` to map a variant to an arbitrary string, +/// including spaces, capitalization, Unicode or the empty string. Mappings must +/// be distinct, with 1 to 256 variants and at most 8192 UTF-8 bytes per string. +/// Generic enums and variants with payloads are not supported. +/// +/// The schema contains only allowed strings. A named environment accessor returns +/// this enum, or `Option` for an optional field; generic `env.get` still +/// returns `Option`. Missing required values and unmapped strings panic +/// with the key name only. Other derives and the enum's ordinary serialization +/// are unaffected. +#[doc(inline)] +pub use spacetimedb_bindings_macro::EnvironmentValue; + /// Read-only access to this database's environment store. /// /// Reads use the current transaction. In a procedure outside a transaction, diff --git a/crates/bindings/src/rt.rs b/crates/bindings/src/rt.rs index 486ba90ba62..4ebb26e0b63 100644 --- a/crates/bindings/src/rt.rs +++ b/crates/bindings/src/rt.rs @@ -917,42 +917,82 @@ pub fn register_case_conversion_policy(policy: CaseConversionPolicy) { }) } -mod environment_value_sealed { - pub trait Sealed {} - - impl Sealed for String {} - impl Sealed for Option {} -} - -/// The compiler resolves declaration types, including aliases, before selecting -/// their metadata and accessor. Sealing keeps the accepted types identical to -/// the host's string and optional-string environment model. +/// Implementation support for `#[env]` and `#[derive(EnvironmentValue)]`. +/// +/// The compiler resolves aliases before selecting metadata and accessors. Custom +/// implementations must keep their declared constraint and decoding in agreement. +/// The host independently validates every published string against that constraint. #[doc(hidden)] -#[diagnostic::on_unimplemented(message = "environment fields must be `String` or `Option`")] -pub trait EnvironmentValue: environment_value_sealed::Sealed + Sized { +#[diagnostic::on_unimplemented( + message = "environment fields must be `String`, an enum deriving `EnvironmentValue`, or an `Option` of either" +)] +pub trait EnvironmentValue: Sized { const OPTIONAL: bool; - fn get(environment: &crate::Environment, key: &str) -> Self; + fn constraint() -> spacetimedb_lib::environment::EnvironmentConstraint; + + /// Decode a checked host result. Errors must identify only the key, never its value. + fn from_environment(value: Option, key: &str) -> Self; + + fn get(environment: &crate::Environment, key: &str) -> Self { + Self::from_environment(environment.get(key), key) + } } +/// Required environment types supported by the optional-value implementation. +/// Derived enums and `String` implement this; `Option` deliberately does not. +#[doc(hidden)] +pub trait RequiredEnvironmentValue: EnvironmentValue {} + impl EnvironmentValue for String { const OPTIONAL: bool = false; - fn get(environment: &crate::Environment, key: &str) -> Self { - environment - .get(key) - .unwrap_or_else(|| panic!("required environment key is missing: {key}")) + fn constraint() -> spacetimedb_lib::environment::EnvironmentConstraint { + spacetimedb_lib::environment::EnvironmentConstraint::AnyString + } + + fn from_environment(value: Option, key: &str) -> Self { + value.unwrap_or_else(|| panic!("required environment key is missing: {key}")) } } -impl EnvironmentValue for Option { +impl RequiredEnvironmentValue for String {} + +impl EnvironmentValue for Option { const OPTIONAL: bool = true; - fn get(environment: &crate::Environment, key: &str) -> Self { - environment.get(key) + fn constraint() -> spacetimedb_lib::environment::EnvironmentConstraint { + T::constraint() + } + + fn from_environment(value: Option, key: &str) -> Self { + value.map(|value| T::from_environment(Some(value), key)) } } +mod string_environment_value_sealed { + pub trait Sealed {} + + impl Sealed for String {} + impl Sealed for Option {} +} + +/// Legacy field-level string constraints cannot override a typed enum's mapping. +#[doc(hidden)] +#[diagnostic::on_unimplemented( + message = "`#[env(values(...))]` requires `String` or `Option`; map enum variants with `#[env(value = \"...\")]` instead" +)] +pub trait StringEnvironmentValue: EnvironmentValue + string_environment_value_sealed::Sealed { + fn with_constraint( + constraint: spacetimedb_lib::environment::EnvironmentConstraint, + ) -> spacetimedb_lib::environment::EnvironmentConstraint { + constraint + } +} + +impl StringEnvironmentValue for String {} +impl StringEnvironmentValue for Option {} + /// Register declarative ENV metadata without reading any environment values. #[doc(hidden)] pub fn register_environment(declarations: fn() -> Vec) { diff --git a/crates/bindings/tests/environment.rs b/crates/bindings/tests/environment.rs index 6636b9160ef..b2b5181bc9d 100644 --- a/crates/bindings/tests/environment.rs +++ b/crates/bindings/tests/environment.rs @@ -3,4 +3,5 @@ fn environment_declaration_accessors_compile_with_exact_types() { let tests = trybuild::TestCases::new(); tests.pass("tests/pass/environment.rs"); tests.compile_fail("tests/ui/environment_types.rs"); + tests.compile_fail("tests/ui/environment_enum.rs"); } diff --git a/crates/bindings/tests/environment_enum_values.rs b/crates/bindings/tests/environment_enum_values.rs new file mode 100644 index 00000000000..ceecb647b5f --- /dev/null +++ b/crates/bindings/tests/environment_enum_values.rs @@ -0,0 +1,78 @@ +use spacetimedb::rt::EnvironmentValue as _; +use spacetimedb::spacetimedb_lib::environment::{EnvironmentConstraint, EnvironmentDeclaration, EnvironmentSchema}; +use std::collections::BTreeMap; + +#[derive(Debug, PartialEq, Eq, spacetimedb::SpacetimeType, spacetimedb::EnvironmentValue)] +enum Mode { + #[env(value = "in progress")] + InProgress, + Ready, + #[env(value = "")] + Empty, + #[env(value = "héllo\0世界")] + Unicode, +} + +#[derive(Debug, PartialEq, Eq, spacetimedb::EnvironmentValue)] +enum Literal { + #[env(value = "only")] + Only, +} + +#[test] +fn typed_mappings_match_exact_schema_strings_and_optional_absence() { + let cases = [ + ("in progress", Mode::InProgress), + ("Ready", Mode::Ready), + ("", Mode::Empty), + ("héllo\0世界", Mode::Unicode), + ]; + assert_eq!( + Mode::constraint(), + EnvironmentConstraint::OneOf(cases.iter().map(|(s, _)| s.to_string()).collect()) + ); + assert_eq!(Option::::constraint(), Mode::constraint()); + let schema = EnvironmentSchema::new(vec![EnvironmentDeclaration { + name: "MODE".into(), + constraint: Mode::constraint(), + optional: false, + }]) + .unwrap(); + for (value, variant) in cases { + schema + .validate_values(&BTreeMap::from([("MODE".into(), value.into())])) + .unwrap(); + assert_eq!(Mode::from_environment(Some(value.into()), "MODE"), variant); + assert_eq!( + Option::::from_environment(Some(value.into()), "MODE"), + Some(variant) + ); + } + assert_eq!(Option::::from_environment(None, "MODE"), None); + assert_eq!(Literal::constraint(), EnvironmentConstraint::Literal("only".into())); + assert_eq!(Literal::from_environment(Some("only".into()), "VALUE"), Literal::Only); + for rejected in ["InProgress", "ready", "in progress ", "private-unmapped-value"] { + assert!(schema + .validate_values(&BTreeMap::from([("MODE".into(), rejected.into())])) + .is_err()); + } +} + +#[test] +fn decode_errors_report_the_key_without_the_supplied_value() { + for value in [None, Some("private-unmapped-value".into())] { + let error = std::panic::catch_unwind(|| Mode::from_environment(value, "MODE")).unwrap_err(); + let message = error.downcast_ref::().unwrap(); + assert!(message.contains("MODE")); + assert!(!message.contains("private-unmapped-value")); + assert!(!message.contains("in progress")); + } +} + +#[test] +fn environment_mapping_does_not_change_ordinary_enum_serialization() { + use spacetimedb::spacetimedb_lib::bsatn; + assert_eq!(bsatn::to_vec(&Mode::InProgress).unwrap(), vec![0]); + assert_eq!(bsatn::to_vec(&Mode::Ready).unwrap(), vec![1]); + assert_eq!(bsatn::from_slice::(&[2]).unwrap(), Mode::Empty); +} diff --git a/crates/bindings/tests/pass/environment.rs b/crates/bindings/tests/pass/environment.rs index c0f80526ea4..136d631dc89 100644 --- a/crates/bindings/tests/pass/environment.rs +++ b/crates/bindings/tests/pass/environment.rs @@ -6,12 +6,27 @@ use std::string::String as RenamedString; type RequiredAlias = RenamedString; type OptionalAlias = Maybe; +#[derive(Debug, PartialEq, spacetimedb::EnvironmentValue)] +pub enum Mode { + #[env(value = "in progress")] + InProgress, + #[env(value = "Ready")] + Ready, +} +type ModeAlias = Mode; +type MaybeMode = Maybe; + +const _: [(); 0] = [(); ::OPTIONAL as usize]; +const _: [(); 1] = [(); ::OPTIONAL as usize]; + const _: [(); 0] = [(); ::OPTIONAL as usize]; const _: [(); 1] = [(); ::OPTIONAL as usize]; #[spacetimedb::env] pub struct Env { pub REQUIRED: RequiredAlias, + pub MODE: ModeAlias, + pub OPTIONAL_MODE: MaybeMode, #[env(values("false", "true"))] pub FLAG: String, #[env(values(""))] @@ -22,6 +37,8 @@ pub struct Env { fn reads(env: spacetimedb::Environment) { let _: String = env.REQUIRED(); + let _: Mode = env.MODE(); + let _: Option = env.OPTIONAL_MODE(); let _: String = env.FLAG(); let _: Option = env.OPTIONAL(); let _: Option = env.get("get"); diff --git a/crates/bindings/tests/ui/environment_enum.rs b/crates/bindings/tests/ui/environment_enum.rs new file mode 100644 index 00000000000..95a6387f5c4 --- /dev/null +++ b/crates/bindings/tests/ui/environment_enum.rs @@ -0,0 +1,54 @@ +#[derive(spacetimedb::EnvironmentValue)] +struct NotEnum; + +#[derive(spacetimedb::EnvironmentValue)] +enum Empty {} + +#[derive(spacetimedb::EnvironmentValue)] +enum Generic { + Value(T), +} + +#[derive(spacetimedb::EnvironmentValue)] +enum Payload { + Value(String), +} + +#[derive(spacetimedb::EnvironmentValue)] +enum Duplicate { + #[env(value = "Same")] + First, + Same, +} + +#[derive(spacetimedb::EnvironmentValue)] +enum DuplicateAttribute { + #[env(value = "x", value = "y")] + Value, +} + +#[derive(spacetimedb::EnvironmentValue)] +enum WrongAttribute { + #[env(values("x"))] + Value, +} + +#[derive(spacetimedb::EnvironmentValue)] +enum WrongLiteral { + #[env(value = 1)] + Value, +} + +#[derive(spacetimedb::EnvironmentValue)] +pub enum Mode { + Ready, +} + +#[spacetimedb::env] +pub struct Env { + #[env(values("other"))] + pub MODE: Mode, + pub NESTED: Option>, +} + +fn main() {} diff --git a/crates/bindings/tests/ui/environment_enum.stderr b/crates/bindings/tests/ui/environment_enum.stderr new file mode 100644 index 00000000000..f3f3d25ddc0 --- /dev/null +++ b/crates/bindings/tests/ui/environment_enum.stderr @@ -0,0 +1,85 @@ +error: EnvironmentValue requires an enum with unit variants + --> tests/ui/environment_enum.rs:2:8 + | +2 | struct NotEnum; + | ^^^^^^^ + +error: environment value enums require 1 to 256 variants + --> tests/ui/environment_enum.rs:5:6 + | +5 | enum Empty {} + | ^^^^^ + +error: environment value enums cannot be generic + --> tests/ui/environment_enum.rs:8:13 + | +8 | enum Generic { + | ^^^ + +error: environment value variants cannot have payloads + --> tests/ui/environment_enum.rs:14:10 + | +14 | Value(String), + | ^^^^^^^^ + +error: environment variants must map to distinct strings + --> tests/ui/environment_enum.rs:21:5 + | +21 | Same, + | ^^^^ + +error: expected one value = "literal" mapping + --> tests/ui/environment_enum.rs:26:24 + | +26 | #[env(value = "x", value = "y")] + | ^^^^^ + +error: expected one value = "literal" mapping + --> tests/ui/environment_enum.rs:32:11 + | +32 | #[env(values("x"))] + | ^^^^^^ + +error: expected string literal + --> tests/ui/environment_enum.rs:38:19 + | +38 | #[env(value = 1)] + | ^ + +error[E0277]: the trait bound `Option: RequiredEnvironmentValue` is not satisfied + --> tests/ui/environment_enum.rs:51:17 + | +51 | pub NESTED: Option>, + | ^^^^^^^^^^^^^^^^^^^^ the trait `RequiredEnvironmentValue` is not implemented for `Option` + | +help: the following other types implement trait `RequiredEnvironmentValue` + --> tests/ui/environment_enum.rs:42:10 + | +42 | #[derive(spacetimedb::EnvironmentValue)] + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ `Mode` + | + ::: src/rt.rs + | + | impl RequiredEnvironmentValue for String {} + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ `std::string::String` + = note: required for `Option>` to implement `EnvironmentValue` + = note: this error originates in the derive macro `spacetimedb::EnvironmentValue` (in Nightly builds, run with -Z macro-backtrace for more info) + +error[E0277]: `#[env(values(...))]` requires `String` or `Option`; map enum variants with `#[env(value = "...")]` instead + --> tests/ui/environment_enum.rs:50:15 + | +50 | pub MODE: Mode, + | ^^^^ unsatisfied trait bound + | +help: the trait `StringEnvironmentValue` is not implemented for `Mode` + --> tests/ui/environment_enum.rs:43:1 + | +43 | pub enum Mode { + | ^^^^^^^^^^^^^ +help: the following other types implement trait `StringEnvironmentValue` + --> src/rt.rs + | + | impl StringEnvironmentValue for String {} + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ `std::string::String` + | impl StringEnvironmentValue for Option {} + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ `Option` diff --git a/crates/bindings/tests/ui/environment_types.rs b/crates/bindings/tests/ui/environment_types.rs index 4fa3e3aae0b..27693688e4f 100644 --- a/crates/bindings/tests/ui/environment_types.rs +++ b/crates/bindings/tests/ui/environment_types.rs @@ -24,15 +24,4 @@ pub struct Unsupported { pub get: u32, } -struct Custom; - -// External code cannot extend the set of supported environment types. -impl spacetimedb::rt::EnvironmentValue for Custom { - const OPTIONAL: bool = false; - - fn get(_: &spacetimedb::Environment, _: &str) -> Self { - Self - } -} - fn main() {} diff --git a/crates/bindings/tests/ui/environment_types.stderr b/crates/bindings/tests/ui/environment_types.stderr index 3c7346dcb4e..f884a635897 100644 --- a/crates/bindings/tests/ui/environment_types.stderr +++ b/crates/bindings/tests/ui/environment_types.stderr @@ -1,57 +1,24 @@ -error[E0277]: the trait bound `Custom: spacetimedb::rt::environment_value_sealed::Sealed` is not satisfied - --> tests/ui/environment_types.rs:30:44 - | -30 | impl spacetimedb::rt::EnvironmentValue for Custom { - | ^^^^^^ unsatisfied trait bound - | -help: the trait `spacetimedb::rt::environment_value_sealed::Sealed` is not implemented for `Custom` - --> tests/ui/environment_types.rs:27:1 - | -27 | struct Custom; - | ^^^^^^^^^^^^^ -help: the following other types implement trait `spacetimedb::rt::environment_value_sealed::Sealed` - --> src/rt.rs - | - | impl Sealed for String {} - | ^^^^^^^^^^^^^^^^^^^^^^ `std::string::String` - | impl Sealed for Option {} - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ `std::option::Option` -note: required by a bound in `EnvironmentValue` - --> src/rt.rs - | - | pub trait EnvironmentValue: environment_value_sealed::Sealed + Sized { - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ required by this bound in `EnvironmentValue` - = note: `EnvironmentValue` is a "sealed trait", because to implement it you also need to implement `spacetimedb::rt::environment_value_sealed::Sealed`, which is not accessible; this is usually done to force you to use one of the provided types that already implement it - = help: the following types implement the trait: - std::string::String - std::option::Option - -error[E0277]: environment fields must be `String` or `Option` - --> tests/ui/environment_types.rs:6:20 - | - 6 | pub VALUE: String, - | ^^^^^^ unsatisfied trait bound - | +error[E0277]: environment fields must be `String`, an enum deriving `EnvironmentValue`, or an `Option` of either + --> tests/ui/environment_types.rs:6:20 + | +6 | pub VALUE: String, + | ^^^^^^ unsatisfied trait bound + | help: the trait `EnvironmentValue` is not implemented for `shadowed_string::String` - --> tests/ui/environment_types.rs:2:5 - | - 2 | pub struct String; - | ^^^^^^^^^^^^^^^^^ + --> tests/ui/environment_types.rs:2:5 + | +2 | pub struct String; + | ^^^^^^^^^^^^^^^^^ help: the following other types implement trait `EnvironmentValue` - --> tests/ui/environment_types.rs:30:1 - | -30 | impl spacetimedb::rt::EnvironmentValue for Custom { - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ `Custom` - | - ::: src/rt.rs - | - | impl EnvironmentValue for String { - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ `std::string::String` + --> src/rt.rs + | + | impl EnvironmentValue for String { + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ `std::string::String` ... - | impl EnvironmentValue for Option { - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ `std::option::Option` + | impl EnvironmentValue for Option { + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ `std::option::Option` -error[E0277]: environment fields must be `String` or `Option` +error[E0277]: environment fields must be `String`, an enum deriving `EnvironmentValue`, or an `Option` of either --> tests/ui/environment_types.rs:15:20 | 15 | pub VALUE: Option, @@ -63,67 +30,53 @@ help: the trait `EnvironmentValue` is not implemented for `shadowed_option::Opti 11 | pub struct Option(T); | ^^^^^^^^^^^^^^^^^^^^ help: the following other types implement trait `EnvironmentValue` - --> tests/ui/environment_types.rs:30:1 - | -30 | impl spacetimedb::rt::EnvironmentValue for Custom { - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ `Custom` - | - ::: src/rt.rs + --> src/rt.rs | | impl EnvironmentValue for String { | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ `std::string::String` ... - | impl EnvironmentValue for Option { - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ `std::option::Option` + | impl EnvironmentValue for Option { + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ `std::option::Option` -error[E0277]: environment fields must be `String` or `Option` +error[E0277]: environment fields must be `String`, an enum deriving `EnvironmentValue`, or an `Option` of either --> tests/ui/environment_types.rs:21:15 | 21 | pub BOOL: bool, | ^^^^ the trait `EnvironmentValue` is not implemented for `bool` | help: the following other types implement trait `EnvironmentValue` - --> tests/ui/environment_types.rs:30:1 - | -30 | impl spacetimedb::rt::EnvironmentValue for Custom { - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ `Custom` - | - ::: src/rt.rs + --> src/rt.rs | | impl EnvironmentValue for String { | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ `std::string::String` ... - | impl EnvironmentValue for Option { - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ `std::option::Option` + | impl EnvironmentValue for Option { + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ `std::option::Option` -error[E0277]: environment fields must be `String` or `Option` +error[E0277]: the trait bound `std::option::Option: RequiredEnvironmentValue` is not satisfied --> tests/ui/environment_types.rs:22:17 | 22 | pub NESTED: Option>, - | ^^^^^^^^^^^^^^^^^^^^^^ the trait `EnvironmentValue` is not implemented for `std::option::Option>` + | ^^^^^^^^^^^^^^^^^^^^^^ the trait `RequiredEnvironmentValue` is not implemented for `std::option::Option` | -help: the trait `EnvironmentValue` is implemented for `std::option::Option` +help: the trait `RequiredEnvironmentValue` is implemented for `std::string::String` --> src/rt.rs | - | impl EnvironmentValue for Option { + | impl RequiredEnvironmentValue for String {} | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + = note: required for `std::option::Option>` to implement `EnvironmentValue` -error[E0277]: environment fields must be `String` or `Option` +error[E0277]: environment fields must be `String`, an enum deriving `EnvironmentValue`, or an `Option` of either --> tests/ui/environment_types.rs:24:14 | 24 | pub get: u32, | ^^^ the trait `EnvironmentValue` is not implemented for `u32` | help: the following other types implement trait `EnvironmentValue` - --> tests/ui/environment_types.rs:30:1 - | -30 | impl spacetimedb::rt::EnvironmentValue for Custom { - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ `Custom` - | - ::: src/rt.rs + --> src/rt.rs | | impl EnvironmentValue for String { | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ `std::string::String` ... - | impl EnvironmentValue for Option { - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ `std::option::Option` + | impl EnvironmentValue for Option { + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ `std::option::Option` diff --git a/crates/cli/src/subcommands/env.rs b/crates/cli/src/subcommands/env.rs index 20bc11a5b9b..01a1f25afe9 100644 --- a/crates/cli/src/subcommands/env.rs +++ b/crates/cli/src/subcommands/env.rs @@ -41,7 +41,7 @@ pub fn cli() -> Command { ), )) .subcommand(target( - Command::new("list").about("List published environment keys (never values)"), + Command::new("list").about("List published environment keys and values"), )) } @@ -53,7 +53,7 @@ enum Query { impl Query { fn sql(&self) -> anyhow::Result { match self { - Self::List => Ok("SELECT key FROM st_env".into()), + Self::List => Ok("SELECT key, value FROM st_env".into()), Self::Get(key) => { // POSIX names cannot contain quotes or SQL syntax. validate_key(key).map_err(|_| anyhow::anyhow!("Invalid environment key name"))?; @@ -117,41 +117,40 @@ async fn fetch(request: reqwest::RequestBuilder, query: Query) -> anyhow::Result } fn render(body: &[u8], query: &Query) -> anyhow::Result { - // Only project the requested single string column; do not dump an error or - // unexpected response which could contain unrequested secret values. + // Validate the requested projection before rendering any response values. let results: Vec>> = serde_json::from_slice(body).map_err(|_| anyhow::anyhow!("Invalid environment read response"))?; ensure!(results.len() == 1, "Invalid environment read result count"); let result = &results[0]; - let expected = match query { - Query::List => "key", - Query::Get(_) => "value", + let expected: &[&str] = match query { + Query::List => &["key", "value"], + Query::Get(_) => &["value"], }; ensure!( - result.schema.elements.len() == 1 - && result.schema.elements[0].name.as_deref() == Some(expected) - && result.schema.elements[0].algebraic_type == spacetimedb_lib::AlgebraicType::String, + result.schema.elements.len() == expected.len() + && result.schema.elements.iter().zip(expected).all(|(column, name)| { + column.name.as_deref() == Some(*name) && column.algebraic_type == spacetimedb_lib::AlgebraicType::String + }), "Invalid environment read projection" ); - let mut values = Vec::new(); + ensure!(result.rows.len() <= MAX_ENV_VARS, "Environment key count exceeds limit"); for row in &result.rows { - ensure!(row.len() == 1, "Invalid environment read row"); + ensure!(row.len() == expected.len(), "Invalid environment read row"); if matches!(query, Query::List) { validate_key(&row[0]).context("Invalid environment key in response")?; } ensure!( - row[0].len() <= MAX_ENV_VALUE_BYTES, + row.last().unwrap().len() <= MAX_ENV_VALUE_BYTES, "Environment read value exceeds limit" ); - values.push(row[0].as_str()); } match query { Query::List => { - ensure!(values.len() <= MAX_ENV_VARS, "Environment key count exceeds limit"); - values.sort_unstable(); - let rows = values - .into_iter() - .map(|key| Ok::<_, std::convert::Infallible>(spacetimedb_lib::sats::product![key])); + let mut rows: Vec<_> = result.rows.iter().collect(); + rows.sort_unstable_by(|a, b| a[0].cmp(&b[0])); + let rows = rows.into_iter().map(|row| { + Ok::<_, std::convert::Infallible>(spacetimedb_lib::sats::product![row[0].as_str(), row[1].as_str()]) + }); let table = sql::build_table( spacetimedb_lib::sats::satn::PsqlClient::SpacetimeDB, &result.schema, @@ -160,8 +159,8 @@ fn render(body: &[u8], query: &Query) -> anyhow::Result { Ok(format!("{table}\n")) } Query::Get(_) => { - ensure!(values.len() == 1, "Environment key is absent"); - Ok(format!("{}\n", values[0])) + ensure!(result.rows.len() == 1, "Environment key is absent"); + Ok(format!("{}\n", result.rows[0][0])) } } } @@ -169,9 +168,13 @@ fn render(body: &[u8], query: &Query) -> anyhow::Result { #[cfg(test)] mod tests { use super::*; - fn body(column: &'static str, rows: Vec>) -> Vec { + fn body(columns: &[&'static str], rows: Vec>) -> Vec { serde_json::to_vec(&[spacetimedb_client_api_messages::http::SqlStmtResult { - schema: spacetimedb_lib::sats::ProductType::from([(column, spacetimedb_lib::AlgebraicType::String)]), + schema: spacetimedb_lib::sats::ProductType::from_iter( + columns + .iter() + .map(|column| (*column, spacetimedb_lib::AlgebraicType::String)), + ), rows, total_duration_micros: 0, stats: Default::default(), @@ -197,7 +200,7 @@ mod tests { let get = matches.subcommand_matches("get").unwrap(); assert_eq!(get.get_one::("database").unwrap(), "db"); assert_eq!(get.get_one::("key").unwrap(), "KEY"); - assert_eq!(Query::List.sql().unwrap(), "SELECT key FROM st_env"); + assert_eq!(Query::List.sql().unwrap(), "SELECT key, value FROM st_env"); assert_eq!( Query::Get("KEY".into()).sql().unwrap(), "SELECT value FROM st_env WHERE key = 'KEY'" @@ -205,22 +208,27 @@ mod tests { assert!(Query::Get("x';DELETE FROM st_env;--".into()).sql().is_err()); } #[test] - fn list_projects_keys_and_rejects_unexpected_secret_columns() { + fn list_renders_sorted_keys_and_values_and_rejects_unexpected_columns() { assert_eq!( - render(&body("key", vec![vec!["Z"], vec!["A"]]), &Query::List) - .unwrap() - .lines() - .map(str::trim) - .collect::>(), - ["key", "-----", "\"A\"", "\"Z\""] + render( + &body(&["key", "value"], vec![vec!["Z", "last"], vec!["A", "first"]]), + &Query::List + ) + .unwrap() + .lines() + .enumerate() + .filter(|(index, _)| *index != 1) + .map(|(_, line)| line.split('|').map(str::trim).collect::>()) + .collect::>(), + [["key", "value"], ["\"A\"", "\"first\""], ["\"Z\"", "\"last\""]] ); - let err = render(&body("value", vec![vec!["generated-secret-sentinel"]]), &Query::List).unwrap_err(); + let err = render(&body(&["value"], vec![vec!["generated-secret-sentinel"]]), &Query::List).unwrap_err(); assert!(!format!("{err:#}").contains("generated-secret-sentinel")); assert_eq!( - render(&body("value", vec![vec![""]]), &Query::Get("A".into())).unwrap(), + render(&body(&["value"], vec![vec![""]]), &Query::Get("A".into())).unwrap(), "\n" ); - assert!(render(&body("value", vec![]), &Query::Get("A".into())).is_err()); + assert!(render(&body(&["value"], vec![]), &Query::Get("A".into())).is_err()); } #[tokio::test] async fn actual_loopback_queries_and_error_redaction() { @@ -229,13 +237,13 @@ mod tests { ( Query::List, "200 OK", - body("key", vec![vec!["KEY"]]), - Some(" key \n-------\n \"KEY\" \n"), + body(&["key", "value"], vec![vec!["KEY", "generated-list-sentinel"]]), + Some("generated-list-sentinel"), ), ( Query::Get("KEY".into()), "200 OK", - body("value", vec![vec!["generated-read-sentinel"]]), + body(&["value"], vec![vec!["generated-read-sentinel"]]), Some("generated-read-sentinel\n"), ), ( @@ -290,8 +298,17 @@ mod tests { .redirect(reqwest::redirect::Policy::none()) .build() .unwrap(); - let result = fetch(client.post(format!("http://{address}/v1/database/owned/sql")), query).await; + let result = fetch( + client.post(format!("http://{address}/v1/database/owned/sql")), + query.clone(), + ) + .await; match expected { + Some(expected) if matches!(query, Query::List) => { + let output = result.unwrap(); + assert!(output.contains("key") && output.contains("value")); + assert!(output.contains("KEY") && output.contains(expected)); + } Some(expected) => assert_eq!(result.unwrap(), expected), None => assert!(!format!("{:#}", result.unwrap_err()).contains("generated-error-secret")), } diff --git a/crates/smoketests/tests/standalone/cli/environment.rs b/crates/smoketests/tests/standalone/cli/environment.rs index 2ba96d8adb9..f044c7a0a11 100644 --- a/crates/smoketests/tests/standalone/cli/environment.rs +++ b/crates/smoketests/tests/standalone/cli/environment.rs @@ -186,10 +186,19 @@ impl Fixture { &[], ); let mut lines = output.lines().map(str::trim); - assert_eq!(lines.next(), Some("key")); + assert_eq!( + lines.next().unwrap().split('|').map(str::trim).collect::>(), + ["key", "value"] + ); lines - .filter(|line| !line.is_empty() && !line.chars().all(|c| c == '-')) - .map(|line| serde_json::from_str::(line).unwrap()) + .filter(|line| !line.is_empty() && !line.chars().all(|c| matches!(c, '-' | '+'))) + .map(|line| { + let (key, value) = line.split_once('|').unwrap(); + let key: String = serde_json::from_str(key.trim()).unwrap(); + let value: String = serde_json::from_str(value.trim()).unwrap(); + assert_eq!(self.get(&key), format!("{value}\n")); + key + }) .collect() } diff --git a/crates/testing/tests/environment.rs b/crates/testing/tests/environment.rs index 96a29b29bf9..5d01ca55ab5 100644 --- a/crates/testing/tests/environment.rs +++ b/crates/testing/tests/environment.rs @@ -483,3 +483,79 @@ fn suspended_procedure_cannot_read_environment_from_a_replacement_program() { }, ); } + +// The actual host keeps ENV as exact strings; generated Rust accessors decode +// the selected enum variant after initial publish and complete replacements. +#[test] +#[serial] +fn rust_environment_enums_preserve_exact_typed_mappings() { + let initial = Values::from([ + ("REQUIRED".into(), "initial-required".into()), + ("MODE".into(), "ready".into()), + ]); + CompiledModule::compile("environment-test", CompilationMode::Debug).with_module_async_with_environment( + DEFAULT_CONFIG, + initial.clone(), + |handle| async move { + let mut values = initial; + for (value, index) in [ + ("ready", 0u8), + ("other", 1), + ("in progress", 2), + ("Ready", 3), + ("", 4), + ("héllo\0世界", 5), + ] { + values.insert("MODE".into(), value.into()); + values.insert("TYPED".into(), value.into()); + let module = publish(&handle, &values).await; + module + .call_reducer( + Identity::ZERO, + None, + None, + None, + None, + "expect_typed_environment", + FunctionArgs::Bsatn(bsatn::to_vec(&product![index, Some(index)]).unwrap().into()), + ) + .await + .unwrap() + .outcome + .into_result() + .unwrap(); + assert_eq!( + read(&module, "MODE").await, + AlgebraicValue::from(Some(value.to_string())) + ); + } + for rejected in ["InProgress", "READY", "in progress "] { + let mut invalid = values.clone(); + invalid.insert("TYPED".into(), rejected.into()); + let result = handle.republish_environment(invalid).await; + assert!(result.as_ref().is_err() || !result.as_ref().unwrap().was_successful()); + assert_eq!( + read(&handle.client.module(), "TYPED").await, + AlgebraicValue::from(Some("héllo\0世界".to_string())) + ); + } + values.remove("TYPED"); + let module = publish(&handle, &values).await; + module + .call_reducer( + Identity::ZERO, + None, + None, + None, + None, + "expect_typed_environment", + FunctionArgs::Bsatn(bsatn::to_vec(&product![5u8, None::]).unwrap().into()), + ) + .await + .unwrap() + .outcome + .into_result() + .unwrap(); + }, + ); +} diff --git a/docs/docs/00200-core-concepts/00100-databases/00700-environment-variables.md b/docs/docs/00200-core-concepts/00100-databases/00700-environment-variables.md index 34161f9095a..dfcf6d245a1 100644 --- a/docs/docs/00200-core-concepts/00100-databases/00700-environment-variables.md +++ b/docs/docs/00200-core-concepts/00100-databases/00700-environment-variables.md @@ -56,16 +56,30 @@ Within an environment declaration, a simple enum specifies allowed strings. Enum -Add one environment declaration to the module: +Declare allowed strings with enums, then add one environment declaration to the module: ```rust +#[derive(spacetimedb::EnvironmentValue)] +pub enum Mode { + #[env(value = "development")] + Development, + #[env(value = "production")] + Production, +} + +#[derive(spacetimedb::EnvironmentValue)] +pub enum LogLevel { + #[env(value = "info")] + Info, + #[env(value = "debug")] + Debug, +} + #[spacetimedb::env] pub struct Env { pub API_KEY: String, - #[env(values("development", "production"))] - pub MODE: String, - #[env(values("info", "debug"))] - pub LOG_LEVEL: Option, + pub MODE: Mode, + pub LOG_LEVEL: Option, } ``` @@ -73,12 +87,14 @@ Inside a reducer, procedure, or view, read values from its context: ```rust let api_key: String = ctx.env.API_KEY(); -let mode: String = ctx.env.MODE(); -let log_level: Option = ctx.env.LOG_LEVEL(); +let mode: Mode = ctx.env.MODE(); +let log_level: Option = ctx.env.LOG_LEVEL(); let checked: Option = ctx.env.get("LOG_LEVEL"); ``` -`String` requires a value, and `Option` permits absence. `#[env(values(...))]` restricts the allowed strings. Supplying one string makes it an exact-value constraint. +`String` accepts any string. An enum restricts values to its variants, and `Option` permits absence. Type aliases work for these types. Without an attribute, a variant accepts its exact Rust name. An explicit mapping such as `#[env(value = "in progress")] InProgress` supports spaces, capitalization, Unicode, or the empty string. Mappings must be unique; variants cannot have payloads. A one-variant enum declares a single allowed string. These mappings affect environment reads and declarations only, not the enum's ordinary serialization. + +Existing `#[env(values(...))]` constraints on `String` and `Option` fields remain supported. Use enum variants to constrain typed enum fields. @@ -198,6 +214,8 @@ Previously stored values are **not** defaults for the next publish. Every publis The same rules apply to precompiled modules published with `--bin-path`. The CLI reads declarations from the artifact being published. +For publishing from an HTTP client or a module procedure, see the [HTTP publish format and example](../../00300-resources/00200-reference/00200-http-api/00300-database.md#publishing-with-environment-values). Supply the module and complete environment in the request body; project configuration and shell overrides are CLI conveniences. + ## Inspect published values The database owner and collaborators with private-table read access can inspect the environment. For the local database above: @@ -207,7 +225,7 @@ spacetime env list env-example --server http://127.0.0.1:3000 spacetime env get env-example MODE --server http://127.0.0.1:3000 ``` -`env list` prints keys only. `env get` prints the requested value and fails if it is absent. Reading a secret with `env get` therefore exposes that secret in the command's output. +`env list` prints a table of keys and values. `env get` prints the requested value and fails if it is absent. Both are explicit inspection commands and include secrets in their output. Automatic publish output still omits values. Values are stored in the private system table `st_env`. Authorized SQL reads are also supported: diff --git a/docs/docs/00300-resources/00200-reference/00100-cli-reference/00100-cli-reference.md b/docs/docs/00300-resources/00200-reference/00100-cli-reference/00100-cli-reference.md index 7c9ee00f3f7..c820d565bb6 100644 --- a/docs/docs/00300-resources/00200-reference/00100-cli-reference/00100-cli-reference.md +++ b/docs/docs/00300-resources/00200-reference/00100-cli-reference/00100-cli-reference.md @@ -150,7 +150,7 @@ Inspect published database environment variables ###### **Subcommands:** * `get` — Read one published environment value -* `list` — List published environment keys (never values) +* `list` — List published environment keys and values @@ -180,7 +180,7 @@ Read one published environment value ## `spacetime env list` -List published environment keys (never values) +List published environment keys and values **Usage:** `spacetime env list [OPTIONS] ` diff --git a/modules/environment-test/src/lib.rs b/modules/environment-test/src/lib.rs index 8d6efa37a11..99846b25301 100644 --- a/modules/environment-test/src/lib.rs +++ b/modules/environment-test/src/lib.rs @@ -6,11 +6,29 @@ static VIEW_TRAP_ENTERED: AtomicBool = AtomicBool::new(false); type RequiredString = String; type OptionalString = Option; +#[derive(Debug, PartialEq, spacetimedb::EnvironmentValue)] +pub enum Mode { + #[env(value = "ready")] + Ready, + #[env(value = "other")] + Other, + #[env(value = "in progress")] + InProgress, + #[env(value = "Ready")] + Capitalized, + #[env(value = "")] + Empty, + #[env(value = "héllo\0世界")] + Unicode, +} + +type OptionalMode = Option; + #[spacetimedb::env] pub struct Env { pub REQUIRED: RequiredString, - #[env(values("ready", "other"))] - pub MODE: String, + pub MODE: Mode, + pub TYPED: OptionalMode, pub MISSING: OptionalString, pub EMPTY: Option, pub UTF8: Option, @@ -24,7 +42,25 @@ pub struct Env { #[spacetimedb::reducer(init)] pub fn init(ctx: &ReducerContext) { assert_eq!(ctx.env.REQUIRED(), "initial-required"); - assert_eq!(ctx.env.MODE(), "ready"); + assert_eq!(ctx.env.MODE(), Mode::Ready); + assert_eq!(ctx.env.TYPED(), None); +} + +#[spacetimedb::reducer] +pub fn expect_typed_environment(ctx: &ReducerContext, required: u8, optional: Option) { + fn index(mode: Mode) -> u8 { + match mode { + Mode::Ready => 0, + Mode::Other => 1, + Mode::InProgress => 2, + Mode::Capitalized => 3, + Mode::Empty => 4, + Mode::Unicode => 5, + } + } + assert_eq!(index(ctx.env.MODE()), required); + assert_eq!(ctx.env.TYPED().map(index), optional); + assert_eq!(index(ctx.as_read_only().env.MODE()), required); } #[spacetimedb::reducer] From 2bd321b989c21d96a111cdbfd50ff73d6463be39 Mon Sep 17 00:00:00 2001 From: Tyler Cloutier Date: Wed, 9 Sep 2026 00:46:22 -0400 Subject: [PATCH 13/34] Fix ENV test CLI inspection and isolated root discovery --- .../tests/standalone/cli/environment.rs | 4 ++ crates/testing/src/lib.rs | 49 +++++++++++++++++++ 2 files changed, 53 insertions(+) diff --git a/crates/smoketests/tests/standalone/cli/environment.rs b/crates/smoketests/tests/standalone/cli/environment.rs index f044c7a0a11..1dde0899f72 100644 --- a/crates/smoketests/tests/standalone/cli/environment.rs +++ b/crates/smoketests/tests/standalone/cli/environment.rs @@ -88,6 +88,10 @@ impl Fixture { .env("NO_PROXY", "*") .env("no_proxy", "*") .envs(shell.iter().copied()) + // Avoid platform directory discovery after clearing the environment, + // including Windows' known-folder lookup for LocalAppData. + .arg("--root-dir") + .arg(self.test.project_dir.path()) .arg("--config-path") .arg(&self.test.config_path) .args(args) diff --git a/crates/testing/src/lib.rs b/crates/testing/src/lib.rs index 13cb9872b96..63799d77e72 100644 --- a/crates/testing/src/lib.rs +++ b/crates/testing/src/lib.rs @@ -49,6 +49,55 @@ pub fn invoke_cli(paths: &SpacetimePaths, args: &[&str]) { } // If CUSTOM_SPACETIMEDB_PATH is missing, fall through to the default behavior. + if cmd == "publish" { + // Publishing inspects exact module bytes in a child process. This + // function is linked into a libtest under target/*/deps, so current_exe + // cannot locate the standalone companion of the actual runtime CLI. + // Use the same explicit build artifacts as the test server guard. + let cli = spacetimedb_guard::ensure_binaries_built(); + let inspector = cli + .with_file_name("spacetimedb-standalone") + .with_extension(std::env::consts::EXE_EXTENSION); + assert!( + inspector.is_file(), + "SDK tests require the standalone schema inspector beside the CLI: {}", + inspector.display() + ); + assert!( + sub_args.get_one::("server").is_some(), + "SDK publication must use its explicit test server" + ); + let root = paths + .to_root_dir() + .expect("SDK tests require an isolated root directory"); + let status = RUNTIME.block_on(async { + let mut child = tokio::process::Command::new(cli) + .arg("--root-dir") + .arg(root) + .arg("--config-path") + .arg(paths.cli_config_dir.cli_toml()) + .args(args) + .arg("--no-config") + .env("SPACETIMEDB_SCHEMA_EXTRACTOR", inspector) + .stdin(std::process::Stdio::null()) + .kill_on_drop(true) + .spawn() + .expect("Failed to start the pre-built publish CLI"); + // Output streams directly to the test log, without accumulating a + // second buffer. The inspector retains its own tighter bounds. + match tokio::time::timeout(std::time::Duration::from_secs(120), child.wait()).await { + Ok(status) => status.expect("Failed to reap the pre-built publish CLI"), + Err(_) => { + let _ = child.start_kill(); + child.wait().await.expect("Failed to reap timed-out publish CLI"); + panic!("SDK module publication timed out"); + } + } + }); + assert!(status.success(), "SDK module publication failed"); + return; + } + // Default: run in-process CLI (fast/path-friendly for tests). let config = Config::new_with_localhost(paths.cli_config_dir.cli_toml()); RUNTIME From 93f89e9099258f65d8a165bf49f53c49aa75d8a8 Mon Sep 17 00:00:00 2001 From: Tyler Cloutier Date: Wed, 9 Sep 2026 01:00:37 -0400 Subject: [PATCH 14/34] Add Rust module-test environment example and runtime coverage --- .../snapshots/codegen__codegen_csharp.snap | 155 +++++++++++++++++ .../snapshots/codegen__codegen_rust.snap | 156 ++++++++++++++++++ .../codegen__codegen_typescript.snap | 44 +++++ crates/testing/tests/environment.rs | 6 + modules/module-test/src/environment.rs | 26 +++ modules/module-test/src/lib.rs | 2 + 6 files changed, 389 insertions(+) create mode 100644 modules/module-test/src/environment.rs diff --git a/crates/codegen/tests/snapshots/codegen__codegen_csharp.snap b/crates/codegen/tests/snapshots/codegen__codegen_csharp.snap index ab899e7f663..21fe014ee8d 100644 --- a/crates/codegen/tests/snapshots/codegen__codegen_csharp.snap +++ b/crates/codegen/tests/snapshots/codegen__codegen_csharp.snap @@ -69,6 +69,84 @@ namespace SpacetimeDB } } ''' +"Procedures/ReadEnvironment.g.cs" = ''' +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +#nullable enable + +using System; +using SpacetimeDB.ClientApi; +using System.Collections.Generic; +using System.Runtime.Serialization; + +namespace SpacetimeDB +{ + public sealed partial class RemoteProcedures : RemoteBase + { + public void ReadEnvironment(string key, ProcedureCallback callback) + { + // Convert the clean callback to the wrapper callback + InternalReadEnvironment(key, (ctx, result) => { + if (result.IsSuccess && result.Value != null) + { + callback(ctx, ProcedureCallbackResult.Success(result.Value.Value)); + } + else + { + callback(ctx, ProcedureCallbackResult.Failure(result.Error!)); + } + }); + } + + private void InternalReadEnvironment(string key, ProcedureCallback callback) + { + conn.InternalCallProcedure(new Procedure.ReadEnvironmentArgs(key), callback); + } + + } + + public abstract partial class Procedure + { + [SpacetimeDB.Type] + [DataContract] + public sealed partial class ReadEnvironment + { + [DataMember(Name = "Value")] + public string? Value; + + public ReadEnvironment(string? Value) + { + this.Value = Value; + } + + public ReadEnvironment() + { + } + } + [SpacetimeDB.Type] + [DataContract] + public sealed partial class ReadEnvironmentArgs : Procedure, IProcedureArgs + { + [DataMember(Name = "key")] + public string Key; + + public ReadEnvironmentArgs(string Key) + { + this.Key = Key; + } + + public ReadEnvironmentArgs() + { + this.Key = ""; + } + + string IProcedureArgs.ProcedureName => "read_environment"; + } + + } +} +''' "Procedures/ReturnValue.g.cs" = ''' // THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE // WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. @@ -683,6 +761,82 @@ namespace SpacetimeDB } } ''' +"Reducers/ExpectEnvironment.g.cs" = ''' +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +#nullable enable + +using System; +using SpacetimeDB.ClientApi; +using System.Collections.Generic; +using System.Runtime.Serialization; + +namespace SpacetimeDB +{ + public sealed partial class RemoteReducers : RemoteBase + { + public delegate void ExpectEnvironmentHandler(ReducerEventContext ctx, string key, string? expected); + public event ExpectEnvironmentHandler? OnExpectEnvironment; + + public void ExpectEnvironment(string key, string? expected) + { + conn.InternalCallReducer(new Reducer.ExpectEnvironment(key, expected)); + } + + public bool InvokeExpectEnvironment(ReducerEventContext ctx, Reducer.ExpectEnvironment args) + { + if (OnExpectEnvironment == null) + { + if (InternalOnUnhandledReducerError != null) + { + switch(ctx.Event.Status) + { + case Status.Failed(var reason): InternalOnUnhandledReducerError(ctx, new Exception(reason)); break; + case Status.OutOfEnergy(var _): InternalOnUnhandledReducerError(ctx, new Exception("out of energy")); break; + } + } + return false; + } + OnExpectEnvironment( + ctx, + args.Key, + args.Expected + ); + return true; + } + } + + public abstract partial class Reducer + { + [SpacetimeDB.Type] + [DataContract] + public sealed partial class ExpectEnvironment : Reducer, IReducerArgs + { + [DataMember(Name = "key")] + public string Key; + [DataMember(Name = "expected")] + public string? Expected; + + public ExpectEnvironment( + string Key, + string? Expected + ) + { + this.Key = Key; + this.Expected = Expected; + } + + public ExpectEnvironment() + { + this.Key = ""; + } + + string IReducerArgs.ReducerName => "expect_environment"; + } + } +} +''' "Reducers/ListOverAge.g.cs" = ''' // THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE // WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. @@ -1681,6 +1835,7 @@ namespace SpacetimeDB Reducer.AssertCallerIdentityIsModuleIdentity args => Reducers.InvokeAssertCallerIdentityIsModuleIdentity(eventContext, args), Reducer.DeletePlayer args => Reducers.InvokeDeletePlayer(eventContext, args), Reducer.DeletePlayersByName args => Reducers.InvokeDeletePlayersByName(eventContext, args), + Reducer.ExpectEnvironment args => Reducers.InvokeExpectEnvironment(eventContext, args), Reducer.ListOverAge args => Reducers.InvokeListOverAge(eventContext, args), Reducer.LogModuleIdentity args => Reducers.InvokeLogModuleIdentity(eventContext, args), Reducer.QueryPrivate args => Reducers.InvokeQueryPrivate(eventContext, args), diff --git a/crates/codegen/tests/snapshots/codegen__codegen_rust.snap b/crates/codegen/tests/snapshots/codegen__codegen_rust.snap index 1cda8c8d75b..af115ee46b5 100644 --- a/crates/codegen/tests/snapshots/codegen__codegen_rust.snap +++ b/crates/codegen/tests/snapshots/codegen__codegen_rust.snap @@ -488,6 +488,88 @@ impl delete_players_by_name for super::RemoteReducers { } } +''' +"expect_environment_reducer.rs" = ''' +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +#![allow(unused, clippy::all)] +use spacetimedb_sdk::__codegen::{ + self as __sdk, + __lib, + __sats, + __ws, +}; + + +#[derive(__lib::ser::Serialize, __lib::de::Deserialize, Clone, PartialEq, Debug)] +#[sats(crate = __lib)] +pub(super) struct ExpectEnvironmentArgs { + pub key: String, + pub expected: Option::, +} + +impl From for super::Reducer { + fn from(args: ExpectEnvironmentArgs) -> Self { + Self::ExpectEnvironment { + key: args.key, + expected: args.expected, +} +} +} + +impl __sdk::InModule for ExpectEnvironmentArgs { + type Module = super::RemoteModule; +} + +#[allow(non_camel_case_types)] +/// Extension trait for access to the reducer `expect_environment`. +/// +/// Implemented for [`super::RemoteReducers`]. +pub trait expect_environment { + /// Request that the remote module invoke the reducer `expect_environment` to run as soon as possible. + /// + /// This method returns immediately, and errors only if we are unable to send the request. + /// The reducer will run asynchronously in the future, + /// and this method provides no way to listen for its completion status. + /// /// Use [`expect_environment:expect_environment_then`] to run a callback after the reducer completes. + fn expect_environment(&self, key: String, +expected: Option::, +) -> __sdk::Result<()> { + self.expect_environment_then(key, expected, |_, _| {}) + } + + /// Request that the remote module invoke the reducer `expect_environment` to run as soon as possible, + /// registering `callback` to run when we are notified that the reducer completed. + /// + /// This method returns immediately, and errors only if we are unable to send the request. + /// The reducer will run asynchronously in the future, + /// and its status can be observed with the `callback`. + fn expect_environment_then( + &self, + key: String, +expected: Option::, + + callback: impl FnOnce(&super::ReducerEventContext, Result, __sdk::InternalError>) + + Send + + 'static, + ) -> __sdk::Result<()>; +} + +impl expect_environment for super::RemoteReducers { + fn expect_environment_then( + &self, + key: String, +expected: Option::, + + callback: impl FnOnce(&super::ReducerEventContext, Result, __sdk::InternalError>) + + Send + + 'static, + ) -> __sdk::Result<()> { + self.imp.invoke_reducer_with_callback(ExpectEnvironmentArgs { key, expected, }, callback) + } +} + ''' "foobar_type.rs" = ''' // THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE @@ -1116,6 +1198,7 @@ pub mod add_private_reducer; pub mod assert_caller_identity_is_module_identity_reducer; pub mod delete_player_reducer; pub mod delete_players_by_name_reducer; +pub mod expect_environment_reducer; pub mod list_over_age_reducer; pub mod log_module_identity_reducer; pub mod query_private_reducer; @@ -1129,6 +1212,7 @@ pub mod test_d_table; pub mod test_f_table; pub mod my_player_table; pub mod get_my_schema_via_http_procedure; +pub mod read_environment_procedure; pub mod return_value_procedure; pub mod sleep_one_second_procedure; pub mod with_tx_procedure; @@ -1163,6 +1247,7 @@ pub use add_private_reducer::add_private; pub use assert_caller_identity_is_module_identity_reducer::assert_caller_identity_is_module_identity; pub use delete_player_reducer::delete_player; pub use delete_players_by_name_reducer::delete_players_by_name; +pub use expect_environment_reducer::expect_environment; pub use list_over_age_reducer::list_over_age; pub use log_module_identity_reducer::log_module_identity; pub use query_private_reducer::query_private; @@ -1170,6 +1255,7 @@ pub use say_hello_reducer::say_hello; pub use test_reducer::test; pub use test_btree_index_args_reducer::test_btree_index_args; pub use get_my_schema_via_http_procedure::get_my_schema_via_http; +pub use read_environment_procedure::read_environment; pub use return_value_procedure::return_value; pub use sleep_one_second_procedure::sleep_one_second; pub use with_tx_procedure::with_tx; @@ -1198,6 +1284,10 @@ pub enum Reducer { } , DeletePlayersByName { name: String, +} , + ExpectEnvironment { + key: String, + expected: Option::, } , ListOverAge { age: u8, @@ -1228,6 +1318,7 @@ impl __sdk::Reducer for Reducer { Reducer::AssertCallerIdentityIsModuleIdentity => "assert_caller_identity_is_module_identity", Reducer::DeletePlayer { .. } => "delete_player", Reducer::DeletePlayersByName { .. } => "delete_players_by_name", + Reducer::ExpectEnvironment { .. } => "expect_environment", Reducer::ListOverAge { .. } => "list_over_age", Reducer::LogModuleIdentity => "log_module_identity", Reducer::QueryPrivate => "query_private", @@ -1268,6 +1359,13 @@ Reducer::DeletePlayer{ name, } => __sats::bsatn::to_vec(&delete_players_by_name_reducer::DeletePlayersByNameArgs { name: name.clone(), +}), + Reducer::ExpectEnvironment{ + key, + expected, +} => __sats::bsatn::to_vec(&expect_environment_reducer::ExpectEnvironmentArgs { + key: key.clone(), + expected: expected.clone(), }), Reducer::ListOverAge{ age, @@ -3296,6 +3394,64 @@ impl query_private for super::RemoteReducers { } } +''' +"read_environment_procedure.rs" = ''' +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +#![allow(unused, clippy::all)] +use spacetimedb_sdk::__codegen::{ + self as __sdk, + __lib, + __sats, + __ws, +}; + + +#[derive(__lib::ser::Serialize, __lib::de::Deserialize, Clone, PartialEq, Debug)] +#[sats(crate = __lib)] + struct ReadEnvironmentArgs { + pub key: String, +} + + +impl __sdk::InModule for ReadEnvironmentArgs { + type Module = super::RemoteModule; +} + +#[allow(non_camel_case_types)] +/// Extension trait for access to the procedure `read_environment`. +/// +/// Implemented for [`super::RemoteProcedures`]. +pub trait read_environment { + fn read_environment(&self, key: String, +) { + self.read_environment_then(key, |_, _| {}); + } + + fn read_environment_then( + &self, + key: String, + + __callback: impl FnOnce(&super::ProcedureEventContext, Result, __sdk::InternalError>) + Send + 'static, + ); +} + +impl read_environment for super::RemoteProcedures { + fn read_environment_then( + &self, + key: String, + + __callback: impl FnOnce(&super::ProcedureEventContext, Result, __sdk::InternalError>) + Send + 'static, + ) { + self.imp.invoke_procedure_with_callback::<_, Option::>( + "read_environment", + ReadEnvironmentArgs { key, }, + __callback, + ); + } +} + ''' "remove_table_type.rs" = ''' // THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE diff --git a/crates/codegen/tests/snapshots/codegen__codegen_typescript.snap b/crates/codegen/tests/snapshots/codegen__codegen_typescript.snap index 9ca1e9926a5..addaeab2185 100644 --- a/crates/codegen/tests/snapshots/codegen__codegen_typescript.snap +++ b/crates/codegen/tests/snapshots/codegen__codegen_typescript.snap @@ -103,6 +103,24 @@ export default { name: __t.string(), }; ''' +"expect_environment_reducer.ts" = ''' +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +/* eslint-disable */ +/* tslint:disable */ +import { + TypeBuilder as __TypeBuilder, + t as __t, + type AlgebraicTypeType as __AlgebraicTypeType, + type Infer as __Infer, +} from "spacetimedb"; + +export default { + key: __t.string(), + expected: __t.option(__t.string()), +}; +''' "get_my_schema_via_http_procedure.ts" = ''' // THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE // WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. @@ -162,6 +180,7 @@ import AddPrivateReducer from "./add_private_reducer"; import AssertCallerIdentityIsModuleIdentityReducer from "./assert_caller_identity_is_module_identity_reducer"; import DeletePlayerReducer from "./delete_player_reducer"; import DeletePlayersByNameReducer from "./delete_players_by_name_reducer"; +import ExpectEnvironmentReducer from "./expect_environment_reducer"; import ListOverAgeReducer from "./list_over_age_reducer"; import LogModuleIdentityReducer from "./log_module_identity_reducer"; import QueryPrivateReducer from "./query_private_reducer"; @@ -171,6 +190,7 @@ import TestBtreeIndexArgsReducer from "./test_btree_index_args_reducer"; // Import all procedure arg schemas import * as GetMySchemaViaHttpProcedure from "./get_my_schema_via_http_procedure"; +import * as ReadEnvironmentProcedure from "./read_environment_procedure"; import * as ReturnValueProcedure from "./return_value_procedure"; import * as SleepOneSecondProcedure from "./sleep_one_second_procedure"; import * as WithTxProcedure from "./with_tx_procedure"; @@ -270,6 +290,7 @@ const reducersSchema = __reducers( __reducerSchema("assert_caller_identity_is_module_identity", AssertCallerIdentityIsModuleIdentityReducer), __reducerSchema("delete_player", DeletePlayerReducer), __reducerSchema("delete_players_by_name", DeletePlayersByNameReducer), + __reducerSchema("expect_environment", ExpectEnvironmentReducer), __reducerSchema("list_over_age", ListOverAgeReducer), __reducerSchema("log_module_identity", LogModuleIdentityReducer), __reducerSchema("query_private", QueryPrivateReducer), @@ -281,6 +302,7 @@ const reducersSchema = __reducers( /** The schema information for all procedures in this module. This is defined the same way as the procedures would have been defined in the server. */ const proceduresSchema = __procedures( __procedureSchema("get_my_schema_via_http", GetMySchemaViaHttpProcedure.params, GetMySchemaViaHttpProcedure.returnType), + __procedureSchema("read_environment", ReadEnvironmentProcedure.params, ReadEnvironmentProcedure.returnType), __procedureSchema("return_value", ReturnValueProcedure.params, ReturnValueProcedure.returnType), __procedureSchema("sleep_one_second", SleepOneSecondProcedure.params, SleepOneSecondProcedure.returnType), __procedureSchema("with_tx", WithTxProcedure.params, WithTxProcedure.returnType), @@ -531,6 +553,23 @@ import { export default {}; ''' +"read_environment_procedure.ts" = ''' +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +/* eslint-disable */ +/* tslint:disable */ +import { + TypeBuilder as __TypeBuilder, + t as __t, + type AlgebraicTypeType as __AlgebraicTypeType, + type Infer as __Infer, +} from "spacetimedb"; + +export const params = { + key: __t.string(), +}; +export const returnType = __t.option(__t.string())''' "return_value_procedure.ts" = ''' // THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE // WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. @@ -827,12 +866,15 @@ import { type Infer as __Infer } from "spacetimedb"; // Import all procedure arg schemas import * as GetMySchemaViaHttpProcedure from "../get_my_schema_via_http_procedure"; +import * as ReadEnvironmentProcedure from "../read_environment_procedure"; import * as ReturnValueProcedure from "../return_value_procedure"; import * as SleepOneSecondProcedure from "../sleep_one_second_procedure"; import * as WithTxProcedure from "../with_tx_procedure"; export type GetMySchemaViaHttpArgs = __Infer; export type GetMySchemaViaHttpResult = __Infer; +export type ReadEnvironmentArgs = __Infer; +export type ReadEnvironmentResult = __Infer; export type ReturnValueArgs = __Infer; export type ReturnValueResult = __Infer; export type SleepOneSecondArgs = __Infer; @@ -856,6 +898,7 @@ import AddPrivateReducer from "../add_private_reducer"; import AssertCallerIdentityIsModuleIdentityReducer from "../assert_caller_identity_is_module_identity_reducer"; import DeletePlayerReducer from "../delete_player_reducer"; import DeletePlayersByNameReducer from "../delete_players_by_name_reducer"; +import ExpectEnvironmentReducer from "../expect_environment_reducer"; import ListOverAgeReducer from "../list_over_age_reducer"; import LogModuleIdentityReducer from "../log_module_identity_reducer"; import QueryPrivateReducer from "../query_private_reducer"; @@ -869,6 +912,7 @@ export type AddPrivateParams = __Infer; export type AssertCallerIdentityIsModuleIdentityParams = __Infer; export type DeletePlayerParams = __Infer; export type DeletePlayersByNameParams = __Infer; +export type ExpectEnvironmentParams = __Infer; export type ListOverAgeParams = __Infer; export type LogModuleIdentityParams = __Infer; export type QueryPrivateParams = __Infer; diff --git a/crates/testing/tests/environment.rs b/crates/testing/tests/environment.rs index 5d01ca55ab5..838b26e43b5 100644 --- a/crates/testing/tests/environment.rs +++ b/crates/testing/tests/environment.rs @@ -363,6 +363,12 @@ fn rust_environment_publish_is_atomic_and_reads_follow_declared_configuration() exercise_fixture("environment-test"); } +#[test] +#[serial] +fn rust_module_test_environment_publish_and_checked_reads() { + exercise_fixture("module-test"); +} + #[test] #[serial] fn typescript_environment_publish_and_checked_reads() { diff --git a/modules/module-test/src/environment.rs b/modules/module-test/src/environment.rs new file mode 100644 index 00000000000..6adce646498 --- /dev/null +++ b/modules/module-test/src/environment.rs @@ -0,0 +1,26 @@ +//! Optional declarations keep this general-purpose test module publishable +//! without configuration, just like the C#, C++, and TypeScript examples. + +use spacetimedb::{ProcedureContext, ReducerContext}; + +#[spacetimedb::env] +pub struct Env { + pub MISSING: Option, + pub EMPTY: Option, + pub UTF8: Option, + pub NUL: Option, + pub MAXIMUM: Option, +} + +#[spacetimedb::reducer] +pub fn expect_environment(ctx: &ReducerContext, key: String, expected: Option) { + assert_eq!(ctx.env.EMPTY(), ctx.env.get("EMPTY")); + assert_eq!(ctx.env.get(&key), expected); +} + +#[spacetimedb::procedure] +pub fn read_environment(ctx: &mut ProcedureContext, key: String) -> Option { + let outside = ctx.env.get(&key); + ctx.with_tx(|tx| assert_eq!(tx.env.get(&key), outside)); + outside +} diff --git a/modules/module-test/src/lib.rs b/modules/module-test/src/lib.rs index fc1851b21b0..9a2d6790d53 100644 --- a/modules/module-test/src/lib.rs +++ b/modules/module-test/src/lib.rs @@ -10,6 +10,8 @@ use spacetimedb::{ }; use spacetimedb::{log, ProcedureContext}; +pub mod environment; + pub type TestAlias = TestA; // ───────────────────────────────────────────────────────────────────────────── From 60992adb61f75daa5d62001f52f59dc3a280463b Mon Sep 17 00:00:00 2001 From: Tyler Cloutier Date: Wed, 9 Sep 2026 01:00:38 -0400 Subject: [PATCH 15/34] Apply ENV system table and test naming review cleanup --- crates/cli/src/spacetime_config/environment.rs | 2 ++ .../src/locking_tx_datastore/committed_state.rs | 9 ++++----- crates/datastore/src/locking_tx_datastore/datastore.rs | 4 ++-- crates/datastore/src/system_tables.rs | 3 ++- crates/datastore/src/system_tables/environment.rs | 3 --- crates/smoketests/tests/standalone/cli/environment.rs | 10 +++++----- 6 files changed, 15 insertions(+), 16 deletions(-) diff --git a/crates/cli/src/spacetime_config/environment.rs b/crates/cli/src/spacetime_config/environment.rs index c4493835376..4fea7a548c7 100644 --- a/crates/cli/src/spacetime_config/environment.rs +++ b/crates/cli/src/spacetime_config/environment.rs @@ -1,4 +1,6 @@ //! Preserve JSON numeric values before the JSON5 deserializer can round them. +//! For example, converting `9007199254740993` through `f64` yields +//! `9007199254740992`, changing an environment value before it reaches the module. //! This also permits existing comments, unquoted names and trailing commas. use serde_json::Value; diff --git a/crates/datastore/src/locking_tx_datastore/committed_state.rs b/crates/datastore/src/locking_tx_datastore/committed_state.rs index 9169a96b45c..3f61903e645 100644 --- a/crates/datastore/src/locking_tx_datastore/committed_state.rs +++ b/crates/datastore/src/locking_tx_datastore/committed_state.rs @@ -29,9 +29,9 @@ use crate::{ locking_tx_datastore::ViewCallInfo, system_tables::{ ST_COLUMN_ACCESSOR_ID, ST_COLUMN_ACCESSOR_IDX, ST_CONNECTION_CREDENTIALS_ID, ST_CONNECTION_CREDENTIALS_IDX, - ST_EVENT_TABLE_ID, ST_EVENT_TABLE_IDX, ST_INDEX_ACCESSOR_ID, ST_INDEX_ACCESSOR_IDX, ST_TABLE_ACCESSOR_ID, - ST_TABLE_ACCESSOR_IDX, ST_VIEW_COLUMN_ID, ST_VIEW_COLUMN_IDX, ST_VIEW_ID, ST_VIEW_IDX, ST_VIEW_PARAM_ID, - ST_VIEW_PARAM_IDX, ST_VIEW_SUB_ID, ST_VIEW_SUB_IDX, + ST_ENV_ID, ST_ENV_IDX, ST_EVENT_TABLE_ID, ST_EVENT_TABLE_IDX, ST_INDEX_ACCESSOR_ID, ST_INDEX_ACCESSOR_IDX, + ST_TABLE_ACCESSOR_ID, ST_TABLE_ACCESSOR_IDX, ST_VIEW_COLUMN_ID, ST_VIEW_COLUMN_IDX, ST_VIEW_ID, ST_VIEW_IDX, + ST_VIEW_PARAM_ID, ST_VIEW_PARAM_IDX, ST_VIEW_SUB_ID, ST_VIEW_SUB_IDX, }, }; use anyhow::anyhow; @@ -357,8 +357,7 @@ impl CommittedState { self.create_table(ST_TABLE_ACCESSOR_ID, schemas[ST_TABLE_ACCESSOR_IDX].clone()); self.create_table(ST_INDEX_ACCESSOR_ID, schemas[ST_INDEX_ACCESSOR_IDX].clone()); self.create_table(ST_COLUMN_ACCESSOR_ID, schemas[ST_COLUMN_ACCESSOR_IDX].clone()); - let env = crate::system_tables::st_env_schema(); - self.create_table(env.table_id, env.into()); + self.create_table(ST_ENV_ID, schemas[ST_ENV_IDX].clone()); // Insert the sequences into `st_sequences` let (st_sequences, blob_store, pool) = diff --git a/crates/datastore/src/locking_tx_datastore/datastore.rs b/crates/datastore/src/locking_tx_datastore/datastore.rs index cf5fd662c63..1082e0ae9c6 100644 --- a/crates/datastore/src/locking_tx_datastore/datastore.rs +++ b/crates/datastore/src/locking_tx_datastore/datastore.rs @@ -1079,7 +1079,6 @@ pub(crate) mod tests { use super::*; use crate::error::IndexError; use crate::locking_tx_datastore::tx_state::PendingSchemaChange; - use crate::system_tables::ST_ENV_ID; use crate::system_tables::{ system_tables, StColumnRow, StConnectionCredentialsFields, StConstraintData, StConstraintFields, StConstraintRow, StEventTableFields, StIndexAlgorithm, StIndexFields, StIndexRow, StRowLevelSecurityFields, @@ -1093,6 +1092,7 @@ pub(crate) mod tests { ST_VIEW_ARG_NAME, ST_VIEW_COLUMN_ID, ST_VIEW_COLUMN_NAME, ST_VIEW_ID, ST_VIEW_NAME, ST_VIEW_PARAM_ID, ST_VIEW_PARAM_NAME, ST_VIEW_SUB_ID, ST_VIEW_SUB_NAME, }; + use crate::system_tables::{ST_ENV_ID, ST_ENV_NAME}; use crate::traits::{IsolationLevel, MutTx}; use crate::Result; use core::{fmt, mem}; @@ -1559,7 +1559,7 @@ pub(crate) mod tests { TableRow { id: ST_TABLE_ACCESSOR_ID.into(), name: ST_TABLE_ACCESSOR_NAME, ty: StTableType::System, access: StAccess::Public, primary_key: None }, TableRow { id: ST_INDEX_ACCESSOR_ID.into(), name: ST_INDEX_ACCESSOR_NAME, ty: StTableType::System, access: StAccess::Public, primary_key: None }, TableRow { id: ST_COLUMN_ACCESSOR_ID.into(), name: ST_COLUMN_ACCESSOR_NAME, ty: StTableType::System, access: StAccess::Public, primary_key: None }, - TableRow { id: ST_ENV_ID.into(), name: "st_env", ty: StTableType::System, access: StAccess::Private, primary_key: Some(ColId(0)) }, + TableRow { id: ST_ENV_ID.into(), name: ST_ENV_NAME, ty: StTableType::System, access: StAccess::Private, primary_key: Some(ColId(0)) }, ])); #[rustfmt::skip] diff --git a/crates/datastore/src/system_tables.rs b/crates/datastore/src/system_tables.rs index d39cd14923c..61dcd11be9b 100644 --- a/crates/datastore/src/system_tables.rs +++ b/crates/datastore/src/system_tables.rs @@ -279,6 +279,7 @@ pub(crate) const ST_EVENT_TABLE_IDX: usize = 16; pub(crate) const ST_TABLE_ACCESSOR_IDX: usize = 17; pub(crate) const ST_INDEX_ACCESSOR_IDX: usize = 18; pub(crate) const ST_COLUMN_ACCESSOR_IDX: usize = 19; +pub(crate) const ST_ENV_IDX: usize = 20; macro_rules! st_fields_enum { ($(#[$attr:meta])* enum $ty_name:ident { $($name:expr, $var:ident = $discr:expr,)* }) => { @@ -700,8 +701,8 @@ fn system_module_def() -> ModuleDef { validate_system_table::(&result, ST_EVENT_TABLE_NAME); validate_system_table::(&result, ST_TABLE_ACCESSOR_NAME); validate_system_table::(&result, ST_INDEX_ACCESSOR_NAME); - environment::validate_table(&result); validate_system_table::(&result, ST_COLUMN_ACCESSOR_NAME); + validate_system_table::(&result, ST_ENV_NAME); result } diff --git a/crates/datastore/src/system_tables/environment.rs b/crates/datastore/src/system_tables/environment.rs index 82395f4bd77..d79f676edb8 100644 --- a/crates/datastore/src/system_tables/environment.rs +++ b/crates/datastore/src/system_tables/environment.rs @@ -30,9 +30,6 @@ pub(super) fn register_table(builder: &mut RawModuleDefV9Builder) { .with_unique_constraint(ColId(0)) .with_index_no_accessor_name(btree(ColId(0))); } -pub(super) fn validate_table(def: &ModuleDef) { - validate_system_table::(def, ST_ENV_NAME); -} pub(crate) fn st_env_schema() -> TableSchema { st_schema(ST_ENV_NAME, ST_ENV_ID) } diff --git a/crates/smoketests/tests/standalone/cli/environment.rs b/crates/smoketests/tests/standalone/cli/environment.rs index 1dde0899f72..957ace39286 100644 --- a/crates/smoketests/tests/standalone/cli/environment.rs +++ b/crates/smoketests/tests/standalone/cli/environment.rs @@ -19,13 +19,13 @@ const KEYS: &[&str] = &[ "SMOKE_FLAG", ]; -struct Fixture { +struct EnvironmentFixture { test: Smoketest, database: String, wasm: PathBuf, } -impl Fixture { +impl EnvironmentFixture { fn new() -> Self { // Private CI supplies remote cluster settings to the same test binary. // This fixture must still create its own server and fresh credentials, @@ -296,7 +296,7 @@ fn bounded_output(mut command: Command) -> Output { #[test] fn cli_environment_layers_shell_and_exact_precompiled_declarations() { - let f = Fixture::new(); + let f = EnvironmentFixture::new(); f.write( "spacetime.json", json!({"database":"unused-parent", "env":{ @@ -362,7 +362,7 @@ fn cli_environment_layers_shell_and_exact_precompiled_declarations() { #[test] fn cli_environment_replacement_rejection_and_read_only_commands() { - let f = Fixture::new(); + let f = EnvironmentFixture::new(); f.config(Some( json!({"SMOKE_REQUIRED":"initial-sentinel","SMOKE_MODE":"ready","SMOKE_OPTIONAL":"remove-me"}), )); @@ -436,7 +436,7 @@ fn cli_environment_replacement_rejection_and_read_only_commands() { #[test] fn cli_environment_initial_rejection_clear_and_omitted_payload() { - let mut f = Fixture::new(); + let mut f = EnvironmentFixture::new(); f.config(None); assert!(!f.publish(&[], &[]).status.success()); f.config(Some(json!({"SMOKE_REQUIRED":"clear-initial","SMOKE_MODE":"ready"}))); From b491ae43867d15ad4e854e5568eef756dc4d31d2 Mon Sep 17 00:00:00 2001 From: Tyler Cloutier Date: Wed, 9 Sep 2026 01:26:59 -0400 Subject: [PATCH 16/34] Simplify ENV schema declaration state and example naming --- crates/schema/src/def.rs | 20 ++++++++++++-------- crates/schema/src/def/validate/v10.rs | 24 +++++++++++------------- crates/schema/src/def/validate/v9.rs | 3 +-- crates/schema/src/error.rs | 4 ++-- modules/module-test-ts/src/index.ts | 6 ++++-- 5 files changed, 30 insertions(+), 27 deletions(-) diff --git a/crates/schema/src/def.rs b/crates/schema/src/def.rs index 2516894ceea..e324d93b0ea 100644 --- a/crates/schema/src/def.rs +++ b/crates/schema/src/def.rs @@ -18,6 +18,7 @@ use std::collections::BTreeMap; use std::fmt::{self, Debug, Write}; use std::hash::Hash; +use std::sync::LazyLock; use crate::error::{IdentifierError, ValidationErrors}; use crate::identifier::{Identifier, NamespacePath, NamespacedIdentifier}; @@ -44,6 +45,7 @@ use spacetimedb_lib::db::raw_def::v9::{ RawUniqueConstraintDataV9, RawViewDefV9, TableAccess, TableType, }; use spacetimedb_lib::db::view::{extract_view_return_product_type_ref, ViewKind}; +use spacetimedb_lib::environment::EnvironmentSchema; use spacetimedb_lib::{ProductType, RawModuleDef}; use spacetimedb_primitives::{ ColId, ColList, ColOrCols, ColSet, HttpHandlerId, ProcedureId, ReducerId, TableId, ViewFnPtr, @@ -180,8 +182,8 @@ pub struct ModuleDef { /// Submodules, keyed by the namespace they are registered under. submodules: IndexMap, - environment: spacetimedb_lib::environment::EnvironmentSchema, - environment_declared: bool, + /// `None` means undeclared; an explicitly empty declaration is `Some(empty)`. + environment: Option, } #[derive(Debug, Clone, Copy, Eq, PartialEq)] @@ -194,13 +196,17 @@ pub enum RawModuleDefVersion { impl ModuleDef { /// The validated root environment schema. Legacy modules have an empty schema. - pub fn environment(&self) -> &spacetimedb_lib::environment::EnvironmentSchema { - &self.environment + pub fn environment(&self) -> &EnvironmentSchema { + static EMPTY: LazyLock = LazyLock::new(EnvironmentSchema::default); + match &self.environment { + Some(schema) => schema, + None => &EMPTY, + } } /// Whether the raw module explicitly required environment support. pub fn environment_declared(&self) -> bool { - self.environment_declared + self.environment.is_some() } /// The raw module definition version this module was authored under. @@ -1012,7 +1018,6 @@ impl From for RawModuleDefV9 { raw_module_def_version: _, submodules: _, environment: _, - environment_declared: _, } = val; // Extract column defaults from tables before consuming tables @@ -1074,11 +1079,10 @@ impl From for RawModuleDefV10 { raw_module_def_version: _, submodules, environment, - environment_declared, } = val; let mut sections = Vec::new(); - if environment_declared { + if let Some(environment) = environment { sections.push(RawModuleDefV10Section::Environment(environment.into_declarations())); } let mut explicit_names = ExplicitNames::default(); diff --git a/crates/schema/src/def/validate/v10.rs b/crates/schema/src/def/validate/v10.rs index f164642a615..084f6a1aac5 100644 --- a/crates/schema/src/def/validate/v10.rs +++ b/crates/schema/src/def/validate/v10.rs @@ -298,13 +298,10 @@ pub fn validate(def: RawModuleDefV10) -> Result { .map(|rls| (rls.sql.clone(), rls.to_owned())) .collect(); - let ( - (tables, types, reducers, procedures, views, (http_handlers, http_routes)), - submodules, - (environment, environment_declared), - ) = (tables_types_reducers_procedures_views, submodules, environment) - .combine_errors() - .map_err(|errors: ValidationErrors| errors.sort_deduplicate())?; + let ((tables, types, reducers, procedures, views, (http_handlers, http_routes)), submodules, environment) = + (tables_types_reducers_procedures_views, submodules, environment) + .combine_errors() + .map_err(|errors: ValidationErrors| errors.sort_deduplicate())?; let typespace_for_generate = typespace_for_generate.finish(); @@ -327,7 +324,6 @@ pub fn validate(def: RawModuleDefV10) -> Result { raw_module_def_version: RawModuleDefVersion::V10, submodules, environment, - environment_declared, }; // Submodules were validated in isolation, so their defs carry root-relative names. @@ -340,20 +336,20 @@ pub fn validate(def: RawModuleDefV10) -> Result { Ok(module_def) } -fn validate_environment(def: &RawModuleDefV10) -> Result<(spacetimedb_lib::environment::EnvironmentSchema, bool)> { +fn validate_environment(def: &RawModuleDefV10) -> Result> { let mut sections = def.sections.iter().filter_map(|section| match section { RawModuleDefV10Section::Environment(declarations) => Some(declarations), _ => None, }); let Some(declarations) = sections.next() else { - return Ok((Default::default(), false)); + return Ok(None); }; if sections.next().is_some() { - return Err(ValidationError::RepeatedEnvironmentSection.into()); + return Err(ValidationError::RepeatedEnvironmentDeclaration.into()); } let schema = spacetimedb_lib::environment::EnvironmentSchema::from_declarations(declarations) .map_err(|error| ValidationError::Environment { error })?; - Ok((schema, true)) + Ok(Some(schema)) } /// Validate that each submodule's namespace is a valid identifier of at most 63 characters, @@ -2844,6 +2840,8 @@ mod environment_tests { let legacy = validate(RawModuleDefV10::default()).unwrap(); assert!(legacy.environment().is_empty()); assert!(!legacy.environment_declared()); + let raw: RawModuleDefV10 = legacy.into(); + assert!(!validate(raw).unwrap().environment_declared()); let explicit = validate(RawModuleDefV10 { sections: vec![RawModuleDefV10Section::Environment(vec![])], }) @@ -2870,7 +2868,7 @@ mod environment_tests { assert!(validate(duplicate) .unwrap_err() .into_iter() - .any(|error| matches!(error, ValidationError::RepeatedEnvironmentSection))); + .any(|error| matches!(error, ValidationError::RepeatedEnvironmentDeclaration))); let nested = RawModuleDefV10 { sections: vec![RawModuleDefV10Section::Submodules(vec![RawSubmoduleV10 { namespace: "outer".into(), diff --git a/crates/schema/src/def/validate/v9.rs b/crates/schema/src/def/validate/v9.rs index 1ba7c03c7b9..c6eef4f74f9 100644 --- a/crates/schema/src/def/validate/v9.rs +++ b/crates/schema/src/def/validate/v9.rs @@ -171,8 +171,7 @@ pub fn validate(def: RawModuleDefV9) -> Result { http_routes: Vec::new(), raw_module_def_version: RawModuleDefVersion::V9OrEarlier, submodules: IndexMap::new(), - environment: Default::default(), - environment_declared: false, + environment: None, }; // Records each def's namespace. V9 has no submodules, so this just resolves everything at diff --git a/crates/schema/src/error.rs b/crates/schema/src/error.rs index 770c44c7a3e..691fef4a3f5 100644 --- a/crates/schema/src/error.rs +++ b/crates/schema/src/error.rs @@ -22,8 +22,8 @@ pub type ValidationErrors = ErrorStream; #[derive(thiserror::Error, Debug, PartialOrd, Ord, PartialEq, Eq)] #[non_exhaustive] pub enum ValidationError { - #[error("module has repeated environment sections")] - RepeatedEnvironmentSection, + #[error("module has repeated environment declarations")] + RepeatedEnvironmentDeclaration, #[error("invalid environment declaration: {error}")] Environment { error: spacetimedb_lib::environment::EnvironmentSchemaError, diff --git a/modules/module-test-ts/src/index.ts b/modules/module-test-ts/src/index.ts index 2d903356b00..000e25be0e4 100644 --- a/modules/module-test-ts/src/index.ts +++ b/modules/module-test-ts/src/index.ts @@ -574,7 +574,8 @@ export const router = spacetimedb.httpRouter( ); // Dedicated environment ABI integration exercised by crates/testing. -export const expect_environment = spacetimedb.reducer( +export const expectEnvironment = spacetimedb.reducer( + { name: 'expect_environment' }, { key: t.string(), expected: t.option(t.string()) }, (ctx, { key, expected }) => { if (libSubmodule.readRootEnvironmentHelper() !== ctx.env.get('EMPTY')) throw new Error('helper environment scope mismatch'); @@ -584,7 +585,8 @@ export const expect_environment = spacetimedb.reducer( } } ); -export const read_environment = spacetimedb.procedure( +export const readEnvironment = spacetimedb.procedure( + { name: 'read_environment' }, { key: t.string() }, t.option(t.string()), (ctx, { key }) => { From b0c498b1346b6c5864b3ff16c4a8f63a767e0aad Mon Sep 17 00:00:00 2001 From: Tyler Cloutier Date: Wed, 9 Sep 2026 01:41:23 -0400 Subject: [PATCH 17/34] Clarify standalone publication locking and ENV recovery invariants --- .../standalone/src/control_db/environment.rs | 14 +++++++++++ crates/standalone/src/lib.rs | 24 ++++++++++++------- 2 files changed, 29 insertions(+), 9 deletions(-) diff --git a/crates/standalone/src/control_db/environment.rs b/crates/standalone/src/control_db/environment.rs index ede423c3935..454dd0e1d70 100644 --- a/crates/standalone/src/control_db/environment.rs +++ b/crates/standalone/src/control_db/environment.rs @@ -1,4 +1,18 @@ //! Private bootstrap inputs, separate from the historical public Database encoding. +//! +//! Creation and reset atomically persist both database indexes, the generation +//! and initial-program binding, the complete ENV input, and the nominated leader. +//! The transaction is flushed before host launch, so a lost request response can +//! recover the same generation through ordinary leader lookup. +//! +//! Reading the input rechecks the persisted owner, program, and generation in +//! the same transaction. A legacy generation with neither metadata nor input has +//! an empty environment; missing input for a recorded generation is an error. +//! +//! The host reads this input only before the database's first initialization. +//! Reopening an initialized database uses its committed program and `st_env`, +//! including later module updates. Reset replaces the bootstrap input, and +//! database deletion removes it atomically with both indexes and the binding. use super::*; use spacetimedb_client_api_messages::publish::PublishRequest; use spacetimedb_lib::Hash; diff --git a/crates/standalone/src/lib.rs b/crates/standalone/src/lib.rs index efe11d860fa..c0ad9638d92 100644 --- a/crates/standalone/src/lib.rs +++ b/crates/standalone/src/lib.rs @@ -188,7 +188,7 @@ impl NodeDelegate for StandaloneEnv { let owner = self.weak_self.upgrade().expect("standalone owner exists during lookup"); tokio::spawn(async move { let _guard = guard; - owner.leader_under_publication(database_id).await + owner.leader_with_publication_lock_held(database_id).await }) .await .map_err(|error| GetLeaderHostError::LaunchError { source: error.into() })? @@ -403,7 +403,11 @@ impl StandaloneEnv { .await? } - async fn leader_under_publication(&self, database_id: u64) -> Result { + /// Look up or start the current leader while the caller holds the publication + /// lock. Retain the read or write guard through this future's completion. + /// Ordinary lookup holds a read guard; publication and reset hold a write + /// guard, so calling `leader()` here would acquire the lock again and deadlock. + async fn leader_with_publication_lock_held(&self, database_id: u64) -> Result { let Some(leader) = self.control_db.get_leader_replica_by_database(database_id) else { return Err(GetLeaderHostError::NoSuchReplica); }; @@ -477,7 +481,7 @@ impl StandaloneEnv { let database_id = database.id; let database_identity = database.database_identity; - let leader = self.leader_under_publication(database_id).await?; + let leader = self.leader_with_publication_lock_held(database_id).await?; let update_result = leader .update_with_environment( database, @@ -574,7 +578,11 @@ impl StandaloneEnv { None => { // A reset without an artifact retains the currently committed // module, not the original bootstrap program or its old values. - let module = self.leader_under_publication(database.id).await?.module().await?; + let module = self + .leader_with_publication_lock_held(database.id) + .await? + .module() + .await?; module .relational_db() .program()? @@ -671,16 +679,14 @@ impl StandaloneEnv { async fn on_insert_replica(&self, instance: &Replica) -> Result<(), anyhow::Error> { if instance.leader { - let database = self - .control_db - .get_database_by_id(instance.database_id)? + self.leader_with_publication_lock_held(instance.database_id) + .await .with_context(|| { format!( - "unknown database: id: {}, instance: {}", + "failed to start leader for database {}, replica {}", instance.database_id, instance.id ) })?; - self.leader_under_publication(database.id).await?; } Ok(()) From 29b778057bc5ad0bca61decbcd4899cc20d61d5c Mon Sep 17 00:00:00 2001 From: Tyler Cloutier Date: Wed, 9 Sep 2026 07:55:21 -0400 Subject: [PATCH 18/34] Format C# ENV and invocation authority imports --- crates/bindings-csharp/Runtime/Internal/FFI.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/bindings-csharp/Runtime/Internal/FFI.cs b/crates/bindings-csharp/Runtime/Internal/FFI.cs index 9dbef3ea2df..c0e30ffe7f7 100644 --- a/crates/bindings-csharp/Runtime/Internal/FFI.cs +++ b/crates/bindings-csharp/Runtime/Internal/FFI.cs @@ -132,11 +132,11 @@ public static unsafe partial CheckedStatus env_get( uint keyLen, out BytesSource source ); + [WasmImportLinkage] [LibraryImport(StdbNamespace10_7)] public static partial uint get_call_auth_flags(); - [NativeMarshalling(typeof(Marshaller))] public struct CheckedStatus { From 9847404241721d24a6fd2bdb98ff03141851d302 Mon Sep 17 00:00:00 2001 From: Tyler Cloutier Date: Wed, 9 Sep 2026 08:07:23 -0400 Subject: [PATCH 19/34] Regenerate CLI docs for private function visibility --- .../00200-reference/00100-cli-reference/00100-cli-reference.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/docs/00300-resources/00200-reference/00100-cli-reference/00100-cli-reference.md b/docs/docs/00300-resources/00200-reference/00100-cli-reference/00100-cli-reference.md index c820d565bb6..1908eef5f6c 100644 --- a/docs/docs/00300-resources/00200-reference/00100-cli-reference/00100-cli-reference.md +++ b/docs/docs/00300-resources/00200-reference/00100-cli-reference/00100-cli-reference.md @@ -445,7 +445,7 @@ Run `spacetime help generate` for more detailed information. Default value: `` * `--dotnet-version ` — Target .NET SDK major version for C# projects (e.g. 8 or 10). Auto-detected when omitted. -* `--include-private` — Include private tables and functions in generated code (types are always included). +* `--include-private` — Include private tables and private/internal non-lifecycle functions (types are always included). Default value: `false` * `-y`, `--yes` — Run non-interactively wherever possible. This will answer "yes" to almost all prompts, but will sometimes answer "no" to preserve non-interactivity (e.g. when prompting whether to log in with spacetimedb.com). From 0b1abdb24f5862041488af5a91a3a7068263fa9a Mon Sep 17 00:00:00 2001 From: Tyler Cloutier Date: Wed, 9 Sep 2026 08:56:26 -0400 Subject: [PATCH 20/34] Regenerate canonical C# environment metadata bindings --- .../Autogen/EnvironmentConstraint.g.cs | 15 +++-- .../Autogen/EnvironmentDeclaration.g.cs | 55 +++++++++++-------- 2 files changed, 43 insertions(+), 27 deletions(-) diff --git a/crates/bindings-csharp/Runtime/Internal/Autogen/EnvironmentConstraint.g.cs b/crates/bindings-csharp/Runtime/Internal/Autogen/EnvironmentConstraint.g.cs index f4226020799..87031068dec 100644 --- a/crates/bindings-csharp/Runtime/Internal/Autogen/EnvironmentConstraint.g.cs +++ b/crates/bindings-csharp/Runtime/Internal/Autogen/EnvironmentConstraint.g.cs @@ -1,11 +1,16 @@ -// Canonical module-definition metadata; declaration constraints contain no runtime values. +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + #nullable enable -namespace SpacetimeDB.Internal; -[SpacetimeDB.Type] -public partial record EnvironmentConstraint - : SpacetimeDB.TaggedEnum<( +using System; + +namespace SpacetimeDB.Internal +{ + [SpacetimeDB.Type] + public partial record EnvironmentConstraint : SpacetimeDB.TaggedEnum<( SpacetimeDB.Unit AnyString, string Literal, System.Collections.Generic.List OneOf )>; +} diff --git a/crates/bindings-csharp/Runtime/Internal/Autogen/EnvironmentDeclaration.g.cs b/crates/bindings-csharp/Runtime/Internal/Autogen/EnvironmentDeclaration.g.cs index 554d77ba55a..504d1a846a9 100644 --- a/crates/bindings-csharp/Runtime/Internal/Autogen/EnvironmentDeclaration.g.cs +++ b/crates/bindings-csharp/Runtime/Internal/Autogen/EnvironmentDeclaration.g.cs @@ -1,29 +1,40 @@ -#nullable enable -namespace SpacetimeDB.Internal; - -[SpacetimeDB.Type] -[System.Runtime.Serialization.DataContract] -public sealed partial class EnvironmentDeclaration -{ - [System.Runtime.Serialization.DataMember(Name = "name")] - public string Name; +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. - [System.Runtime.Serialization.DataMember(Name = "constraint")] - public EnvironmentConstraint Constraint; +#nullable enable - [System.Runtime.Serialization.DataMember(Name = "optional")] - public bool Optional; +using System; +using System.Collections.Generic; +using System.Runtime.Serialization; - public EnvironmentDeclaration(string Name, EnvironmentConstraint Constraint, bool Optional) +namespace SpacetimeDB.Internal +{ + [SpacetimeDB.Type] + [DataContract] + public sealed partial class EnvironmentDeclaration { - this.Name = Name; - this.Constraint = Constraint; - this.Optional = Optional; - } + [DataMember(Name = "name")] + public string Name; + [DataMember(Name = "constraint")] + public EnvironmentConstraint Constraint; + [DataMember(Name = "optional")] + public bool Optional; - public EnvironmentDeclaration() - { - Name = ""; - Constraint = new EnvironmentConstraint.AnyString(default); + public EnvironmentDeclaration( + string Name, + EnvironmentConstraint Constraint, + bool Optional + ) + { + this.Name = Name; + this.Constraint = Constraint; + this.Optional = Optional; + } + + public EnvironmentDeclaration() + { + this.Name = ""; + this.Constraint = null!; + } } } From d8379a641701111a0e5c56592e0fcfa7a04f5bdb Mon Sep 17 00:00:00 2001 From: Tyler Cloutier Date: Wed, 9 Sep 2026 09:44:01 -0400 Subject: [PATCH 21/34] Align ABI regression fixtures with V10 schema metadata --- crates/bindings/tests/ui/tables.stderr | 16 ++++++++-------- .../examples~/regression-tests/server/Lib.cs | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/crates/bindings/tests/ui/tables.stderr b/crates/bindings/tests/ui/tables.stderr index 18b61f49224..7609d9ba378 100644 --- a/crates/bindings/tests/ui/tables.stderr +++ b/crates/bindings/tests/ui/tables.stderr @@ -209,13 +209,13 @@ error[E0277]: `&'a Alpha` cannot appear as an argument to an index filtering ope = note: The allowed set of types are limited to integers, bool, strings, `Identity`, `Uuid`, `Timestamp`, `ConnectionId`, `Hash` and no-payload enums which derive `SpacetimeType`, = help: the following other types implement trait `FilterableValue`: &ConnectionId - &ContainerMode &FunctionVisibility &Identity &Lifecycle - &PortExposure - &PortProtocol - &RestartPolicy + &TableAccess + &TableType + &bool + ðnum::int::I256 and $N others note: required by a bound in `UniqueColumn::::ColType, Col>::find` --> src/table.rs @@ -241,13 +241,13 @@ help: the trait `FilterableValue` is not implemented for `Alpha` | ^^^^^^^^^^^^ = help: the following other types implement trait `FilterableValue`: &ConnectionId - &ContainerMode &FunctionVisibility &Identity &Lifecycle - &PortExposure - &PortProtocol - &RestartPolicy + &TableAccess + &TableType + &bool + ðnum::int::I256 and $N others = note: required for `Alpha` to implement `IndexScanRangeBounds<(Alpha,), SingleBound>` note: required by a bound in `RangedIndex::::filter` diff --git a/sdks/csharp/examples~/regression-tests/server/Lib.cs b/sdks/csharp/examples~/regression-tests/server/Lib.cs index e3391b710ff..64e4f164da4 100644 --- a/sdks/csharp/examples~/regression-tests/server/Lib.cs +++ b/sdks/csharp/examples~/regression-tests/server/Lib.cs @@ -831,7 +831,7 @@ public static string ReadMySchemaViaHttp(ProcedureContext ctx) try { var moduleIdentity = ProcedureContext.Identity; - var uri = $"http://localhost:3000/v1/database/{moduleIdentity}/schema?version=9"; + var uri = $"http://localhost:3000/v1/database/{moduleIdentity}/schema?version=10"; var res = ctx.Http.Get(uri, System.TimeSpan.FromSeconds(2)); return res switch { From a420f86058c8ecdc538c43d6afa25e2b2fbf4ee4 Mon Sep 17 00:00:00 2001 From: Tyler Cloutier Date: Wed, 9 Sep 2026 11:24:25 -0400 Subject: [PATCH 22/34] Fix C++ and C# procedure tests to request V10 schemas --- modules/sdk-test-procedure-cpp/src/lib.cpp | 2 +- modules/sdk-test-procedure-cs/Lib.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/sdk-test-procedure-cpp/src/lib.cpp b/modules/sdk-test-procedure-cpp/src/lib.cpp index da1278ccdca..b5310efac63 100644 --- a/modules/sdk-test-procedure-cpp/src/lib.cpp +++ b/modules/sdk-test-procedure-cpp/src/lib.cpp @@ -151,7 +151,7 @@ SPACETIMEDB_PROCEDURE(std::string, read_my_schema, ProcedureContext ctx, std::st LOG_INFO("read_my_schema using identity: " + identity_hex); // Make HTTP GET request to the schema endpoint (matches Rust) - std::string url = server_url + "/v1/database/" + identity_hex + "/schema?version=9"; + std::string url = server_url + "/v1/database/" + identity_hex + "/schema?version=10"; auto result = ctx.http.get(url); if (!result.is_ok()) { diff --git a/modules/sdk-test-procedure-cs/Lib.cs b/modules/sdk-test-procedure-cs/Lib.cs index 2e405c3a9bc..30c4a8ed4ac 100644 --- a/modules/sdk-test-procedure-cs/Lib.cs +++ b/modules/sdk-test-procedure-cs/Lib.cs @@ -70,7 +70,7 @@ public static string ReadMySchema(ProcedureContext ctx, string serverUrl) { var moduleIdentity = ProcedureContextBase.Identity; serverUrl = serverUrl.TrimEnd('/'); - var result = ctx.Http.Get($"{serverUrl}/v1/database/{moduleIdentity}/schema?version=9"); + var result = ctx.Http.Get($"{serverUrl}/v1/database/{moduleIdentity}/schema?version=10"); return result.Match( response => response.Body.ToStringUtf8Lossy(), error => throw new Exception($"HTTP request failed: {error}") From 87169bd3fd331ec136228f7682682c75eba3f582 Mon Sep 17 00:00:00 2001 From: Tyler Cloutier Date: Wed, 9 Sep 2026 20:41:46 -0400 Subject: [PATCH 23/34] Clarify environment usage and lower rejected SQL logging to debug --- crates/client-api/src/lib.rs | 2 +- .../00100-databases/00600-submodules.md | 6 +++ .../00700-environment-variables.md | 43 +++++++++++++++---- .../00200-http-api/00300-database.md | 43 +++++++------------ 4 files changed, 56 insertions(+), 38 deletions(-) diff --git a/crates/client-api/src/lib.rs b/crates/client-api/src/lib.rs index 9423849ddf8..7d3175e751b 100644 --- a/crates/client-api/src/lib.rs +++ b/crates/client-api/src/lib.rs @@ -166,7 +166,7 @@ impl Host { .await .map_err(|e| { // Parser diagnostics can quote values. Return them only to the caller. - log::warn!("SQL request rejected"); + log::debug!("SQL request rejected"); (StatusCode::BAD_REQUEST, e.to_string()) })?; diff --git a/docs/docs/00200-core-concepts/00100-databases/00600-submodules.md b/docs/docs/00200-core-concepts/00100-databases/00600-submodules.md index 1bfb9a2a3a4..78b1db00454 100644 --- a/docs/docs/00200-core-concepts/00100-databases/00600-submodules.md +++ b/docs/docs/00200-core-concepts/00100-databases/00600-submodules.md @@ -251,6 +251,12 @@ export default authSchema; +## Environment variables + +Only the root module can declare a nonempty [environment](./00700-environment-variables.md). Including a submodule with environment declarations causes publication to fail. A module with such declarations can still be published independently as a root module. + +Submodules have no separate environment-variable namespace, and their host-dispatched entry points cannot read the root module's environment. Root module code can pass configuration values to helpers explicitly. Ordinary helper calls retain the calling entry point's access, including calls to helpers defined in submodules. + ## Client Subscriptions Client subscriptions use the same namespace structure as server-side access. Submodule tables and views are queried as `.`. diff --git a/docs/docs/00200-core-concepts/00100-databases/00700-environment-variables.md b/docs/docs/00200-core-concepts/00100-databases/00700-environment-variables.md index dfcf6d245a1..b1fc7fe9d9f 100644 --- a/docs/docs/00200-core-concepts/00100-databases/00700-environment-variables.md +++ b/docs/docs/00200-core-concepts/00100-databases/00700-environment-variables.md @@ -53,6 +53,12 @@ const checked: string | null = ctx.env.get('LOG_LEVEL'); Within an environment declaration, a simple enum specifies allowed strings. Enums used elsewhere in the module retain their usual tagged representation. An enum with one case restricts the value to that string. Enums with payloads cannot be used as environment constraints. +The string-key getter checks the same declaration and permissions as named accessors. Reading an undeclared key fails. Optional named accessors return `undefined`; `ctx.env.get` returns `null` for an absent optional value. + +Key names are exact and case-sensitive. The name `get` is reserved for the getter; read a declaration named `get` with `ctx.env.get('get')`. A module with no environment declarations accepts no keys. + +Reads inside a transaction use that transaction's snapshot. In a procedure outside a transaction, each read uses a separate snapshot. To read several keys consistently, group the reads in `withTx`. + @@ -96,6 +102,12 @@ let checked: Option = ctx.env.get("LOG_LEVEL"); Existing `#[env(values(...))]` constraints on `String` and `Option` fields remain supported. Use enum variants to constrain typed enum fields. +The string-key getter checks the same declaration and permissions as named accessors. Reading an undeclared key fails. Optional named accessors and `ctx.env.get` return `None` for an absent optional value. + +Key names are exact and case-sensitive. The name `get` is reserved for the getter; read a declaration named `get` with `ctx.env.get("get")`. A module with no environment declarations accepts no keys. + +Reads inside a transaction use that transaction's snapshot. In a procedure outside a transaction, each read uses a separate snapshot. To read several keys consistently, group the reads in `with_tx`. + @@ -126,6 +138,12 @@ string? checkedValue = ctx.Env.Get("LOG_LEVEL"); `string` requires a value, and `string?` permits absence. `[SpacetimeDB.EnvValues(...)]` restricts the allowed strings. Supplying one string makes it an exact-value constraint. +The string-key getter checks the same declaration and permissions as named accessors. Reading an undeclared key fails. Optional named accessors and `ctx.Env.Get` return `null` for an absent optional value. + +Key names are exact and case-sensitive. Names that collide with `Get`, `ModuleEnvironment`, or inherited `Object` methods are available through the string-key getter, for example `ctx.Env.Get("GetType")`. A module with no environment declarations accepts no keys. + +Reads inside a transaction use that transaction's snapshot. In a procedure outside a transaction, each read uses a separate snapshot. To read several keys consistently, group the reads in `WithTx`. + @@ -161,14 +179,14 @@ std::optional checked = ctx.env.get("LOG_LEVEL"); `std::string` requires a value, and `std::optional` permits absence. The optional third element restricts the allowed strings. Supplying one string makes it an exact-value constraint. - - +The string-key getter checks the same declaration and permissions as named accessors. Reading an undeclared key fails. Optional named accessors and `ctx.env.get` return `std::nullopt` for an absent optional value. -The string-key getter checks the same declaration and permissions as named accessors. Reading an undeclared key fails; it does not return an absent value. Named optional accessors return `None`, `undefined`, `null`, or `std::nullopt`, depending on the language. The TypeScript string-key getter uses `null` for absence. +Key names are exact and case-sensitive. Names reserved by the generated accessor type, including `get`, remain available through the string-key getter, for example `ctx.env.get("get")`. A module with no environment declarations accepts no keys. -Key names are exact and case-sensitive. The getter name `get`, or `Get` in C#, is reserved: a key with that name remains accessible through the string-key getter. A module with no environment declarations accepts no keys. +Reads inside a transaction use that transaction's snapshot. In a procedure outside a transaction, each read uses a separate snapshot. To read several keys consistently, group the reads in `with_tx`. -Reads inside a transaction use that transaction's snapshot. In a procedure outside a transaction, each read uses a separate snapshot. To read several keys consistently, group the reads in `with_tx` in Rust or C++, `withTx` in TypeScript, or `WithTx` in C#. + + ## Supply values when publishing @@ -192,7 +210,14 @@ With a local server running on port 3000, publish from the directory containing API_KEY='development-only-key' spacetime publish ``` -For real credentials, supply the value through the publishing process's environment or an appropriately ignored local configuration file. Keep secrets out of checked-in configuration and module source. +For real credentials, supply the value through the publishing process's environment, `spacetime.local.json`, or `spacetime.{environment}.local.json`. Keep checked-in `spacetime.json` and `spacetime.{environment}.json` limited to non-secret defaults. Ensure the local files are ignored by Git: + +```gitignore +spacetime.local.json +spacetime.*.local.json +``` + +The `.local` naming convention does not itself prevent a file from being committed. The CLI resolves each declared key in this order: @@ -204,7 +229,7 @@ A shell value overrides JSON even when it is an empty string or the key is absen The configuration files `spacetime.json`, `spacetime.local.json`, `spacetime.{environment}.json`, and `spacetime.{environment}.local.json` apply in increasing precedence, where _environment_ is the environment selected with `--env`. Their `env` maps merge by key, as do maps inherited by child database targets. A higher-precedence value replaces that key while preserving unrelated keys. An empty map does not erase inherited keys. -JSON strings pass through unchanged. Booleans and numbers are converted to strings, so `false` supplies `"false"`; declarations still validate strings. Use JSON strings when exact numeric spelling matters. Arrays, objects, and `null` are rejected, as are JSON keys the module has not declared. An invalid effective value rejects the publish rather than falling back to a lower-precedence value. +JSON strings pass through unchanged. Booleans and numbers are converted to strings, so `false` supplies `"false"`; declarations still validate strings. Use JSON strings when exact numeric spelling matters. Arrays, objects, and `null` are rejected, as are JSON keys the module has not declared. An invalid effective value rejects the publish rather than falling back to a lower-precedence value. For an absent optional value, omit its property from the JSON object instead of setting it to `null`. Also remove any inherited or shell value for that key, as described below. ### Every publish replaces the complete environment @@ -238,7 +263,7 @@ SQL writes to `st_env`, module-side writes, and separate CLI setters are not sup ## Access and limits -Reducers, procedures, views, and HTTP handlers entered by the host in the root module can read its declared environment. Host-dispatched submodule entry points cannot read it, and submodules cannot declare a nonempty environment. Ordinary helper calls retain their calling entry point's access, including helpers defined in libraries or submodules. Root code can also pass a value to a helper explicitly. +Reducers, procedures, views, and HTTP handlers entered by the host in the root module can read its declared environment. Host-dispatched submodule entry points cannot read it, and submodules cannot declare a nonempty environment. There is no separate environment-variable namespace to configure for a submodule. A module with environment declarations can be published independently as a root module, but cannot be included as a submodule with those declarations. See [Submodules](./00600-submodules.md) for this restriction. Ordinary helper calls retain their calling entry point's access, including helpers defined in libraries or submodules. Root code can also pass a value to a helper explicitly. A procedure suspended across a publish cannot read values belonging to a replacement program. Environment reads in views participate in dependency tracking, so publishing changed values refreshes affected views. Module code remains responsible for what it returns or logs: returning a secret from a public view exposes that value to clients. @@ -250,4 +275,4 @@ Use an ordinary [private table](../00300-tables/00400-access-permissions.md) whe Update that table through reducers that explicitly authorize the caller. Keeping a table private controls direct client reads; it does not authorize calls to a reducer that modifies or returns its contents. Apply the same care to views, procedure results, and logs. Private tables follow the database's normal private-table permissions, including administrative reads. -Unlike environment variables, these values follow the table's ordinary update and migration behavior. They do not receive environment schema validation or complete replacement on every publish. +Both approaches are supported. Environment declarations additionally guarantee that required values are validated and available before `init` or migration runs. Private-table values follow the table's ordinary update and migration behavior. They do not receive environment schema validation or complete replacement on every publish. diff --git a/docs/docs/00300-resources/00200-reference/00200-http-api/00300-database.md b/docs/docs/00300-resources/00200-reference/00200-http-api/00300-database.md index 2066a7682c2..52faf7df576 100644 --- a/docs/docs/00300-resources/00200-reference/00200-http-api/00300-database.md +++ b/docs/docs/00300-resources/00200-reference/00200-http-api/00300-database.md @@ -114,36 +114,23 @@ Both publish endpoints accept `Content-Type: application/vnd.spacetimedb.publish `module` uses standard padded Base64. `environment` maps declared names to strings. The server validates the complete map against the module's declarations and installs both in one transaction. Missing required values reject the publish; omitted optional values are removed. Omitting `environment` is equivalent to `{}`, including when publishing unchanged module bytes. -For example, this Python script publishes a Wasm module to a local server. Set `SPACETIME_TOKEN` to a token authorized to publish and `API_KEY` to the complete configuration's required value. Change the module path to the artifact you built. - -```python -import base64 -import json -import os -from pathlib import Path -from urllib.request import Request, urlopen - -body = json.dumps({ - "module": base64.b64encode(Path("module.wasm").read_bytes()).decode("ascii"), - "environment": { - "API_KEY": os.environ["API_KEY"], - "MODE": "development", - }, -}).encode("utf-8") - -request = Request( - "http://127.0.0.1:3000/v1/database/env-example?host_type=wasm", - data=body, - method="PUT", - headers={ - "Authorization": "Bearer " + os.environ["SPACETIME_TOKEN"], - "Content-Type": "application/vnd.spacetimedb.publish+json", - }, -) -with urlopen(request, timeout=60) as response: - print(response.read().decode("utf-8")) +For example, use `curl`, `jq`, and `base64` to publish a Wasm module to a local server. Export `SPACETIME_TOKEN` with a token authorized to publish and `API_KEY` with the required value. Change `module.wasm` to the artifact you built. + +```bash +base64 < module.wasm | + jq --raw-input --slurp '{ + module: gsub("[\\r\\n]"; ""), + environment: { API_KEY: env.API_KEY, MODE: "development" } + }' | + curl --fail-with-body --request PUT \ + 'http://127.0.0.1:3000/v1/database/env-example?host_type=wasm' \ + --header "Authorization: Bearer $SPACETIME_TOKEN" \ + --header 'Content-Type: application/vnd.spacetimedb.publish+json' \ + --data-binary @- ``` +`jq` handles JSON escaping for the supplied value, including quotes and newlines. For a procedure or another HTTP client, construct the same JSON object with that language's JSON serializer and Base64 encoder. + Direct HTTP callers, including module procedures, use this same format; the server does not load project configuration or shell values for them. See [Environment Variables](../../../00200-core-concepts/00100-databases/00700-environment-variables.md) for declaration syntax and value limits. The decoded module is limited to 128 MiB and the complete encoded request to 192 MiB. Raw module bodies continue to work and supply an empty environment. Use `application/octet-stream` for that format. This preserves compatibility with older servers for modules that do not require ENV support; older servers do not support the JSON publish format. From 291e02d6ce84f929de54d9eee8e915439c2267e7 Mon Sep 17 00:00:00 2001 From: Tyler Cloutier Date: Wed, 9 Sep 2026 20:42:31 -0400 Subject: [PATCH 24/34] Avoid inherited member collisions in C# environment accessors --- .../Codegen.Tests/EnvironmentTests.cs | 46 +++++++++++++++++++ crates/bindings-csharp/Codegen/Environment.cs | 15 +++++- 2 files changed, 59 insertions(+), 2 deletions(-) diff --git a/crates/bindings-csharp/Codegen.Tests/EnvironmentTests.cs b/crates/bindings-csharp/Codegen.Tests/EnvironmentTests.cs index d7dd4d4382e..efb8997828a 100644 --- a/crates/bindings-csharp/Codegen.Tests/EnvironmentTests.cs +++ b/crates/bindings-csharp/Codegen.Tests/EnvironmentTests.cs @@ -25,6 +25,8 @@ public static class Host { return key switch { "REQUIRED" => Reads.ToString(), "OPTIONAL" => null, "MODE" => "prod", "Get" => "reserved", "class" => "keyword", + "ModuleEnvironment" or "Equals" or "GetHashCode" or "ToString" or + "Finalize" or "GetType" or "MemberwiseClone" => key, _ => throw new System.InvalidOperationException("undeclared environment key") }; } @@ -104,6 +106,50 @@ declarations[0].Constraint is not SpacetimeDB.Internal.EnvironmentConstraint.Any Assert.Null(assembly.GetType("SpacetimeDB.ModuleEnvironment")!.GetProperty("Get")); } + [Theory] + [InlineData("Get")] + [InlineData("ModuleEnvironment")] + [InlineData("Equals")] + [InlineData("GetHashCode")] + [InlineData("ToString")] + [InlineData("Finalize")] + [InlineData("GetType")] + [InlineData("MemberwiseClone")] + public static void ReservedAccessorNamesRemainAvailableThroughCheckedGet(string name) + { + var (compilation, result) = Generate( + $$""" + [SpacetimeDB.Env] public struct Declarations { + public string {{name}}; + } + public static class Usage { + public static void Check() { + var env = new SpacetimeDB.ModuleEnvironment(); + if (env.Get("{{name}}") != "{{(name == "Get" ? "reserved" : name)}}") + throw new System.Exception("bad reserved accessor"); + var declarations = SpacetimeDB.Internal.Module.Declarations; + if (declarations.Count != 1 || declarations[0].Name != "{{name}}") + throw new System.Exception("missing reserved declaration"); + } + } + """ + ); + Assert.Empty(result.Diagnostics); + Assert.DoesNotContain( + compilation.GetDiagnostics(), + diagnostic => + diagnostic.Severity is DiagnosticSeverity.Warning or DiagnosticSeverity.Error + && diagnostic.Location.SourceTree is { } tree + && result.GeneratedTrees.Contains(tree) + ); + using var stream = new MemoryStream(); + var emitted = compilation.Emit(stream); + Assert.True(emitted.Success, string.Join("\n", emitted.Diagnostics)); + var assembly = Assembly.Load(stream.ToArray()); + assembly.GetType("Usage")!.GetMethod("Check")!.Invoke(null, null); + Assert.Null(assembly.GetType("SpacetimeDB.ModuleEnvironment")!.GetProperty(name)); + } + [Theory] [InlineData("public int BAD;")] [InlineData("public static string BAD;")] diff --git a/crates/bindings-csharp/Codegen/Environment.cs b/crates/bindings-csharp/Codegen/Environment.cs index e049abb5fda..a9f0992da9c 100644 --- a/crates/bindings-csharp/Codegen/Environment.cs +++ b/crates/bindings-csharp/Codegen/Environment.cs @@ -123,8 +123,19 @@ void Report(ISymbol symbol, string message) => $"global::SpacetimeDB.Internal.Module.RegisterEnvironment(new({Literal(name)}, {constraint}, {(optional ? "true" : "false")}));" ); // Preserve the checked generic method, including a key literally - // named Get. Keywords are escaped without renaming stored keys. - if (name is "Get" or "ModuleEnvironment" or "Equals" or "GetHashCode" or "ToString") + // named Get, and inherited object members. Keywords are escaped + // without renaming stored keys. + if ( + name + is "Get" + or "ModuleEnvironment" + or "Equals" + or "GetHashCode" + or "ToString" + or "Finalize" + or "GetType" + or "MemberwiseClone" + ) continue; var read = $"Get({Literal(name)})"; if (!optional) From 81a4190ab36c43717f93fa41b80fdb2bf25a26b8 Mon Sep 17 00:00:00 2001 From: Tyler Cloutier Date: Wed, 9 Sep 2026 21:29:45 -0400 Subject: [PATCH 25/34] Separate generic view handling from ENV changes --- crates/core/src/host/module_host.rs | 127 ++---------------- .../src/host/wasm_common/module_host_actor.rs | 9 +- crates/testing/tests/environment.rs | 126 +++++------------ modules/environment-test/src/lib.rs | 15 --- 4 files changed, 49 insertions(+), 228 deletions(-) diff --git a/crates/core/src/host/module_host.rs b/crates/core/src/host/module_host.rs index 685a6046e13..495f6a61fa9 100644 --- a/crates/core/src/host/module_host.rs +++ b/crates/core/src/host/module_host.rs @@ -1630,18 +1630,6 @@ impl fmt::Debug for ViewCallResult { } impl ViewCallResult { - fn into_materialized_tx(self, db: &RelationalDB, trapped: bool) -> Result { - let error = match self.outcome { - ViewOutcome::Success if !trapped => return Ok(self.tx), - ViewOutcome::Success => "View instance trapped during materialization".to_owned(), - ViewOutcome::Failed(error) => error, - ViewOutcome::BudgetExceeded => "View terminated due to insufficient budget".to_owned(), - }; - let (_, metrics, reducer) = db.rollback_mut_tx(self.tx); - db.report_mut_tx_metrics(reducer, metrics, None); - Err(ViewCallError::InternalError(error)) - } - pub fn default(tx: MutTxId) -> Self { Self { outcome: ViewOutcome::Success, @@ -1721,9 +1709,6 @@ pub enum ClientConnectedError { pub struct RefInstance<'a, I: WasmInstance> { pub common: &'a mut InstanceCommon, pub instance: &'a mut I, - // Invocation-local disposal state survives errors propagated through SQL or - // subscription helpers, whose Result error does not carry a success tuple. - pub(crate) trapped: bool, } macro_rules! call_view_command_method { @@ -2961,26 +2946,17 @@ impl ModuleHost { /// Passing [`Workload::Sql`] will update the instance's last-used timestamp. /// Passing [`Workload::Subscribe`] will also increment the subscriber's refcount. pub fn materialize_views( - tx: MutTxId, + mut tx: MutTxId, instance: &mut RefInstance<'_, I>, view_collector: &impl CollectViews, caller: Identity, workload: Workload, ) -> Result<(MutTxId, bool), ViewCallError> { use FunctionArgs::*; - let db = instance.instance.replica_ctx().relational_db().clone(); - // Keep all earlier view materializations and subscription refcounts in - // the same rollback boundary if any later view fails. - let mut tx = scopeguard::guard(Some(tx), |tx| { - if let Some(tx) = tx { - let (_, metrics, reducer) = db.rollback_mut_tx(tx); - db.report_mut_tx_metrics(reducer, metrics, None); - } - }); let mut view_ids = HashSet::new(); view_collector.collect_views(&mut view_ids); for view_id in view_ids { - let st_view_row = tx.as_ref().unwrap().lookup_st_view(view_id)?; + let st_view_row = tx.lookup_st_view(view_id)?; let view_name: NamespacedIdentifier = st_view_row.view_name.into(); let view_id = st_view_row.view_id; let table_id = st_view_row.table_id.ok_or(ViewCallError::TableDoesNotExist(view_id))?; @@ -2992,30 +2968,25 @@ impl ModuleHost { }; let view_call = ViewCallInfo::from_args(view_id, args); let sender = args.sender(); - let is_materialized = tx.as_ref().unwrap().is_view_materialized(&view_call)?; + let is_materialized = tx.is_view_materialized(&view_call)?; if !is_materialized { - let (res, trapped) = Self::call_view( - instance, - tx.take().unwrap(), - &view_name, - view_id, - table_id, - Nullary, - caller, - sender, - )?; - *tx = Some(res.into_materialized_tx(&db, trapped)?); + let (res, trapped) = + Self::call_view(instance, tx, &view_name, view_id, table_id, Nullary, caller, sender)?; + tx = res.tx; + if trapped { + return Ok((tx, true)); + } } - let tx = tx.as_mut().unwrap(); - // These changes commit only after every requested view succeeds. + // If this is a sql call, we only update this view's "last called" timestamp if let Workload::Sql = workload { tx.update_view_timestamp(view_call.clone(), args)?; } + // If this is a subscribe call, we also increment this view's subscriber count if let Workload::Subscribe = workload { tx.subscribe_view(view_call, args, caller)?; } } - Ok((ScopeGuard::into_inner(tx).unwrap(), false)) + Ok((tx, false)) } /// Refreshes every view made stale by `tx`. @@ -3162,11 +3133,6 @@ impl ModuleHost { sender: Option, timestamp: Timestamp, ) -> Result<(ViewCallResult, bool), ViewCallError> { - let db = instance.instance.replica_ctx().relational_db().clone(); - let tx = scopeguard::guard(tx, |tx| { - let (_, metrics, reducer) = db.rollback_mut_tx(tx); - db.report_mut_tx_metrics(reducer, metrics, None); - }); let module_def = &instance.common.info().module_def; let (global_fn_ptr, view_def, owning_def) = module_def .view_by_name_with_global_fn_ptr(view_name) @@ -3178,7 +3144,7 @@ impl ModuleHost { Ok(Self::call_view_inner( instance, - ScopeGuard::into_inner(tx), + tx, view_name, view_id, table_id, @@ -3220,9 +3186,7 @@ impl ModuleHost { view_typespace, }; - let (result, trapped) = instance.common.call_view_with_tx(tx, params, instance.instance); - instance.trapped |= trapped; - (result, trapped) + instance.common.call_view_with_tx(tx, params, instance.instance) } pub async fn init_database(&self, program: Program) -> Result { @@ -3724,69 +3688,6 @@ mod tests { use spacetimedb_sats::product; use std::sync::Arc; - #[test] - fn failed_view_materialization_rolls_back_prior_rows_and_subscriber_counts() -> anyhow::Result<()> { - use super::{ViewCallResult, ViewOutcome}; - use crate::db::relational_db::tests_utils::begin_mut_tx; - use spacetimedb_datastore::locking_tx_datastore::{ViewCallInfo, ViewInstanceArgs}; - use spacetimedb_lib::db::raw_def::v10::RawModuleDefV10Builder; - use spacetimedb_lib::ProductType; - use spacetimedb_schema::def::ModuleDef; - - let db = TestDB::in_memory()?; - let mut builder = RawModuleDefV10Builder::new(); - let row = builder.add_algebraic_type( - [], - "Row", - AlgebraicType::Product(ProductType::from_iter([("value", AlgebraicType::U8)])), - true, - ); - builder.add_view( - "earlier", - 0, - true, - true, - ProductType::unit(), - AlgebraicType::array(row.into()), - ); - let module: ModuleDef = builder.finish().try_into()?; - let mut tx = begin_mut_tx(&db); - let (view_id, table_id) = db.create_view(&mut tx, &module, module.view("earlier").unwrap())?; - db.commit_tx(tx)?; - let call = ViewCallInfo::anonymous(view_id); - - for (outcome, trapped) in [ - (ViewOutcome::Failed("denied".into()), false), - (ViewOutcome::Failed("trap".into()), true), - (ViewOutcome::BudgetExceeded, true), - (ViewOutcome::Success, true), - ] { - let mut tx = begin_mut_tx(&db); - // A preceding view succeeded in this same multi-view request. - db.materialize_view_call(&mut tx, table_id, call.clone(), vec![product![9_u8]])?; - tx.subscribe_view(call.clone(), ViewInstanceArgs::Anonymous, Identity::ZERO)?; - assert_eq!(tx.active_subscribers_for_view(view_id), vec![(Identity::ZERO, 1)]); - let mut result = ViewCallResult::default(tx); - result.outcome = outcome; - assert!(result.into_materialized_tx(&db, trapped).is_err()); - - let tx = begin_mut_tx(&db); - assert!(tx.active_subscribers_for_view(view_id).is_empty()); - assert!(!tx.is_view_materialized(&call)?); - assert_eq!(db.iter_mut(&tx, table_id)?.count(), 0); - let _ = db.rollback_mut_tx(tx); - } - // Success retains the owned transaction for the normal commit path. - let mut tx = begin_mut_tx(&db); - db.materialize_view_call(&mut tx, table_id, call.clone(), vec![product![9_u8]])?; - tx.subscribe_view(call, ViewInstanceArgs::Anonymous, Identity::ZERO)?; - db.commit_tx(ViewCallResult::default(tx).into_materialized_tx(&db, false)?)?; - let tx = begin_mut_tx(&db); - assert_eq!(tx.active_subscribers_for_view(view_id), vec![(Identity::ZERO, 1)]); - let _ = db.rollback_mut_tx(tx); - Ok(()) - } - fn v2_client_config() -> ClientConfig { ClientConfig { protocol: Protocol::Binary, diff --git a/crates/core/src/host/wasm_common/module_host_actor.rs b/crates/core/src/host/wasm_common/module_host_actor.rs index 2733d4cefe3..7abc0aaf186 100644 --- a/crates/core/src/host/wasm_common/module_host_actor.rs +++ b/crates/core/src/host/wasm_common/module_host_actor.rs @@ -1210,7 +1210,6 @@ impl InstanceCommon { let mut inst = RefInstance { instance: inst, common: self, - trapped: false, }; let (res, trapped) = match cmds { ViewCommand::AddSingleSubscription { @@ -1297,7 +1296,7 @@ impl InstanceCommon { if let Err(err) = &res { error_target.send(&info.subscriptions, err); } - (res, trapped || inst.trapped) + (res, trapped) } pub(in crate::host) fn handle_sql_cmd( @@ -1308,7 +1307,6 @@ impl InstanceCommon { let mut inst = RefInstance { instance: inst, common: self, - trapped: false, }; let SqlCommand { db, @@ -1325,9 +1323,9 @@ impl InstanceCommon { result: Ok(result), head, }, - trapped || inst.trapped, + trapped, ), - Err(err) => (SqlCommandResult { result: Err(err), head }, inst.trapped), + Err(err) => (SqlCommandResult { result: Err(err), head }, false), } } @@ -1494,7 +1492,6 @@ impl InstanceCommon { let mut instance = RefInstance { common: self, instance: inst, - trapped: false, }; ModuleHost::call_views_with_tx_at(tx, &mut instance, caller, timestamp) } diff --git a/crates/testing/tests/environment.rs b/crates/testing/tests/environment.rs index 838b26e43b5..a8187619fca 100644 --- a/crates/testing/tests/environment.rs +++ b/crates/testing/tests/environment.rs @@ -95,30 +95,18 @@ async fn check_submodule_scope(handle: &mut ModuleHandle, values: &mut Values) { &mut vec![], ) .await; - let error = format!( - "{:#}", - result.expect_err("view bypassed the environment read interface") - ); - assert!(!error.contains("not found"), "view failed before dispatch: {error}"); - // A valid view and a forbidden view share one actual subscription - // request. It must fail as a whole, without an initial-success message. - let request_id = 880; - let subscribe = ws_v1::ClientMessage::::Subscribe(ws_v1::Subscribe { - query_strings: ["SELECT * FROM my_player".into(), format!("SELECT * FROM {view}").into()].into(), - request_id, - }); - let _ = handle.send(bsatn::to_vec(&subscribe).unwrap()).await; - let message = next_message(handle).await; - let OutboundMessage::V1(SerializableMessage::Subscription(message)) = message else { - panic!("failed view subscription returned a success or unexpected message: {message:?}"); - }; - assert_eq!(message.request_id, Some(request_id)); - let spacetimedb::client::messages::SubscriptionResult::Error(error) = message.result else { - panic!("forbidden view subscription returned rows"); - }; - assert!(!error.message.is_empty()); - assert!(!error.message.contains("not found")); + // Host access must be denied, independently of the general view-error + // transport contract tracked in #5912. A failed view may yield no rows. + match result { + Ok(result) => assert!(result.rows.is_empty(), "forbidden view exposed rows"), + Err(error) => { + let error = format!("{error:#}"); + assert!(!error.contains("not found"), "view failed before dispatch: {error}"); + assert!(!error.contains("root-visible"), "view error exposed environment data"); + } + } } + // HTTP routes are root entries today. Calling an exported child callback // as an ordinary helper retains that root entry's authority. assert_eq!( @@ -127,63 +115,25 @@ async fn check_submodule_scope(handle: &mut ModuleHandle, values: &mut Values) { ); } -// SQL, subscription materialization, and ordinary reducers all use the same -// main Wasmtime instance. The fixture view sets a guest-global marker and traps; -// the next reducer checks the marker is absent in the replacement instance. -async fn check_wasm_trap_disposal(handle: &mut ModuleHandle) { - let module = handle.client.module(); - let view = "SELECT * FROM environment_trap"; - let result = spacetimedb::sql::execute::run( - module.relational_db().clone(), - view.to_string(), - AuthCtx::for_current(Identity::ZERO), - Some(module.info.subscriptions.clone()), - Some(module.clone()), - &mut vec![], - ) - .await; - let error = format!("{:#}", result.expect_err("trapped view returned SQL success")); - assert!(!error.contains("not found"), "view failed before dispatch: {error}"); - expect_clean_wasm_instance(&module).await; - - // Keep the existing successful ENV-view subscription while adding this - // separate query. The failing request must report an error, not rows. - let request_id = 890; - let subscribe = ws_v1::ClientMessage::::SubscribeSingle(ws_v1::SubscribeSingle { - query: view.into(), - request_id, - query_id: ws_v1::QueryId::new(890), - }); - let _ = handle.send(bsatn::to_vec(&subscribe).unwrap()).await; - let message = next_message(handle).await; - let OutboundMessage::V1(SerializableMessage::Subscription(message)) = message else { - panic!("trapped view subscription returned success or an unexpected message: {message:?}"); +// Qualification can pin locally built inputs without invoking a nested build. +// Ordinary test runs keep the existing compilation path when no pin is supplied. +fn compiled_fixture(name: &str) -> CompiledModule { + use spacetimedb::messages::control_db::HostType; + let input = match name { + "environment-test" => Some(("SPACETIMEDB_ENV_RUST_MODULE", HostType::Wasm)), + "module-test-ts" => Some(("SPACETIMEDB_ENV_TYPESCRIPT_MODULE", HostType::Js)), + "module-test-cs" => Some(("SPACETIMEDB_ENV_CSHARP_MODULE", HostType::Wasm)), + _ => None, }; - assert_eq!(message.request_id, Some(request_id)); - let spacetimedb::client::messages::SubscriptionResult::Error(error) = message.result else { - panic!("trapped view subscription returned rows"); - }; - assert!(!error.message.is_empty()); - assert!(!error.message.contains("not found")); - expect_clean_wasm_instance(&module).await; -} - -async fn expect_clean_wasm_instance(module: &ModuleHost) { - module - .call_reducer( - Identity::ZERO, - None, - None, - None, - None, - "expect_environment", - FunctionArgs::Bsatn(bsatn::to_vec(&product!["MISSING", None::]).unwrap().into()), - ) - .await - .unwrap() - .outcome - .into_result() - .unwrap(); + if let Some((key, host_type)) = input + && let Some(path) = std::env::var_os(key) + { + let path = std::path::PathBuf::from(path); + assert!(path.is_absolute() && path.is_file(), "invalid explicit module artifact"); + CompiledModule::from_artifact(name, host_type, path) + } else { + CompiledModule::compile(name, CompilationMode::Debug) + } } fn exercise_fixture(name: &str) { @@ -195,18 +145,7 @@ fn exercise_fixture(name: &str) { } else { Values::new() }; - // NativeAOT's WebAssembly compiler requires a supported compiler host. - // Allow this one fixture to consume the exact artifact built there while - // exercising all normal publication and runtime paths below. - let artifact = (name == "module-test-cs") - .then(|| std::env::var_os("SPACETIMEDB_ENV_CSHARP_MODULE")) - .flatten(); - let compiled = match artifact { - Some(path) => { - CompiledModule::from_artifact(name, spacetimedb::messages::control_db::HostType::Wasm, path.into()) - } - None => CompiledModule::compile(name, CompilationMode::Debug), - }; + let compiled = compiled_fixture(name); compiled.with_module_async_with_environment(DEFAULT_CONFIG, initial.clone(), |mut handle| async move { let mut values = initial; for (key, expected) in [ @@ -333,7 +272,6 @@ fn exercise_fixture(name: &str) { .into_result() .unwrap(); } - check_wasm_trap_disposal(&mut handle).await; } if name == "module-test-ts" { check_submodule_scope(&mut handle, &mut values).await; @@ -398,7 +336,7 @@ fn suspended_procedure_cannot_read_environment_from_a_replacement_program() { ("REQUIRED".into(), "initial-required".into()), ("MODE".into(), "ready".into()), ]); - CompiledModule::compile("environment-test", CompilationMode::Debug).with_module_async_with_environment( + compiled_fixture("environment-test").with_module_async_with_environment( DEFAULT_CONFIG, initial.clone(), |handle| async move { @@ -499,7 +437,7 @@ fn rust_environment_enums_preserve_exact_typed_mappings() { ("REQUIRED".into(), "initial-required".into()), ("MODE".into(), "ready".into()), ]); - CompiledModule::compile("environment-test", CompilationMode::Debug).with_module_async_with_environment( + compiled_fixture("environment-test").with_module_async_with_environment( DEFAULT_CONFIG, initial.clone(), |handle| async move { diff --git a/modules/environment-test/src/lib.rs b/modules/environment-test/src/lib.rs index 99846b25301..c72d1bd8625 100644 --- a/modules/environment-test/src/lib.rs +++ b/modules/environment-test/src/lib.rs @@ -1,7 +1,4 @@ use spacetimedb::{AnonymousViewContext, ProcedureContext, ReducerContext, SpacetimeType}; -use std::sync::atomic::{AtomicBool, Ordering}; - -static VIEW_TRAP_ENTERED: AtomicBool = AtomicBool::new(false); type RequiredString = String; type OptionalString = Option; @@ -65,10 +62,6 @@ pub fn expect_typed_environment(ctx: &ReducerContext, required: u8, optional: Op #[spacetimedb::reducer] pub fn expect_environment(ctx: &ReducerContext, key: String, expected: Option) { - assert!( - !VIEW_TRAP_ENTERED.load(Ordering::Relaxed), - "trapped Wasm view instance was reused" - ); assert_eq!(ctx.env.get(&key), expected); assert_eq!(ctx.as_read_only().env.get(&key), expected); assert_eq!(ctx.as_anonymous_read_only().env.get(&key), expected); @@ -111,14 +104,6 @@ pub fn environment_value(ctx: &AnonymousViewContext) -> Option Some(EnvironmentValue { value }) } -/// A Rust panic is a Wasm trap. The next main-instance reducer verifies that -/// the guest-global marker did not survive the failed SQL/subscription call. -#[spacetimedb::view(accessor = environment_trap, public)] -pub fn environment_trap(_ctx: &AnonymousViewContext) -> Option { - VIEW_TRAP_ENTERED.store(true, Ordering::Relaxed); - panic!("intentional environment view trap"); -} - /// Hand-written ABI callers cannot retain unbounded host allocations. #[spacetimedb::reducer] pub fn bounded_environment_sources(_ctx: &ReducerContext) { From 776689bdd487348fadbdf8d739d04fe35559121d Mon Sep 17 00:00:00 2001 From: Tyler Cloutier Date: Thu, 10 Sep 2026 06:24:10 -0400 Subject: [PATCH 26/34] Check canonical procedure names in V10 schema HTTP test --- .../procedure-client/src/test_handlers.rs | 23 +++++++++++-------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/sdks/rust/tests/procedure-client/src/test_handlers.rs b/sdks/rust/tests/procedure-client/src/test_handlers.rs index e81cf5b3724..54e4d875ab0 100644 --- a/sdks/rust/tests/procedure-client/src/test_handlers.rs +++ b/sdks/rust/tests/procedure-client/src/test_handlers.rs @@ -1,7 +1,7 @@ use crate::module_bindings::*; use anyhow::Context; use core::time::Duration; -use spacetimedb_lib::db::raw_def::v10::{RawModuleDefV10, RawModuleDefV10Section}; +use spacetimedb_lib::db::raw_def::v10::{ExplicitNameEntry, RawModuleDefV10}; use spacetimedb_sdk::{DbConnectionBuilder, DbContext, Table}; use test_counter::{server_url, TestCounter}; @@ -265,14 +265,19 @@ async fn exec_procedure_http_ok(db_name: &str) { let module_def: RawModuleDefV10 = spacetimedb_lib::de::serde::deserialize_from( &mut serde_json::Deserializer::from_str(&res.unwrap()), )?; - anyhow::ensure!(module_def.sections.iter().any(|section| { - if let RawModuleDefV10Section::Procedures(procedures) = section { - procedures - .iter() - .any(|procedure| &*procedure.source_name == "read_my_schema") - } else { - false - } + // The schema endpoint exports source-to-canonical name mappings. + // C# uses `ReadMySchema` in source and `read_my_schema` on the wire. + let names = module_def.explicit_names().cloned().unwrap_or_default().into_entries(); + anyhow::ensure!(names.iter().any(|entry| { + let ExplicitNameEntry::Function(mapping) = entry else { + return false; + }; + &*mapping.canonical_name == "read_my_schema" + && module_def + .procedures() + .into_iter() + .flatten() + .any(|procedure| procedure.source_name == mapping.source_name) })); Ok(()) })(), From a7c7799705ea918717bbaeacf0f26c3dcc0a7213 Mon Sep 17 00:00:00 2001 From: Jason Larabie Date: Thu, 10 Sep 2026 08:34:10 -0700 Subject: [PATCH 27/34] Add to C++ HandlerContext --- crates/bindings-cpp/include/spacetimedb/handler_context.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/bindings-cpp/include/spacetimedb/handler_context.h b/crates/bindings-cpp/include/spacetimedb/handler_context.h index cb9b4abe84c..80f42d821c5 100644 --- a/crates/bindings-cpp/include/spacetimedb/handler_context.h +++ b/crates/bindings-cpp/include/spacetimedb/handler_context.h @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -23,6 +24,7 @@ namespace SpacetimeDB { struct HandlerContext { Timestamp timestamp; + Environment env; HttpClient http; private: From e98ef53982b73694d437b3cbed2161c534acbd6f Mon Sep 17 00:00:00 2001 From: Tyler Cloutier Date: Thu, 10 Sep 2026 19:13:25 -0400 Subject: [PATCH 28/34] Preserve Windows inspector PATH and clarify secret visibility --- crates/cli/src/schema_extract.rs | 20 ++++++--- crates/cli/src/schema_extract/tests.rs | 45 +++++++++++++++++++ .../00700-environment-variables.md | 4 ++ 3 files changed, 64 insertions(+), 5 deletions(-) diff --git a/crates/cli/src/schema_extract.rs b/crates/cli/src/schema_extract.rs index c00481ae40f..76b2ec64159 100644 --- a/crates/cli/src/schema_extract.rs +++ b/crates/cli/src/schema_extract.rs @@ -97,6 +97,16 @@ impl Observation { } } +fn configure_inspector_environment(command: &mut tokio::process::Command) { + command.env_clear(); + // Windows needs PATH to resolve the inspector's dependent DLLs. Keep + // other inherited variables, including publish-time secrets, isolated. + #[cfg(windows)] + if let Some(path) = std::env::var_os("PATH") { + command.env("PATH", path); + } +} + async fn inspect_observed( extractor: PathBuf, program: Vec, @@ -114,18 +124,18 @@ async fn inspect_observed( tokio::fs::write(&module, program) .await .context("Cannot prepare module inspection input")?; - let mut child = tokio::process::Command::new(extractor) + let mut command = tokio::process::Command::new(extractor); + command .arg("extract-schema") .arg(&module) .arg("--host-type") .arg(host_type.to_ascii_lowercase()) - .env_clear() .stdin(Stdio::null()) .stdout(Stdio::piped()) .stderr(Stdio::null()) - .kill_on_drop(true) - .spawn() - .context("Cannot start local module schema inspection")?; + .kill_on_drop(true); + configure_inspector_environment(&mut command); + let mut child = command.spawn().context("Cannot start local module schema inspection")?; observation.started(child.id()); let mut output = Vec::new(); let mut stdout = child diff --git a/crates/cli/src/schema_extract/tests.rs b/crates/cli/src/schema_extract/tests.rs index d8c6daea04a..7738fb8a5f3 100644 --- a/crates/cli/src/schema_extract/tests.rs +++ b/crates/cli/src/schema_extract/tests.rs @@ -3,6 +3,51 @@ use spacetimedb_lib::environment::{ EnvironmentConstraint as Constraint, EnvironmentDeclaration as Declaration, EnvironmentSchema, }; +#[tokio::test] +async fn inspector_environment_excludes_secrets_and_preserves_windows_dll_search() { + let mut command = tokio::process::Command::new(std::env::current_exe().unwrap()); + command + .args([ + "--exact", + "schema_extract::tests::inspector_environment_child", + "--ignored", + ]) + .env("STDB_INSPECTOR_TEST_SECRET", "must-not-reach-inspector"); + configure_inspector_environment(&mut command); + let explicit: Vec<_> = command.as_std().get_envs().collect(); + #[cfg(windows)] + { + let path = std::env::var_os("PATH"); + let expected: Vec<_> = path + .as_deref() + .map(|value| (std::ffi::OsStr::new("PATH"), Some(value))) + .into_iter() + .collect(); + assert_eq!(explicit, expected); + } + #[cfg(not(windows))] + assert!(explicit.is_empty()); + let output = command.output().await.unwrap(); + assert!( + output.status.success(), + "isolated inspector process failed: {}", + String::from_utf8_lossy(&output.stdout) + ); +} + +#[test] +#[ignore = "invoked by the isolated inspector environment regression"] +fn inspector_environment_child() { + // CoreFoundation adds this variable during process initialization on macOS, + // even when the parent supplies an empty environment. + #[cfg(target_os = "macos")] + assert!(std::env::vars_os().all(|(key, _)| key == "__CF_USER_TEXT_ENCODING")); + #[cfg(windows)] + assert!(std::env::vars_os().all(|(key, _)| key.as_encoded_bytes().eq_ignore_ascii_case(b"PATH"))); + #[cfg(all(not(windows), not(target_os = "macos")))] + assert!(std::env::vars_os().next().is_none(), "unexpected inherited variable"); +} + fn schema() -> EnvironmentSchema { EnvironmentSchema::new(vec![ Declaration { diff --git a/docs/docs/00200-core-concepts/00100-databases/00700-environment-variables.md b/docs/docs/00200-core-concepts/00100-databases/00700-environment-variables.md index b1fc7fe9d9f..d05c96ed526 100644 --- a/docs/docs/00200-core-concepts/00100-databases/00700-environment-variables.md +++ b/docs/docs/00200-core-concepts/00100-databases/00700-environment-variables.md @@ -275,4 +275,8 @@ Use an ordinary [private table](../00300-tables/00400-access-permissions.md) whe Update that table through reducers that explicitly authorize the caller. Keeping a table private controls direct client reads; it does not authorize calls to a reducer that modifies or returns its contents. Apply the same care to views, procedure results, and logs. Private tables follow the database's normal private-table permissions, including administrative reads. +:::warning Secret visibility +Private tables can store dynamically editable secrets, but changing a table to public can expose its contents to clients. Environment variables have no public-table visibility setting. With either approach, module code can still expose secrets through return values or logs. +::: + Both approaches are supported. Environment declarations additionally guarantee that required values are validated and available before `init` or migration runs. Private-table values follow the table's ordinary update and migration behavior. They do not receive environment schema validation or complete replacement on every publish. From b7adb97a93d33bc76ac18e5682d0d356cb68bb24 Mon Sep 17 00:00:00 2001 From: Tyler Cloutier Date: Fri, 11 Sep 2026 22:06:22 -0400 Subject: [PATCH 29/34] Update persistent environment publication and management --- crates/cli/src/subcommands/publish.rs | 112 +++++++++- .../src/subcommands/publish/environment.rs | 90 +++++++- .../subcommands/publish/environment/tests.rs | 45 +++- .../cli/src/subcommands/publish/wire_tests.rs | 78 ++++++- crates/client-api-messages/src/publish.rs | 76 ++++++- crates/client-api/src/lib.rs | 40 +++- crates/client-api/src/routes/database.rs | 196 ++++++++++++++++- .../routes/database/publish_environment.rs | 52 ++++- crates/core/src/db/environment.rs | 2 +- crates/core/src/host/host_controller.rs | 81 ++++++- crates/core/src/host/mod.rs | 7 +- .../src/host/wasm_common/module_host_actor.rs | 55 +++++ crates/lib/src/environment.rs | 198 +++++++++++++++++- .../tests/standalone/cli/environment.rs | 121 ++++++++++- .../standalone/src/control_db/environment.rs | 5 +- crates/standalone/src/environment_tests.rs | 47 ++++- crates/standalone/src/lib.rs | 30 ++- crates/testing/src/modules.rs | 6 + crates/testing/tests/environment.rs | 2 +- .../00700-environment-variables.md | 57 +++-- .../00100-cli-reference.md | 5 +- .../00200-http-api/00300-database.md | 21 +- 22 files changed, 1217 insertions(+), 109 deletions(-) diff --git a/crates/cli/src/subcommands/publish.rs b/crates/cli/src/subcommands/publish.rs index ab4133144c0..443943fb45d 100644 --- a/crates/cli/src/subcommands/publish.rs +++ b/crates/cli/src/subcommands/publish.rs @@ -107,6 +107,9 @@ pub fn build_publish_schema(command: &clap::Command) -> Result, + replace: bool, +} + +impl EnvironmentOptions { + fn from_args(args: &ArgMatches) -> anyhow::Result { + let options = Self { + only: args.get_flag("env_only"), + remove: args + .get_many::("unset_env") + .map(|keys| keys.cloned().collect()) + .unwrap_or_default(), + replace: args.get_flag("replace_env"), + }; + ensure!( + options.remove.len() <= spacetimedb_lib::environment::MAX_ENV_VARS, + "Too many environment removals" + ); + for key in &options.remove { + spacetimedb_lib::environment::validate_key(key)?; + } + Ok(options) + } + + fn validate_values(&self, values: &std::collections::BTreeMap) -> anyhow::Result<()> { + ensure!( + !self.replace || self.remove.is_empty(), + "--replace-env cannot be combined with --unset-env" + ); + for key in &self.remove { + ensure!( + !values.contains_key(key), + "Environment key {key:?} is both supplied and removed" + ); + } + Ok(()) + } } fn publication_body( module: &spacetimedb_schema::def::ModuleDef, bytes: Vec, environment: std::collections::BTreeMap, + options: &EnvironmentOptions, ) -> anyhow::Result<(&'static str, Vec)> { - if module.environment_declared() { + options.validate_values(&environment)?; + if module.environment_declared() || !environment.is_empty() || !options.remove.is_empty() || options.replace { let body = spacetimedb_client_api_messages::publish::PublishRequest { - module: bytes, + module: Some(bytes), environment, + environment_remove: options.remove.clone(), + environment_replace: options.replace, + expected_module_version: None, } .encode()?; Ok((spacetimedb_client_api_messages::publish::CONTENT_TYPE, body)) } else { - anyhow::ensure!(environment.is_empty(), "Module does not declare environment keys"); - // Preserve older servers for ordinary modules. Explicit empty ENV - // declarations still use the envelope, expressing replacement intent. + // Preserve older servers for ordinary modules without environment changes. Ok(("application/octet-stream", bytes)) } } @@ -462,6 +531,7 @@ pub async fn exec_with_options( .copied() .unwrap_or(ClearMode::Never); let yes = yes_flags_from_args(args); + let environment_options = EnvironmentOptions::from_args(args)?; let config_dir = loaded_config_ref.map(|lc| lc.config_dir.as_path()); execute_publish_configs( @@ -471,6 +541,7 @@ pub async fn exec_with_options( config_dir, clear_database, yes, + &environment_options, ) .await } @@ -491,7 +562,16 @@ pub async fn exec_from_entry( let yes = if force { YesFlags::all() } else { YesFlags::default() }; - execute_publish_configs(&mut config, vec![command_config], true, config_dir, clear_database, yes).await + execute_publish_configs( + &mut config, + vec![command_config], + true, + config_dir, + clear_database, + yes, + &EnvironmentOptions::default(), + ) + .await } async fn execute_publish_configs<'a>( @@ -501,6 +581,7 @@ async fn execute_publish_configs<'a>( config_dir: Option<&std::path::Path>, clear_database: ClearMode, yes: YesFlags, + environment_options: &EnvironmentOptions, ) -> Result<(), anyhow::Error> { // Execute publish for each config for command_config in publish_configs { @@ -510,6 +591,20 @@ async fn execute_publish_configs<'a>( let name_or_identity_opt = command_config.get_one::("database")?; let name_or_identity = name_or_identity_opt.as_deref(); let anon_identity = command_config.get_one::("anon_identity")?.unwrap_or(false); + if environment_options.only { + ensure!(clear_database == ClearMode::Never, "--env-only cannot reset a database"); + environment::publish_only( + config, + server, + name_or_identity.context("--env-only requires an existing database name or identity")?, + anon_identity, + yes, + command_config.get_config_value("env"), + environment_options, + ) + .await?; + continue; + } let wasm_file = command_config.get_one::("wasm_file")?; let js_file = command_config.get_one::("js_file")?; let resolved_module_path = command_config.get_resolved_path("module_path", config_dir)?; @@ -670,7 +765,8 @@ async fn execute_publish_configs<'a>( // Set the host type. builder = builder.query(&[("host_type", host_type)]); - let (content_type, payload) = publication_body(&module_schema, program_bytes, environment.values)?; + let (content_type, payload) = + publication_body(&module_schema, program_bytes, environment.values, environment_options)?; let res = builder .header(reqwest::header::CONTENT_TYPE, content_type) .body(payload) diff --git a/crates/cli/src/subcommands/publish/environment.rs b/crates/cli/src/subcommands/publish/environment.rs index 4427ae52a65..626c909ddc7 100644 --- a/crates/cli/src/subcommands/publish/environment.rs +++ b/crates/cli/src/subcommands/publish/environment.rs @@ -1,9 +1,9 @@ -//! Resolve a complete, declared environment without consulting stored values. +//! Resolve explicit overrides without fetching stored secrets. use std::collections::BTreeMap; use std::ffi::OsString; pub(super) use crate::schema_extract::{inspect, read_program}; -use anyhow::{ensure, Context}; +use anyhow::Context; use serde_json::Value; use spacetimedb_lib::environment::EnvironmentSchema; @@ -49,10 +49,7 @@ pub(super) fn resolve( if let Some(config) = config { let config = config.as_object().context("Environment config must be an object")?; for (name, value) in config { - ensure!( - schema.get(name).is_some(), - "Environment key {name:?}: key is not declared" - ); + spacetimedb_lib::environment::validate_key(name)?; let value = match value { Value::String(value) => value.clone(), Value::Bool(value) => value.to_string(), @@ -73,9 +70,88 @@ pub(super) fn resolve( resolved.sources.insert(declaration.name.clone(), Source::Shell); } } - schema.validate_values(&resolved.values)?; + schema.validate_supplied_values(&resolved.values)?; Ok(resolved) } #[cfg(test)] mod tests; + +/// Update configuration without inspecting or building a local module. +pub(super) async fn publish_only( + config: &mut crate::config::Config, + server: Option<&str>, + database: &str, + anonymous: bool, + yes: super::YesFlags, + input: Option<&Value>, + options: &super::EnvironmentOptions, +) -> anyhow::Result<()> { + use crate::util::{add_auth_header_opt, get_auth_header, y_or_n}; + use spacetimedb_client_api_messages::publish::{EnvironmentMetadata, PublishRequest, CONTENT_TYPE}; + + let host = config.get_host_url(server)?; + let server_url = reqwest::Url::parse(&host)?; + let hostname = server_url.host_str().context("Server URL has no hostname")?; + if hostname != "localhost" && hostname != "127.0.0.1" { + println!("You are about to publish environment values to a non-local server: {hostname}"); + anyhow::ensure!( + y_or_n(yes.publish_to_remote, "Are you sure you want to proceed?")?, + "Publish aborted by user" + ); + } + let auth = get_auth_header(config, anonymous, server, !yes.skip_login).await?; + let encoded = percent_encoding::percent_encode( + database.as_bytes(), + const { &percent_encoding::NON_ALPHANUMERIC.remove(b'_').remove(b'-') }, + ) + .to_string(); + let url = format!("{host}/v1/database/{encoded}"); + // Neither credentials nor publish bodies may be forwarded to redirect destinations. + let client = reqwest::Client::builder() + .redirect(reqwest::redirect::Policy::none()) + .build()?; + let response = add_auth_header_opt(client.get(format!("{url}/environment")), &auth) + .send() + .await?; + anyhow::ensure!( + response.status().is_success(), + "Cannot read environment schema: HTTP {}", + response.status() + ); + let metadata: EnvironmentMetadata = response.json().await.context("Invalid environment metadata")?; + let schema = EnvironmentSchema::new(metadata.declarations)?; + let resolved = resolve(&schema, input, |key| std::env::var_os(key))?; + options.validate_values(&resolved.values)?; + print!("{}", resolved.display()); + let request = PublishRequest { + module: None, + environment: resolved.values, + environment_remove: options.remove.clone(), + environment_replace: options.replace, + expected_module_version: Some(metadata.module_version), + }; + let response = add_auth_header_opt(client.put(url), &auth) + .header(reqwest::header::CONTENT_TYPE, CONTENT_TYPE) + .body(request.encode()?) + .send() + .await?; + anyhow::ensure!( + response.status().is_success(), + "Environment publish failed with HTTP {}", + response.status() + ); + match response + .json::() + .await + .map_err(|_| anyhow::anyhow!("Invalid publish response"))? + { + spacetimedb_client_api_messages::name::PublishResult::Success { database_identity, .. } => { + println!("Updated environment for database {database_identity}"); + Ok(()) + } + spacetimedb_client_api_messages::name::PublishResult::PermissionDenied { .. } => { + anyhow::bail!("Permission denied publishing environment values") + } + } +} diff --git a/crates/cli/src/subcommands/publish/environment/tests.rs b/crates/cli/src/subcommands/publish/environment/tests.rs index 7ed2bca6f2e..41e9cdf130e 100644 --- a/crates/cli/src/subcommands/publish/environment/tests.rs +++ b/crates/cli/src/subcommands/publish/environment/tests.rs @@ -29,7 +29,7 @@ fn schema() -> EnvironmentSchema { } #[test] -fn declared_shell_overrides_are_complete_and_redacted() { +fn declared_shell_overrides_are_redacted() { let mut checked = Vec::new(); let resolved = resolve( &schema(), @@ -64,7 +64,7 @@ fn declared_shell_overrides_are_complete_and_redacted() { } #[test] -fn missing_required_never_reuses_old_values_and_optional_disappears() { +fn omitted_inputs_are_left_for_host_to_resolve() { let config = serde_json::json!({"A":"first","B":true,"C":"first","OPTIONAL":"old"}); let first = resolve(&schema(), Some(&config), |_| None).unwrap(); assert!(first.values.contains_key("OPTIONAL")); @@ -75,10 +75,9 @@ fn missing_required_never_reuses_old_values_and_optional_disappears() { ) .unwrap(); assert!(!second.values.contains_key("OPTIONAL")); - let error = resolve(&schema(), Some(&serde_json::json!({"A":"first","B":true})), |_| None) - .err() - .unwrap(); - assert!(error.to_string().contains('C')); + let partial = resolve(&schema(), Some(&serde_json::json!({"A":"first","B":true})), |_| None).unwrap(); + assert!(!partial.values.contains_key("C")); + assert!(resolve(&schema(), None, |_| None).unwrap().values.is_empty()); } #[test] @@ -97,12 +96,15 @@ fn invalid_inputs_fail_without_values_or_lower_priority_fallback() { let error = format!("{error:#}"); assert!(error.contains('B')); assert!(!error.contains("invalid-shell-secret") && !error.contains("private-sentinel")); - let error = resolve(&schema(), Some(&serde_json::json!({"UNDECLARED":"secret"})), |_| { - panic!("must fail first") + let mut looked_up = Vec::new(); + let resolved = resolve(&schema(), Some(&serde_json::json!({"UNDECLARED":"secret"})), |key| { + looked_up.push(key.to_owned()); + None }) - .err() .unwrap(); - assert!(error.to_string().contains("UNDECLARED")); + assert_eq!(resolved.values["UNDECLARED"], "secret"); + assert!(!looked_up.iter().any(|key| key == "UNDECLARED")); + assert!(!resolved.display().contains("secret")); } #[test] @@ -176,5 +178,26 @@ async fn actual_precompiled_declarations_are_inspected_without_server_or_values( let resolved = resolve(schema, Some(&config), |_| None).unwrap(); assert_eq!(resolved.values.len(), 2); assert!(!resolved.display().contains("generated-local-inspection-sentinel")); - assert!(resolve(schema, None, |_| None).is_err()); + assert!(resolve(schema, None, |_| None).unwrap().values.is_empty()); +} + +#[test] +fn undeclared_inputs_still_obey_storage_limits_and_redact_invalid_keys() { + for input in [ + serde_json::json!({"private-marker\ninvalid":null}), + serde_json::json!({"UNDECLARED":"x".repeat(8193)}), + ] { + let error = resolve(&EnvironmentSchema::default(), Some(&input), |_| { + panic!("no declared shell lookup") + }) + .err() + .unwrap(); + assert!(!format!("{error:#}").contains("private-marker")); + } + let input = Value::Object( + (0..257) + .map(|i| (format!("K{i}"), Value::String(String::new()))) + .collect(), + ); + assert!(resolve(&EnvironmentSchema::default(), Some(&input), |_| None).is_err()); } diff --git a/crates/cli/src/subcommands/publish/wire_tests.rs b/crates/cli/src/subcommands/publish/wire_tests.rs index 70781417800..4708bffb081 100644 --- a/crates/cli/src/subcommands/publish/wire_tests.rs +++ b/crates/cli/src/subcommands/publish/wire_tests.rs @@ -17,25 +17,40 @@ fn schema(declared: bool) -> ModuleDef { #[test] fn ordinary_and_explicit_empty_declarations_choose_distinct_wire_formats() { let bytes = b"exact selected module bytes\0\xff".to_vec(); - let (kind, body) = publication_body(&schema(false), bytes.clone(), BTreeMap::new()).unwrap(); + let (kind, body) = publication_body( + &schema(false), + bytes.clone(), + BTreeMap::new(), + &EnvironmentOptions::default(), + ) + .unwrap(); assert_eq!(kind, "application/octet-stream"); assert_eq!(body, bytes); - let (kind, body) = publication_body(&schema(true), bytes.clone(), BTreeMap::new()).unwrap(); + let (kind, body) = publication_body( + &schema(true), + bytes.clone(), + BTreeMap::new(), + &EnvironmentOptions::default(), + ) + .unwrap(); assert_eq!(kind, spacetimedb_client_api_messages::publish::CONTENT_TYPE); let envelope = spacetimedb_client_api_messages::publish::PublishRequest::decode(&body).unwrap(); - assert_eq!(envelope.module, bytes); + assert_eq!(envelope.module, Some(bytes.clone())); assert!(envelope.environment.is_empty()); - let error = publication_body( + let (kind, body) = publication_body( &schema(false), bytes, BTreeMap::from([("KEY".into(), "secret-sentinel".into())]), + &EnvironmentOptions::default(), ) - .unwrap_err(); - assert!(!format!("{error:#}").contains("secret-sentinel")); + .unwrap(); + assert_eq!(kind, spacetimedb_client_api_messages::publish::CONTENT_TYPE); + let envelope = spacetimedb_client_api_messages::publish::PublishRequest::decode(&body).unwrap(); + assert_eq!(envelope.environment["KEY"], "secret-sentinel"); } #[test] -fn short_help_is_concise_and_long_help_explains_environment_replacement() { +fn short_help_is_concise_and_long_help_explains_environment_modes() { let short = cli().render_help().to_string(); let long = cli() .render_long_help() @@ -46,11 +61,56 @@ fn short_help_is_concise_and_long_help_explains_environment_replacement() { assert!(!short.contains("Every publish replaces")); assert!(short.contains("spacetime help publish")); for text in [ - "Every publish replaces the complete declared environment", + "Publishing preserves unspecified environment values", "including empty strings", - "Optional values omitted", + "--replace-env replaces all stored values", "--env selects config file layers", ] { assert!(long.contains(text), "missing long-help guidance: {text}"); } } + +#[test] +fn environment_flags_are_explicit_and_incompatible_modes_are_rejected() { + for args in [ + vec!["publish", "db", "--replace-env", "--unset-env", "KEY"], + vec!["publish", "db", "--env-only", "--bin-path", "module.wasm"], + vec!["publish", "db", "--env-only", "--clear-database"], + ] { + assert!(cli().try_get_matches_from(args).is_err()); + } + let args = cli() + .try_get_matches_from(["publish", "db", "--env-only", "--unset-env", "A", "--unset-env", "B"]) + .unwrap(); + let options = EnvironmentOptions::from_args(&args).unwrap(); + assert!(options.only); + assert_eq!(options.remove, ["A", "B"]); + assert!(options + .validate_values(&BTreeMap::from([("A".into(), "secret-sentinel".into())])) + .is_err()); + let args = cli() + .try_get_matches_from(["publish", "db", "--env-only", "--replace-env"]) + .unwrap(); + let options = EnvironmentOptions::from_args(&args).unwrap(); + assert!(options.only && options.replace); +} + +#[test] +fn removal_and_replacement_use_json_even_for_legacy_modules() { + for options in [ + EnvironmentOptions { + remove: vec!["OLD".into()], + ..Default::default() + }, + EnvironmentOptions { + replace: true, + ..Default::default() + }, + ] { + let (kind, body) = publication_body(&schema(false), vec![1], BTreeMap::new(), &options).unwrap(); + assert_eq!(kind, spacetimedb_client_api_messages::publish::CONTENT_TYPE); + let request = spacetimedb_client_api_messages::publish::PublishRequest::decode(&body).unwrap(); + assert_eq!(request.environment_remove, options.remove); + assert_eq!(request.environment_replace, options.replace); + } +} diff --git a/crates/client-api-messages/src/publish.rs b/crates/client-api-messages/src/publish.rs index fea29732139..c77a21d56a8 100644 --- a/crates/client-api-messages/src/publish.rs +++ b/crates/client-api-messages/src/publish.rs @@ -1,4 +1,4 @@ -//! Complete publish input. Environment values travel only in the request body. +//! Atomic publish input. Environment values travel only in the request body. use serde::{Deserialize, Deserializer, Serialize}; use serde_with::{base64::Base64, serde_as}; use spacetimedb_lib::environment::{validate_key, validate_value, MAX_ENV_VARS}; @@ -12,13 +12,28 @@ pub const MAX_REQUEST_BYTES: usize = 192 * 1024 * 1024; /// Values deliberately have no Debug representation. Omission is an empty map, /// including for a publish of an unchanged module. #[serde_as] -#[derive(Clone, Serialize, Deserialize)] +#[derive(Clone, Default, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct PublishRequest { - #[serde_as(as = "Base64")] - pub module: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[serde_as(as = "Option")] + pub module: Option>, #[serde(default, deserialize_with = "deserialize_environment")] pub environment: BTreeMap, + #[serde(default)] + pub environment_remove: Vec, + #[serde(default)] + pub environment_replace: bool, + #[serde(default)] + pub expected_module_version: Option, +} + +/// Authorized environment metadata. Values never leave the database in this response. +#[derive(Clone, Serialize, Deserialize)] +pub struct EnvironmentMetadata { + pub module_version: String, + pub declarations: Vec, + pub stored_keys: Vec, } #[derive(Debug, Clone, Copy, thiserror::Error)] @@ -46,9 +61,28 @@ impl PublishRequest { } fn validate(&self) -> Result<(), PublishRequestError> { - if self.module.len() > MAX_MODULE_BYTES || self.environment.len() > MAX_ENV_VARS { + if self + .module + .as_ref() + .is_some_and(|module| module.len() > MAX_MODULE_BYTES) + || self.environment.len() > MAX_ENV_VARS + { return Err(PublishRequestError::TooLarge); } + spacetimedb_lib::environment::EnvironmentUpdate { + values: self.environment.clone(), + remove: self.environment_remove.clone(), + replace: self.environment_replace, + } + .validate() + .map_err(|_| PublishRequestError::Invalid)?; + if self + .expected_module_version + .as_ref() + .is_some_and(|hash| hash.len() != 64 || !hash.bytes().all(|byte| byte.is_ascii_hexdigit())) + { + return Err(PublishRequestError::Invalid); + } for (key, value) in &self.environment { validate_key(key).map_err(|_| PublishRequestError::Invalid)?; validate_value(value).map_err(|_| PublishRequestError::TooLarge)?; @@ -62,7 +96,7 @@ fn deserialize_environment<'de, D: Deserializer<'de>>(de: D) -> Result serde::de::Visitor<'de> for Visitor { type Value = BTreeMap; fn expecting(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.write_str("a complete map of environment strings") + f.write_str("a map of supplied environment strings") } fn visit_map>(self, mut map: A) -> Result { use serde::de::Error; @@ -89,8 +123,9 @@ mod tests { #[test] fn roundtrip_and_omission_preserve_complete_string_input() { let request = PublishRequest { - module: vec![0, 1, 255], + module: Some(vec![0, 1, 255]), environment: BTreeMap::from([("EMPTY".into(), "".into()), ("TOKEN".into(), "雪\0false".into())]), + ..Default::default() }; let decoded = PublishRequest::decode(&request.encode().unwrap()).unwrap(); assert_eq!(decoded.module, request.module); @@ -100,6 +135,33 @@ mod tests { .environment .is_empty()); } + #[test] + fn environment_only_mutation_roundtrips_and_rejects_conflicting_operations() { + let request = PublishRequest { + environment: BTreeMap::from([("FUTURE".into(), "secret-marker".into())]), + environment_remove: vec!["OPTIONAL".into()], + expected_module_version: Some("ab".repeat(32)), + ..Default::default() + }; + let bytes = request.encode().unwrap(); + assert!(serde_json::from_slice::(&bytes) + .unwrap() + .get("module") + .is_none()); + let decoded = PublishRequest::decode(&bytes).unwrap(); + assert_eq!(decoded.environment_remove, request.environment_remove); + assert_eq!(decoded.expected_module_version, request.expected_module_version); + for body in [ + r#"{"environment_replace":true,"environment_remove":["KEY"]}"#, + r#"{"environment":{"KEY":"secret-marker"},"environment_remove":["KEY"]}"#, + r#"{"environment_remove":["KEY","KEY"]}"#, + r#"{"expected_module_version":"secret-marker"}"#, + ] { + let error = PublishRequest::decode(body.as_bytes()).err().expect("must reject"); + assert!(!error.to_string().contains("secret-marker")); + } + } + #[test] fn malformed_inputs_and_duplicate_keys_are_rejected_without_values() { for body in [ diff --git a/crates/client-api/src/lib.rs b/crates/client-api/src/lib.rs index 7d3175e751b..97ef8a286d2 100644 --- a/crates/client-api/src/lib.rs +++ b/crates/client-api/src/lib.rs @@ -207,6 +207,39 @@ impl Host { .await } + pub async fn with_publication_lock(&self, operation: F) -> anyhow::Result + where + T: Send + 'static, + F: FnOnce(ModuleHost) -> Fut + Send + 'static, + Fut: std::future::Future> + Send + 'static, + { + self.host_controller + .with_publication_lock(self.replica_id, operation) + .await + } + + pub async fn update_with_environment_options( + &self, + database: Database, + host_type: HostType, + program_bytes: Box<[u8]>, + policy: MigrationPolicy, + environment: spacetimedb_lib::environment::EnvironmentUpdate, + expected_module_version: Option, + ) -> anyhow::Result { + self.host_controller + .update_module_host_with_environment_options( + database, + host_type, + self.replica_id, + program_bytes, + policy, + environment, + expected_module_version, + ) + .await + } + pub async fn update_with_environment( &self, database: Database, @@ -235,8 +268,11 @@ pub struct DatabaseDef { pub database_identity: Identity, /// The compiled program of the database module. pub program_bytes: Bytes, - /// Complete publish input, never persisted in the public Database record. + /// Supplied overrides, never persisted in the public Database record. pub environment: std::collections::BTreeMap, + pub environment_remove: Vec, + pub environment_replace: bool, + pub expected_module_version: Option, /// The desired number of replicas the database shall have. /// /// If `None`, the edition default is used. @@ -255,6 +291,8 @@ pub struct DatabaseResetDef { pub database_identity: Identity, pub program_bytes: Option, pub environment: std::collections::BTreeMap, + pub environment_remove: Vec, + pub environment_replace: bool, pub num_replicas: Option, pub host_type: Option, } diff --git a/crates/client-api/src/routes/database.rs b/crates/client-api/src/routes/database.rs index 5d98a065976..e6bf150acde 100644 --- a/crates/client-api/src/routes/database.rs +++ b/crates/client-api/src/routes/database.rs @@ -579,6 +579,83 @@ impl From for DatabaseResponse { } } +fn environment_validation_error(error: &anyhow::Error) -> Option { + use spacetimedb::db::environment::EnvironmentError; + use spacetimedb::host::module_host::InitDatabaseError; + use spacetimedb_lib::environment::{validate_key, EnvironmentSchemaError, EnvironmentSchemaErrorKind}; + if let Some(InitDatabaseError::Other(error)) = error.downcast_ref::() { + return environment_validation_error(error); + } + let error = + error + .downcast_ref::() + .or_else(|| match error.downcast_ref::() { + Some(EnvironmentError::Schema(error)) => Some(error), + _ => None, + })?; + // Only typed host validation can ask the caller for a secret. Never infer + // missing keys from module failures or arbitrary diagnostic text. + if error.kind == EnvironmentSchemaErrorKind::MissingRequired + && let Some(key) = error.key.as_deref().filter(|key| validate_key(key).is_ok()) + { + return Some( + ( + StatusCode::BAD_REQUEST, + axum::Json(serde_json::json!({ + "error": "missing_required_environment", + "key": key, + })), + ) + .into(), + ); + } + Some((StatusCode::BAD_REQUEST, error.to_string()).into()) +} + +fn publish_error(error: anyhow::Error) -> axum::response::ErrorResponse { + if let Some(response) = environment_validation_error(&error) { + return response; + } + if let Some(error) = error.downcast_ref::() { + return (StatusCode::CONFLICT, error.to_string()).into(); + } + log_and_500(error) +} + +fn publish_migration_error(error: anyhow::Error) -> axum::response::ErrorResponse { + environment_validation_error(&error) + .unwrap_or_else(|| bad_request(format!("Failed to create or update the database: {error}").into())) +} + +pub async fn environment_metadata( + State(ctx): State, + Extension(ResolvedDatabase(database)): Extension, + Extension(auth): Extension, +) -> axum::response::Result +where + S: ControlStateDelegate + NodeDelegate + Authorization, +{ + ctx.authorize_action(auth.claims.identity, database.database_identity, Action::UpdateDatabase) + .await?; + let leader = find_database_leader(&ctx, &database).await?; + let metadata = leader + .with_publication_lock(|module| async move { + let stored_keys = module + .relational_db() + .with_read_only(spacetimedb_datastore::execution_context::Workload::Internal, |tx| { + spacetimedb::db::environment::snapshot(tx).map(|values| values.into_keys().collect()) + })?; + Ok(spacetimedb_client_api_messages::publish::EnvironmentMetadata { + module_version: module.info.module_hash.to_string(), + declarations: module.info.module_def.environment().declarations().cloned().collect(), + stored_keys, + }) + }) + .await + .map_err(log_and_500)?; + Ok(([(http::header::CACHE_CONTROL, "no-store")], axum::Json(metadata))) +} + pub async fn db_info( Extension(ResolvedDatabase(database)): Extension, ) -> axum::response::Result { @@ -841,8 +918,18 @@ pub async fn reset( PublishBody { program_bytes, environment, + environment_remove, + environment_replace, + expected_module_version, + environment_only, }: PublishBody, ) -> axum::response::Result> { + if expected_module_version.is_some() { + return Err(bad_request( + "expected_module_version is not supported for database reset".into(), + )); + } + let _ = environment_only; let database_identity = database.database_identity; ctx.authorize_action(auth.claims.identity, database.database_identity, Action::ResetDatabase) @@ -863,12 +950,14 @@ pub async fn reset( database_identity, program_bytes, environment, + environment_remove, + environment_replace, num_replicas, host_type: Some(host_type), }, ) .await - .map_err(log_and_500)?; + .map_err(publish_error)?; Ok(axum::Json(PublishResult::Success { domain: name_or_identity.name().cloned(), @@ -943,8 +1032,27 @@ pub async fn publish( PublishBody { program_bytes, environment, + environment_remove, + environment_replace, + expected_module_version, + environment_only, }: PublishBody, ) -> axum::response::Result> { + if environment_only && (clear || parent.is_some() || organization.is_some() || num_replicas.is_some()) { + return Err(bad_request( + "environment-only publication cannot change database configuration or reset data".into(), + )); + } + if environment_only && expected_module_version.is_none() { + return Err(bad_request( + "environment-only publication requires expected_module_version".into(), + )); + } + if environment_only && name_or_identity.is_none() { + return Err(bad_request( + "environment-only publication requires an existing database".into(), + )); + } // If `clear`, check that the database exists and delegate to `reset`. // If it doesn't exist, ignore the `clear` parameter. // TODO: Replace with actual redirect at the next possible version bump. @@ -976,6 +1084,10 @@ pub async fn publish( PublishBody { program_bytes, environment, + environment_remove, + environment_replace, + expected_module_version, + environment_only, }, ) .await; @@ -984,7 +1096,12 @@ pub async fn publish( } let program_bytes = program_bytes.unwrap_or_default(); - let (database_identity, db_name) = get_or_create_identity_and_name(&ctx, &auth, name_or_identity.as_ref()).await?; + let (database_identity, db_name) = if environment_only { + let name = name_or_identity.as_ref().expect("validated existing database name"); + (name.resolve(&ctx).await?, name.name()) + } else { + get_or_create_identity_and_name(&ctx, &auth, name_or_identity.as_ref()).await? + }; let maybe_parent_database_identity = match parent.as_ref() { None => None, Some(parent) => parent.resolve(&ctx).await.map(Some)?, @@ -1004,6 +1121,13 @@ pub async fn publish( .get_database_by_identity(&database_identity) .await .map_err(log_and_500)?; + if environment_only && existing.is_none() { + return Err(( + StatusCode::NOT_FOUND, + "environment-only publication requires an existing database", + ) + .into()); + } match existing.as_ref() { None => { allow_creation(&auth)?; @@ -1048,6 +1172,9 @@ pub async fn publish( database_identity, program_bytes, environment, + environment_remove, + environment_replace, + expected_module_version, num_replicas, host_type, parent, @@ -1056,7 +1183,7 @@ pub async fn publish( schema_migration_policy, ) .await - .map_err(log_and_500)?; + .map_err(publish_error)?; let success = || { axum::Json(PublishResult::Success { @@ -1069,9 +1196,7 @@ pub async fn publish( Some(UpdateDatabaseResult::AutoMigrateError(errs)) => { Err(bad_request(format!("Database update rejected: {errs}").into())) } - Some(UpdateDatabaseResult::ErrorExecutingMigration(err)) => Err(bad_request( - format!("Failed to create or update the database: {err}").into(), - )), + Some(UpdateDatabaseResult::ErrorExecutingMigration(err)) => Err(publish_migration_error(err)), None | Some(UpdateDatabaseResult::NoUpdateNeeded) => Ok(success()), Some( UpdateDatabaseResult::UpdatePerformed { @@ -1252,6 +1377,9 @@ pub async fn pre_publish database_identity, program_bytes, environment: Default::default(), + environment_remove: Default::default(), + environment_replace: false, + expected_module_version: None, num_replicas: None, host_type, parent: None, @@ -1491,6 +1619,7 @@ pub struct DatabaseRoutes { pub call_reducer_procedure_post: MethodRouter, /// GET: /database/:name_or_identity/schema pub schema_get: MethodRouter, + pub environment_get: MethodRouter, /// GET: /database/:name_or_identity/logs pub logs_get: MethodRouter, /// POST: /database/:name_or_identity/sql @@ -1533,6 +1662,7 @@ where subscribe_get: get(handle_websocket::), call_reducer_procedure_post: post(call::), schema_get: get(schema::), + environment_get: get(environment_metadata::), logs_get: get(logs::), sql_post: post(sql::), mcp_post: post(crate::routes::mcp::mcp::), @@ -1565,6 +1695,7 @@ where .route("/names", self.names_put) .route("/call/:reducer", self.call_reducer_procedure_post) .route("/schema", self.schema_get) + .route("/environment", self.environment_get) .route("/logs", self.logs_get) .route("/sql", self.sql_post) .route("/mcp", self.mcp_post) @@ -1720,6 +1851,59 @@ mod tests { use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use tower::util::ServiceExt; + #[tokio::test] + async fn publish_environment_error_identifies_only_typed_missing_required_keys() { + use spacetimedb_lib::environment::{EnvironmentSchemaError, EnvironmentSchemaErrorKind}; + let missing = || EnvironmentSchemaError { + key: Some("API_KEY".into()), + kind: EnvironmentSchemaErrorKind::MissingRequired, + }; + for response in [ + publish_error(anyhow::Error::new(missing()).context("publication failed")), + publish_migration_error(spacetimedb::db::environment::EnvironmentError::Schema(missing()).into()), + publish_error( + spacetimedb::host::module_host::InitDatabaseError::Other( + spacetimedb::db::environment::EnvironmentError::Schema(missing()).into(), + ) + .into(), + ), + publish_migration_error( + spacetimedb::host::module_host::InitDatabaseError::Other( + spacetimedb::db::environment::EnvironmentError::Schema(missing()).into(), + ) + .into(), + ), + ] { + let response = Err::<(), _>(response).into_response(); + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + assert_eq!(response.headers()[http::header::CONTENT_TYPE], "application/json"); + let body = axum::body::to_bytes(response.into_body(), 1024).await.unwrap(); + assert_eq!( + serde_json::from_slice::(&body).unwrap(), + serde_json::json!({ + "error": "missing_required_environment", "key": "API_KEY", + }) + ); + } + for error in [ + anyhow::anyhow!("environment key API_KEY: required value is missing"), + EnvironmentSchemaError { + key: Some("API_KEY".into()), + kind: EnvironmentSchemaErrorKind::ConstraintMismatch, + } + .into(), + EnvironmentSchemaError { + key: Some("INVALID-KEY".into()), + kind: EnvironmentSchemaErrorKind::MissingRequired, + } + .into(), + ] { + let response = Err::<(), _>(publish_migration_error(error)).into_response(); + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + assert_ne!(response.headers()[http::header::CONTENT_TYPE], "application/json"); + } + } + #[derive(Clone, Default)] struct DummyValidator; diff --git a/crates/client-api/src/routes/database/publish_environment.rs b/crates/client-api/src/routes/database/publish_environment.rs index 0287d226688..86d8d30f9cb 100644 --- a/crates/client-api/src/routes/database/publish_environment.rs +++ b/crates/client-api/src/routes/database/publish_environment.rs @@ -9,6 +9,10 @@ use std::collections::BTreeMap; pub struct PublishBody { pub program_bytes: Option, pub environment: BTreeMap, + pub environment_remove: Vec, + pub environment_replace: bool, + pub expected_module_version: Option, + pub environment_only: bool, } async fn bounded_body(request: Request, limit: usize) -> Result { @@ -47,15 +51,31 @@ impl FromRequest for PublishBody { if envelope { let request = PublishRequest::decode(&bytes) .map_err(|_| (StatusCode::BAD_REQUEST, "invalid publish request body").into_response())?; + if request.module.as_ref().is_some_and(|module| module.is_empty()) { + return Err((StatusCode::BAD_REQUEST, "module artifact must not be empty").into_response()); + } + let environment_only = request.module.is_none(); Ok(Self { - program_bytes: (!request.module.is_empty()).then_some(request.module.into()), + program_bytes: request.module.map(Into::into), environment: request.environment, + environment_remove: request.environment_remove, + environment_replace: request.environment_replace, + expected_module_version: request + .expected_module_version + .map(spacetimedb_lib::Hash::from_hex) + .transpose() + .map_err(|_| (StatusCode::BAD_REQUEST, "invalid expected module version").into_response())?, + environment_only, }) } else { - // An absent reset body retains the program, but never retains env values. + // An empty legacy reset body retains the program; reset clears all data. Ok(Self { program_bytes: (!bytes.is_empty()).then_some(bytes), environment: BTreeMap::new(), + environment_remove: Vec::new(), + environment_replace: false, + expected_module_version: None, + environment_only: false, }) } } @@ -90,8 +110,9 @@ mod tests { assert!(empty.program_bytes.is_none()); assert!(empty.environment.is_empty()); let input = PublishRequest { - module: vec![1, 2, 3], + module: Some(vec![1, 2, 3]), environment: BTreeMap::from([("TOKEN".into(), "雪\0".into())]), + ..Default::default() }; let request = Request::builder() .header(header::CONTENT_TYPE, CONTENT_TYPE) @@ -99,10 +120,11 @@ mod tests { .unwrap(); let extracted = PublishBody::from_request(request, &()).await.unwrap(); assert_eq!(extracted.environment, input.environment); - assert_eq!(extracted.program_bytes.unwrap(), input.module); + assert_eq!(extracted.program_bytes.unwrap(), input.module.unwrap()); let reset = PublishRequest { - module: vec![], + module: None, environment: input.environment, + ..Default::default() }; let request = Request::builder() .header(header::CONTENT_TYPE, CONTENT_TYPE) @@ -113,6 +135,26 @@ mod tests { assert_eq!(extracted.environment, reset.environment); } + #[tokio::test] + async fn empty_artifact_is_rejected_but_omission_is_environment_only() { + for (body, accepted) in [(r#"{"module":""}"#, false), (r#"{}"#, true)] { + let request = Request::builder() + .header(header::CONTENT_TYPE, CONTENT_TYPE) + .body(Body::from(body)) + .unwrap(); + match PublishBody::from_request(request, &()).await { + Ok(body) => { + assert!(accepted); + assert!(body.environment_only); + } + Err(error) => { + assert!(!accepted); + assert_eq!(error.status(), StatusCode::BAD_REQUEST); + } + } + } + } + #[tokio::test] async fn streamed_limits_apply_without_global_body_limit_and_errors_are_redacted() { let stream = futures::stream::iter([ diff --git a/crates/core/src/db/environment.rs b/crates/core/src/db/environment.rs index 10718637d13..a8492d227f1 100644 --- a/crates/core/src/db/environment.rs +++ b/crates/core/src/db/environment.rs @@ -140,7 +140,7 @@ mod tests { BTreeMap::new(), BTreeMap::from([ ("REQUIRED".into(), "new".into()), - ("UNKNOWN".into(), "secret-marker".into()), + ("INVALID-NAME".into(), "secret-marker".into()), ]), BTreeMap::from([("REQUIRED".into(), "x".repeat(8193))]), ] { diff --git a/crates/core/src/host/host_controller.rs b/crates/core/src/host/host_controller.rs index 0226230fcf2..9bfe02498f1 100644 --- a/crates/core/src/host/host_controller.rs +++ b/crates/core/src/host/host_controller.rs @@ -92,6 +92,10 @@ where pub type ProgramStorage = Arc; +#[derive(Debug, thiserror::Error)] +#[error("database program changed before publication; reload environment metadata and retry")] +pub struct EnvironmentVersionConflict; + /// Private complete configuration for a not-yet-initialized database generation. /// Implementations must verify the exact persisted database identity, program and /// bootstrap generation. This source is never consulted during ordinary reopen. @@ -609,6 +613,35 @@ impl HostController { policy: MigrationPolicy, environment: std::collections::BTreeMap, ) -> anyhow::Result { + self.update_module_host_with_environment_options( + database, + host_type, + replica_id, + program_bytes, + policy, + environment.into(), + None, + ) + .await + } + + #[allow(clippy::too_many_arguments)] + pub async fn update_module_host_with_environment_options( + &self, + database: Database, + host_type: HostType, + replica_id: u64, + program_bytes: Box<[u8]>, + policy: MigrationPolicy, + environment: spacetimedb_lib::environment::EnvironmentUpdate, + expected_module_version: Option, + ) -> anyhow::Result { + environment.validate()?; + let environment_only = program_bytes.is_empty(); + anyhow::ensure!( + !environment_only || expected_module_version.is_some(), + "environment-only publication requires expected_module_version" + ); let program = Program::from_bytes(host_type.into(), program_bytes); trace!( "update module host {}/{}: genesis={} update-to={}", @@ -652,8 +685,28 @@ impl HostController { host } }; - let update_result = host - .update_module( + let update_result = async { + let module = host.module.borrow().clone(); + if let Some(expected) = expected_module_version + && module.info.module_hash != expected + { + return Err(EnvironmentVersionConflict.into()); + } + let previous = host + .replica_ctx + .relational_db() + .with_read_only(Workload::Internal, |tx| crate::db::environment::snapshot(tx))?; + let environment = environment.resulting_values(&previous)?; + if environment_only || program.hash == module.info.module_hash { + let program = module + .relational_db() + .program()? + .context("database program is not initialized")?; + return module + .update_database_with_environment(program, module.info.clone(), policy, environment) + .await; + } + host.update_module( this.runtimes.clone(), program, policy, @@ -662,7 +715,9 @@ impl HostController { this.db_cores.take(), environment, ) - .await; + .await + } + .await; // Rejected publication leaves the existing host usable. Restore it // before propagating validation or migration failure to the caller. @@ -792,6 +847,26 @@ impl HostController { .ok_or(NoSuchModule) } + /// Run a publication operation while retaining the controller write lock. + /// Cancellation of the caller cannot release the lock before the operation finishes. + pub async fn with_publication_lock(&self, replica_id: u64, operation: F) -> anyhow::Result + where + T: Send + 'static, + F: FnOnce(ModuleHost) -> Fut + Send + 'static, + Fut: std::future::Future> + Send + 'static, + { + let guard = self + .acquire_write_lock(replica_id) + .await + .map_err(|_| anyhow::anyhow!("unable to lock database for publication"))?; + let module = guard.as_ref().ok_or(NoSuchModule)?.module.borrow().clone(); + tokio::spawn(async move { + let _guard = guard; + operation(module).await + }) + .await? + } + /// Subscribe to updates of the [`ModuleHost`] identified by `replica_id`, /// or return an error if it is not registered with the controller. /// diff --git a/crates/core/src/host/mod.rs b/crates/core/src/host/mod.rs index d0a1f23c560..01e4dee6309 100644 --- a/crates/core/src/host/mod.rs +++ b/crates/core/src/host/mod.rs @@ -25,9 +25,10 @@ mod wasm_common; pub use disk_storage::DiskStorage; pub use host_controller::{ - extract_schema, BootstrapCompletion, CallProcedureReturn, CallResult, ExternalDurability, ExternalStorage, - HostController, HostRuntimeConfig, InitialEnvironmentSource, MigratePlanResult, ModuleHostWithBootstrap, - ProcedureCallResult, ProgramStorage, ReducerCallResult, ReducerCallResultWithTxOffset, ReducerOutcome, + extract_schema, BootstrapCompletion, CallProcedureReturn, CallResult, EnvironmentVersionConflict, + ExternalDurability, ExternalStorage, HostController, HostRuntimeConfig, InitialEnvironmentSource, + MigratePlanResult, ModuleHostWithBootstrap, ProcedureCallResult, ProgramStorage, ReducerCallResult, + ReducerCallResultWithTxOffset, ReducerOutcome, }; pub use module_host::{ InitDatabaseResult, ModuleHost, NoSuchModule, ProcedureCallError, ReducerCallError, UpdateDatabaseResult, diff --git a/crates/core/src/host/wasm_common/module_host_actor.rs b/crates/core/src/host/wasm_common/module_host_actor.rs index 7abc0aaf186..519b098a017 100644 --- a/crates/core/src/host/wasm_common/module_host_actor.rs +++ b/crates/core/src/host/wasm_common/module_host_actor.rs @@ -703,6 +703,9 @@ impl InstanceCommon { environment: std::collections::BTreeMap, inst: &mut I, ) -> Result { + if program.hash == old_module_info.module_hash { + return self.update_environment(environment, inst); + } let replica_ctx = inst.replica_ctx().clone(); let system_logger = replica_ctx.logger.system_logger(); let stdb = &replica_ctx.relational_db(); @@ -816,6 +819,58 @@ impl InstanceCommon { } } + /// Apply an environment publication using the installed module instance. No + /// initialization, migration, program replacement, or scheduler restart occurs. + fn update_environment( + &mut self, + environment: std::collections::BTreeMap, + inst: &mut I, + ) -> anyhow::Result { + let replica_ctx = inst.replica_ctx().clone(); + let db = replica_ctx.relational_db(); + let tx = db.begin_mut_tx(IsolationLevel::Serializable, Workload::Internal); + let (tx, _) = db.with_auto_rollback(tx, |tx| -> anyhow::Result<()> { + use spacetimedb_datastore::system_tables::{read_hash_from_col, StModuleFields, ST_MODULE_ID}; + let row = tx + .iter(ST_MODULE_ID)? + .next() + .context("database program is not initialized")?; + anyhow::ensure!( + read_hash_from_col(row, StModuleFields::ProgramHash)? == self.info.module_hash, + "database program changed before publication" + ); + crate::db::environment::replace(db, tx, self.info.module_def.environment(), &environment)?; + Ok(()) + })?; + let (out, _, trapped) = self.evaluate_subscribed_views(tx, inst)?; + if trapped || out.outcome != ViewOutcome::Success { + let (_, metrics, reducer) = db.rollback_mut_tx(out.tx); + db.report_mut_tx_metrics(reducer, metrics, None); + return Ok(UpdateDatabaseResult::ErrorExecutingMigration(anyhow::anyhow!( + "view evaluation failed during environment publication" + ))); + } + let event = ModuleEvent { + timestamp: Timestamp::now(), + caller_identity: self.info.owner_identity, + caller_connection_id: None, + function_call: ModuleFunctionCall::update(), + status: EventStatus::Committed(DatabaseUpdate::default()), + reducer_return_value: None, + execution_budget_used: out.execution_budget_used, + host_execution_duration: out.total_duration, + request_id: None, + timer: None, + }; + let durable_offset = db.durable_tx_offset(); + let CommitAndBroadcastEventSuccess { tx_offset, .. } = + commit_and_broadcast_event(&self.info.subscriptions, None, event, out.tx); + Ok(UpdateDatabaseResult::UpdatePerformed { + tx_offset, + durable_offset, + }) + } + /// Re-evaluates all materialized view instances tracked in view lifecycle state. fn evaluate_subscribed_views( &mut self, diff --git a/crates/lib/src/environment.rs b/crates/lib/src/environment.rs index 8b6f6db38d8..75e5730ddcf 100644 --- a/crates/lib/src/environment.rs +++ b/crates/lib/src/environment.rs @@ -66,6 +66,7 @@ mod tests { /// Host-validated string constraints. Values remain strings in every module SDK. #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, crate::SpacetimeType)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] #[sats(crate = crate)] pub enum EnvironmentConstraint { AnyString, @@ -75,6 +76,7 @@ pub enum EnvironmentConstraint { /// Declaration metadata, never an environment value supplied during publishing. #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, crate::SpacetimeType)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] #[sats(crate = crate)] pub struct EnvironmentDeclaration { pub name: String, @@ -92,6 +94,8 @@ pub struct EnvironmentSchema { pub enum EnvironmentSchemaErrorKind { InvalidName, TooManyDeclarations, + TooManyValues, + ConflictingUpdate, DuplicateDeclaration, EmptyUnion, TooManyUnionEntries, @@ -118,6 +122,8 @@ impl std::fmt::Display for EnvironmentSchemaError { f.write_str(match self.kind { EnvironmentSchemaErrorKind::InvalidName => "invalid name", EnvironmentSchemaErrorKind::TooManyDeclarations => "too many declarations", + EnvironmentSchemaErrorKind::TooManyValues => "too many stored values", + EnvironmentSchemaErrorKind::ConflictingUpdate => "conflicting environment update operations", EnvironmentSchemaErrorKind::DuplicateDeclaration => "duplicate declaration", EnvironmentSchemaErrorKind::EmptyUnion => "string union must not be empty", EnvironmentSchemaErrorKind::TooManyUnionEntries => "string union has too many entries", @@ -222,12 +228,28 @@ impl EnvironmentSchema { self.declarations.is_empty() } - /// Validate a complete publish input. Existing stored values are not inputs. + /// Validate the complete resulting store, including required declarations. pub fn validate_values( &self, values: &std::collections::BTreeMap, + ) -> Result<(), EnvironmentSchemaError> { + self.validate_supplied_values(values)?; + self.validate_required(values) + } + + /// Validate supplied values without requiring every required key in this input. + /// Undeclared values are stored strings but are not readable by module code. + pub fn validate_supplied_values( + &self, + values: &std::collections::BTreeMap, ) -> Result<(), EnvironmentSchemaError> { use EnvironmentSchemaErrorKind as Kind; + if values.len() > MAX_ENV_VARS { + return Err(EnvironmentSchemaError { + key: None, + kind: Kind::TooManyValues, + }); + } for (name, value) in values { validate_key(name).map_err(|_| EnvironmentSchemaError { key: None, @@ -237,8 +259,8 @@ impl EnvironmentSchema { key: Some(name.clone()), kind, }; - let declaration = self.get(name).ok_or_else(|| error(Kind::Undeclared))?; validate_value(value).map_err(|_| error(Kind::ValueTooLarge))?; + let Some(declaration) = self.get(name) else { continue }; let matches = match &declaration.constraint { EnvironmentConstraint::AnyString => true, EnvironmentConstraint::Literal(expected) => value == expected, @@ -248,11 +270,18 @@ impl EnvironmentSchema { return Err(error(Kind::ConstraintMismatch)); } } + Ok(()) + } + + fn validate_required( + &self, + values: &std::collections::BTreeMap, + ) -> Result<(), EnvironmentSchemaError> { for declaration in self.declarations() { if !declaration.optional && !values.contains_key(&declaration.name) { return Err(EnvironmentSchemaError { key: Some(declaration.name.clone()), - kind: Kind::MissingRequired, + kind: EnvironmentSchemaErrorKind::MissingRequired, }); } } @@ -260,6 +289,77 @@ impl EnvironmentSchema { } } +/// An environment mutation. Deliberately does not implement Debug because values are secrets. +#[derive(Clone, Default)] +pub struct EnvironmentUpdate { + pub values: std::collections::BTreeMap, + pub remove: Vec, + pub replace: bool, +} + +impl From> for EnvironmentUpdate { + fn from(values: std::collections::BTreeMap) -> Self { + Self { + values, + ..Self::default() + } + } +} + +impl EnvironmentUpdate { + /// Validate operations before any mutation. Diagnostics never include values. + pub fn validate(&self) -> Result<(), EnvironmentSchemaError> { + use EnvironmentSchemaErrorKind as Kind; + EnvironmentSchema::default().validate_supplied_values(&self.values)?; + if self.remove.len() > MAX_ENV_VARS { + return Err(EnvironmentSchemaError { + key: None, + kind: Kind::TooManyValues, + }); + } + if self.replace && !self.remove.is_empty() { + return Err(EnvironmentSchemaError { + key: None, + kind: Kind::ConflictingUpdate, + }); + } + let mut seen = std::collections::BTreeSet::new(); + for key in &self.remove { + validate_key(key).map_err(|_| EnvironmentSchemaError { + key: None, + kind: Kind::InvalidName, + })?; + if self.values.contains_key(key) || !seen.insert(key) { + return Err(EnvironmentSchemaError { + key: Some(key.clone()), + kind: Kind::ConflictingUpdate, + }); + } + } + Ok(()) + } + + /// Resolve the new store without mutating the old store. Schema validation follows + /// against the deployed schema or the schema from the proposed new module. + pub fn resulting_values( + &self, + previous: &std::collections::BTreeMap, + ) -> Result, EnvironmentSchemaError> { + self.validate()?; + let mut values = if self.replace { + Default::default() + } else { + previous.clone() + }; + for key in &self.remove { + values.remove(key); + } + values.extend(self.values.clone()); + EnvironmentSchema::default().validate_supplied_values(&values)?; + Ok(values) + } +} + #[cfg(test)] mod schema_tests { use super::*; @@ -301,10 +401,90 @@ mod schema_tests { ); values.insert("UNDECLARED".into(), "secret-marker".into()); let error = schema.validate_values(&values).unwrap_err(); - assert_eq!(error.kind, EnvironmentSchemaErrorKind::Undeclared); + assert_eq!(error.kind, EnvironmentSchemaErrorKind::MissingRequired); assert!(!format!("{error:?}: {error}").contains("secret-marker")); } + #[test] + fn publishing_preserves_values_and_validates_the_resulting_declared_subset() { + let old = BTreeMap::from([("REQUIRED".into(), "ready".into()), ("UNUSED".into(), "secret".into())]); + let schema = EnvironmentSchema::new(vec![declaration( + "REQUIRED", + EnvironmentConstraint::Literal("ready".into()), + false, + )]) + .unwrap(); + let unchanged = EnvironmentUpdate::default().resulting_values(&old).unwrap(); + assert_eq!(unchanged, old); + schema.validate_values(&unchanged).unwrap(); + let removed = EnvironmentUpdate { + remove: vec!["REQUIRED".into()], + ..Default::default() + } + .resulting_values(&old) + .unwrap(); + assert_eq!( + schema.validate_values(&removed).unwrap_err().kind, + EnvironmentSchemaErrorKind::MissingRequired + ); + let replace = EnvironmentUpdate { + replace: true, + values: BTreeMap::from([("REQUIRED".into(), "ready".into())]), + ..Default::default() + } + .resulting_values(&old) + .unwrap(); + schema.validate_values(&replace).unwrap(); + assert!(!replace.contains_key("UNUSED")); + let newly_declared = EnvironmentSchema::new(vec![declaration( + "UNUSED", + EnvironmentConstraint::Literal("other".into()), + false, + )]) + .unwrap(); + assert_eq!( + newly_declared.validate_values(&old).unwrap_err().kind, + EnvironmentSchemaErrorKind::ConstraintMismatch + ); + let corrected = EnvironmentUpdate::from(BTreeMap::from([("UNUSED".into(), "other".into())])) + .resulting_values(&old) + .unwrap(); + newly_declared.validate_values(&corrected).unwrap(); + assert_eq!(old["UNUSED"], "secret"); + } + + #[test] + fn mutation_conflicts_and_resulting_store_limit_are_rejected() { + let stored = (0..MAX_ENV_VARS).map(|i| (format!("KEY{i}"), String::new())).collect(); + assert_eq!( + EnvironmentUpdate::from(BTreeMap::from([("EXTRA".into(), String::new())])) + .resulting_values(&stored) + .unwrap_err() + .kind, + EnvironmentSchemaErrorKind::TooManyValues + ); + for update in [ + EnvironmentUpdate { + replace: true, + remove: vec!["KEY".into()], + ..Default::default() + }, + EnvironmentUpdate { + values: BTreeMap::from([("KEY".into(), "secret-marker".into())]), + remove: vec!["KEY".into()], + ..Default::default() + }, + EnvironmentUpdate { + remove: vec!["KEY".into(), "KEY".into()], + ..Default::default() + }, + ] { + let error = update.validate().unwrap_err(); + assert_eq!(error.kind, EnvironmentSchemaErrorKind::ConflictingUpdate); + assert!(!error.to_string().contains("secret-marker")); + } + } + #[test] fn declaration_limits_count_absent_optionals_and_reject_invalid_metadata() { let optional = declaration("A", EnvironmentConstraint::AnyString, true); @@ -349,13 +529,9 @@ mod schema_tests { ); } assert!(EnvironmentSchema::default().validate_values(&BTreeMap::new()).is_ok()); - assert_eq!( - EnvironmentSchema::default() - .validate_values(&BTreeMap::from([("A".into(), "".into())])) - .unwrap_err() - .kind, - EnvironmentSchemaErrorKind::Undeclared - ); + EnvironmentSchema::default() + .validate_values(&BTreeMap::from([("A".into(), "".into())])) + .unwrap(); } #[test] diff --git a/crates/smoketests/tests/standalone/cli/environment.rs b/crates/smoketests/tests/standalone/cli/environment.rs index 957ace39286..ab4b1666111 100644 --- a/crates/smoketests/tests/standalone/cli/environment.rs +++ b/crates/smoketests/tests/standalone/cli/environment.rs @@ -150,6 +150,19 @@ impl EnvironmentFixture { self.command(&args, shell) } + fn publish_environment(&self, shell: &[(&str, &str)], extra: &[&str]) -> Output { + let mut args = vec![ + "publish", + &self.database, + "--env-only", + "--server", + &self.test.server_url, + "--yes", + ]; + args.extend_from_slice(extra); + self.command(&args, shell) + } + fn published(&self, shell: &[(&str, &str)], extra: &[&str]) -> String { let before = fs::read(&self.wasm).unwrap(); let output = self.publish(shell, extra); @@ -366,11 +379,11 @@ fn cli_environment_replacement_rejection_and_read_only_commands() { f.config(Some( json!({"SMOKE_REQUIRED":"initial-sentinel","SMOKE_MODE":"ready","SMOKE_OPTIONAL":"remove-me"}), )); - f.published(&[], &[]); + f.published(&[], &["--replace-env"]); f.config(Some( json!({"SMOKE_REQUIRED":"replacement-sentinel","SMOKE_MODE":"other"}), )); - f.published(&[], &[]); + f.published(&[], &["--replace-env"]); f.typed("replacement-sentinel", "other", [None; 4]); assert_eq!(f.list(), ["SMOKE_MODE", "SMOKE_REQUIRED"]); assert!(!f @@ -391,11 +404,10 @@ fn cli_environment_replacement_rejection_and_read_only_commands() { for input in [ json!({"SMOKE_MODE":"ready"}), json!({"SMOKE_REQUIRED":"rejected-sentinel","SMOKE_MODE":"invalid-sentinel"}), - json!({"SMOKE_REQUIRED":"rejected-sentinel","SMOKE_MODE":"ready","UNKNOWN":"unknown-sentinel"}), json!({"SMOKE_REQUIRED":"rejected-sentinel","SMOKE_MODE":"ready","SMOKE_OPTIONAL":{}}), ] { f.config(Some(input)); - let output = f.publish(&[], &[]); + let output = f.publish(&[], &["--replace-env"]); assert!(!output.status.success()); for value in ["rejected-sentinel", "invalid-sentinel", "unknown-sentinel"] { assert!(!String::from_utf8_lossy(&output.stdout).contains(value)); @@ -456,3 +468,104 @@ fn cli_environment_initial_rejection_clear_and_omitted_payload() { f.published(&[("SMOKE_REQUIRED", "must-not-be-ambient")], &["--delete-data"]); assert!(f.list().is_empty()); } + +#[test] +fn cli_environment_preservation_and_environment_only_updates() { + let f = EnvironmentFixture::new(); + f.config(Some( + json!({"SMOKE_REQUIRED":"persisted-required", "SMOKE_MODE":"ready", "SMOKE_OPTIONAL":"keep-optional"}), + )); + f.published(&[], &[]); + let client = reqwest::blocking::Client::builder() + .no_proxy() + .redirect(reqwest::redirect::Policy::none()) + .timeout(Duration::from_secs(20)) + .build() + .unwrap(); + let url = format!("{}/v1/database/{}", f.test.server_url, f.database); + let denied = client.get(format!("{url}/environment")).send().unwrap(); + assert!(matches!(denied.status().as_u16(), 401 | 403)); + // This config and token were created by the fixture's isolated local login. + let config: toml::Value = toml::from_str(&fs::read_to_string(&f.test.config_path).unwrap()).unwrap(); + let token = config["spacetimedb_token"].as_str().unwrap(); + let metadata = client + .get(format!("{url}/environment")) + .bearer_auth(token) + .send() + .unwrap(); + assert!(metadata.status().is_success()); + assert_eq!(metadata.headers()["cache-control"], "no-store"); + let metadata = metadata.text().unwrap(); + assert!(!metadata.contains("persisted-required") && !metadata.contains("keep-optional")); + let metadata: Value = serde_json::from_str(&metadata).unwrap(); + assert!(metadata["stored_keys"] + .as_array() + .unwrap() + .contains(&json!("SMOKE_REQUIRED"))); + for (body, status) in [ + ( + json!({"environment":{"SMOKE_REQUIRED":"stale-replacement"},"expected_module_version":"00".repeat(32)}), + 409, + ), + (json!({"environment":{"SMOKE_REQUIRED":"missing-version"}}), 400), + (json!({"module":""}), 400), + ] { + let response = client + .put(&url) + .bearer_auth(token) + .header("Content-Type", "application/vnd.spacetimedb.publish+json") + .body(body.to_string()) + .send() + .unwrap(); + assert_eq!(response.status().as_u16(), status); + } + f.config(Some(json!({"FUTURE_KEY":"stored-future"}))); + f.published(&[("FUTURE_KEY", "must-not-import-undeclared")], &[]); + assert_eq!(f.get("FUTURE_KEY"), "stored-future\n"); + f.typed("persisted-required", "ready", [Some("keep-optional"), None, None, None]); + + // No local bundle or valid module source is available for env-only publishing. + fs::remove_file(&f.wasm).unwrap(); + f.config(Some(json!({"SMOKE_MODE":"other", "UNDECLARED":"new-value"}))); + let output = f.publish_environment(&[], &["--unset-env", "SMOKE_OPTIONAL"]); + assert!(output.status.success(), "{}", String::from_utf8_lossy(&output.stderr)); + f.typed("persisted-required", "other", [None; 4]); + assert_eq!(f.get("UNDECLARED"), "new-value\n"); + assert_eq!(f.get("FUTURE_KEY"), "stored-future\n"); + let initial = f.sql("SELECT required, mode FROM initial_environment"); + assert!(initial.status.success()); + let initial = String::from_utf8(initial.stdout).unwrap(); + assert!(initial.contains("persisted-required") && initial.contains("ready")); + assert!(!initial.contains("other"), "env-only reran init"); + + f.config(None); + for flags in [vec!["--unset-env", "SMOKE_REQUIRED"], vec!["--replace-env"]] { + assert!(!f.publish_environment(&[], &flags).status.success()); + f.typed("persisted-required", "other", [None; 4]); + assert_eq!(f.get("UNDECLARED"), "new-value\n"); + } + f.config(Some(json!({"SMOKE_REQUIRED":"replace-required", "SMOKE_MODE":"ready"}))); + assert!(f.publish_environment(&[], &["--replace-env"]).status.success()); + assert_eq!(f.list(), ["SMOKE_MODE", "SMOKE_REQUIRED"]); + f.typed("replace-required", "ready", [None; 4]); +} + +#[test] +fn cli_environment_preloaded_values_are_checked_when_declared() { + let mut f = EnvironmentFixture::new(); + let declared_module = f.wasm.clone(); + f.wasm = modules::precompiled_module("noop"); + f.config(Some( + json!({"SMOKE_REQUIRED":"preloaded", "SMOKE_MODE":"not-an-allowed-mode"}), + )); + f.published(&[], &[]); + f.config(None); + f.wasm = declared_module; + assert!(!f.publish(&[], &[]).status.success()); + assert_eq!(f.get("SMOKE_MODE"), "not-an-allowed-mode\n"); + f.config(Some(json!({"SMOKE_MODE":"ready"}))); + assert!(f.publish_environment(&[], &[]).status.success()); + f.config(None); + f.published(&[], &[]); + f.typed("preloaded", "ready", [None; 4]); +} diff --git a/crates/standalone/src/control_db/environment.rs b/crates/standalone/src/control_db/environment.rs index 454dd0e1d70..d0f623f60c0 100644 --- a/crates/standalone/src/control_db/environment.rs +++ b/crates/standalone/src/control_db/environment.rs @@ -109,7 +109,7 @@ impl ControlDb { None => Ok(BTreeMap::new()), Some(bytes) => { let request = PublishRequest::decode(&bytes).map_err(|_| invalid())?; - if !request.module.is_empty() { + if request.module.as_ref().is_some_and(|module| !module.is_empty()) { return Err(invalid()); } Ok(request.environment) @@ -140,8 +140,9 @@ impl ControlDb { leader: true, }; let input = PublishRequest { - module: Vec::new(), + module: None, environment, + ..Default::default() } .encode() .map_err(|_| invalid())?; diff --git a/crates/standalone/src/environment_tests.rs b/crates/standalone/src/environment_tests.rs index abeb5a84193..070140193d2 100644 --- a/crates/standalone/src/environment_tests.rs +++ b/crates/standalone/src/environment_tests.rs @@ -26,7 +26,7 @@ async fn read(env: &StandaloneEnv, database: u64, key: &str) -> anyhow::Result anyhow::Result<()> { +async fn real_module_reopen_and_environment_only_publication_preserve_values() -> anyhow::Result<()> { let module_path = std::path::PathBuf::from( std::env::var_os("SPACETIMEDB_ENV_STANDALONE_TEST_MODULE") .context("SPACETIMEDB_ENV_STANDALONE_TEST_MODULE must name the owned local fixture")?, @@ -66,6 +66,9 @@ async fn real_module_reopen_and_no_artifact_reset_preserve_complete_environment_ database_identity: Identity::ZERO, program_bytes: bytes.clone(), environment, + environment_remove: Vec::new(), + environment_replace: false, + expected_module_version: None, num_replicas: None, host_type: HostType::Wasm, parent: None, @@ -79,8 +82,10 @@ async fn real_module_reopen_and_no_artifact_reset_preserve_complete_environment_ let database = env.control_db.get_database_by_identity(&Identity::ZERO)?.unwrap(); let replica = env.control_db.get_leader_replica_by_database(database.id).unwrap(); log::info!("ENV standalone fixture: rejected publication preserves live host"); + let mut invalid = spec(Values::new()); + invalid.environment_replace = true; let rejected = env - .publish_database(&Identity::ZERO, spec(Values::new()), MigrationPolicy::Compatible) + .publish_database(&Identity::ZERO, invalid, MigrationPolicy::Compatible) .await; assert!( rejected.as_ref().is_err() @@ -94,6 +99,40 @@ async fn real_module_reopen_and_no_artifact_reset_preserve_complete_environment_ read(&env, database.id, "REQUIRED").await?, AlgebraicValue::from(Some("initial-required".to_owned())) ); + // An omitted input preserves required values. Environment-only requests + // also retain the module instance and cannot invoke init again. + assert!(env + .publish_database(&Identity::ZERO, spec(Values::new()), MigrationPolicy::Compatible) + .await? + .unwrap() + .was_successful()); + let previous_module = env.leader(database.id).await?.module().await?; + let version = previous_module.info.module_hash; + let mut env_only = spec(Values::from([("FUTURE".into(), "undeclared".into())])); + env_only.program_bytes = Default::default(); + env_only.expected_module_version = Some(version); + assert!(env + .publish_database(&Identity::ZERO, env_only, MigrationPolicy::Compatible) + .await? + .unwrap() + .was_successful()); + let current = env.leader(database.id).await?.module().await?; + assert!(Arc::ptr_eq(&previous_module.info, ¤t.info)); + let stored = current + .relational_db() + .with_read_only(spacetimedb_datastore::execution_context::Workload::Internal, |tx| { + spacetimedb::db::environment::snapshot(tx) + })?; + assert_eq!(stored["FUTURE"], "undeclared"); + assert_eq!(stored["REQUIRED"], "initial-required"); + assert!(read(&env, database.id, "FUTURE").await.is_err()); + let mut stale = spec(Values::from([("REQUIRED".into(), "wrong".into())])); + stale.program_bytes = Default::default(); + stale.expected_module_version = Some(spacetimedb_lib::Hash::from_hex("00".repeat(32))?); + assert!(env + .publish_database(&Identity::ZERO, stale, MigrationPolicy::Compatible) + .await + .is_err()); log::info!("ENV standalone fixture: same-program update"); let mut updated = initial.clone(); updated.insert("REQUIRED".into(), "republished".into()); @@ -130,6 +169,8 @@ async fn real_module_reopen_and_no_artifact_reset_preserve_complete_environment_ DatabaseResetDef { database_identity: Identity::ZERO, program_bytes: None, + environment_remove: Default::default(), + environment_replace: false, environment: Values::new(), num_replicas: None, host_type: None, @@ -156,6 +197,8 @@ async fn real_module_reopen_and_no_artifact_reset_preserve_complete_environment_ DatabaseResetDef { database_identity: Identity::ZERO, program_bytes: None, + environment_remove: Default::default(), + environment_replace: false, environment: reset, num_replicas: None, host_type: None, diff --git a/crates/standalone/src/lib.rs b/crates/standalone/src/lib.rs index c0ad9638d92..822b23d0e1a 100644 --- a/crates/standalone/src/lib.rs +++ b/crates/standalone/src/lib.rs @@ -432,12 +432,23 @@ impl StandaloneEnv { ) -> anyhow::Result> { let existing_db = self.control_db.get_database_by_identity(&spec.database_identity)?; + let update = spacetimedb_lib::environment::EnvironmentUpdate { + values: spec.environment, + remove: spec.environment_remove, + replace: spec.environment_replace, + }; + update.validate()?; // standalone does not support replication. let num_replicas = 1; match existing_db { // The database does not already exist, so we'll create it. None => { + anyhow::ensure!( + !spec.program_bytes.is_empty() && spec.expected_module_version.is_none(), + "initial publication requires a module and cannot require an existing version" + ); + let environment = update.resulting_values(&Default::default())?; let program = Program::from_bytes(spec.host_type.into(), &spec.program_bytes[..]); let database = Database { @@ -454,7 +465,7 @@ impl StandaloneEnv { // Instantiate a temporary database in order to check that the module is valid. // This will e.g. typecheck RLS filters. self.host_controller - .check_module_validity_with_environment(database.clone(), program, spec.environment.clone()) + .check_module_validity_with_environment(database.clone(), program, environment.clone()) .await?; let program_hash = self.program_store.put(&spec.program_bytes).await?; @@ -463,7 +474,7 @@ impl StandaloneEnv { let (database, replica) = self.control_db - .install_database_with_environment(database, None, spec.environment, &[])?; + .install_database_with_environment(database, None, environment, &[])?; // The leader nomination and input are durable already. If this // waiter is cancelled, ordinary lookup resumes the same input. self.on_insert_replica(&replica).await?; @@ -483,12 +494,13 @@ impl StandaloneEnv { let leader = self.leader_with_publication_lock_held(database_id).await?; let update_result = leader - .update_with_environment( + .update_with_environment_options( database, spec.host_type, spec.program_bytes.to_vec().into(), policy, - spec.environment, + update, + spec.expected_module_version, ) .await?; if update_result.was_successful() { @@ -569,6 +581,12 @@ impl StandaloneEnv { previous.owner_identity == *caller_identity, "database ownership changed before reset" ); + let environment = spacetimedb_lib::environment::EnvironmentUpdate { + values: spec.environment, + remove: spec.environment_remove, + replace: spec.environment_replace, + } + .resulting_values(&Default::default())?; let mut database = previous.clone(); let program = match spec.program_bytes { Some(bytes) => { @@ -592,7 +610,7 @@ impl StandaloneEnv { database.host_type = HostType::from(program.kind); database.initial_program = program.hash; self.host_controller - .check_module_validity_with_environment(database.clone(), program.clone(), spec.environment.clone()) + .check_module_validity_with_environment(database.clone(), program.clone(), environment.clone()) .await?; let stored = self.program_store.put(&program.bytes).await?; anyhow::ensure!(stored == program.hash, "stored reset program changed"); @@ -605,7 +623,7 @@ impl StandaloneEnv { let (_, replica) = self.control_db.install_database_with_environment( database, Some(&previous), - spec.environment, + environment, &previous_replicas, )?; self.on_insert_replica(&replica).await?; diff --git a/crates/testing/src/modules.rs b/crates/testing/src/modules.rs index 09d35280562..5f7c2b23db9 100644 --- a/crates/testing/src/modules.rs +++ b/crates/testing/src/modules.rs @@ -97,6 +97,9 @@ impl ModuleHandle { database_identity: self.db_identity, program_bytes, environment, + environment_remove: Vec::new(), + environment_replace: true, + expected_module_version: None, num_replicas: None, host_type, parent: None, @@ -421,6 +424,9 @@ impl CompiledModule { database_identity: db_identity, program_bytes: self.program_bytes(), environment, + environment_remove: Vec::new(), + environment_replace: false, + expected_module_version: None, num_replicas: None, host_type: self.host_type, parent: None, diff --git a/crates/testing/tests/environment.rs b/crates/testing/tests/environment.rs index a8187619fca..a79b0b8467a 100644 --- a/crates/testing/tests/environment.rs +++ b/crates/testing/tests/environment.rs @@ -186,7 +186,7 @@ fn exercise_fixture(name: &str) { } } if name == "environment-test" { - // Required values cannot be inherited from the previous publish, + // Explicit complete replacement cannot inherit required values, // and an invalid literal cannot replace the previous configuration. for invalid in [ Values::new(), diff --git a/docs/docs/00200-core-concepts/00100-databases/00700-environment-variables.md b/docs/docs/00200-core-concepts/00100-databases/00700-environment-variables.md index d05c96ed526..3dbcc9a7309 100644 --- a/docs/docs/00200-core-concepts/00100-databases/00700-environment-variables.md +++ b/docs/docs/00200-core-concepts/00100-databases/00700-environment-variables.md @@ -8,9 +8,9 @@ import TabItem from '@theme/TabItem'; # Environment Variables -Environment variables store configuration and secrets for a database, such as API keys and deployment settings. A module declares the names it accepts and any allowed values. Each publish supplies the complete set of values for that module. Module code reads them through `ctx.env`, or `ctx.Env` in C#. +Environment variables store configuration and secrets for a database, such as API keys and deployment settings. A module declares the names it reads and any allowed values. Publishing preserves stored values unless you explicitly replace or delete them. Undeclared values can be stored before a module starts using them. Module code reads them through `ctx.env`, or `ctx.Env` in C#. -Use environment variables for configuration that changes when a module is published. For secrets or configuration that must change without publishing, use a [private table](#dynamically-editable-or-untyped-secrets). +Use environment-only publishing to change configuration without uploading a module. Use a [private table](#dynamically-editable-or-untyped-secrets) when module code must edit values or read arbitrary keys. This guide assumes a module set up using a quickstart, such as the [Rust quickstart](../../00100-intro/00200-quickstarts/00500-rust.md), and familiarity with [publishing](./00300-spacetime-publish.md). @@ -55,7 +55,7 @@ Within an environment declaration, a simple enum specifies allowed strings. Enum The string-key getter checks the same declaration and permissions as named accessors. Reading an undeclared key fails. Optional named accessors return `undefined`; `ctx.env.get` returns `null` for an absent optional value. -Key names are exact and case-sensitive. The name `get` is reserved for the getter; read a declaration named `get` with `ctx.env.get('get')`. A module with no environment declarations accepts no keys. +Key names are exact and case-sensitive. The name `get` is reserved for the getter; read a declaration named `get` with `ctx.env.get('get')`. A module with no environment declarations cannot read stored environment values. Reads inside a transaction use that transaction's snapshot. In a procedure outside a transaction, each read uses a separate snapshot. To read several keys consistently, group the reads in `withTx`. @@ -104,7 +104,7 @@ Existing `#[env(values(...))]` constraints on `String` and `Option` fiel The string-key getter checks the same declaration and permissions as named accessors. Reading an undeclared key fails. Optional named accessors and `ctx.env.get` return `None` for an absent optional value. -Key names are exact and case-sensitive. The name `get` is reserved for the getter; read a declaration named `get` with `ctx.env.get("get")`. A module with no environment declarations accepts no keys. +Key names are exact and case-sensitive. The name `get` is reserved for the getter; read a declaration named `get` with `ctx.env.get("get")`. A module with no environment declarations cannot read stored environment values. Reads inside a transaction use that transaction's snapshot. In a procedure outside a transaction, each read uses a separate snapshot. To read several keys consistently, group the reads in `with_tx`. @@ -140,7 +140,7 @@ string? checkedValue = ctx.Env.Get("LOG_LEVEL"); The string-key getter checks the same declaration and permissions as named accessors. Reading an undeclared key fails. Optional named accessors and `ctx.Env.Get` return `null` for an absent optional value. -Key names are exact and case-sensitive. Names that collide with `Get`, `ModuleEnvironment`, or inherited `Object` methods are available through the string-key getter, for example `ctx.Env.Get("GetType")`. A module with no environment declarations accepts no keys. +Key names are exact and case-sensitive. Names that collide with `Get`, `ModuleEnvironment`, or inherited `Object` methods are available through the string-key getter, for example `ctx.Env.Get("GetType")`. A module with no environment declarations cannot read stored environment values. Reads inside a transaction use that transaction's snapshot. In a procedure outside a transaction, each read uses a separate snapshot. To read several keys consistently, group the reads in `WithTx`. @@ -181,7 +181,7 @@ std::optional checked = ctx.env.get("LOG_LEVEL"); The string-key getter checks the same declaration and permissions as named accessors. Reading an undeclared key fails. Optional named accessors and `ctx.env.get` return `std::nullopt` for an absent optional value. -Key names are exact and case-sensitive. Names reserved by the generated accessor type, including `get`, remain available through the string-key getter, for example `ctx.env.get("get")`. A module with no environment declarations accepts no keys. +Key names are exact and case-sensitive. Names reserved by the generated accessor type, including `get`, remain available through the string-key getter, for example `ctx.env.get("get")`. A module with no environment declarations cannot read stored environment values. Reads inside a transaction use that transaction's snapshot. In a procedure outside a transaction, each read uses a separate snapshot. To read several keys consistently, group the reads in `with_tx`. @@ -207,10 +207,10 @@ Add non-secret defaults to the selected database target in `spacetime.json`: With a local server running on port 3000, publish from the directory containing that configuration. This example uses a disposable development value: ```bash -API_KEY='development-only-key' spacetime publish +API_KEY='development-only-key' spacetime publish --server http://127.0.0.1:3000 ``` -For real credentials, supply the value through the publishing process's environment, `spacetime.local.json`, or `spacetime.{environment}.local.json`. Keep checked-in `spacetime.json` and `spacetime.{environment}.json` limited to non-secret defaults. Ensure the local files are ignored by Git: +For an existing database, you can enter credentials once through its website settings and reuse them on later publishes. For initial publishing, supply the value through the publishing process's environment, `spacetime.local.json`, or `spacetime.{environment}.local.json`. Keep checked-in `spacetime.json` and `spacetime.{environment}.json` limited to non-secret defaults. Ensure the local files are ignored by Git: ```gitignore spacetime.local.json @@ -223,23 +223,40 @@ The CLI resolves each declared key in this order: 1. A value in the publishing process's environment. 2. A value in the resolved configuration's `env` map. -3. Absence, which is accepted only for an optional declaration. +3. The stored database value, unless `--replace-env` is specified. +4. Absence, which is accepted only for an optional declaration. A shell value overrides JSON even when it is an empty string or the key is absent from JSON. Already-exported variables behave the same as inline assignments. Unrelated shell variables are ignored unless the module declares their names. The CLI displays supplied key names and their sources, without printing their values. The configuration files `spacetime.json`, `spacetime.local.json`, `spacetime.{environment}.json`, and `spacetime.{environment}.local.json` apply in increasing precedence, where _environment_ is the environment selected with `--env`. Their `env` maps merge by key, as do maps inherited by child database targets. A higher-precedence value replaces that key while preserving unrelated keys. An empty map does not erase inherited keys. -JSON strings pass through unchanged. Booleans and numbers are converted to strings, so `false` supplies `"false"`; declarations still validate strings. Use JSON strings when exact numeric spelling matters. Arrays, objects, and `null` are rejected, as are JSON keys the module has not declared. An invalid effective value rejects the publish rather than falling back to a lower-precedence value. For an absent optional value, omit its property from the JSON object instead of setting it to `null`. Also remove any inherited or shell value for that key, as described below. +JSON strings pass through unchanged. Booleans and numbers are converted to strings, so `false` supplies `"false"`; declarations still validate strings. Use JSON strings when exact numeric spelling matters. Arrays, objects, and `null` are rejected. Explicit JSON keys the module has not declared are accepted as stored strings; module code cannot read them. A matching shell variable is not forwarded until the key is declared. An invalid effective value rejects the publish rather than falling back to a lower-precedence value. Omitting an optional key preserves any stored value. To clear it, use `--unset-env` and remove it from effective JSON and shell inputs. An empty string is a value, not a deletion instruction. -### Every publish replaces the complete environment +### Preserve, edit, or replace values -Publishing validates the supplied values and installs them atomically with the module, before initialization or migration. Invalid environment configuration leaves the previous module and values unchanged. Changing only values still requires publishing, and does not rerun `init` on an existing database. +After a successful publish, every required declaration has a stored value, and every present declared value satisfies its constraint. Publishing validates the resulting environment atomically before initialization or migration. You can supply a replacement value in the same publish that changes its declaration. Failure leaves the previous module and values unchanged. -Previously stored values are **not** defaults for the next publish. Every publish must supply each required value again. An optional value omitted from all effective inputs is removed. To remove `LOG_LEVEL` in the example, remove it from every applicable configuration layer and unset any exported `LOG_LEVEL` before publishing. An empty string is a value, not a deletion instruction. +An ordinary publish retains unspecified values, including values whose declarations were removed. You can explicitly supply undeclared keys in JSON or through website editing before deploying a declaration. Later publishing validates those stored strings against the new declaration. Removing a declaration does not delete its value, but module reads of that key then fail. -The same rules apply to precompiled modules published with `--bin-path`. The CLI reads declarations from the artifact being published. +Update an existing database without building or uploading a module: -For publishing from an HTTP client or a module procedure, see the [HTTP publish format and example](../../00300-resources/00200-reference/00200-http-api/00300-database.md#publishing-with-environment-values). Supply the module and complete environment in the request body; project configuration and shell overrides are CLI conveniences. +```bash +API_KEY='development-only-replacement' spacetime publish env-example --env-only --server http://127.0.0.1:3000 +``` + +This uses the deployed schema for declared shell overrides and accepts explicit JSON values for undeclared keys. It does not run initialization or migration. A stale module version rejects the request; refresh and retry. For a new database, supply required values together with the initial module publish so they are available in `init`. + +Delete an optional or undeclared value explicitly, after removing it from effective JSON and shell inputs: + +```bash +spacetime publish env-example --env-only --unset-env LOG_LEVEL --server http://127.0.0.1:3000 +``` + +Deleting a required value is rejected. Repeat `--unset-env` to delete multiple keys. To replace the entire store with the supplied values, use `--replace-env` with either normal publishing or `--env-only`. Replacement deletes every omitted declared and undeclared key, and fails atomically if required values are missing. It cannot be combined with `--unset-env`. + +The destructive `--delete-data` operation still clears all database data, including environment values. Supply required secrets again when resetting a database. `--env-only` cannot be combined with a reset. + +The same preservation rules apply to precompiled modules published with `--bin-path`. The CLI reads declarations from the artifact being published. HTTP clients and module procedures can use the [HTTP publish format](../../00300-resources/00200-reference/00200-http-api/00300-database.md#publishing-with-environment-values); project configuration and shell overrides are CLI conveniences. ## Inspect published values @@ -259,19 +276,21 @@ SELECT key FROM st_env; SELECT value FROM st_env WHERE key = 'MODE'; ``` -SQL writes to `st_env`, module-side writes, and separate CLI setters are not supported. Changes go through a publish with the complete desired environment. +SQL writes to `st_env`, module-side writes, and separate CLI setters are not supported. Changes go through a validated module or environment-only publish. ## Access and limits +Publishing environment values uses the same permission as publishing the module. This includes Developer-role collaborators where they can publish modules. Environment changes add no separate Admin requirement; a destructive reset or container configuration change still requires its normal permissions. + Reducers, procedures, views, and HTTP handlers entered by the host in the root module can read its declared environment. Host-dispatched submodule entry points cannot read it, and submodules cannot declare a nonempty environment. There is no separate environment-variable namespace to configure for a submodule. A module with environment declarations can be published independently as a root module, but cannot be included as a submodule with those declarations. See [Submodules](./00600-submodules.md) for this restriction. Ordinary helper calls retain their calling entry point's access, including helpers defined in libraries or submodules. Root code can also pass a value to a helper explicitly. A procedure suspended across a publish cannot read values belonging to a replacement program. Environment reads in views participate in dependency tracking, so publishing changed values refreshes affected views. Module code remains responsible for what it returns or logs: returning a secret from a public view exposes that value to clients. -Keys must match `[A-Za-z_][A-Za-z0-9_]*`. The limits are 256 bytes per key, 8 KiB per value, and 256 declarations per database. Length limits count UTF-8 bytes. Values may be empty if their declaration accepts an empty string. +Keys must match `[A-Za-z_][A-Za-z0-9_]*`. The limits are 256 bytes per key, 8 KiB per value, 256 declarations per database, and 256 stored values including undeclared keys. Length limits count UTF-8 bytes. Values may be empty if their declaration accepts an empty string. ## Dynamically editable or untyped secrets -Use an ordinary [private table](../00300-tables/00400-access-permissions.md) when a secret must change without republishing, or when its keys and allowed values should not be declared in the environment schema. For example, a table with a string primary-key column and a string value column can store arbitrary secret names and values. The table itself still has typed columns; its individual keys need no environment declarations or constraints. +Use an ordinary [private table](../00300-tables/00400-access-permissions.md) when module code must edit secrets or read keys without declaring them. For example, a table with a string primary-key column and a string value column can store arbitrary secret names and values. The table itself still has typed columns; its individual keys need no environment declarations or constraints. Update that table through reducers that explicitly authorize the caller. Keeping a table private controls direct client reads; it does not authorize calls to a reducer that modifies or returns its contents. Apply the same care to views, procedure results, and logs. Private tables follow the database's normal private-table permissions, including administrative reads. @@ -279,4 +298,4 @@ Update that table through reducers that explicitly authorize the caller. Keeping Private tables can store dynamically editable secrets, but changing a table to public can expose its contents to clients. Environment variables have no public-table visibility setting. With either approach, module code can still expose secrets through return values or logs. ::: -Both approaches are supported. Environment declarations additionally guarantee that required values are validated and available before `init` or migration runs. Private-table values follow the table's ordinary update and migration behavior. They do not receive environment schema validation or complete replacement on every publish. +Both approaches are supported. Environment declarations additionally guarantee that required values are validated and available before `init` or migration runs. Private-table values follow the table's ordinary update and migration behavior. They do not receive environment schema validation or environment publish semantics. diff --git a/docs/docs/00300-resources/00200-reference/00100-cli-reference/00100-cli-reference.md b/docs/docs/00300-resources/00200-reference/00100-cli-reference/00100-cli-reference.md index 474e9a5526b..cf11039d77c 100644 --- a/docs/docs/00300-resources/00200-reference/00100-cli-reference/00100-cli-reference.md +++ b/docs/docs/00300-resources/00200-reference/00100-cli-reference/00100-cli-reference.md @@ -86,7 +86,7 @@ Create and update a SpacetimeDB database **Usage:** `spacetime publish [OPTIONS] [name|identity]` -Every publish replaces the complete declared environment. Put an env map in spacetime.json; declared shell variables override config values (including empty strings). The CLI displays supplied keys and sources, never values. Optional values omitted from every input are removed. --env selects config file layers. Run `spacetime help publish` for more detailed information. +Publishing preserves unspecified environment values. Put an env map in spacetime.json; explicit undeclared keys are allowed, and declared shell variables override config values (including empty strings). The CLI displays supplied keys and sources, never values. --env-only updates an existing database without a module. --unset-env explicitly deletes a value; required values cannot be removed. --replace-env replaces all stored values with the supplied set, including deleting unspecified undeclared keys, and cannot be combined with --unset-env. The host validates the resulting environment atomically. --env selects config file layers. ###### **Arguments:** @@ -138,6 +138,9 @@ Every publish replaces the complete declared environment. Put an env map in spac * `--env ` — Environment name for config file layering (e.g., dev, staging) * `--native-aot` — Use NativeAOT-LLVM compilation for C# modules (experimental; supported on Windows, and on Linux with .NET 10) * `--dotnet-version ` — Target .NET SDK major version for C# projects (e.g. 8 or 10). Auto-detected when omitted. +* `--env-only` — Update environment values without building or uploading a module. +* `--unset-env ` — Delete an environment value. Repeat for multiple keys. +* `--replace-env` — Replace all environment values, deleting every unspecified key. diff --git a/docs/docs/00300-resources/00200-reference/00200-http-api/00300-database.md b/docs/docs/00300-resources/00200-reference/00200-http-api/00300-database.md index 52faf7df576..ab5cd1b01f7 100644 --- a/docs/docs/00300-resources/00200-reference/00200-http-api/00300-database.md +++ b/docs/docs/00300-resources/00200-reference/00200-http-api/00300-database.md @@ -112,7 +112,24 @@ Both publish endpoints accept `Content-Type: application/vnd.spacetimedb.publish } ``` -`module` uses standard padded Base64. `environment` maps declared names to strings. The server validates the complete map against the module's declarations and installs both in one transaction. Missing required values reject the publish; omitted optional values are removed. Omitting `environment` is equivalent to `{}`, including when publishing unchanged module bytes. +`module` uses standard padded Base64. `environment` supplies string overrides for declared or undeclared names. Unspecified stored values survive by default. The server validates the resulting environment against the module's declarations and installs both in one transaction: all required values must exist and all present declared values must satisfy their constraints. An empty or omitted map preserves stored values, including when publishing unchanged module bytes. + +A missing required value returns HTTP 400 with `Content-Type: application/json` and `{"error":"missing_required_environment","key":"API_KEY"}` identifying the missing key. Clients can prompt for that value and retry the publish. Other validation and module failures do not use this error code. + +Optional fields `environment_remove` (an array of keys) and `environment_replace` (a boolean, default `false`) request explicit deletion or complete replacement. A key cannot be both supplied and removed. Replacement uses only the supplied map, deleting every unspecified declared and undeclared key, and rejects any nonempty removal list. Invalid updates leave the database unchanged. + +To update an existing database without a module, omit `module` and provide `expected_module_version` from `GET /v1/database/{name_or_identity}/environment`. That authorized endpoint returns `module_version`, `declarations`, and `stored_keys`, without secret values. Each declaration contains `name`, `optional`, and `constraint`: `"AnyString"`, `{"Literal":"value"}`, or `{"OneOf":["a","b"]}`. Metadata comes from one database version. For example: + +```json +{ + "expected_module_version": "", + "environment": { "FUTURE_KEY": "development-only-value" }, + "environment_remove": ["OLD_OPTIONAL_KEY"], + "environment_replace": false +} +``` + +Send this body to the existing database's PUT endpoint with the same content type and publish authorization. Environment-only updates validate against the deployed schema, never invoke initialization or migration, and reject stale module versions. A new database still requires a module and all initially required values in its initial publish. For example, use `curl`, `jq`, and `base64` to publish a Wasm module to a local server. Export `SPACETIME_TOKEN` with a token authorized to publish and `API_KEY` with the required value. Change `module.wasm` to the artifact you built. @@ -133,7 +150,7 @@ base64 < module.wasm | Direct HTTP callers, including module procedures, use this same format; the server does not load project configuration or shell values for them. See [Environment Variables](../../../00200-core-concepts/00100-databases/00700-environment-variables.md) for declaration syntax and value limits. The decoded module is limited to 128 MiB and the complete encoded request to 192 MiB. -Raw module bodies continue to work and supply an empty environment. Use `application/octet-stream` for that format. This preserves compatibility with older servers for modules that do not require ENV support; older servers do not support the JSON publish format. +Raw module bodies continue to work and supply no environment overrides, retaining stored values on ordinary publishes. A destructive clear/reset still removes stored values and requires all initially required values again. Use `application/octet-stream` for that format. This preserves compatibility with older servers for modules that do not require ENV support; older servers do not support the JSON publish format. ## `GET /v1/database/:name_or_identity` From 642a50bc13712da7c9450189050ed5f131f065c6 Mon Sep 17 00:00:00 2001 From: Tyler Cloutier Date: Fri, 11 Sep 2026 22:26:35 -0400 Subject: [PATCH 30/34] Clarify stored environment keys and publish precedence --- crates/cli/src/subcommands/env.rs | 2 +- .../00100-databases/00700-environment-variables.md | 4 ++-- .../00100-cli-reference/00100-cli-reference.md | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/cli/src/subcommands/env.rs b/crates/cli/src/subcommands/env.rs index 01a1f25afe9..c0092bad377 100644 --- a/crates/cli/src/subcommands/env.rs +++ b/crates/cli/src/subcommands/env.rs @@ -37,7 +37,7 @@ pub fn cli() -> Command { Arg::new("key") .index(2) .required(true) - .help("The declared environment key to read"), + .help("The stored environment key to read"), ), )) .subcommand(target( diff --git a/docs/docs/00200-core-concepts/00100-databases/00700-environment-variables.md b/docs/docs/00200-core-concepts/00100-databases/00700-environment-variables.md index 3dbcc9a7309..8dda2a4a545 100644 --- a/docs/docs/00200-core-concepts/00100-databases/00700-environment-variables.md +++ b/docs/docs/00200-core-concepts/00100-databases/00700-environment-variables.md @@ -16,7 +16,7 @@ This guide assumes a module set up using a quickstart, such as the [Rust quickst ## Declare and read variables -Declare every environment key in the module. All values are strings. A declaration can accept any string, one exact string, or a set of allowed strings. Optional declarations permit an absent value. +Declare every environment key the module reads. All values are strings. A declaration can accept any string, one exact string, or a set of allowed strings. Optional declarations permit an absent value. These examples declare a required `API_KEY`, a required `MODE` restricted to `development` or `production`, and an optional `LOG_LEVEL` restricted to `info` or `debug`. @@ -219,7 +219,7 @@ spacetime.*.local.json The `.local` naming convention does not itself prevent a file from being committed. -The CLI resolves each declared key in this order: +Publishing uses the first available value for each declared key in this order: 1. A value in the publishing process's environment. 2. A value in the resolved configuration's `env` map. diff --git a/docs/docs/00300-resources/00200-reference/00100-cli-reference/00100-cli-reference.md b/docs/docs/00300-resources/00200-reference/00100-cli-reference/00100-cli-reference.md index cf11039d77c..f9a483ddd6d 100644 --- a/docs/docs/00300-resources/00200-reference/00100-cli-reference/00100-cli-reference.md +++ b/docs/docs/00300-resources/00200-reference/00100-cli-reference/00100-cli-reference.md @@ -165,7 +165,7 @@ Read one published environment value ###### **Arguments:** -* `` — The declared environment key to read +* `` — The stored environment key to read * `` — The database name, identity, or configured target ###### **Options:** From a10bbd8d04f3a6b66a162657b333dea5ea51014d Mon Sep 17 00:00:00 2001 From: Tyler Cloutier Date: Sat, 12 Sep 2026 18:17:35 -0400 Subject: [PATCH 31/34] Remove superseded workflow coordinator changes --- .../commands/workflow-coordinator/src/main.rs | 95 +------------------ 1 file changed, 3 insertions(+), 92 deletions(-) diff --git a/tools/ci/commands/workflow-coordinator/src/main.rs b/tools/ci/commands/workflow-coordinator/src/main.rs index f8d9c23fcfa..6b76deae096 100644 --- a/tools/ci/commands/workflow-coordinator/src/main.rs +++ b/tools/ci/commands/workflow-coordinator/src/main.rs @@ -197,8 +197,6 @@ struct Repository { #[derive(Clone, Debug, Deserialize, PartialEq, Eq)] struct PullRequestRef { sha: String, - #[serde(rename = "ref")] - branch_name: String, repo: Option, } @@ -346,29 +344,10 @@ fn related_private_pr(public_pr_number: Option) -> Result 1 { - Some(pull_request(PUBLIC_REPO, public_pr_number)?.head.branch_name) - } else { - None - }; - select_related_private_pr(pulls, public_branch.as_deref()) -} - -fn select_related_private_pr(mut pulls: Vec, public_branch: Option<&str>) -> Result> { - if pulls.len() <= 1 { - return Ok(pulls.pop()); - } - - // Timeline references include historical links and links to other layers of - // a PR stack. A unique shared branch name identifies the companion PR; the - // exact public-submodule SHA is still checked before selecting its CI run. - if let Some(public_branch) = public_branch.filter(|branch| !branch.is_empty()) { - pulls.retain(|pull| pull.head.branch_name == public_branch); - if pulls.len() == 1 { - return Ok(pulls.pop()); - } + if pulls.len() > 1 { + bail!("found multiple open linked private PRs"); } - bail!("found multiple open linked private PRs without a unique matching head branch") + Ok(pulls.pop()) } fn mentions_public_pr(body: Option<&str>, public_pr_number: u64) -> bool { @@ -616,7 +595,6 @@ mod tests { state: "open".to_owned(), head: PullRequestRef { sha: "private-sha".to_owned(), - branch_name: "tyler/environment-variables".to_owned(), repo: Some(Repository { full_name: PRIVATE_REPO.to_owned(), }), @@ -625,73 +603,6 @@ mod tests { } } - #[test] - fn related_private_pr_prefers_the_unique_exact_public_head_branch() { - let matching = pull(); - let mut downstream = pull(); - downstream.number = 43; - downstream.head.branch_name = "tyler/environment-variables-followup".to_owned(); - for candidates in [ - vec![matching.clone(), downstream.clone()], - vec![downstream, matching.clone()], - ] { - assert_eq!( - select_related_private_pr(candidates, Some("tyler/environment-variables")).unwrap(), - Some(matching.clone()) - ); - } - } - - #[test] - fn related_private_pr_preserves_absent_and_single_candidate_behavior() { - assert_eq!(select_related_private_pr(Vec::new(), None).unwrap(), None); - for public_branch in [None, Some("unrelated-branch")] { - assert_eq!( - select_related_private_pr(vec![pull()], public_branch).unwrap(), - Some(pull()) - ); - } - } - - #[test] - fn related_private_pr_rejects_multiple_matching_branches() { - let mut duplicate = pull(); - duplicate.number = 43; - assert!(select_related_private_pr(vec![pull(), duplicate], Some("tyler/environment-variables")).is_err()); - } - - #[test] - fn related_private_pr_rejects_missing_or_unmatched_public_branch() { - let mut downstream = pull(); - downstream.number = 43; - downstream.head.branch_name = "tyler/v10-abi-extensions".to_owned(); - for public_branch in [None, Some(""), Some("tyler/unrelated")] { - assert!(select_related_private_pr(vec![pull(), downstream.clone()], public_branch).is_err()); - } - } - - #[test] - fn selected_companion_still_requires_the_exact_public_submodule() { - let selected = select_related_private_pr(vec![pull()], None).unwrap().unwrap(); - assert!(ensure_public_submodule_matches(selected.number, "old-public-sha", "requested-public-sha").is_err()); - ensure_public_submodule_matches(selected.number, "requested-public-sha", "requested-public-sha").unwrap(); - } - - #[test] - fn pull_request_head_branch_uses_the_github_ref_field() { - let parsed: PullRequest = serde_json::from_value(serde_json::json!({ - "number": 42, - "state": "open", - "head": { - "sha": "private-sha", - "ref": "tyler/environment-variables", - "repo": { "full_name": PRIVATE_REPO } - } - })) - .unwrap(); - assert_eq!(parsed, pull()); - } - fn run(id: u64, title: &str, created_at: &str) -> WorkflowRun { WorkflowRun { id, From ea49bdadfd3eac14454ac0f40ac9cb0f887e53f6 Mon Sep 17 00:00:00 2001 From: Tyler Cloutier Date: Sun, 13 Sep 2026 14:29:34 -0400 Subject: [PATCH 32/34] Drop redundant pnpm script workaround after CI pinning fix --- crates/bindings-typescript/package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/bindings-typescript/package.json b/crates/bindings-typescript/package.json index 49b18bc090c..53031c3e8a8 100644 --- a/crates/bindings-typescript/package.json +++ b/crates/bindings-typescript/package.json @@ -25,14 +25,14 @@ "scripts": { "build:js": "tsup", "build:types": "tsc -p tsconfig.build.json", - "build": "pnpm run build:js && pnpm run build:types", + "build": "pnpm -s build:js && pnpm -s build:types", "format": "prettier . --write --ignore-path ../../.prettierignore", "lint": "eslint . && prettier . --check --ignore-path ../../.prettierignore", "test": "vitest run", "test:typecheck": "vitest typecheck --run", "coverage": "vitest run --coverage", "brotli-size": "brotli-size dist/index.js", - "size": "pnpm run build && size-limit", + "size": "pnpm -s build && size-limit", "generate:moduledef": "cargo run -p spacetimedb-codegen --example regen-typescript-moduledef && prettier --write src/lib/autogen", "generate:client-api": "cargo run -p generate-client-api && prettier --write src/sdk/client_api", "generate:test-app": "pnpm --filter @clockworklabs/test-app generate", From a140882626a5469e5b53c89075a662770d5ed485 Mon Sep 17 00:00:00 2001 From: Tyler Cloutier Date: Fri, 18 Sep 2026 10:11:45 -0400 Subject: [PATCH 33/34] Separate deferred function visibility from V10 authentication --- crates/bindings-cpp/README.md | 24 +- .../include/spacetimedb/function_visibility.h | 6 - .../internal/autogen/FunctionVisibility.g.h | 2 - .../spacetimedb/internal/v10_builder.h | 5 +- .../bindings-cpp/include/spacetimedb/macros.h | 10 +- .../bindings-cpp/src/internal/v10_builder.cpp | 43 ++- crates/bindings-cpp/tests/unit/CMakeLists.txt | 13 +- .../unit/function_visibility_unit_tests.cpp | 115 ------- crates/bindings-csharp/Codegen.Tests/Tests.cs | 60 ---- .../diag/snapshots/Module#FFI.verified.cs | 4 +- .../server/snapshots/Module#FFI.verified.cs | 2 +- crates/bindings-csharp/Codegen/Diag.cs | 9 - crates/bindings-csharp/Codegen/Module.cs | 52 +-- crates/bindings-csharp/README.md | 25 +- .../Runtime.Tests/FunctionVisibilityTests.cs | 91 ------ crates/bindings-csharp/Runtime/Attrs.cs | 12 - .../Internal/Autogen/FunctionVisibility.g.cs | 2 - .../Runtime/Internal/Module.cs | 33 +- crates/bindings-macro/src/procedure.rs | 37 +-- crates/bindings-macro/src/reducer.rs | 89 ------ crates/bindings-typescript/README.md | 22 +- .../src/lib/autogen/types.ts | 2 - .../src/server/function_visibility.ts | 24 -- .../bindings-typescript/src/server/index.ts | 1 - .../src/server/procedures.ts | 16 +- .../src/server/reducers.ts | 18 +- .../bindings-typescript/src/server/schema.ts | 10 +- .../tests/hosted_auth.test.ts | 180 +---------- crates/bindings/src/rt.rs | 17 +- .../tests/pass/function_visibility.rs | 62 ---- crates/cli/src/subcommands/generate.rs | 2 +- crates/codegen/src/util.rs | 88 ++--- .../host_controller/invocation_flags_tests.rs | 33 +- crates/lib/src/db/raw_def/v10.rs | 90 +----- crates/schema/src/auto_migrate.rs | 61 ---- crates/schema/src/auto_migrate/formatter.rs | 9 - .../src/auto_migrate/termcolor_formatter.rs | 9 - crates/schema/src/def.rs | 34 +- crates/schema/src/def/validate/v10.rs | 300 +----------------- crates/schema/src/def/validate/v9.rs | 2 +- crates/schema/src/error.rs | 2 - crates/testing/tests/invocation_flags.rs | 14 +- .../00100-cli-reference.md | 2 +- modules/invocation-flags-test/src/lib.rs | 16 +- modules/module-test-ts/src/index.ts | 2 +- modules/module-test/src/lib.rs | 2 +- modules/sdk-test-procedure-cpp/src/lib.cpp | 2 +- modules/sdk-test-procedure-cs/Lib.cs | 2 +- modules/sdk-test-procedure-ts/src/index.ts | 2 +- modules/sdk-test-procedure/src/lib.rs | 2 +- .../examples~/regression-tests/server/Lib.cs | 2 +- .../identity_connected_reducer.rs | 62 ++++ .../identity_disconnected_reducer.rs | 62 ++++ .../src/module_bindings/mod.rs | 15 +- .../procedure-client/src/test_handlers.rs | 25 +- 55 files changed, 296 insertions(+), 1530 deletions(-) delete mode 100644 crates/bindings-cpp/include/spacetimedb/function_visibility.h delete mode 100644 crates/bindings-cpp/tests/unit/function_visibility_unit_tests.cpp delete mode 100644 crates/bindings-csharp/Runtime.Tests/FunctionVisibilityTests.cs delete mode 100644 crates/bindings-typescript/src/server/function_visibility.ts delete mode 100644 crates/bindings/tests/pass/function_visibility.rs create mode 100644 sdks/rust/tests/connect_disconnect_client/src/module_bindings/identity_connected_reducer.rs create mode 100644 sdks/rust/tests/connect_disconnect_client/src/module_bindings/identity_disconnected_reducer.rs diff --git a/crates/bindings-cpp/README.md b/crates/bindings-cpp/README.md index 9ccfcc32a97..3aba197cb05 100644 --- a/crates/bindings-cpp/README.md +++ b/crates/bindings-cpp/README.md @@ -2,29 +2,9 @@ The SpacetimeDB C++ Module Library provides a modern C++20 API for building SpacetimeDB modules that run inside the database as WebAssembly. -## Function visibility and invocation authentication +## Invocation authentication -Apply `SPACETIMEDB_FUNCTION_VISIBILITY(name, Public)`, `Private`, or `Internal` -to a reducer or procedure after its definition: - -```cpp -SPACETIMEDB_REDUCER(process_jobs, ReducerContext ctx) { - return Ok(); -} -SPACETIMEDB_FUNCTION_VISIBILITY(process_jobs, Internal); -``` - -Omission means public for ordinary functions and private for scheduled functions. -An explicit choice is preserved when the function is scheduled. Lifecycle -reducers permit only omission or `Internal` and can only run for their host -lifecycle event. Internal functions require verified internal authority. Private -functions also admit the owner, and public functions admit any client. - -`ctx.sender_auth().is_internal()` captures the host's invocation authority. It is -independent of connection and JWT presence, so an internal call can have a JWT. -JWT identity is the verified sender supplied by the host. Procedures preserve -this authentication in `with_tx` and `try_with_tx`. Newly compiled modules emit -schema V10 and advertise `hosted_auth_v1`, requiring a compatible host. +`ctx.sender_auth().is_internal()` captures host-verified invocation authority. JWT identity is the verified sender supplied by the host. Procedures preserve authentication in `with_tx` and `try_with_tx`. Internal authority is independent of connection and JWT presence. Newly compiled modules advertise `hosted_auth_v1` and require a compatible host. Function visibility and scheduled defaults are unchanged. ## Current State diff --git a/crates/bindings-cpp/include/spacetimedb/function_visibility.h b/crates/bindings-cpp/include/spacetimedb/function_visibility.h deleted file mode 100644 index 9bd36e19e48..00000000000 --- a/crates/bindings-cpp/include/spacetimedb/function_visibility.h +++ /dev/null @@ -1,6 +0,0 @@ -#pragma once - -namespace SpacetimeDB { -// Omission preserves the host default: Public ordinarily, Private when scheduled. -enum class FunctionVisibility { Public, Private, Internal }; -} diff --git a/crates/bindings-cpp/include/spacetimedb/internal/autogen/FunctionVisibility.g.h b/crates/bindings-cpp/include/spacetimedb/internal/autogen/FunctionVisibility.g.h index 9795b3bd2d7..423276de9b4 100644 --- a/crates/bindings-cpp/include/spacetimedb/internal/autogen/FunctionVisibility.g.h +++ b/crates/bindings-cpp/include/spacetimedb/internal/autogen/FunctionVisibility.g.h @@ -18,7 +18,5 @@ namespace SpacetimeDB::Internal { enum class FunctionVisibility : uint8_t { Private = 0, ClientCallable = 1, - Internal = 2, - ExplicitClientCallable = 3, }; } // namespace SpacetimeDB::Internal diff --git a/crates/bindings-cpp/include/spacetimedb/internal/v10_builder.h b/crates/bindings-cpp/include/spacetimedb/internal/v10_builder.h index f398a4eb9c2..235b5e5f680 100644 --- a/crates/bindings-cpp/include/spacetimedb/internal/v10_builder.h +++ b/crates/bindings-cpp/include/spacetimedb/internal/v10_builder.h @@ -13,7 +13,6 @@ #include #include "../bsatn/bsatn.h" #include "../database.h" -#include "../function_visibility.h" #include "autogen/CaseConversionPolicy.g.h" #include "autogen/ExplicitNameEntry.g.h" #include "autogen/NameMapping.g.h" @@ -50,7 +49,6 @@ void fail_reducer(std::string message); namespace Internal { -// Builds the V10 module definition with explicit function visibility. class V10Builder { public: V10Builder() = default; @@ -439,7 +437,7 @@ class V10Builder { RawReducerDefV10 reducer_def{ reducer_name, ProductType{}, - FunctionVisibility::Internal, + FunctionVisibility::Private, MakeUnitAlgebraicType(), MakeStringAlgebraicType(), }; @@ -648,7 +646,6 @@ class V10Builder { void RegisterExplicitTableName(const std::string& source_name, const std::string& canonical_name); void RegisterExplicitFunctionName(const std::string& source_name, const std::string& canonical_name); - void SetFunctionVisibility(const std::string& source_name, ::SpacetimeDB::FunctionVisibility visibility); void RegisterExplicitIndexName(const std::string& source_name, const std::string& canonical_name); RawModuleDefV10 BuildModuleDef() const; diff --git a/crates/bindings-cpp/include/spacetimedb/macros.h b/crates/bindings-cpp/include/spacetimedb/macros.h index b4ac3ba0d9c..2807be4333b 100644 --- a/crates/bindings-cpp/include/spacetimedb/macros.h +++ b/crates/bindings-cpp/include/spacetimedb/macros.h @@ -609,15 +609,6 @@ inline std::vector parseParameterNames(const std::string& param_lis // VISIBILITY FILTER MACRO // ============================================================================= -// Apply to a registered reducer or procedure. Runs after function registration; -// lifecycle reducers only accept Internal. Scheduling preserves this choice. -#define SPACETIMEDB_FUNCTION_VISIBILITY(function_name, visibility) \ - extern "C" __attribute__((export_name("__preinit__40_visibility_" #function_name))) \ - void CONCAT(__spacetimedb_function_visibility_, function_name)() { \ - ::SpacetimeDB::Internal::getV10Builder().SetFunctionVisibility( \ - #function_name, ::SpacetimeDB::FunctionVisibility::visibility); \ - } - /** * @brief Set module case conversion policy using a fixed preinit registration symbol. * @@ -926,3 +917,4 @@ inline std::vector parseParameterNames(const std::string& param_lis #endif // SPACETIMEDB_MACROS_H + diff --git a/crates/bindings-cpp/src/internal/v10_builder.cpp b/crates/bindings-cpp/src/internal/v10_builder.cpp index f5726307b65..e03917e8561 100644 --- a/crates/bindings-cpp/src/internal/v10_builder.cpp +++ b/crates/bindings-cpp/src/internal/v10_builder.cpp @@ -220,31 +220,6 @@ RawConstraintDefV10 V10Builder::CreateUniqueConstraint(const std::string& table_ }; } -void V10Builder::SetFunctionVisibility(const std::string& name, ::SpacetimeDB::FunctionVisibility visibility) { - FunctionVisibility declared; - switch (visibility) { - case ::SpacetimeDB::FunctionVisibility::Public: declared = FunctionVisibility::ExplicitClientCallable; break; - case ::SpacetimeDB::FunctionVisibility::Private: declared = FunctionVisibility::Private; break; - case ::SpacetimeDB::FunctionVisibility::Internal: declared = FunctionVisibility::Internal; break; - default: - SetConstraintRegistrationError("INVALID_FUNCTION_VISIBILITY", "function='" + name + "'"); - return; - } - for (const auto& lifecycle : lifecycle_reducers_) { - if (lifecycle.function_name == name && declared != FunctionVisibility::Internal) { - SetConstraintRegistrationError("INVALID_LIFECYCLE_VISIBILITY", "function='" + name + "' must be Internal"); - return; - } - } - for (auto& reducer : reducers_) { - if (reducer.source_name == name) { reducer.visibility = declared; return; } - } - for (auto& procedure : procedures_) { - if (procedure.source_name == name) { procedure.visibility = declared; return; } - } - SetConstraintRegistrationError("UNKNOWN_FUNCTION_VISIBILITY", "function='" + name + "' is not a reducer or procedure"); -} - RawModuleDefV10 V10Builder::BuildModuleDef() const { RawModuleDefV10 v10_module; @@ -253,6 +228,24 @@ RawModuleDefV10 V10Builder::BuildModuleDef() const { std::vector reducers = reducers_; std::vector procedures = procedures_; + std::unordered_set internal_functions; + for (const auto& lifecycle : lifecycle_reducers_) { + internal_functions.insert(lifecycle.function_name); + } + for (const auto& schedule : schedules_) { + internal_functions.insert(schedule.function_name); + } + for (auto& reducer : reducers) { + if (internal_functions.find(reducer.source_name) != internal_functions.end()) { + reducer.visibility = FunctionVisibility::Private; + } + } + for (auto& procedure : procedures) { + if (internal_functions.find(procedure.source_name) != internal_functions.end()) { + procedure.visibility = FunctionVisibility::Private; + } + } + RawModuleDefV10Section section_typespace; section_typespace.set<0>(typespace_); v10_module.sections.push_back(section_typespace); diff --git a/crates/bindings-cpp/tests/unit/CMakeLists.txt b/crates/bindings-cpp/tests/unit/CMakeLists.txt index a26014fdd56..f43799931b7 100644 --- a/crates/bindings-cpp/tests/unit/CMakeLists.txt +++ b/crates/bindings-cpp/tests/unit/CMakeLists.txt @@ -9,21 +9,10 @@ if(NOT CMAKE_SYSTEM_NAME STREQUAL "Emscripten") endif() add_executable(bindings_cpp_unit_tests + hosted_auth_unit_tests.cpp main.cpp http_unit_tests.cpp environment_unit_tests.cpp - hosted_auth_unit_tests.cpp - function_visibility_unit_tests.cpp -) - -# Exercise the real module builder without the standalone WASI shims, which -# replace the Node test runner's standard I/O and process lifecycle functions. -target_sources(bindings_cpp_unit_tests PRIVATE - ../../src/internal/Module.cpp - ../../src/internal/AlgebraicType.cpp - ../../src/internal/v9_builder.cpp - ../../src/internal/v10_builder.cpp - ../../src/internal/module_type_registration.cpp ) target_include_directories(bindings_cpp_unit_tests PRIVATE diff --git a/crates/bindings-cpp/tests/unit/function_visibility_unit_tests.cpp b/crates/bindings-cpp/tests/unit/function_visibility_unit_tests.cpp deleted file mode 100644 index 95597eb5ff6..00000000000 --- a/crates/bindings-cpp/tests/unit/function_visibility_unit_tests.cpp +++ /dev/null @@ -1,115 +0,0 @@ -#include "test_harness.h" -#include "spacetimedb/reducer_error.h" -#include "spacetimedb/procedure_context.h" -#include "spacetimedb/internal/v10_builder.h" -#include "spacetimedb/internal/autogen/RawModuleDef.g.h" -#include "spacetimedb/macros.h" - -using namespace SpacetimeDB; -using namespace SpacetimeDB::Internal; - -namespace { -ReducerResult noop(ReducerContext) { return Ok(); } -uint32_t procedure(ProcedureContext) { return 7; } -} - -SPACETIMEDB_FUNCTION_VISIBILITY(visibility_macro_target, Internal); - -TEST_CASE(visibility_macro_applies_after_function_registration) { - auto& builder = getV10Builder(); - builder.RegisterReducer("visibility_macro_target", &noop, {}); - __spacetimedb_function_visibility_visibility_macro_target(); - bool found = false; - for (const auto& section : builder.BuildModuleDef().sections) { - if (section.get_tag() != 3) continue; - for (const auto& reducer : section.get<3>()) { - if (reducer.source_name != "visibility_macro_target") continue; - ASSERT_EQ(SpacetimeDB::Internal::FunctionVisibility::Internal, reducer.visibility); - found = true; - } - } - ASSERT_TRUE(found); -} - -TEST_CASE(v10_retains_explicit_visibility_and_schedule_default) { - V10Builder builder; - builder.RegisterReducer("omitted", &noop, {}); - builder.RegisterReducer("public", &noop, {}); - builder.RegisterReducer("private", &noop, {}); - builder.RegisterReducer("internal", &noop, {}); - builder.SetFunctionVisibility("public", SpacetimeDB::FunctionVisibility::Public); - builder.SetFunctionVisibility("private", SpacetimeDB::FunctionVisibility::Private); - builder.SetFunctionVisibility("internal", SpacetimeDB::FunctionVisibility::Internal); - builder.RegisterSchedule("jobs", 0, "public"); - builder.RegisterSchedule("other_jobs", 0, "omitted"); - builder.RegisterProcedure("procedure", &procedure); - builder.SetFunctionVisibility("procedure", SpacetimeDB::FunctionVisibility::Internal); - - RawModuleDef versioned; - versioned.set<2>(builder.BuildModuleDef()); - std::vector bytes; - bsatn::Writer writer(bytes); - bsatn::serialize(writer, versioned); - ASSERT_EQ(uint8_t{2}, bytes.at(0)); - ASSERT_EQ(uint8_t{2}, versioned.get_tag()); - bool saw_reducers = false, saw_procedure = false, saw_environment = false, saw_capability = false; - for (const auto& section : versioned.get<2>().sections) { - if (section.get_tag() == 3) { - const auto& reducers = section.get<3>(); - ASSERT_EQ(size_t{4}, reducers.size()); - ASSERT_EQ(SpacetimeDB::Internal::FunctionVisibility::ClientCallable, reducers[0].visibility); - ASSERT_EQ(SpacetimeDB::Internal::FunctionVisibility::ExplicitClientCallable, reducers[1].visibility); - ASSERT_EQ(SpacetimeDB::Internal::FunctionVisibility::Private, reducers[2].visibility); - ASSERT_EQ(SpacetimeDB::Internal::FunctionVisibility::Internal, reducers[3].visibility); - saw_reducers = true; - } else if (section.get_tag() == 4) { - ASSERT_EQ(SpacetimeDB::Internal::FunctionVisibility::Internal, section.get<4>().at(0).visibility); - saw_procedure = true; - } else if (section.get_tag() == 15) { - ASSERT_TRUE(section.get<15>().empty()); - saw_environment = true; - } else if (section.get_tag() == 16) { - ASSERT_EQ(std::vector{"hosted_auth_v1"}, section.get<16>()); - saw_capability = true; - } - } - ASSERT_TRUE(saw_reducers && saw_procedure && saw_environment && saw_capability); -} - -TEST_CASE(v10_visibility_extends_enum_without_changing_reducer_field_layout) { - V10Builder builder; - builder.RegisterReducer("r", &noop, {}); - auto reducer = builder.GetReducers().at(0); - for (uint8_t tag = 0; tag <= 3; ++tag) { - reducer.visibility = static_cast(tag); - std::vector bytes; - bsatn::Writer writer(bytes); - bsatn::serialize(writer, reducer); - const std::vector expected{1, 0, 0, 0, 'r', 0, 0, 0, 0, tag, 2, 0, 0, 0, 0, 4}; - ASSERT_EQ(expected, bytes); - const RawProcedureDefV10 procedure_def{ - "p", ProductType{}, reducer.ok_return_type, reducer.visibility, - }; - std::vector procedure_bytes; - bsatn::Writer procedure_writer(procedure_bytes); - bsatn::serialize(procedure_writer, procedure_def); - const std::vector expected_procedure{ - 1, 0, 0, 0, 'p', 0, 0, 0, 0, 2, 0, 0, 0, 0, tag, - }; - ASSERT_EQ(expected_procedure, procedure_bytes); - } -} - -TEST_CASE(v10_environment_and_capabilities_have_distinct_appended_wire_tags) { - RawModuleDefV10Section environment; - environment.set<15>(std::vector{}); - RawModuleDefV10Section capabilities; - capabilities.set<16>(std::vector{}); - for (const auto& section : {environment, capabilities}) { - std::vector bytes; - bsatn::Writer writer(bytes); - bsatn::serialize(writer, section); - const std::vector expected{section.get_tag(), 0, 0, 0, 0}; - ASSERT_EQ(expected, bytes); - } -} diff --git a/crates/bindings-csharp/Codegen.Tests/Tests.cs b/crates/bindings-csharp/Codegen.Tests/Tests.cs index 39957bbef0c..5ecccedb31b 100644 --- a/crates/bindings-csharp/Codegen.Tests/Tests.cs +++ b/crates/bindings-csharp/Codegen.Tests/Tests.cs @@ -352,66 +352,6 @@ public static void @params(ProcedureContext ctx) Assert.Empty(GetCompilationErrors(compilationAfterGen)); } - [Fact] - public static async Task ExplicitFunctionVisibilityCompilesAndRejectsExternalLifecycle() - { - var fixture = await Fixture.Compile("server"); - const string source = """ - using SpacetimeDB; - public static partial class VisibilityFunctions - { - [Reducer(Visibility = FunctionVisibility.Public)] - public static void PublicJob(ReducerContext ctx) {} - [Reducer(Visibility = FunctionVisibility.Private)] - public static void PrivateJob(ReducerContext ctx) {} - [Reducer(Visibility = FunctionVisibility.Internal)] - public static void InternalJob(ReducerContext ctx) {} - [Procedure(Visibility = FunctionVisibility.Internal)] - public static int InternalProcedure(ProcedureContext ctx) => 1; - } - """; - var parseOptions = fixture.ParseOptions; - var tree = CSharpSyntaxTree.ParseText(source, parseOptions); - var compilation = fixture.SampleCompilation.AddSyntaxTrees(tree); - var driver = CSharpGeneratorDriver.Create( - [ - new SpacetimeDB.Codegen.Type().AsSourceGenerator(), - new SpacetimeDB.Codegen.Module().AsSourceGenerator(), - new EnvironmentGenerator().AsSourceGenerator(), - ], - parseOptions: parseOptions - ); - var result = driver.RunGenerators(compilation).GetRunResult(); - Assert.Empty(result.Diagnostics.Where(d => d.Severity == DiagnosticSeverity.Error)); - Assert.Empty(GetCompilationErrors(compilation.AddSyntaxTrees(result.GeneratedTrees))); - var generated = string.Join("\n", result.GeneratedTrees.Select(t => t.ToString())); - Assert.Contains( - "Visibility: SpacetimeDB.Internal.FunctionVisibility.ExplicitClientCallable", - generated - ); - Assert.Contains("Visibility: SpacetimeDB.Internal.FunctionVisibility.Private", generated); - Assert.Contains("Visibility: SpacetimeDB.Internal.FunctionVisibility.Internal", generated); - - var invalid = CSharpSyntaxTree.ParseText( - """ - using SpacetimeDB; - public static partial class BadVisibility - { - [Reducer(ReducerKind.Init, Visibility = FunctionVisibility.Public)] - public static void InvalidLifecycle(ReducerContext ctx) {} - } - """, - parseOptions - ); - var rejected = driver - .RunGenerators(fixture.SampleCompilation.AddSyntaxTrees(invalid)) - .GetRunResult(); - Assert.Contains( - rejected.Diagnostics, - diagnostic => diagnostic.GetMessage().Contains("Lifecycle reducers only permit") - ); - } - [Fact] public static async Task TestDiagnostics() { diff --git a/crates/bindings-csharp/Codegen.Tests/fixtures/diag/snapshots/Module#FFI.verified.cs b/crates/bindings-csharp/Codegen.Tests/fixtures/diag/snapshots/Module#FFI.verified.cs index 8a5cb58534d..e81f122020d 100644 --- a/crates/bindings-csharp/Codegen.Tests/fixtures/diag/snapshots/Module#FFI.verified.cs +++ b/crates/bindings-csharp/Codegen.Tests/fixtures/diag/snapshots/Module#FFI.verified.cs @@ -3304,7 +3304,7 @@ SpacetimeDB.BSATN.ITypeRegistrar registrar new( SourceName: nameof(TestDuplicateReducerKind1), Params: [], - Visibility: SpacetimeDB.Internal.FunctionVisibility.Internal, + Visibility: SpacetimeDB.Internal.FunctionVisibility.ClientCallable, OkReturnType: SpacetimeDB.BSATN.AlgebraicType.Unit, ErrReturnType: new SpacetimeDB.BSATN.AlgebraicType.String(default) ); @@ -3325,7 +3325,7 @@ SpacetimeDB.BSATN.ITypeRegistrar registrar new( SourceName: nameof(TestDuplicateReducerKind2), Params: [], - Visibility: SpacetimeDB.Internal.FunctionVisibility.Internal, + Visibility: SpacetimeDB.Internal.FunctionVisibility.ClientCallable, OkReturnType: SpacetimeDB.BSATN.AlgebraicType.Unit, ErrReturnType: new SpacetimeDB.BSATN.AlgebraicType.String(default) ); diff --git a/crates/bindings-csharp/Codegen.Tests/fixtures/server/snapshots/Module#FFI.verified.cs b/crates/bindings-csharp/Codegen.Tests/fixtures/server/snapshots/Module#FFI.verified.cs index 82db7c6742d..95bbf9c0516 100644 --- a/crates/bindings-csharp/Codegen.Tests/fixtures/server/snapshots/Module#FFI.verified.cs +++ b/crates/bindings-csharp/Codegen.Tests/fixtures/server/snapshots/Module#FFI.verified.cs @@ -2348,7 +2348,7 @@ SpacetimeDB.BSATN.ITypeRegistrar registrar new( SourceName: nameof(Init), Params: [], - Visibility: SpacetimeDB.Internal.FunctionVisibility.Internal, + Visibility: SpacetimeDB.Internal.FunctionVisibility.ClientCallable, OkReturnType: SpacetimeDB.BSATN.AlgebraicType.Unit, ErrReturnType: new SpacetimeDB.BSATN.AlgebraicType.String(default) ); diff --git a/crates/bindings-csharp/Codegen/Diag.cs b/crates/bindings-csharp/Codegen/Diag.cs index 5d6f3259d56..c2374db5b8e 100644 --- a/crates/bindings-csharp/Codegen/Diag.cs +++ b/crates/bindings-csharp/Codegen/Diag.cs @@ -361,13 +361,4 @@ string type $"View '{ctx.method.Identifier}' declares primary key '{ctx.primaryKey}', but its type '{ctx.type}' is not supported for view primary keys.", ctx => ctx.primaryKeySyntax ); - - public static readonly ErrorDescriptor InvalidFunctionVisibility = - new( - group, - "Invalid function visibility", - _ => - $"Visibility must be Default, Public, Private, or Internal. Lifecycle reducers only permit Default or Internal.", - method => method.Identifier - ); } diff --git a/crates/bindings-csharp/Codegen/Module.cs b/crates/bindings-csharp/Codegen/Module.cs index e90b494df50..29fc16d2cfc 100644 --- a/crates/bindings-csharp/Codegen/Module.cs +++ b/crates/bindings-csharp/Codegen/Module.cs @@ -1500,46 +1500,13 @@ public static byte[] Invoke( } /// -/// Validates a declared function visibility and maps it to the V10 schema. +/// Represents a reducer method declaration in a module. /// -static class FunctionVisibilityDeclaration -{ - internal static string Resolve( - FunctionVisibility visibility, - bool lifecycle, - MethodDeclarationSyntax method, - DiagReporter diag - ) - { - if ( - ( - lifecycle - && visibility is not (FunctionVisibility.Default or FunctionVisibility.Internal) - ) || !Enum.IsDefined(typeof(FunctionVisibility), visibility) - ) - { - diag.Report(ErrorDescriptor.InvalidFunctionVisibility, method); - return "SpacetimeDB.Internal.FunctionVisibility.Internal"; - } - return visibility switch - { - FunctionVisibility.Public => - "SpacetimeDB.Internal.FunctionVisibility.ExplicitClientCallable", - FunctionVisibility.Private => "SpacetimeDB.Internal.FunctionVisibility.Private", - FunctionVisibility.Internal => "SpacetimeDB.Internal.FunctionVisibility.Internal", - _ => lifecycle - ? "SpacetimeDB.Internal.FunctionVisibility.Internal" - : "SpacetimeDB.Internal.FunctionVisibility.ClientCallable", - }; - } -} - record ReducerDeclaration { public readonly string Name; public readonly string? CanonicalName; public readonly ReducerKind Kind; - public readonly string Visibility; public readonly string FullName; public readonly EquatableArray Args; public readonly Scope Scope; @@ -1578,12 +1545,6 @@ public ReducerDeclaration(GeneratorAttributeSyntaxContext context, DiagReporter } Kind = attr.Kind; - Visibility = FunctionVisibilityDeclaration.Resolve( - attr.Visibility, - Kind != ReducerKind.UserDefined, - methodSyntax, - diag - ); CanonicalName = attr.Name; FullName = SymbolToName(method); Args = new( @@ -1612,7 +1573,7 @@ sealed class {{Identifier}}: SpacetimeDB.Internal.IReducer { public SpacetimeDB.Internal.RawReducerDefV10 MakeReducerDef(SpacetimeDB.BSATN.ITypeRegistrar registrar) => new ( SourceName: nameof({{Identifier}}), Params: [{{MemberDeclaration.GenerateDefs(Args)}}], - Visibility: {{Visibility}}, + Visibility: SpacetimeDB.Internal.FunctionVisibility.ClientCallable, OkReturnType: SpacetimeDB.BSATN.AlgebraicType.Unit, ErrReturnType: new SpacetimeDB.BSATN.AlgebraicType.String(default) ); @@ -1669,7 +1630,6 @@ record ProcedureDeclaration { public readonly string Name; public readonly string? CanonicalName; - public readonly string Visibility; public readonly string FullName; public readonly EquatableArray Args; public readonly Scope Scope; @@ -1686,12 +1646,6 @@ public ProcedureDeclaration(GeneratorAttributeSyntaxContext context, DiagReporte var methodSyntax = (MethodDeclarationSyntax)context.TargetNode; var method = (IMethodSymbol)context.TargetSymbol; var attr = context.Attributes.Single().ParseAs(); - Visibility = FunctionVisibilityDeclaration.Resolve( - attr.Visibility, - false, - methodSyntax, - diag - ); if ( method.Parameters.FirstOrDefault()?.Type @@ -1850,7 +1804,7 @@ sealed class {{{Identifier}}} : SpacetimeDB.Internal.IProcedure { SourceName: nameof({{{Identifier}}}), Params: [{{{MemberDeclaration.GenerateDefs(Args)}}}], ReturnType: {{{returnTypeExpr}}}, - Visibility: {{{Visibility}}} + Visibility: SpacetimeDB.Internal.FunctionVisibility.ClientCallable ); public static byte[] Invoke(BinaryReader reader, SpacetimeDB.Internal.IProcedureContext ctx) { diff --git a/crates/bindings-csharp/README.md b/crates/bindings-csharp/README.md index 2aa4f8e4e9a..06c7446836c 100644 --- a/crates/bindings-csharp/README.md +++ b/crates/bindings-csharp/README.md @@ -6,27 +6,6 @@ See the [C# module library reference](https://spacetimedb.com/docs/modules/c-sha ## Internal documentation -### Function visibility and invocation authentication - -Reducers and procedures can declare `Visibility = FunctionVisibility.Public`, -`Private`, or `Internal` in their attributes. Omission (`Default`) means public -for ordinary functions and private for scheduled functions. An explicit choice -is preserved when the function is scheduled. Lifecycle reducers permit only -omission or `Internal` and can only run for their host lifecycle event. - -Internal functions require verified internal authority. Private functions also -admit the owner, and public functions admit any client. For example: - -```csharp -[Reducer(Visibility = FunctionVisibility.Internal)] -public static void ProcessJobs(ReducerContext ctx) { } -``` - -`ctx.SenderAuth.IsInternal` comes from the host's invocation authority. It is -independent of connection and JWT presence, so an internal call can have a JWT. -JWT identity is the verified sender supplied by the host. Newly compiled modules -emit schema V10 and advertise `hosted_auth_v1`, requiring a compatible host. - These projects contain the SpacetimeDB SATS typesystem, codegen and runtime bindings for SpacetimeDB WebAssembly modules. It also contains serialization code for SpacetimeDB C# clients. @@ -42,6 +21,10 @@ The [`Codegen`](./Codegen/) and [`Runtime`](./Runtime/) libraries are used: They provide all of the functionality needed to write SpacetimeDB modules in C#. See their READMEs for more information. +### Invocation authentication + +`ctx.SenderAuth.IsInternal` captures host-verified invocation authority. JWT identity is the verified sender supplied by the host. Internal authority is independent of connection and JWT presence. Newly compiled modules advertise `hosted_auth_v1` and require a compatible host. Function visibility and scheduled defaults are unchanged. + ### Declared environment A module may declare one `[SpacetimeDB.Env]` struct. `string` is required and diff --git a/crates/bindings-csharp/Runtime.Tests/FunctionVisibilityTests.cs b/crates/bindings-csharp/Runtime.Tests/FunctionVisibilityTests.cs deleted file mode 100644 index c92a47ad4ec..00000000000 --- a/crates/bindings-csharp/Runtime.Tests/FunctionVisibilityTests.cs +++ /dev/null @@ -1,91 +0,0 @@ -namespace Runtime.Tests; - -using SpacetimeDB.BSATN; -using SpacetimeDB.Internal; - -public class FunctionVisibilityTests -{ - [Theory] - [InlineData(FunctionVisibility.Private, 0)] - [InlineData(FunctionVisibility.ClientCallable, 1)] - [InlineData(FunctionVisibility.Internal, 2)] - [InlineData(FunctionVisibility.ExplicitClientCallable, 3)] - public void V10RetainsVisibilityEnumEncoding(FunctionVisibility visibility, byte tag) - { - var bytes = IStructuralReadWrite.ToBytes( - new SpacetimeDB.BSATN.Enum(), - visibility - ); - Assert.Equal(new byte[] { tag }, bytes); - } - - [Theory] - [InlineData(FunctionVisibility.ExplicitClientCallable)] - [InlineData(FunctionVisibility.ClientCallable)] - [InlineData(FunctionVisibility.Private)] - [InlineData(FunctionVisibility.Internal)] - public void SchedulingPreservesVisibility(FunctionVisibility visibility) - { - var module = new RawModuleDefV10(); - var reducer = new RawReducerDefV10( - "run_job", - [], - visibility, - AlgebraicType.Unit, - new AlgebraicType.String(default) - ); - module.RegisterReducer(reducer, null); - module.RegisterTable( - new RawTableDefV10 { SourceName = "jobs" }, - new RawScheduleDefV10(null, "jobs", 0, "run_job") - ); - var raw = module.BuildModuleDefinition(); - var reducers = Assert.Single(raw.Sections.OfType()); - Assert.Equal(visibility, Assert.Single(reducers.Reducers_).Visibility); - Assert.Empty( - Assert.Single(raw.Sections.OfType()).Environment_ - ); - var capabilities = Assert.Single( - raw.Sections.OfType() - ); - Assert.Contains("hosted_auth_v1", capabilities.Capabilities_); - } - - [Fact] - public void EnvironmentAndCapabilitiesUseDistinctAppendedV10WireTags() - { - var serializer = new RawModuleDefV10Section.BSATN(); - Assert.Equal( - new byte[] { 15, 0, 0, 0, 0 }, - IStructuralReadWrite.ToBytes( - serializer, - new RawModuleDefV10Section.Environment([]) - ) - ); - Assert.Equal( - new byte[] { 16, 0, 0, 0, 0 }, - IStructuralReadWrite.ToBytes( - serializer, - new RawModuleDefV10Section.Capabilities([]) - ) - ); - } - - [Theory] - [InlineData(FunctionVisibility.ClientCallable)] - [InlineData(FunctionVisibility.ExplicitClientCallable)] - public void LifecycleRejectsExternalVisibility(FunctionVisibility visibility) - { - var module = new RawModuleDefV10(); - var reducer = new RawReducerDefV10( - "initialize", - [], - visibility, - AlgebraicType.Unit, - new AlgebraicType.String(default) - ); - Assert.Throws( - () => module.RegisterReducer(reducer, Lifecycle.Init) - ); - } -} diff --git a/crates/bindings-csharp/Runtime/Attrs.cs b/crates/bindings-csharp/Runtime/Attrs.cs index b06340d8aa1..afcfcc0688e 100644 --- a/crates/bindings-csharp/Runtime/Attrs.cs +++ b/crates/bindings-csharp/Runtime/Attrs.cs @@ -210,30 +210,18 @@ public enum ReducerKind ClientDisconnected, } - /// Invocation admission for reducers and procedures. - public enum FunctionVisibility - { - /// Public for ordinary functions, Private for scheduled functions. - Default, - Public, - Private, - Internal, - } - [AttributeUsage(AttributeTargets.Method, Inherited = false)] public sealed class ReducerAttribute(ReducerKind kind = ReducerKind.UserDefined) : Attribute { public ReducerKind Kind => kind; public string? Name { get; init; } - public FunctionVisibility Visibility { get; init; } } [AttributeUsage(AttributeTargets.Method, Inherited = false)] public sealed class ProcedureAttribute() : Attribute { public string? Name { get; init; } - public FunctionVisibility Visibility { get; init; } } [AttributeUsage(AttributeTargets.Method, Inherited = false)] diff --git a/crates/bindings-csharp/Runtime/Internal/Autogen/FunctionVisibility.g.cs b/crates/bindings-csharp/Runtime/Internal/Autogen/FunctionVisibility.g.cs index 29adc856f78..2f9772dd591 100644 --- a/crates/bindings-csharp/Runtime/Internal/Autogen/FunctionVisibility.g.cs +++ b/crates/bindings-csharp/Runtime/Internal/Autogen/FunctionVisibility.g.cs @@ -12,7 +12,5 @@ public enum FunctionVisibility { Private, ClientCallable, - Internal, - ExplicitClientCallable, } } diff --git a/crates/bindings-csharp/Runtime/Internal/Module.cs b/crates/bindings-csharp/Runtime/Internal/Module.cs index cb2cfc7c426..260a0ec3265 100644 --- a/crates/bindings-csharp/Runtime/Internal/Module.cs +++ b/crates/bindings-csharp/Runtime/Internal/Module.cs @@ -57,23 +57,13 @@ internal AlgebraicType.Ref RegisterType(Func l.FunctionName) + .Concat(scheduleDefs.Select(s => s.FunctionName)) + .ToHashSet(StringComparer.Ordinal); + + foreach (var reducer in reducerDefs) + { + if (internalFunctions.Contains(reducer.SourceName)) + { + reducer.Visibility = FunctionVisibility.Private; + } + } + + foreach (var procedure in procedureDefs) + { + if (internalFunctions.Contains(procedure.SourceName)) + { + procedure.Visibility = FunctionVisibility.Private; + } + } + var sections = new List { new RawModuleDefV10Section.Typespace(typespace), diff --git a/crates/bindings-macro/src/procedure.rs b/crates/bindings-macro/src/procedure.rs index 129b32cc2fe..9f76e5b547f 100644 --- a/crates/bindings-macro/src/procedure.rs +++ b/crates/bindings-macro/src/procedure.rs @@ -1,5 +1,4 @@ use crate::reducer::{assert_only_lifetime_generics, extract_typed_args, generate_explicit_names_impl}; -use crate::reducer::{parse_visibility, DeclaredVisibility}; use crate::sym; use crate::util::{check_duplicate, ident_to_litstr, match_meta}; use proc_macro2::TokenStream; @@ -11,16 +10,12 @@ use syn::{ItemFn, LitStr}; pub(crate) struct ProcedureArgs { /// For consistency with reducers: allow specifying a different export name than the Rust function name. name: Option, - visibility: Option, } impl ProcedureArgs { pub(crate) fn parse(input: TokenStream) -> syn::Result { let mut args = Self::default(); syn::meta::parser(|meta| { - if parse_visibility(&meta, &mut args.visibility)? { - return Ok(()); - } match_meta!(match meta { sym::name => { check_duplicate(&args.name, &meta)?; @@ -34,11 +29,10 @@ impl ProcedureArgs { } } -pub(crate) fn procedure_impl(args: ProcedureArgs, original_function: &ItemFn) -> syn::Result { +pub(crate) fn procedure_impl(_args: ProcedureArgs, original_function: &ItemFn) -> syn::Result { let func_name = &original_function.sig.ident; let vis = &original_function.vis; - let explicit_name = args.name.as_ref(); - let visibility = args.visibility.map(DeclaredVisibility::tokens).into_iter(); + let explicit_name = _args.name.as_ref(); let procedure_name = ident_to_litstr(func_name); @@ -123,7 +117,6 @@ pub(crate) fn procedure_impl(args: ProcedureArgs, original_function: &ItemFn) -> /// The name of this function const NAME: &'static str = #procedure_name; - #(const DECLARED_VISIBILITY: Option = Some(#visibility);)* /// The parameter names of this function const ARG_NAMES: &'static [Option<&'static str>] = &[#(#opt_arg_names),*]; @@ -140,29 +133,3 @@ pub(crate) fn procedure_impl(args: ProcedureArgs, original_function: &ItemFn) -> #generate_explicit_names }) } - -#[cfg(test)] -mod tests { - use super::*; - #[test] - fn procedure_visibility_rejects_duplicates_and_emits_selection() { - assert!(ProcedureArgs::parse(quote!(private, public)).is_err()); - assert!(ProcedureArgs::parse(quote!(internal, internal)).is_err()); - let function: ItemFn = syn::parse_quote!( - fn example(ctx: &mut ProcedureContext) -> u64 { - 0 - } - ); - for (input, expected) in [ - (quote!(internal), "Internal"), - (quote!(private), "Private"), - (quote!(public), "ClientCallable"), - ] { - let tokens = procedure_impl(ProcedureArgs::parse(input).unwrap(), &function) - .unwrap() - .to_string(); - assert!(tokens.contains("DECLARED_VISIBILITY")); - assert!(tokens.contains(&format!("FunctionVisibility :: {expected}"))); - } - } -} diff --git a/crates/bindings-macro/src/reducer.rs b/crates/bindings-macro/src/reducer.rs index 3093a51397a..ac261ced35f 100644 --- a/crates/bindings-macro/src/reducer.rs +++ b/crates/bindings-macro/src/reducer.rs @@ -10,44 +10,6 @@ use syn::{FnArg, Ident, ItemFn, LitStr, PatType}; pub(crate) struct ReducerArgs { name: Option, lifecycle: Option, - visibility: Option, -} - -#[derive(Clone, Copy, PartialEq, Eq)] -pub(crate) enum DeclaredVisibility { - Internal, - Private, - Public, -} - -impl DeclaredVisibility { - pub(crate) fn tokens(self) -> TokenStream { - let variant = match self { - Self::Internal => "Internal", - Self::Private => "Private", - Self::Public => "ClientCallable", - }; - let variant = Ident::new(variant, Span::call_site()); - quote!(spacetimedb::rt::FunctionVisibility::#variant) - } -} - -pub(crate) fn parse_visibility( - meta: &syn::meta::ParseNestedMeta<'_>, - visibility: &mut Option, -) -> syn::Result { - let value = if meta.path.is_ident("internal") { - DeclaredVisibility::Internal - } else if meta.path.is_ident("private") { - DeclaredVisibility::Private - } else if meta.path.is_ident("public") { - DeclaredVisibility::Public - } else { - return Ok(false); - }; - check_duplicate_msg(visibility, meta, "already specified a function visibility")?; - *visibility = Some(value); - Ok(true) } enum LifecycleReducer { @@ -75,9 +37,6 @@ impl ReducerArgs { pub(crate) fn parse(input: TokenStream) -> syn::Result { let mut args = Self::default(); syn::meta::parser(|meta| { - if parse_visibility(&meta, &mut args.visibility)? { - return Ok(()); - } let mut set_lifecycle = |kind: fn(Span) -> _| -> syn::Result<()> { check_duplicate_msg(&args.lifecycle, &meta, "already specified a lifecycle reducer kind")?; args.lifecycle = Some(kind(meta.path.span())); @@ -96,12 +55,6 @@ impl ReducerArgs { Ok(()) }) .parse2(input)?; - if args.lifecycle.is_some() && args.visibility.is_some_and(|v| v != DeclaredVisibility::Internal) { - return Err(syn::Error::new( - Span::call_site(), - "lifecycle reducers must have internal visibility", - )); - } Ok(args) } } @@ -148,7 +101,6 @@ pub(crate) fn reducer_impl(args: ReducerArgs, original_function: &ItemFn) -> syn assert_only_lifetime_generics(original_function, "reducers")?; let lifecycle = args.lifecycle.iter().filter_map(|lc| lc.to_lifecycle_value()); - let visibility = args.visibility.map(DeclaredVisibility::tokens).into_iter(); let typed_args = extract_typed_args(original_function)?; @@ -213,7 +165,6 @@ pub(crate) fn reducer_impl(args: ReducerArgs, original_function: &ItemFn) -> syn /// The function kind, which will cause scheduled tables to accept reducers. type FnKind = spacetimedb::rt::FnKindReducer; const NAME: &'static str = #reducer_name; - #(const DECLARED_VISIBILITY: Option = Some(#visibility);)* #(const LIFECYCLE: Option = Some(#lifecycle);)* const ARG_NAMES: &'static [Option<&'static str>] = &[#(#opt_arg_names),*]; const INVOKE: Self::Invoke = #func_name::invoke; @@ -251,43 +202,3 @@ pub(crate) fn generate_explicit_names_impl( } } } - -#[cfg(test)] -mod tests { - use super::*; - #[test] - fn visibility_declarations_are_unambiguous() { - for input in [ - quote!(private, public), - quote!(internal, internal), - quote!(init, private), - quote!(public, client_connected), - ] { - assert!(ReducerArgs::parse(input).is_err()); - } - for input in [ - quote!(), - quote!(public), - quote!(private), - quote!(internal), - quote!(init, internal), - ] { - assert!(ReducerArgs::parse(input).is_ok()); - } - } - #[test] - fn rust_item_visibility_does_not_select_database_visibility() { - let function: ItemFn = syn::parse_quote!( - pub fn example(ctx: &ReducerContext) {} - ); - let implicit = reducer_impl(ReducerArgs::parse(quote!()).unwrap(), &function) - .unwrap() - .to_string(); - assert!(!implicit.contains("DECLARED_VISIBILITY")); - let explicit = reducer_impl(ReducerArgs::parse(quote!(internal)).unwrap(), &function) - .unwrap() - .to_string(); - assert!(explicit.contains("DECLARED_VISIBILITY")); - assert!(explicit.contains("FunctionVisibility :: Internal")); - } -} diff --git a/crates/bindings-typescript/README.md b/crates/bindings-typescript/README.md index ddf546c296e..6ac1d49dcf5 100644 --- a/crates/bindings-typescript/README.md +++ b/crates/bindings-typescript/README.md @@ -18,25 +18,9 @@ You can use the package in the browser, using a bundler like vite/parcel/rsbuild ### Usage -#### Module function visibility and invocation authentication - -Reducer and procedure options accept `visibility: 'public'`, `'private'`, or -`'internal'`. For example, `spacetime.reducer({ visibility: 'internal' }, ctx => {})` -declares an internal reducer. Omission means public for ordinary functions and -private for scheduled functions. An explicit choice is preserved when the -function is scheduled. Lifecycle reducers permit only omission or `'internal'` -and can only run for their host lifecycle event. - -Internal functions require verified internal authority. Private functions also -admit the owner, and public functions admit any client. `ctx.senderAuth.isInternal` -captures the host's invocation authority independently of connection and JWT -presence, so an internal call can have a JWT. `ctx.senderAuth.jwt.identity` is the -verified sender supplied by the host. Procedure transactions preserve this -authentication. Newly compiled modules retain schema V10 and advertise -`hosted_auth_v1`. The extended visibility values and capability section require -a compatible host; older V10 definitions retain their existing defaults. - -#### Client SDK +#### Module invocation authentication + +`ctx.senderAuth.isInternal` captures host-verified invocation authority independently of connection and JWT presence. `ctx.senderAuth.jwt.identity` is the verified sender. Procedure transactions preserve authentication. Newly compiled modules advertise `hosted_auth_v1` and require a compatible host. Function visibility and scheduled defaults are unchanged. In order to connect to a database you have to generate module bindings for your database. diff --git a/crates/bindings-typescript/src/lib/autogen/types.ts b/crates/bindings-typescript/src/lib/autogen/types.ts index 40ab70f9ae7..6e47663aa70 100644 --- a/crates/bindings-typescript/src/lib/autogen/types.ts +++ b/crates/bindings-typescript/src/lib/autogen/types.ts @@ -76,8 +76,6 @@ export type ExplicitNames = __Infer; export const FunctionVisibility = __t.enum('FunctionVisibility', { Private: __t.unit(), ClientCallable: __t.unit(), - Internal: __t.unit(), - ExplicitClientCallable: __t.unit(), }); export type FunctionVisibility = __Infer; diff --git a/crates/bindings-typescript/src/server/function_visibility.ts b/crates/bindings-typescript/src/server/function_visibility.ts deleted file mode 100644 index 658fb1dad0d..00000000000 --- a/crates/bindings-typescript/src/server/function_visibility.ts +++ /dev/null @@ -1,24 +0,0 @@ -import { FunctionVisibility as RawFunctionVisibility } from '../lib/autogen/types'; - -/** Internal functions require verified internal authority. Private functions also - * admit the owner. Public functions admit any authenticated client. */ -export type FunctionVisibility = 'public' | 'private' | 'internal'; - -export function rawVisibility( - visibility: FunctionVisibility | undefined -): RawFunctionVisibility { - switch (visibility) { - case undefined: - // Preserve V10's existing context-dependent default, including scheduled - // private functions, without changing the raw definition's field layout. - return RawFunctionVisibility.ClientCallable; - case 'public': - return RawFunctionVisibility.ExplicitClientCallable; - case 'private': - return RawFunctionVisibility.Private; - case 'internal': - return RawFunctionVisibility.Internal; - default: - throw new TypeError('Invalid function visibility'); - } -} diff --git a/crates/bindings-typescript/src/server/index.ts b/crates/bindings-typescript/src/server/index.ts index fdf3f45a224..3ac3e8f0fbb 100644 --- a/crates/bindings-typescript/src/server/index.ts +++ b/crates/bindings-typescript/src/server/index.ts @@ -10,7 +10,6 @@ export { table } from '../lib/table'; export { SenderError, SpacetimeHostError, errors } from './errors'; export type { Reducer, ReducerCtx, JwtClaims, AuthCtx } from '../lib/reducers'; export type { ReducerExport } from './reducers'; -export type { FunctionVisibility } from './function_visibility'; export { type DbView } from './db_view'; export * from './query'; export type { diff --git a/crates/bindings-typescript/src/server/procedures.ts b/crates/bindings-typescript/src/server/procedures.ts index 55fbe1c5fcb..863cd6ce62b 100644 --- a/crates/bindings-typescript/src/server/procedures.ts +++ b/crates/bindings-typescript/src/server/procedures.ts @@ -5,7 +5,7 @@ import { type Deserializer, type Serializer, } from '../lib/algebraic_type'; -import { rawVisibility, type FunctionVisibility } from './function_visibility'; +import { FunctionVisibility } from '../lib/autogen/types'; import BinaryReader from '../lib/binary_reader'; import BinaryWriter from '../lib/binary_writer'; import type { ConnectionId } from '../lib/connection_id'; @@ -59,19 +59,21 @@ export function makeProcedureExport< ret: Ret, fn: ProcedureFn ): ProcedureExport { + const name = opts?.name; + const procedureExport: ProcedureExport = (...args) => fn(...args); procedureExport[exportContext] = ctx; procedureExport[registerExport] = (ctx, exportName) => { - registerProcedure(ctx, exportName, params, ret, fn, opts); + registerProcedure(ctx, name ?? exportName, params, ret, fn); ctx.functionExports.set( procedureExport as ProcedureExport, - exportName + name ?? exportName ); if (opts?.onSchedule !== undefined) { ctx.pendingSchedules.push({ table: opts.onSchedule, - functionName: opts.name ?? exportName, + functionName: name ?? exportName, }); } }; @@ -89,9 +91,7 @@ export interface ProcedureOpts< Params extends ParamsObj = ParamsObj, Ret extends TypeBuilder = TypeBuilder, > { - name?: string; - /** Defaults to public, or private when scheduled. */ - visibility?: FunctionVisibility; + name: string; onSchedule?: Ret extends ReturnType ? ScheduleTableForParams : never; @@ -157,7 +157,7 @@ function registerProcedure< sourceName: exportName, params: paramsType, returnType, - visibility: rawVisibility(opts?.visibility), + visibility: FunctionVisibility.ClientCallable, }); if (opts?.name != null) { diff --git a/crates/bindings-typescript/src/server/reducers.ts b/crates/bindings-typescript/src/server/reducers.ts index 25de0c98820..ea5f770faf8 100644 --- a/crates/bindings-typescript/src/server/reducers.ts +++ b/crates/bindings-typescript/src/server/reducers.ts @@ -1,6 +1,5 @@ import { AlgebraicType } from '../lib/algebraic_type'; -import { type Lifecycle } from '../lib/autogen/types'; -import { rawVisibility, type FunctionVisibility } from './function_visibility'; +import { FunctionVisibility, type Lifecycle } from '../lib/autogen/types'; import type { ParamsObj, Reducer } from '../lib/reducers'; import { type UntypedSchemaDef } from '../lib/schema'; import type { ScheduleTableForParams } from '../lib/table_schema'; @@ -20,9 +19,7 @@ export interface ReducerExport< ModuleExport {} export interface ReducerOpts { - name?: string; - /** Defaults to public, or private when scheduled. Lifecycle hooks are internal. */ - visibility?: FunctionVisibility; + name: string; onSchedule?: ScheduleTableForParams; } @@ -87,19 +84,12 @@ export function registerReducer( const ref = ctx.registerTypesRecursively(params); const paramsType = ctx.resolveType(ref).value; const isLifecycle = lifecycle != null; - if ( - isLifecycle && - opts?.visibility != null && - opts.visibility !== 'internal' - ) { - throw new TypeError('Lifecycle reducers only support internal visibility'); - } ctx.moduleDef.reducers.push({ sourceName: exportName, params: paramsType, - // Keep the legacy default distinct from an explicit public declaration. - visibility: rawVisibility(opts?.visibility), + //ModuleDef validation code is responsible to mark private reducers + visibility: FunctionVisibility.ClientCallable, //Hardcoded for now - reducers do not return values yet okReturnType: AlgebraicType.Product({ elements: [] }), errReturnType: AlgebraicType.String, diff --git a/crates/bindings-typescript/src/server/schema.ts b/crates/bindings-typescript/src/server/schema.ts index d88c49da605..95688b7f3cd 100644 --- a/crates/bindings-typescript/src/server/schema.ts +++ b/crates/bindings-typescript/src/server/schema.ts @@ -409,10 +409,7 @@ export class Schema implements ModuleDefaultExport { case 2: { let arg1; [arg1, fn] = args; - if ( - typeof arg1.name === 'string' || - typeof arg1.visibility === 'string' - ) + if (typeof arg1.name === 'string') opts = arg1 as ReducerOptsWithOptionalName; else params = arg1 as Params; break; @@ -651,10 +648,7 @@ export class Schema implements ModuleDefaultExport { case 3: { let arg1; [arg1, ret, fn] = args; - if ( - typeof arg1.name === 'string' || - typeof arg1.visibility === 'string' - ) + if (typeof arg1.name === 'string') opts = arg1 as ProcedureOptsWithOptionalName; else params = arg1 as Params; break; diff --git a/crates/bindings-typescript/tests/hosted_auth.test.ts b/crates/bindings-typescript/tests/hosted_auth.test.ts index 02d01c0809c..6eeb1af4e83 100644 --- a/crates/bindings-typescript/tests/hosted_auth.test.ts +++ b/crates/bindings-typescript/tests/hosted_auth.test.ts @@ -32,16 +32,6 @@ import { Timestamp } from '../src/lib/timestamp'; import { schema, exportContext, registerExport } from '../src/server/schema'; import { callProcedure } from '../src/server/procedures'; import { t } from '../src/lib/type_builders'; -import { - AlgebraicType, - FunctionVisibility, - ProductType, - RawModuleDef, - RawModuleDefV10Section, - RawReducerDefV10, -} from '../src/lib/autogen/types'; -import BinaryReader from '../src/lib/binary_reader'; -import BinaryWriter from '../src/lib/binary_writer'; beforeEach(() => { Object.assign(host, { flags: 0, payload: '', jwtReads: 0, flagReads: 0 }); @@ -141,169 +131,13 @@ describe('verified invocation authentication', () => { }); }); -describe('V10 explicit function visibility', () => { - it('preserves existing visibility tags and appends the new variants and capability section', () => { - const legacyVisibility = t.enum('LegacyFunctionVisibility', { - Private: t.unit(), - ClientCallable: t.unit(), - }); - const variants = [ - FunctionVisibility.Private, - FunctionVisibility.ClientCallable, - FunctionVisibility.Internal, - FunctionVisibility.ExplicitClientCallable, - ]; - for (const [tag, visibility] of variants.entries()) { - const writer = new BinaryWriter(8); - FunctionVisibility.serialize(writer, visibility); - expect([...writer.getBuffer()]).toEqual([tag]); - const reader = new BinaryReader(writer.getBuffer()); - if (tag < 2) { - expect(legacyVisibility.deserialize(reader).tag).toBe(visibility.tag); - } - } - const writer = new BinaryWriter(8); - RawModuleDefV10Section.serialize(writer, { - tag: 'Capabilities', - value: [], - }); - expect([...writer.getBuffer()]).toEqual([16, 0, 0, 0, 0]); - const environmentWriter = new BinaryWriter(8); - RawModuleDefV10Section.serialize(environmentWriter, { - tag: 'Environment', - value: [], - }); - expect([...environmentWriter.getBuffer()]).toEqual([15, 0, 0, 0, 0]); - }); - - it('retains the V10 reducer field layout without an optional visibility wrapper', () => { - const module = schema({}); - const reducer = module.reducer({ visibility: 'public' }, () => {}); - const inner = reducer[exportContext]!; - reducer[registerExport](inner, 'public_reducer'); - const definition = inner.moduleDef.reducers[0]; - const writer = new BinaryWriter(128); - RawReducerDefV10.serialize(writer, definition); - const expected = new BinaryWriter(128); - expected.writeString(definition.sourceName); - ProductType.serialize(expected, definition.params); - expected.writeByte(3); - AlgebraicType.serialize(expected, definition.okReturnType); - AlgebraicType.serialize(expected, definition.errReturnType); - expect(writer.getBuffer()).toEqual(expected.getBuffer()); - }); - - it('serializes omission separately from explicit visibility and advertises hosted auth', () => { - const module = schema({}); - const omitted = module.reducer(() => {}); - const explicitlyPublic = module.reducer({ visibility: 'public' }, () => {}); - const privateReducer = module.reducer({ visibility: 'private' }, () => {}); - const internalReducer = module.reducer( - { visibility: 'internal' }, - () => {} - ); - const inner = omitted[exportContext]!; - for (const [name, reducer] of Object.entries({ - omitted, - explicitlyPublic, - privateReducer, - internalReducer, - })) { - reducer[registerExport](inner, name); - } - // Being scheduled must not erase a public choice or manufacture an explicit - // choice for the default. The host resolves the latter to Private. - for (const name of [ - 'omitted', - 'explicitlyPublic', - 'privateReducer', - 'internalReducer', - ]) { - inner.moduleDef.schedules.push({ - sourceName: undefined, - tableName: `jobs_${name}`, - scheduleAtCol: 0, - functionName: name, - }); - } - const raw = RawModuleDef.V10(inner.rawModuleDefV10()); - const writer = new BinaryWriter(128); - RawModuleDef.serialize(writer, raw); - expect(writer.getBuffer()[0]).toBe(2); - const decoded = RawModuleDef.deserialize( - new BinaryReader(writer.getBuffer()) - ); - const roundTrip = new BinaryWriter(128); - RawModuleDef.serialize(roundTrip, decoded); - expect(roundTrip.getBuffer()).toEqual(writer.getBuffer()); - expect(decoded.tag).toBe('V10'); - if (decoded.tag !== 'V10') throw new Error('Expected V10'); - const reducers = decoded.value.sections.find( - section => section.tag === 'Reducers' - ); - expect(reducers?.value.map(reducer => reducer.visibility.tag)).toEqual([ - 'ClientCallable', - 'ExplicitClientCallable', - 'Private', - 'Internal', - ]); - expect( - inner.moduleDef.reducers.map(reducer => reducer.visibility.tag) - ).toEqual([ - 'ClientCallable', - 'ExplicitClientCallable', - 'Private', - 'Internal', - ]); - expect(inner.moduleDef.capabilities).toEqual(['hosted_auth_v1']); - }); - - it('retains procedure names and explicit visibility, including a visibility parameter', () => { +describe('hosted authentication capability', () => { + it('advertises the updated bindings without changing function visibility', () => { const module = schema({}); - const proc = module.procedure( - { name: 'public_name', visibility: 'internal' }, - t.unit(), - () => ({}) - ); - const reducer = module.reducer({ visibility: t.string() }, () => {}); - const inner = proc[exportContext]!; - proc[registerExport](inner, 'source_name'); - reducer[registerExport](inner, 'accept_visibility'); - expect(inner.moduleDef.procedures[0].sourceName).toBe('source_name'); - expect(inner.moduleDef.procedures[0].visibility.tag).toBe('Internal'); - expect(inner.moduleDef.explicitNames.entries).toContainEqual({ - tag: 'Function', - value: { sourceName: 'source_name', canonicalName: 'public_name' }, - }); - expect(inner.moduleDef.reducers[0].params.elements[0].name).toBe( - 'visibility' - ); + const run = module.reducer(() => {}); + const inner = run[exportContext]!; + run[registerExport](inner, 'run'); + expect(inner.moduleDef.capabilities).toContain('hosted_auth_v1'); + expect(inner.moduleDef.reducers[0].visibility.tag).toBe('ClientCallable'); }); - - it.each(['private', 'public'] as const)( - 'rejects explicit %s lifecycle declarations', - visibility => { - const module = schema({}); - const invalid = module.init({ visibility }, () => {}); - expect(() => - invalid[registerExport](invalid[exportContext]!, 'invalid_init') - ).toThrow('Lifecycle reducers only support internal visibility'); - } - ); - - it.each([undefined, 'internal'] as const)( - 'preserves permitted lifecycle declaration %s for host event dispatch', - visibility => { - const module = schema({}); - const valid = module.init({ visibility }, () => {}); - valid[registerExport](valid[exportContext]!, 'valid_init'); - const inner = valid[exportContext]!; - expect(inner.moduleDef.reducers[0].visibility.tag).toBe( - visibility === undefined ? 'ClientCallable' : 'Internal' - ); - expect(inner.moduleDef.lifeCycleReducers).toEqual([ - { lifecycleSpec: { tag: 'Init' }, functionName: 'valid_init' }, - ]); - } - ); }); diff --git a/crates/bindings/src/rt.rs b/crates/bindings/src/rt.rs index a26cb221e1d..89e29ae5891 100644 --- a/crates/bindings/src/rt.rs +++ b/crates/bindings/src/rt.rs @@ -1,7 +1,5 @@ #![deny(unsafe_op_in_unsafe_fn)] -pub use spacetimedb_lib::db::raw_def::v10::FunctionVisibility; - use crate::query_builder::{FromWhere, HasCols, LeftSemiJoin, RawQuery, RightSemiJoin, Table as QbTable}; use crate::table::IndexAlgo; use crate::{sys, AnonymousViewContext, IterBuf, ReducerContext, ReducerResult, SpacetimeType, Table, ViewContext}; @@ -161,9 +159,6 @@ pub trait FnInfo: ExplicitNames { /// The lifecycle of the function, if there is one. const LIFECYCLE: Option = None; - /// Explicit SpacetimeDB visibility; Rust item visibility is independent. - const DECLARED_VISIBILITY: Option = None; - /// A description of the parameter names of the function. const ARG_NAMES: &'static [Option<&'static str>]; @@ -805,13 +800,9 @@ pub fn register_reducer<'a, A: Args<'a>, I: FnInfo>(_: impl register_describer(|module| { let params = A::schema::(&mut module.inner); if let Some(lifecycle) = I::LIFECYCLE { - module - .inner - .add_lifecycle_reducer_with_visibility(lifecycle, I::NAME, params, I::DECLARED_VISIBILITY); + module.inner.add_lifecycle_reducer(lifecycle, I::NAME, params); } else { - module - .inner - .add_reducer_with_visibility(I::NAME, params, I::DECLARED_VISIBILITY); + module.inner.add_reducer(I::NAME, params); } module.reducers.push(I::INVOKE); @@ -828,9 +819,7 @@ where register_describer(|module| { let params = A::schema::(&mut module.inner); let ret_ty = ::make_type(&mut module.inner); - module - .inner - .add_procedure_with_visibility(I::NAME, params, ret_ty, I::DECLARED_VISIBILITY); + module.inner.add_procedure(I::NAME, params, ret_ty); module.procedures.push(I::INVOKE); module.inner.add_explicit_names(I::explicit_names()); diff --git a/crates/bindings/tests/pass/function_visibility.rs b/crates/bindings/tests/pass/function_visibility.rs deleted file mode 100644 index 7f53af5717f..00000000000 --- a/crates/bindings/tests/pass/function_visibility.rs +++ /dev/null @@ -1,62 +0,0 @@ -#![deny(warnings)] - -use spacetimedb::rt::{FnInfo, FunctionVisibility}; -use spacetimedb::{ProcedureContext, ReducerContext}; - -#[spacetimedb::reducer(internal)] -pub fn internal_reducer(_ctx: &ReducerContext) {} - -#[spacetimedb::reducer(private)] -fn private_reducer(_ctx: &ReducerContext) {} - -#[spacetimedb::reducer(public)] -fn public_reducer(_ctx: &ReducerContext) {} - -#[spacetimedb::reducer(init, internal)] -fn initialize(_ctx: &ReducerContext) {} - -#[spacetimedb::procedure(internal)] -fn internal_procedure(_ctx: &mut ProcedureContext) -> u64 { - 0 -} - -#[spacetimedb::procedure(private)] -fn private_procedure(_ctx: &mut ProcedureContext) -> u64 { - 0 -} - -#[spacetimedb::procedure(public)] -fn public_procedure(_ctx: &mut ProcedureContext) -> u64 { - 0 -} - -fn main() { - assert!(matches!( - internal_reducer::DECLARED_VISIBILITY, - Some(FunctionVisibility::Internal) - )); - assert!(matches!( - private_reducer::DECLARED_VISIBILITY, - Some(FunctionVisibility::Private) - )); - assert!(matches!( - public_reducer::DECLARED_VISIBILITY, - Some(FunctionVisibility::ClientCallable) - )); - assert!(matches!( - initialize::DECLARED_VISIBILITY, - Some(FunctionVisibility::Internal) - )); - assert!(matches!( - internal_procedure::DECLARED_VISIBILITY, - Some(FunctionVisibility::Internal) - )); - assert!(matches!( - private_procedure::DECLARED_VISIBILITY, - Some(FunctionVisibility::Private) - )); - assert!(matches!( - public_procedure::DECLARED_VISIBILITY, - Some(FunctionVisibility::ClientCallable) - )); -} diff --git a/crates/cli/src/subcommands/generate.rs b/crates/cli/src/subcommands/generate.rs index e1b6caff37e..14b39212458 100644 --- a/crates/cli/src/subcommands/generate.rs +++ b/crates/cli/src/subcommands/generate.rs @@ -266,7 +266,7 @@ pub fn cli() -> clap::Command { .long("include-private") .action(SetTrue) .default_value("false") - .help("Include private tables and private/internal non-lifecycle functions (types are always included)."), + .help("Include private tables and functions in generated code (types are always included)."), ) .arg(common_args::yes()) .arg( diff --git a/crates/codegen/src/util.rs b/crates/codegen/src/util.rs index fdb0e7fcce9..5a62afdd06f 100644 --- a/crates/codegen/src/util.rs +++ b/crates/codegen/src/util.rs @@ -10,8 +10,8 @@ use convert_case::{Case, Casing}; use itertools::Itertools; use spacetimedb_lib::db::raw_def::v9::TableAccess; use spacetimedb_lib::sats::layout::PrimitiveType; -use spacetimedb_lib::sats::AlgebraicTypeRef; use spacetimedb_lib::version; +use spacetimedb_lib::{db::raw_def::v9::Lifecycle, sats::AlgebraicTypeRef}; use spacetimedb_primitives::ColList; use spacetimedb_schema::{def::ViewDef, type_for_generate::ProductTypeDef}; use spacetimedb_schema::{ @@ -99,20 +99,31 @@ pub(super) fn is_reducer_invokable(reducer: &ReducerDef) -> bool { reducer.lifecycle.is_none() } -/// Non-lifecycle reducer entry points in declaration order. Default clients see -/// only public functions; IncludePrivate adds Private and Internal methods. +/// Iterate over all the [`ReducerDef`]s defined by the module, in alphabetical order by name. +/// +/// Skipping the `init` reducer and internal [`FunctionVisibiity::Internal`] reducers because +/// they should not be directly invokable. +/// Sorting is not necessary for reducers because they are already stored in an IndexMap. pub(super) fn iter_reducers(module: &ModuleDef, visibility: CodegenVisibility) -> impl Iterator { module .reducers() - .filter(|reducer| reducer.lifecycle.is_none()) + // `RawModuleDefV10` already marks all lifecycle reducers as private, but we keep + // this filter for backward compatibility with older versions where `init` + // reducers were not private. + .filter(|reducer| reducer.lifecycle != Some(Lifecycle::Init)) + // Prior to `RawModuleDefV10`, all reducers were public by default. Filtering out + // internal reducers here does not break SDKs built against older versions. .filter(move |reducer| match visibility { CodegenVisibility::IncludePrivate => true, - CodegenVisibility::OnlyPublic => reducer.visibility.is_client_callable(), + CodegenVisibility::OnlyPublic => !reducer.visibility.is_private(), }) } -/// Procedure entry points in alphabetical order. Default clients see only Public -/// functions; IncludePrivate also generates Private and Internal methods. +/// Iterate over all the [`ProcedureDef`]s defined by the module, in alphabetical order by name. +/// +/// Skipping internal [`FunctionVisibiity::Internal`] procedures because they should not be +/// directly invokable. +/// Sorting is necessary to have deterministic reproducible codegen. pub(super) fn iter_procedures( module: &ModuleDef, visibility: CodegenVisibility, @@ -122,7 +133,7 @@ pub(super) fn iter_procedures( .sorted_by_key(|procedure| &procedure.name) .filter(move |procedure| match visibility { CodegenVisibility::IncludePrivate => true, - CodegenVisibility::OnlyPublic => procedure.visibility.is_client_callable(), + CodegenVisibility::OnlyPublic => !procedure.visibility.is_private(), }) } @@ -212,64 +223,3 @@ pub(super) fn iter_constraints(table: &TableDef) -> impl Iterator impl Iterator { module.types().sorted_by_key(|table| &table.accessor_name) } - -#[cfg(test)] -mod visibility_tests { - use super::*; - use spacetimedb_lib::db::raw_def::{ - v10::{FunctionVisibility, RawModuleDefV10Builder}, - v9::Lifecycle, - }; - use spacetimedb_lib::{AlgebraicType, ProductType}; - - #[test] - fn public_codegen_excludes_internal_private_and_every_lifecycle() { - let mut builder = RawModuleDefV10Builder::new(); - builder.add_reducer("ordinary", ProductType::unit()); - for (name, visibility) in [ - ("public_function", FunctionVisibility::ClientCallable), - ("private_function", FunctionVisibility::Private), - ("internal_function", FunctionVisibility::Internal), - ] { - builder.add_reducer_with_visibility(name, ProductType::unit(), Some(visibility)); - builder.add_procedure_with_visibility( - format!("{name}_procedure"), - ProductType::unit(), - AlgebraicType::unit(), - Some(visibility), - ); - } - for (name, lifecycle) in [ - ("init", Lifecycle::Init), - ("connect", Lifecycle::OnConnect), - ("disconnect", Lifecycle::OnDisconnect), - ] { - builder.add_lifecycle_reducer(lifecycle, name, ProductType::unit()); - } - let module: ModuleDef = builder.finish().try_into().unwrap(); - let names = |visibility| { - iter_reducers(&module, visibility) - .map(|r| &r.name[..]) - .collect::>() - }; - assert_eq!(names(CodegenVisibility::OnlyPublic), ["ordinary", "public_function"]); - assert_eq!( - names(CodegenVisibility::IncludePrivate), - ["ordinary", "public_function", "private_function", "internal_function"] - ); - let names = |visibility| { - iter_procedures(&module, visibility) - .map(|p| &p.name[..]) - .collect::>() - }; - assert_eq!(names(CodegenVisibility::OnlyPublic), ["public_function_procedure"]); - assert_eq!( - names(CodegenVisibility::IncludePrivate), - [ - "internal_function_procedure", - "private_function_procedure", - "public_function_procedure" - ] - ); - } -} diff --git a/crates/core/src/host/host_controller/invocation_flags_tests.rs b/crates/core/src/host/host_controller/invocation_flags_tests.rs index 5abd02e1c6f..a140cf4d7d2 100644 --- a/crates/core/src/host/host_controller/invocation_flags_tests.rs +++ b/crates/core/src/host/host_controller/invocation_flags_tests.rs @@ -12,16 +12,20 @@ fn program() -> Program { let mut schema = RawModuleDefV10Builder::new(); schema.add_lifecycle_reducer(Lifecycle::Init, "init", ProductType::unit()); schema.add_reducer("external", ProductType::unit()); - schema.add_reducer_with_visibility("internal", ProductType::unit(), Some(FunctionVisibility::Internal)); - schema.add_reducer_with_visibility("private", ProductType::unit(), Some(FunctionVisibility::Private)); + schema.add_reducer("private", ProductType::unit()); schema.add_procedure("external_procedure", ProductType::unit(), AlgebraicType::U8); - schema.add_procedure_with_visibility( - "internal_procedure", - ProductType::unit(), - AlgebraicType::U8, - Some(FunctionVisibility::Internal), - ); - let schema = spacetimedb_lib::bsatn::to_vec(&spacetimedb_lib::RawModuleDef::V10(schema.finish())).unwrap(); + schema.add_procedure("system_procedure", ProductType::unit(), AlgebraicType::U8); + let mut schema = schema.finish(); + for section in &mut schema.sections { + if let spacetimedb_lib::db::raw_def::v10::RawModuleDefV10Section::Reducers(reducers) = section { + reducers + .iter_mut() + .find(|r| &*r.source_name == "private") + .unwrap() + .visibility = FunctionVisibility::Private; + } + } + let schema = spacetimedb_lib::bsatn::to_vec(&spacetimedb_lib::RawModuleDef::V10(schema)).unwrap(); Program::from_bytes( ModuleKind::JS, format!( @@ -47,7 +51,7 @@ fn program() -> Program { } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn invocation_flags_are_host_owned_and_internal_visibility_is_enforced() { +async fn invocation_flags_are_host_owned_and_lifecycle_calls_remain_restricted() { let directory = tempfile::tempdir().unwrap(); let data = Arc::new(ServerDataDir::from_path_unchecked(directory.path().to_owned())); let program = program(); @@ -91,7 +95,7 @@ async fn invocation_flags_are_host_owned_and_internal_visibility_is_enforced() { .outcome .into_result() .unwrap(); - for name in ["internal", "init"] { + for name in ["init"] { assert!(module .call_reducer(sender, None, None, None, None, name, FunctionArgs::Nullary) .await @@ -101,11 +105,6 @@ async fn invocation_flags_are_host_owned_and_internal_visibility_is_enforced() { .call_procedure(sender, None, None, "external_procedure", FunctionArgs::Nullary) .await; assert_eq!(result.result.unwrap().return_val, AlgebraicValue::U8(0)); - assert!(module - .call_procedure(sender, None, None, "internal_procedure", FunctionArgs::Nullary) - .await - .result - .is_err()); assert_eq!( module .call_reducer(sender, None, None, None, None, "private", FunctionArgs::Nullary) @@ -118,7 +117,7 @@ async fn invocation_flags_are_host_owned_and_internal_visibility_is_enforced() { // observes zero again even if the procedure instance is reused. let result = module .call_procedure_with_params( - "internal_procedure", + "system_procedure", CallProcedureParams::from_system( Timestamp::now(), database.database_identity, diff --git a/crates/lib/src/db/raw_def/v10.rs b/crates/lib/src/db/raw_def/v10.rs index a06d1ea2171..c011551ff34 100644 --- a/crates/lib/src/db/raw_def/v10.rs +++ b/crates/lib/src/db/raw_def/v10.rs @@ -338,9 +338,6 @@ pub struct RawReducerDefV10 { } /// The visibility of a function (reducer or procedure). -/// -/// New variants MUST be appended to preserve existing BSATN tags. Older hosts -/// reject unknown tags, so new restrictions cannot be silently discarded. #[derive(Debug, Copy, Clone, SpacetimeType)] #[sats(crate = crate)] #[cfg_attr(feature = "test", derive(PartialEq, Eq, PartialOrd, Ord))] @@ -350,31 +347,11 @@ pub enum FunctionVisibility { /// Still callable by the module owner, collaborators, /// and internal module code. /// - /// The default for scheduled functions. Older lifecycle definitions also use - /// this tag; lifecycle assignments always enforce host-event-only invocation. + /// Enabled for lifecycle reducers and scheduled functions by default. Private, - /// Callable from client code, with the historical contextual defaults. - /// Scheduled functions become Private; lifecycle reducers remain host event handlers. + /// Callable from client code. ClientCallable, - - /// Callable only by a host-verified internal invocation. - Internal, - - /// Explicitly callable from client code, including when scheduled. - /// This separate tag preserves the meaning of existing ClientCallable definitions. - ExplicitClientCallable, -} - -impl FunctionVisibility { - /// Encode a source declaration without changing historical contextual defaults. - pub fn from_declaration(declared: Option, default: Self) -> Self { - match declared { - Some(Self::ClientCallable | Self::ExplicitClientCallable) => Self::ExplicitClientCallable, - Some(visibility) => visibility, - None => default, - } - } } /// A schedule definition. @@ -1143,20 +1120,10 @@ impl RawModuleDefV10Builder { /// This is because `SpacetimeType` is not implemented for `ReducerContext`, /// so it can never act like an ordinary argument.) pub fn add_reducer(&mut self, source_name: impl Into, params: ProductType) { - self.add_reducer_with_visibility(source_name, params, None); - } - - /// Add a reducer with an optional explicit visibility declaration. - pub fn add_reducer_with_visibility( - &mut self, - source_name: impl Into, - params: ProductType, - visibility: Option, - ) { self.reducers_mut().push(RawReducerDefV10 { source_name: source_name.into(), params, - visibility: FunctionVisibility::from_declaration(visibility, FunctionVisibility::ClientCallable), + visibility: FunctionVisibility::ClientCallable, ok_return_type: reducer_default_ok_return_type(), err_return_type: reducer_default_err_return_type(), }); @@ -1177,23 +1144,12 @@ impl RawModuleDefV10Builder { source_name: impl Into, params: ProductType, return_type: AlgebraicType, - ) { - self.add_procedure_with_visibility(source_name, params, return_type, None); - } - - /// Add a procedure with an optional explicit visibility declaration. - pub fn add_procedure_with_visibility( - &mut self, - source_name: impl Into, - params: ProductType, - return_type: AlgebraicType, - visibility: Option, ) { self.procedures_mut().push(RawProcedureDefV10 { source_name: source_name.into(), params, return_type, - visibility: FunctionVisibility::from_declaration(visibility, FunctionVisibility::ClientCallable), + visibility: FunctionVisibility::ClientCallable, }) } @@ -1237,19 +1193,6 @@ impl RawModuleDefV10Builder { lifecycle_spec: Lifecycle, function_name: impl Into, params: ProductType, - ) { - self.add_lifecycle_reducer_with_visibility(lifecycle_spec, function_name, params, None); - } - - /// Add a lifecycle reducer with an optional visibility declaration. - /// Source bindings must reject explicit Private or public lifecycle annotations. - /// The raw Private tag remains accepted for compatibility with existing modules. - pub fn add_lifecycle_reducer_with_visibility( - &mut self, - lifecycle_spec: Lifecycle, - function_name: impl Into, - params: ProductType, - visibility: Option, ) { let function_name = function_name.into(); self.lifecycle_reducers_mut().push(RawLifeCycleReducerDefV10 { @@ -1260,7 +1203,7 @@ impl RawModuleDefV10Builder { self.reducers_mut().push(RawReducerDefV10 { source_name: function_name, params, - visibility: FunctionVisibility::from_declaration(visibility, FunctionVisibility::Private), + visibility: FunctionVisibility::Private, ok_return_type: reducer_default_ok_return_type(), err_return_type: reducer_default_err_return_type(), }); @@ -1655,8 +1598,6 @@ mod compatibility_tests { for (visibility, expected) in [ (FunctionVisibility::Private, 0), (FunctionVisibility::ClientCallable, 1), - (FunctionVisibility::Internal, 2), - (FunctionVisibility::ExplicitClientCallable, 3), ] { assert_eq!(bsatn::to_vec(&visibility).unwrap(), [expected]); } @@ -1695,26 +1636,7 @@ mod compatibility_tests { } #[test] - fn older_hosts_reject_new_visibility_and_capabilities() { - for visibility in [FunctionVisibility::Internal, FunctionVisibility::ExplicitClientCallable] { - for procedure in [false, true] { - let mut builder = RawModuleDefV10Builder::new(); - if procedure { - builder.add_procedure_with_visibility( - "run", - ProductType::unit(), - AlgebraicType::unit(), - Some(visibility), - ); - } else { - builder.add_reducer_with_visibility("run", ProductType::unit(), Some(visibility)); - } - let bytes = bsatn::to_vec(&RawModuleDef::V10(builder.finish())).unwrap(); - assert_eq!(bytes[0], 2); - assert!(bsatn::from_slice::(&bytes).is_err()); - assert!(bsatn::from_slice::(&bytes).is_ok()); - } - } + fn older_hosts_reject_new_capabilities() { let mut builder = RawModuleDefV10Builder::new(); builder.add_capability("hosted_auth_v1"); let bytes = bsatn::to_vec(&RawModuleDef::V10(builder.finish())).unwrap(); diff --git a/crates/schema/src/auto_migrate.rs b/crates/schema/src/auto_migrate.rs index 4a52e079670..97fff428830 100644 --- a/crates/schema/src/auto_migrate.rs +++ b/crates/schema/src/auto_migrate.rs @@ -220,31 +220,6 @@ pub struct AutoMigratePlan<'def> { } impl AutoMigratePlan<'_> { - /// Function authority changes include every namespace in the published API. - pub fn function_visibility_changes( - &self, - ) -> impl Iterator { - let reducers = self - .old - .all_reducers_with_prefix() - .into_iter() - .filter(|(_, _, old)| old.lifecycle.is_none()) - .filter_map(|(_, _, old)| { - let name = old.name.to_string(); - let (_, new) = self.new.reducer_by_name(&name)?; - (old.visibility != new.visibility).then_some((name, &old.visibility, &new.visibility)) - }); - let procedures = self - .old - .all_procedures_with_prefix() - .into_iter() - .filter_map(|(prefix, _, old)| { - let name = format!("{prefix}{}", old.name); - let (_, new) = self.new.procedure_by_name(&name)?; - (old.visibility != new.visibility).then_some((name, &old.visibility, &new.visibility)) - }); - reducers.chain(procedures) - } fn any_step(&self, f: impl Fn(&AutoMigrateStep<'_>) -> bool) -> bool { self.steps.iter().any(f) } @@ -518,15 +493,6 @@ pub fn ponder_auto_migrate<'def>(old: &'def ModuleDef, new: &'def ModuleDef) -> prechecks: Vec::new(), }; - let restricts_function_access = plan.function_visibility_changes().any(|(_, old, new)| { - [false, true] - .into_iter() - .any(|owner| old.allows_invocation(false, owner) && !new.allows_invocation(false, owner)) - }); - if restricts_function_access { - plan.ensure_disconnect_all_users(); - } - let views_ok = auto_migrate_views(&mut plan); let tables_ok = auto_migrate_tables(&mut plan); @@ -2943,33 +2909,6 @@ mod tests { raw.try_into().expect("should be a valid module definition") } - #[test] - fn submodule_visibility_restrictions_disconnect_and_report_qualified_names() { - use spacetimedb_lib::db::raw_def::v10::FunctionVisibility as RawVisibility; - let module = |visibility| { - create_module_def_with_submodules( - |_| {}, - vec![make_submodule("lib", |builder| { - builder.add_reducer_with_visibility("job", ProductType::unit(), Some(visibility)); - builder.add_procedure_with_visibility( - "read", - ProductType::unit(), - AlgebraicType::U8, - Some(visibility), - ); - })], - ) - }; - let old = module(RawVisibility::ExplicitClientCallable); - let restricted = module(RawVisibility::Internal); - let plan = ponder_auto_migrate(&old, &restricted).unwrap(); - assert!(plan.steps.contains(&AutoMigrateStep::DisconnectAllUsers)); - let names: Vec<_> = plan.function_visibility_changes().map(|(name, _, _)| name).collect(); - assert_eq!(names, ["lib.job", "lib.read"]); - let relaxed = ponder_auto_migrate(&restricted, &old).unwrap(); - assert!(!relaxed.steps.contains(&AutoMigrateStep::DisconnectAllUsers)); - } - #[test] fn submodule_table_unchanged() { let submodule = || { diff --git a/crates/schema/src/auto_migrate/formatter.rs b/crates/schema/src/auto_migrate/formatter.rs index 6c684121ca8..7d079f04a62 100644 --- a/crates/schema/src/auto_migrate/formatter.rs +++ b/crates/schema/src/auto_migrate/formatter.rs @@ -20,9 +20,6 @@ use thiserror::Error; pub fn format_plan(f: &mut F, plan: &AutoMigratePlan) -> Result<(), FormattingErrors> { f.format_header()?; - for (name, old, new) in plan.function_visibility_changes() { - f.format_function_visibility(&name, old, new)?; - } for step in &plan.steps { format_step(f, step, plan)?; @@ -183,12 +180,6 @@ pub enum Action { /// It allows for different implementations, such as ANSI formatting or plain text formatting. pub trait MigrationFormatter { fn format_header(&mut self) -> io::Result<()>; - fn format_function_visibility( - &mut self, - name: &str, - old: &crate::def::FunctionVisibility, - new: &crate::def::FunctionVisibility, - ) -> io::Result<()>; fn format_add_table(&mut self, table_info: &TableInfo) -> io::Result<()>; fn format_remove_table(&mut self, table_name: &NamespacedIdentifier) -> io::Result<()>; fn format_view(&mut self, view_info: &ViewInfo, action: Action) -> io::Result<()>; diff --git a/crates/schema/src/auto_migrate/termcolor_formatter.rs b/crates/schema/src/auto_migrate/termcolor_formatter.rs index ab27935cddf..811c04b1860 100644 --- a/crates/schema/src/auto_migrate/termcolor_formatter.rs +++ b/crates/schema/src/auto_migrate/termcolor_formatter.rs @@ -157,15 +157,6 @@ impl TermColorFormatter { } impl MigrationFormatter for TermColorFormatter { - fn format_function_visibility( - &mut self, - name: &str, - old: &crate::def::FunctionVisibility, - new: &crate::def::FunctionVisibility, - ) -> io::Result<()> { - self.write_bullet(&format!("Function {name} visibility: {old} -> {new}")) - } - fn format_header(&mut self) -> io::Result<()> { let line = "━".repeat(60); self.write_line(&line)?; diff --git a/crates/schema/src/def.rs b/crates/schema/src/def.rs index 3817885ea78..ad37d3e12b8 100644 --- a/crates/schema/src/def.rs +++ b/crates/schema/src/def.rs @@ -1180,15 +1180,7 @@ impl From for RawModuleDefV10 { RawIdentifier::from(rd.accessor_name.clone()), RawIdentifier::from(rd.name.local().clone()), ); - let public_scheduled = rd.visibility.is_client_callable() - && schedules - .iter() - .any(|schedule| schedule.function_name == RawIdentifier::from(rd.name.clone())); - let mut raw: RawReducerDefV10 = rd.into(); - if public_scheduled { - raw.visibility = RawFunctionVisibility::ExplicitClientCallable; - } - raw + rd.into() }) .collect(); if !raw_reducers.is_empty() { @@ -1203,15 +1195,7 @@ impl From for RawModuleDefV10 { RawIdentifier::from(pd.accessor_name.clone()), RawIdentifier::from(pd.name.clone()), ); - let public_scheduled = pd.visibility.is_client_callable() - && schedules - .iter() - .any(|schedule| schedule.function_name == RawIdentifier::from(pd.name.clone())); - let mut raw: RawProcedureDefV10 = pd.into(); - if public_scheduled { - raw.visibility = RawFunctionVisibility::ExplicitClientCallable; - } - raw + pd.into() }) .collect(); if !raw_procedures.is_empty() { @@ -2372,9 +2356,6 @@ pub enum FunctionVisibility { /// Callable from client code. ClientCallable, - - /// Callable only by a host-verified internal invocation. - Internal, } impl fmt::Display for FunctionVisibility { @@ -2382,7 +2363,6 @@ impl fmt::Display for FunctionVisibility { f.write_str(match self { Self::Private => "Private", Self::ClientCallable => "Public", - Self::Internal => "Internal", }) } } @@ -2391,13 +2371,9 @@ impl FunctionVisibility { pub fn is_client_callable(&self) -> bool { matches!(self, Self::ClientCallable) } - pub fn is_internal(&self) -> bool { - matches!(self, Self::Internal) - } /// Lifecycle event dispatch is a separate restriction from this predicate. pub fn allows_invocation(&self, is_internal: bool, is_authorized_private_caller: bool) -> bool { match self { - Self::Internal => is_internal, Self::Private => is_internal || is_authorized_private_caller, Self::ClientCallable => true, } @@ -2412,10 +2388,7 @@ impl From for FunctionVisibility { fn from(val: RawFunctionVisibility) -> Self { match val { RawFunctionVisibility::Private => FunctionVisibility::Private, - RawFunctionVisibility::ClientCallable | RawFunctionVisibility::ExplicitClientCallable => { - FunctionVisibility::ClientCallable - } - RawFunctionVisibility::Internal => FunctionVisibility::Internal, + RawFunctionVisibility::ClientCallable => FunctionVisibility::ClientCallable, } } } @@ -2431,7 +2404,6 @@ impl From for RawFunctionVisibility { match val { FunctionVisibility::Private => Self::Private, FunctionVisibility::ClientCallable => Self::ClientCallable, - FunctionVisibility::Internal => Self::Internal, } } } diff --git a/crates/schema/src/def/validate/v10.rs b/crates/schema/src/def/validate/v10.rs index 2e4369821c2..4e7872fd44d 100644 --- a/crates/schema/src/def/validate/v10.rs +++ b/crates/schema/src/def/validate/v10.rs @@ -105,20 +105,6 @@ pub fn validate(def: RawModuleDefV10) -> Result { } } } - // Retain the raw distinction until schedules are attached. Tag 1 has the - // historical contextual default; tag 3 is an explicit public declaration. - let raw_visibility: HashMap<_, _> = def - .reducers() - .into_iter() - .flatten() - .map(|function| (function.source_name.clone(), function.visibility)) - .chain( - def.procedures() - .into_iter() - .flatten() - .map(|function| (function.source_name.clone(), function.visibility)), - ) - .collect(); let environment = validate_environment(&def); let mut typespace = def.typespace().cloned().unwrap_or_else(|| Typespace::EMPTY.clone()); let known_type_definitions = def.types().into_iter().flatten().map(|def| def.ty); @@ -318,12 +304,7 @@ pub fn validate(def: RawModuleDefV10) -> Result { attach_schedules_to_tables(&mut tables, schedules)?; check_scheduled_functions_exist(&mut tables, &reducers, &procedures)?; - change_scheduled_functions_and_lifetimes_visibility( - &tables, - &mut reducers, - &mut procedures, - &raw_visibility, - )?; + change_scheduled_functions_and_lifetimes_visibility(&tables, &mut reducers, &mut procedures)?; attach_view_primary_keys(&mut views, view_primary_keys)?; assign_query_view_primary_keys(&tables, &mut views); @@ -453,13 +434,12 @@ fn validate_submodules(submodules: Vec) -> Result, reducers: &mut IndexMap, procedures: &mut IndexMap, - raw_visibility: &HashMap, ) -> Result<()> { for sched_def in tables.iter().filter_map(|(_, t)| t.schedule.as_ref()) { match sched_def.function_kind { @@ -471,12 +451,7 @@ fn change_scheduled_functions_and_lifetimes_visibility( } })?; - if matches!( - raw_visibility.get(&RawIdentifier::from(def.accessor_name.clone())), - Some(RawFunctionVisibility::ClientCallable) - ) { - def.visibility = crate::def::FunctionVisibility::Private; - } + def.visibility = crate::def::FunctionVisibility::Private; } FunctionKind::Procedure => { @@ -487,12 +462,7 @@ fn change_scheduled_functions_and_lifetimes_visibility( } })?; - if matches!( - raw_visibility.get(&RawIdentifier::from(def.accessor_name.clone())), - Some(RawFunctionVisibility::ClientCallable) - ) { - def.visibility = crate::def::FunctionVisibility::Private; - } + def.visibility = crate::def::FunctionVisibility::Private; } FunctionKind::Unknown => {} @@ -501,16 +471,7 @@ fn change_scheduled_functions_and_lifetimes_visibility( for red_def in reducers.iter_mut().map(|(_, r)| r) { if red_def.lifecycle.is_some() { - if matches!( - raw_visibility.get(&RawIdentifier::from(red_def.accessor_name.clone())), - Some(RawFunctionVisibility::ExplicitClientCallable) - ) { - return Err(ValidationError::InvalidLifecycleVisibility { - function: red_def.accessor_name.clone().into(), - } - .into()); - } - red_def.visibility = crate::def::FunctionVisibility::Internal; + red_def.visibility = crate::def::FunctionVisibility::Private; } } @@ -1550,7 +1511,7 @@ mod tests { def.reducers[&check_deliveries_name].visibility, FunctionVisibility::Private, ); - assert_eq!(def.reducers[&init_name].visibility, FunctionVisibility::Internal); + assert_eq!(def.reducers[&init_name].visibility, FunctionVisibility::Private); assert_eq!( def.reducers[&extra_reducer_name].visibility, FunctionVisibility::ClientCallable @@ -2888,218 +2849,9 @@ mod tests { } #[cfg(test)] -mod visibility_tests { +mod capability_tests { use super::*; - use crate::def::FunctionVisibility; - use spacetimedb_lib::db::raw_def::v10; - use spacetimedb_lib::{db::raw_def::v9, RawModuleDef, ScheduleAt}; - use spacetimedb_sats::{AlgebraicType, ProductType}; - use v10::{FunctionVisibility as Declared, RawModuleDefV10Builder}; - - fn scheduled_module(visibility: Option, procedure: bool) -> ModuleDef { - let mut builder = RawModuleDefV10Builder::new(); - let at = builder.add_type::(); - let row = builder - .build_table_with_new_type( - "Jobs", - ProductType::from([("id", AlgebraicType::U64), ("at", at)]), - true, - ) - .with_auto_inc_primary_key(0) - .with_index_no_accessor_name(v9::btree(0), "jobs_id_idx") - .finish(); - let params = ProductType::from([("job", AlgebraicType::Ref(row))]); - if procedure { - builder.add_procedure_with_visibility("run_job", params, AlgebraicType::unit(), visibility); - } else { - builder.add_reducer_with_visibility("run_job", params, visibility); - } - builder.add_schedule("Jobs", 1, "run_job"); - builder.finish().try_into().unwrap() - } - - #[test] - fn explicit_scheduled_visibility_overrides_the_private_default() { - for procedure in [false, true] { - for (selection, expected) in [ - (None, FunctionVisibility::Private), - (Some(Declared::Private), FunctionVisibility::Private), - (Some(Declared::Internal), FunctionVisibility::Internal), - (Some(Declared::ClientCallable), FunctionVisibility::ClientCallable), - ] { - let module = scheduled_module(selection, procedure); - let visibility = if procedure { - &module.procedure("run_job").unwrap().visibility - } else { - &module.reducer("run_job").unwrap().visibility - }; - assert_eq!(visibility, &expected); - assert_eq!(module.raw_module_def_version(), RawModuleDefVersion::V10); - } - } - } - - #[test] - fn ordinary_defaults_and_lifecycle_restrictions() { - let mut builder = RawModuleDefV10Builder::new(); - builder.add_reducer("ordinary", ProductType::unit()); - builder.add_procedure("ordinary_procedure", ProductType::unit(), AlgebraicType::unit()); - builder.add_lifecycle_reducer(v9::Lifecycle::Init, "initialize", ProductType::unit()); - let module: ModuleDef = builder.finish().try_into().unwrap(); - assert!(module.reducer("ordinary").unwrap().visibility.is_client_callable()); - assert!(module - .procedure("ordinary_procedure") - .unwrap() - .visibility - .is_client_callable()); - assert!(module.reducer("initialize").unwrap().visibility.is_internal()); - let exported: RawModuleDefV10 = module.into(); - assert!(exported - .reducers() - .into_iter() - .flatten() - .all(|function| matches!(function.visibility, Declared::ClientCallable | Declared::Private))); - assert!(exported - .procedures() - .into_iter() - .flatten() - .all(|function| matches!(function.visibility, Declared::ClientCallable))); - for selection in [Declared::ClientCallable, Declared::ExplicitClientCallable] { - let mut builder = RawModuleDefV10Builder::new(); - builder.add_lifecycle_reducer_with_visibility( - v9::Lifecycle::Init, - "initialize", - ProductType::unit(), - Some(selection), - ); - assert!(ModuleDef::try_from(builder.finish()) - .unwrap_err() - .to_string() - .contains("must have Internal visibility")); - } - let mut builder = RawModuleDefV10Builder::new(); - builder.add_lifecycle_reducer_with_visibility( - v9::Lifecycle::Init, - "initialize", - ProductType::unit(), - Some(Declared::Internal), - ); - assert!(ModuleDef::try_from(builder.finish()).is_ok()); - } - - #[test] - fn duplicate_definitions_sections_and_lifecycles_are_rejected() { - let mut builder = RawModuleDefV10Builder::new(); - builder.add_reducer("same", ProductType::unit()); - builder.add_procedure("same", ProductType::unit(), AlgebraicType::unit()); - assert!(ModuleDef::try_from(builder.finish()).is_err()); - let raw = v10::RawModuleDefV10 { - sections: vec![ - v10::RawModuleDefV10Section::Capabilities(vec![]), - v10::RawModuleDefV10Section::Capabilities(vec![]), - ], - }; - assert!(ModuleDef::try_from(raw) - .unwrap_err() - .to_string() - .contains("repeated V10 section")); - let mut builder = RawModuleDefV10Builder::new(); - builder.add_lifecycle_reducer(v9::Lifecycle::Init, "a", ProductType::unit()); - builder.add_lifecycle_reducer(v9::Lifecycle::Init, "b", ProductType::unit()); - assert!(ModuleDef::try_from(builder.finish()).is_err()); - } - - #[test] - fn resolved_v10_roundtrips_without_reapplying_defaults_and_rejects_v9_exports() { - for procedure in [false, true] { - for selection in [ - None, - Some(Declared::Private), - Some(Declared::Internal), - Some(Declared::ClientCallable), - ] { - let module = scheduled_module(selection, procedure); - assert!(v9::RawModuleDefV9::try_from(module.clone()).is_err()); - let RawModuleDef::V10(raw) = module.clone().into_raw() else { - panic!("lost source version") - }; - if matches!(selection, Some(Declared::ClientCallable)) { - assert!(raw - .reducers() - .into_iter() - .flatten() - .map(|function| &function.visibility) - .chain( - raw.procedures() - .into_iter() - .flatten() - .map(|function| &function.visibility) - ) - .all(|visibility| matches!(visibility, Declared::ExplicitClientCallable))); - } - let bytes = spacetimedb_lib::bsatn::to_vec(&RawModuleDef::V10(raw)).unwrap(); - let roundtrip: RawModuleDef = spacetimedb_lib::bsatn::from_slice(&bytes).unwrap(); - let roundtrip: ModuleDef = roundtrip.try_into().unwrap(); - if procedure { - assert_eq!( - roundtrip.procedure("run_job").unwrap().visibility, - module.procedure("run_job").unwrap().visibility - ); - } else { - assert_eq!( - roundtrip.reducer("run_job").unwrap().visibility, - module.reducer("run_job").unwrap().visibility - ); - } - assert_eq!(roundtrip.raw_module_def_version(), RawModuleDefVersion::V10); - } - } - } - - #[test] - fn legacy_v9_schedules_stay_public_and_v10_schedules_stay_private() { - let mut builder = v9::RawModuleDefV9Builder::new(); - let at = builder.add_type::(); - let row = builder - .build_table_with_new_type( - "jobs", - ProductType::from([("id", AlgebraicType::U64), ("at", at)]), - true, - ) - .with_auto_inc_primary_key(0) - .with_index(v9::btree(0), "jobs_id_idx") - .with_schedule("run_job", 1) - .finish(); - builder.add_reducer("run_job", ProductType::from([("job", row.into())]), None); - let v9: ModuleDef = builder.finish().try_into().unwrap(); - assert!(v9.reducer("run_job").unwrap().visibility.is_client_callable()); - let upgraded: RawModuleDefV10 = v9.clone().into(); - assert!(matches!( - upgraded.reducers().unwrap()[0].visibility, - Declared::ExplicitClientCallable - )); - let upgraded: ModuleDef = upgraded.try_into().unwrap(); - assert!(upgraded.reducer("run_job").unwrap().visibility.is_client_callable()); - assert!(matches!(v9.into_raw(), RawModuleDef::V9(_))); - - let mut builder = v10::RawModuleDefV10Builder::new(); - let at = builder.add_type::(); - let row = builder - .build_table_with_new_type( - "jobs", - ProductType::from([("id", AlgebraicType::U64), ("at", at)]), - true, - ) - .with_auto_inc_primary_key(0) - .with_index_no_accessor_name(v9::btree(0), "jobs_id_idx") - .finish(); - builder.add_reducer("run_job", ProductType::from([("job", row.into())])); - builder.add_schedule("jobs", 1, "run_job"); - let v10: ModuleDef = builder.finish().try_into().unwrap(); - assert!(v10.reducer("run_job").unwrap().visibility.is_private()); - assert!(matches!(v10.into_raw(), RawModuleDef::V10(_))); - } - + use spacetimedb_lib::db::raw_def::v10::RawModuleDefV10Builder; #[test] fn capabilities_are_explicit_bounded_and_preserved() { let bare: ModuleDef = RawModuleDefV10Builder::new().finish().try_into().unwrap(); @@ -3125,40 +2877,6 @@ mod visibility_tests { assert!(ModuleDef::try_from(builder.finish()).is_err()); } } - - #[test] - fn narrowing_function_visibility_is_a_reported_client_break() { - let module = |visibility| { - let mut builder = RawModuleDefV10Builder::new(); - builder.add_reducer_with_visibility("run_now", ProductType::unit(), Some(visibility)); - ModuleDef::try_from(builder.finish()).unwrap() - }; - let public = module(Declared::ClientCallable); - let internal = module(Declared::Internal); - let plan = crate::auto_migrate::ponder_migrate(&public, &internal).unwrap(); - assert!(plan.breaks_client()); - let display = plan - .pretty_print(crate::auto_migrate::PrettyPrintStyle::NoColor) - .unwrap(); - assert!(display.contains("run_now")); - assert!(display.contains("Internal")); - assert!(!crate::auto_migrate::ponder_migrate(&internal, &public) - .unwrap() - .breaks_client()); - } - - #[test] - fn visibility_authority_is_cumulative_without_elevating_the_owner() { - for (visibility, external, owner, internal) in [ - (FunctionVisibility::Internal, false, false, true), - (FunctionVisibility::Private, false, true, true), - (FunctionVisibility::ClientCallable, true, true, true), - ] { - assert_eq!(visibility.allows_invocation(false, false), external); - assert_eq!(visibility.allows_invocation(false, true), owner); - assert_eq!(visibility.allows_invocation(true, false), internal); - } - } } #[cfg(test)] diff --git a/crates/schema/src/def/validate/v9.rs b/crates/schema/src/def/validate/v9.rs index 1b546fbcfe7..b8b23e4d37e 100644 --- a/crates/schema/src/def/validate/v9.rs +++ b/crates/schema/src/def/validate/v9.rs @@ -392,7 +392,7 @@ impl ModuleValidatorV9<'_> { }, lifecycle, visibility: if lifecycle.is_some() { - FunctionVisibility::Internal + FunctionVisibility::Private } else { FunctionVisibility::ClientCallable }, diff --git a/crates/schema/src/error.rs b/crates/schema/src/error.rs index 301678b0b01..52ee4996155 100644 --- a/crates/schema/src/error.rs +++ b/crates/schema/src/error.rs @@ -26,8 +26,6 @@ pub enum ValidationError { UnsupportedModuleVersion, #[error("invalid module capabilities: at most 32 unique names of 1..64 lowercase ASCII letters, digits or underscores are allowed")] InvalidModuleCapabilities, - #[error("lifecycle reducer `{function}` must have Internal visibility")] - InvalidLifecycleVisibility { function: RawIdentifier }, #[error("module contains repeated V10 section `{section}`")] DuplicateModuleSection { section: String }, #[error("module has repeated environment declarations")] diff --git a/crates/testing/tests/invocation_flags.rs b/crates/testing/tests/invocation_flags.rs index 16581de6662..44d098efa5c 100644 --- a/crates/testing/tests/invocation_flags.rs +++ b/crates/testing/tests/invocation_flags.rs @@ -20,7 +20,7 @@ fn wasm_invocation_flags_do_not_infer_authority_from_identity_or_connection_abse .outcome .into_result() .unwrap(); - for name in ["internal", "init", "scheduled"] { + for name in ["init"] { assert!(module .call_reducer(sender, None, None, None, None, name, FunctionArgs::Nullary) .await @@ -30,18 +30,6 @@ fn wasm_invocation_flags_do_not_infer_authority_from_identity_or_connection_abse .call_procedure(sender, None, None, "external_procedure", FunctionArgs::Nullary) .await; assert_eq!(result.result.unwrap().return_val, AlgebraicValue::Bool(true)); - assert!(module - .call_procedure(sender, None, None, "internal_procedure", FunctionArgs::Nullary) - .await - .result - .is_err()); - assert_eq!( - module - .call_reducer(sender, None, None, None, None, "private", FunctionArgs::Nullary) - .await - .is_ok(), - sender == Identity::ZERO, - ); } module .call_reducer( diff --git a/docs/docs/00300-resources/00200-reference/00100-cli-reference/00100-cli-reference.md b/docs/docs/00300-resources/00200-reference/00100-cli-reference/00100-cli-reference.md index fdbb2822237..b293fd95c20 100644 --- a/docs/docs/00300-resources/00200-reference/00100-cli-reference/00100-cli-reference.md +++ b/docs/docs/00300-resources/00200-reference/00100-cli-reference/00100-cli-reference.md @@ -450,7 +450,7 @@ Run `spacetime help generate` for more detailed information. Default value: `` * `--dotnet-version ` — Target .NET SDK major version for C# projects (e.g. 8 or 10). Auto-detected when omitted. -* `--include-private` — Include private tables and private/internal non-lifecycle functions (types are always included). +* `--include-private` — Include private tables and functions in generated code (types are always included). Default value: `false` * `-y`, `--yes` — Run non-interactively wherever possible. This will answer "yes" to almost all prompts, but will sometimes answer "no" to preserve non-interactivity (e.g. when prompting whether to log in with spacetimedb.com). diff --git a/modules/invocation-flags-test/src/lib.rs b/modules/invocation-flags-test/src/lib.rs index 9621ef95f59..d4f3000b0dd 100644 --- a/modules/invocation-flags-test/src/lib.rs +++ b/modules/invocation-flags-test/src/lib.rs @@ -14,14 +14,6 @@ pub fn external(ctx: &ReducerContext) { assert!(!ctx.sender_auth().has_jwt()); } -#[spacetimedb::reducer(internal)] -pub fn internal(ctx: &ReducerContext) { - assert!(ctx.sender_auth().is_internal()); -} - -#[spacetimedb::reducer(private)] -pub fn private(_ctx: &ReducerContext) {} - #[spacetimedb::procedure] pub fn external_procedure(ctx: &mut ProcedureContext) -> bool { assert!(!ctx.sender_auth().is_internal()); @@ -35,12 +27,6 @@ pub fn external_procedure(ctx: &mut ProcedureContext) -> bool { true } -#[spacetimedb::procedure(internal)] -pub fn internal_procedure(ctx: &mut ProcedureContext) -> bool { - assert!(ctx.sender_auth().is_internal()); - true -} - #[spacetimedb::table(accessor = jobs, scheduled(scheduled))] pub struct Job { #[primary_key] @@ -63,7 +49,7 @@ pub fn schedule(ctx: &ReducerContext) { }); } -#[spacetimedb::reducer(internal)] +#[spacetimedb::reducer] pub fn scheduled(ctx: &ReducerContext, job: Job) { assert!(ctx.sender_auth().is_internal()); assert_eq!(ctx.sender(), ctx.database_identity()); diff --git a/modules/module-test-ts/src/index.ts b/modules/module-test-ts/src/index.ts index 45dfa555ea5..000e25be0e4 100644 --- a/modules/module-test-ts/src/index.ts +++ b/modules/module-test-ts/src/index.ts @@ -522,7 +522,7 @@ export const getMySchemaViaHttp = spacetimedb.procedure(t.string(), ctx => { const module_identity = ctx.databaseIdentity; try { const response = ctx.http.fetch( - `http://localhost:3000/v1/database/${module_identity}/schema?version=10` + `http://localhost:3000/v1/database/${module_identity}/schema?version=9` ); return response.text(); } catch (e) { diff --git a/modules/module-test/src/lib.rs b/modules/module-test/src/lib.rs index 4f5508cc00f..9a2d6790d53 100644 --- a/modules/module-test/src/lib.rs +++ b/modules/module-test/src/lib.rs @@ -548,7 +548,7 @@ fn with_tx(ctx: &mut ProcedureContext) { fn get_my_schema_via_http(ctx: &mut ProcedureContext) -> String { let module_identity = ctx.database_identity(); match ctx.http.get(format!( - "http://localhost:3000/v1/database/{module_identity}/schema?version=10" + "http://localhost:3000/v1/database/{module_identity}/schema?version=9" )) { Ok(result) => result.into_body().into_string_lossy(), Err(e) => format!("{e}"), diff --git a/modules/sdk-test-procedure-cpp/src/lib.cpp b/modules/sdk-test-procedure-cpp/src/lib.cpp index b5310efac63..da1278ccdca 100644 --- a/modules/sdk-test-procedure-cpp/src/lib.cpp +++ b/modules/sdk-test-procedure-cpp/src/lib.cpp @@ -151,7 +151,7 @@ SPACETIMEDB_PROCEDURE(std::string, read_my_schema, ProcedureContext ctx, std::st LOG_INFO("read_my_schema using identity: " + identity_hex); // Make HTTP GET request to the schema endpoint (matches Rust) - std::string url = server_url + "/v1/database/" + identity_hex + "/schema?version=10"; + std::string url = server_url + "/v1/database/" + identity_hex + "/schema?version=9"; auto result = ctx.http.get(url); if (!result.is_ok()) { diff --git a/modules/sdk-test-procedure-cs/Lib.cs b/modules/sdk-test-procedure-cs/Lib.cs index 30c4a8ed4ac..2e405c3a9bc 100644 --- a/modules/sdk-test-procedure-cs/Lib.cs +++ b/modules/sdk-test-procedure-cs/Lib.cs @@ -70,7 +70,7 @@ public static string ReadMySchema(ProcedureContext ctx, string serverUrl) { var moduleIdentity = ProcedureContextBase.Identity; serverUrl = serverUrl.TrimEnd('/'); - var result = ctx.Http.Get($"{serverUrl}/v1/database/{moduleIdentity}/schema?version=10"); + var result = ctx.Http.Get($"{serverUrl}/v1/database/{moduleIdentity}/schema?version=9"); return result.Match( response => response.Body.ToStringUtf8Lossy(), error => throw new Exception($"HTTP request failed: {error}") diff --git a/modules/sdk-test-procedure-ts/src/index.ts b/modules/sdk-test-procedure-ts/src/index.ts index 76efe296c17..1885eafd156 100644 --- a/modules/sdk-test-procedure-ts/src/index.ts +++ b/modules/sdk-test-procedure-ts/src/index.ts @@ -97,7 +97,7 @@ export const read_my_schema = spacetimedb.procedure( const module_identity = ctx.databaseIdentity; const base_url = server_url.replace(/\/+$/, ''); const response = ctx.http.fetch( - `${base_url}/v1/database/${module_identity}/schema?version=10` + `${base_url}/v1/database/${module_identity}/schema?version=9` ); return response.text(); } diff --git a/modules/sdk-test-procedure/src/lib.rs b/modules/sdk-test-procedure/src/lib.rs index c9af396f4f2..5eb2f848ad5 100644 --- a/modules/sdk-test-procedure/src/lib.rs +++ b/modules/sdk-test-procedure/src/lib.rs @@ -46,7 +46,7 @@ fn read_my_schema(ctx: &mut ProcedureContext, server_url: String) -> String { let server_url = server_url.trim_end_matches('/'); match ctx .http - .get(format!("{server_url}/v1/database/{module_identity}/schema?version=10")) + .get(format!("{server_url}/v1/database/{module_identity}/schema?version=9")) { Ok(result) => result.into_body().into_string_lossy(), Err(e) => panic!("{e}"), diff --git a/sdks/csharp/examples~/regression-tests/server/Lib.cs b/sdks/csharp/examples~/regression-tests/server/Lib.cs index 64e4f164da4..e3391b710ff 100644 --- a/sdks/csharp/examples~/regression-tests/server/Lib.cs +++ b/sdks/csharp/examples~/regression-tests/server/Lib.cs @@ -831,7 +831,7 @@ public static string ReadMySchemaViaHttp(ProcedureContext ctx) try { var moduleIdentity = ProcedureContext.Identity; - var uri = $"http://localhost:3000/v1/database/{moduleIdentity}/schema?version=10"; + var uri = $"http://localhost:3000/v1/database/{moduleIdentity}/schema?version=9"; var res = ctx.Http.Get(uri, System.TimeSpan.FromSeconds(2)); return res switch { diff --git a/sdks/rust/tests/connect_disconnect_client/src/module_bindings/identity_connected_reducer.rs b/sdks/rust/tests/connect_disconnect_client/src/module_bindings/identity_connected_reducer.rs new file mode 100644 index 00000000000..cfbf1d03d30 --- /dev/null +++ b/sdks/rust/tests/connect_disconnect_client/src/module_bindings/identity_connected_reducer.rs @@ -0,0 +1,62 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +#![allow(unused, clippy::all)] +use spacetimedb_sdk::__codegen::{self as __sdk, __lib, __sats, __ws}; + +#[derive(__lib::ser::Serialize, __lib::de::Deserialize, Clone, PartialEq, Debug)] +#[sats(crate = __lib)] +pub(super) struct IdentityConnectedArgs {} + +impl From for super::Reducer { + fn from(args: IdentityConnectedArgs) -> Self { + Self::IdentityConnected + } +} + +impl __sdk::InModule for IdentityConnectedArgs { + type Module = super::RemoteModule; +} + +#[allow(non_camel_case_types)] +/// Extension trait for access to the reducer `identity_connected`. +/// +/// Implemented for [`super::RemoteReducers`]. +pub trait identity_connected { + /// Request that the remote module invoke the reducer `identity_connected` to run as soon as possible. + /// + /// This method returns immediately, and errors only if we are unable to send the request. + /// The reducer will run asynchronously in the future, + /// and this method provides no way to listen for its completion status. + /// /// Use [`identity_connected:identity_connected_then`] to run a callback after the reducer completes. + fn identity_connected(&self) -> __sdk::Result<()> { + self.identity_connected_then(|_, _| {}) + } + + /// Request that the remote module invoke the reducer `identity_connected` to run as soon as possible, + /// registering `callback` to run when we are notified that the reducer completed. + /// + /// This method returns immediately, and errors only if we are unable to send the request. + /// The reducer will run asynchronously in the future, + /// and its status can be observed with the `callback`. + fn identity_connected_then( + &self, + + callback: impl FnOnce(&super::ReducerEventContext, Result, __sdk::InternalError>) + + Send + + 'static, + ) -> __sdk::Result<()>; +} + +impl identity_connected for super::RemoteReducers { + fn identity_connected_then( + &self, + + callback: impl FnOnce(&super::ReducerEventContext, Result, __sdk::InternalError>) + + Send + + 'static, + ) -> __sdk::Result<()> { + self.imp + .invoke_reducer_with_callback(IdentityConnectedArgs {}, callback) + } +} diff --git a/sdks/rust/tests/connect_disconnect_client/src/module_bindings/identity_disconnected_reducer.rs b/sdks/rust/tests/connect_disconnect_client/src/module_bindings/identity_disconnected_reducer.rs new file mode 100644 index 00000000000..8cec050f73e --- /dev/null +++ b/sdks/rust/tests/connect_disconnect_client/src/module_bindings/identity_disconnected_reducer.rs @@ -0,0 +1,62 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +#![allow(unused, clippy::all)] +use spacetimedb_sdk::__codegen::{self as __sdk, __lib, __sats, __ws}; + +#[derive(__lib::ser::Serialize, __lib::de::Deserialize, Clone, PartialEq, Debug)] +#[sats(crate = __lib)] +pub(super) struct IdentityDisconnectedArgs {} + +impl From for super::Reducer { + fn from(args: IdentityDisconnectedArgs) -> Self { + Self::IdentityDisconnected + } +} + +impl __sdk::InModule for IdentityDisconnectedArgs { + type Module = super::RemoteModule; +} + +#[allow(non_camel_case_types)] +/// Extension trait for access to the reducer `identity_disconnected`. +/// +/// Implemented for [`super::RemoteReducers`]. +pub trait identity_disconnected { + /// Request that the remote module invoke the reducer `identity_disconnected` to run as soon as possible. + /// + /// This method returns immediately, and errors only if we are unable to send the request. + /// The reducer will run asynchronously in the future, + /// and this method provides no way to listen for its completion status. + /// /// Use [`identity_disconnected:identity_disconnected_then`] to run a callback after the reducer completes. + fn identity_disconnected(&self) -> __sdk::Result<()> { + self.identity_disconnected_then(|_, _| {}) + } + + /// Request that the remote module invoke the reducer `identity_disconnected` to run as soon as possible, + /// registering `callback` to run when we are notified that the reducer completed. + /// + /// This method returns immediately, and errors only if we are unable to send the request. + /// The reducer will run asynchronously in the future, + /// and its status can be observed with the `callback`. + fn identity_disconnected_then( + &self, + + callback: impl FnOnce(&super::ReducerEventContext, Result, __sdk::InternalError>) + + Send + + 'static, + ) -> __sdk::Result<()>; +} + +impl identity_disconnected for super::RemoteReducers { + fn identity_disconnected_then( + &self, + + callback: impl FnOnce(&super::ReducerEventContext, Result, __sdk::InternalError>) + + Send + + 'static, + ) -> __sdk::Result<()> { + self.imp + .invoke_reducer_with_callback(IdentityDisconnectedArgs {}, callback) + } +} diff --git a/sdks/rust/tests/connect_disconnect_client/src/module_bindings/mod.rs b/sdks/rust/tests/connect_disconnect_client/src/module_bindings/mod.rs index b6ae9d07827..bae06001c05 100644 --- a/sdks/rust/tests/connect_disconnect_client/src/module_bindings/mod.rs +++ b/sdks/rust/tests/connect_disconnect_client/src/module_bindings/mod.rs @@ -10,11 +10,15 @@ pub mod connected_table; pub mod connected_type; pub mod disconnected_table; pub mod disconnected_type; +pub mod identity_connected_reducer; +pub mod identity_disconnected_reducer; pub use connected_table::*; pub use connected_type::Connected; pub use disconnected_table::*; pub use disconnected_type::Disconnected; +pub use identity_connected_reducer::identity_connected; +pub use identity_disconnected_reducer::identity_disconnected; #[derive(Clone, PartialEq, Debug)] @@ -23,7 +27,10 @@ pub use disconnected_type::Disconnected; /// Contained within a [`__sdk::ReducerEvent`] in [`EventContext`]s for reducer events /// to indicate which reducer caused the event. -pub enum Reducer {} +pub enum Reducer { + IdentityConnected, + IdentityDisconnected, +} impl __sdk::InModule for Reducer { type Module = RemoteModule; @@ -32,12 +39,18 @@ impl __sdk::InModule for Reducer { impl __sdk::Reducer for Reducer { fn reducer_name(&self) -> &'static str { match self { + Reducer::IdentityConnected => "identity_connected", + Reducer::IdentityDisconnected => "identity_disconnected", _ => unreachable!(), } } #[allow(clippy::clone_on_copy)] fn args_bsatn(&self) -> Result, __sats::bsatn::EncodeError> { match self { + Reducer::IdentityConnected => __sats::bsatn::to_vec(&identity_connected_reducer::IdentityConnectedArgs {}), + Reducer::IdentityDisconnected => { + __sats::bsatn::to_vec(&identity_disconnected_reducer::IdentityDisconnectedArgs {}) + } _ => unreachable!(), } } diff --git a/sdks/rust/tests/procedure-client/src/test_handlers.rs b/sdks/rust/tests/procedure-client/src/test_handlers.rs index 54e4d875ab0..fdfc417cd9b 100644 --- a/sdks/rust/tests/procedure-client/src/test_handlers.rs +++ b/sdks/rust/tests/procedure-client/src/test_handlers.rs @@ -1,7 +1,7 @@ use crate::module_bindings::*; use anyhow::Context; use core::time::Duration; -use spacetimedb_lib::db::raw_def::v10::{ExplicitNameEntry, RawModuleDefV10}; +use spacetimedb_lib::db::raw_def::v9::{RawMiscModuleExportV9, RawModuleDefV9}; use spacetimedb_sdk::{DbConnectionBuilder, DbContext, Table}; use test_counter::{server_url, TestCounter}; @@ -247,7 +247,7 @@ async fn exec_insert_with_tx_rollback(db_name: &str) { /// Test that a procedure can perform an HTTP request and return a string derived from the response. /// /// Invoke the procedure `read_my_schema`, -/// which does an HTTP request to the `/database/schema` route and returns a JSON-ified [`RawModuleDefV10`], +/// which does an HTTP request to the `/database/schema` route and returns a JSON-ified [`RawModuleDefV9`], /// then (in the client) deserialize the response and assert that it contains a description of that procedure. async fn exec_procedure_http_ok(db_name: &str) { let test_counter = TestCounter::new(); @@ -262,22 +262,15 @@ async fn exec_procedure_http_ok(db_name: &str) { #[allow(clippy::redundant_closure_call)] (|| { anyhow::ensure!(res.is_ok(), "Expected Ok result but got {res:?}"); - let module_def: RawModuleDefV10 = spacetimedb_lib::de::serde::deserialize_from( + let module_def: RawModuleDefV9 = spacetimedb_lib::de::serde::deserialize_from( &mut serde_json::Deserializer::from_str(&res.unwrap()), )?; - // The schema endpoint exports source-to-canonical name mappings. - // C# uses `ReadMySchema` in source and `read_my_schema` on the wire. - let names = module_def.explicit_names().cloned().unwrap_or_default().into_entries(); - anyhow::ensure!(names.iter().any(|entry| { - let ExplicitNameEntry::Function(mapping) = entry else { - return false; - }; - &*mapping.canonical_name == "read_my_schema" - && module_def - .procedures() - .into_iter() - .flatten() - .any(|procedure| procedure.source_name == mapping.source_name) + anyhow::ensure!(module_def.misc_exports.iter().any(|misc_export| { + if let RawMiscModuleExportV9::Procedure(procedure_def) = misc_export { + &*procedure_def.name == "read_my_schema" + } else { + false + } })); Ok(()) })(), From 5324546138f7780932e9f8202dbf5bac18440ff9 Mon Sep 17 00:00:00 2001 From: Tyler Cloutier Date: Fri, 18 Sep 2026 10:27:51 -0400 Subject: [PATCH 34/34] Keep capability-aware schema fixtures with authentication --- modules/module-test-ts/src/index.ts | 2 +- modules/module-test/src/lib.rs | 2 +- modules/sdk-test-procedure-cpp/src/lib.cpp | 2 +- modules/sdk-test-procedure-cs/Lib.cs | 2 +- modules/sdk-test-procedure-ts/src/index.ts | 2 +- modules/sdk-test-procedure/src/lib.rs | 2 +- .../examples~/regression-tests/server/Lib.cs | 2 +- .../procedure-client/src/test_handlers.rs | 25 ++++++++++++------- 8 files changed, 23 insertions(+), 16 deletions(-) diff --git a/modules/module-test-ts/src/index.ts b/modules/module-test-ts/src/index.ts index 000e25be0e4..45dfa555ea5 100644 --- a/modules/module-test-ts/src/index.ts +++ b/modules/module-test-ts/src/index.ts @@ -522,7 +522,7 @@ export const getMySchemaViaHttp = spacetimedb.procedure(t.string(), ctx => { const module_identity = ctx.databaseIdentity; try { const response = ctx.http.fetch( - `http://localhost:3000/v1/database/${module_identity}/schema?version=9` + `http://localhost:3000/v1/database/${module_identity}/schema?version=10` ); return response.text(); } catch (e) { diff --git a/modules/module-test/src/lib.rs b/modules/module-test/src/lib.rs index 9a2d6790d53..4f5508cc00f 100644 --- a/modules/module-test/src/lib.rs +++ b/modules/module-test/src/lib.rs @@ -548,7 +548,7 @@ fn with_tx(ctx: &mut ProcedureContext) { fn get_my_schema_via_http(ctx: &mut ProcedureContext) -> String { let module_identity = ctx.database_identity(); match ctx.http.get(format!( - "http://localhost:3000/v1/database/{module_identity}/schema?version=9" + "http://localhost:3000/v1/database/{module_identity}/schema?version=10" )) { Ok(result) => result.into_body().into_string_lossy(), Err(e) => format!("{e}"), diff --git a/modules/sdk-test-procedure-cpp/src/lib.cpp b/modules/sdk-test-procedure-cpp/src/lib.cpp index da1278ccdca..b5310efac63 100644 --- a/modules/sdk-test-procedure-cpp/src/lib.cpp +++ b/modules/sdk-test-procedure-cpp/src/lib.cpp @@ -151,7 +151,7 @@ SPACETIMEDB_PROCEDURE(std::string, read_my_schema, ProcedureContext ctx, std::st LOG_INFO("read_my_schema using identity: " + identity_hex); // Make HTTP GET request to the schema endpoint (matches Rust) - std::string url = server_url + "/v1/database/" + identity_hex + "/schema?version=9"; + std::string url = server_url + "/v1/database/" + identity_hex + "/schema?version=10"; auto result = ctx.http.get(url); if (!result.is_ok()) { diff --git a/modules/sdk-test-procedure-cs/Lib.cs b/modules/sdk-test-procedure-cs/Lib.cs index 2e405c3a9bc..30c4a8ed4ac 100644 --- a/modules/sdk-test-procedure-cs/Lib.cs +++ b/modules/sdk-test-procedure-cs/Lib.cs @@ -70,7 +70,7 @@ public static string ReadMySchema(ProcedureContext ctx, string serverUrl) { var moduleIdentity = ProcedureContextBase.Identity; serverUrl = serverUrl.TrimEnd('/'); - var result = ctx.Http.Get($"{serverUrl}/v1/database/{moduleIdentity}/schema?version=9"); + var result = ctx.Http.Get($"{serverUrl}/v1/database/{moduleIdentity}/schema?version=10"); return result.Match( response => response.Body.ToStringUtf8Lossy(), error => throw new Exception($"HTTP request failed: {error}") diff --git a/modules/sdk-test-procedure-ts/src/index.ts b/modules/sdk-test-procedure-ts/src/index.ts index 1885eafd156..76efe296c17 100644 --- a/modules/sdk-test-procedure-ts/src/index.ts +++ b/modules/sdk-test-procedure-ts/src/index.ts @@ -97,7 +97,7 @@ export const read_my_schema = spacetimedb.procedure( const module_identity = ctx.databaseIdentity; const base_url = server_url.replace(/\/+$/, ''); const response = ctx.http.fetch( - `${base_url}/v1/database/${module_identity}/schema?version=9` + `${base_url}/v1/database/${module_identity}/schema?version=10` ); return response.text(); } diff --git a/modules/sdk-test-procedure/src/lib.rs b/modules/sdk-test-procedure/src/lib.rs index 5eb2f848ad5..c9af396f4f2 100644 --- a/modules/sdk-test-procedure/src/lib.rs +++ b/modules/sdk-test-procedure/src/lib.rs @@ -46,7 +46,7 @@ fn read_my_schema(ctx: &mut ProcedureContext, server_url: String) -> String { let server_url = server_url.trim_end_matches('/'); match ctx .http - .get(format!("{server_url}/v1/database/{module_identity}/schema?version=9")) + .get(format!("{server_url}/v1/database/{module_identity}/schema?version=10")) { Ok(result) => result.into_body().into_string_lossy(), Err(e) => panic!("{e}"), diff --git a/sdks/csharp/examples~/regression-tests/server/Lib.cs b/sdks/csharp/examples~/regression-tests/server/Lib.cs index e3391b710ff..64e4f164da4 100644 --- a/sdks/csharp/examples~/regression-tests/server/Lib.cs +++ b/sdks/csharp/examples~/regression-tests/server/Lib.cs @@ -831,7 +831,7 @@ public static string ReadMySchemaViaHttp(ProcedureContext ctx) try { var moduleIdentity = ProcedureContext.Identity; - var uri = $"http://localhost:3000/v1/database/{moduleIdentity}/schema?version=9"; + var uri = $"http://localhost:3000/v1/database/{moduleIdentity}/schema?version=10"; var res = ctx.Http.Get(uri, System.TimeSpan.FromSeconds(2)); return res switch { diff --git a/sdks/rust/tests/procedure-client/src/test_handlers.rs b/sdks/rust/tests/procedure-client/src/test_handlers.rs index fdfc417cd9b..54e4d875ab0 100644 --- a/sdks/rust/tests/procedure-client/src/test_handlers.rs +++ b/sdks/rust/tests/procedure-client/src/test_handlers.rs @@ -1,7 +1,7 @@ use crate::module_bindings::*; use anyhow::Context; use core::time::Duration; -use spacetimedb_lib::db::raw_def::v9::{RawMiscModuleExportV9, RawModuleDefV9}; +use spacetimedb_lib::db::raw_def::v10::{ExplicitNameEntry, RawModuleDefV10}; use spacetimedb_sdk::{DbConnectionBuilder, DbContext, Table}; use test_counter::{server_url, TestCounter}; @@ -247,7 +247,7 @@ async fn exec_insert_with_tx_rollback(db_name: &str) { /// Test that a procedure can perform an HTTP request and return a string derived from the response. /// /// Invoke the procedure `read_my_schema`, -/// which does an HTTP request to the `/database/schema` route and returns a JSON-ified [`RawModuleDefV9`], +/// which does an HTTP request to the `/database/schema` route and returns a JSON-ified [`RawModuleDefV10`], /// then (in the client) deserialize the response and assert that it contains a description of that procedure. async fn exec_procedure_http_ok(db_name: &str) { let test_counter = TestCounter::new(); @@ -262,15 +262,22 @@ async fn exec_procedure_http_ok(db_name: &str) { #[allow(clippy::redundant_closure_call)] (|| { anyhow::ensure!(res.is_ok(), "Expected Ok result but got {res:?}"); - let module_def: RawModuleDefV9 = spacetimedb_lib::de::serde::deserialize_from( + let module_def: RawModuleDefV10 = spacetimedb_lib::de::serde::deserialize_from( &mut serde_json::Deserializer::from_str(&res.unwrap()), )?; - anyhow::ensure!(module_def.misc_exports.iter().any(|misc_export| { - if let RawMiscModuleExportV9::Procedure(procedure_def) = misc_export { - &*procedure_def.name == "read_my_schema" - } else { - false - } + // The schema endpoint exports source-to-canonical name mappings. + // C# uses `ReadMySchema` in source and `read_my_schema` on the wire. + let names = module_def.explicit_names().cloned().unwrap_or_default().into_entries(); + anyhow::ensure!(names.iter().any(|entry| { + let ExplicitNameEntry::Function(mapping) = entry else { + return false; + }; + &*mapping.canonical_name == "read_my_schema" + && module_def + .procedures() + .into_iter() + .flatten() + .any(|procedure| procedure.source_name == mapping.source_name) })); Ok(()) })(),