Module unit testing

.github/workflows/ci.yml

@@ -532,6 +532,34 @@ jobs:
       - name: Run bindgen tests
         run: cargo ci wasm-bindings
 
+  portable_datastore:
+    name: Check portable datastore
+    runs-on: spacetimedb-new-runner-2
+    env:
+      CARGO_TARGET_DIR: ${{ github.workspace }}/target
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: dsherret/rust-toolchain-file@v1
+      - name: Set default rust toolchain
+        run: rustup default $(rustup show active-toolchain | cut -d' ' -f1)
+      - run: echo ::add-matcher::.github/workflows/rust_matcher.json
+
+      - name: Install wasm target
+        run: rustup target add wasm32-unknown-unknown
+
+      - name: Cache Rust dependencies
+        uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: ${{ github.workspace }}
+          shared-key: spacetimedb
+          # Let the smoketests job save the cache since it builds the most things
+          save-if: false
+          prefix-key: v1
+
+      - name: Run portable datastore checks
+        run: cargo ci portable-datastore
+
   publish_checks:
     needs: [merge_queue_noop]
     if: ${{ needs.merge_queue_noop.outputs.skip != 'true' }}

Cargo.toml

@@ -25,6 +25,8 @@ members = [
   "crates/paths",
   "crates/pg",
   "crates/physical-plan",
+  "crates/portable-datastore",
+  "crates/portable-datastore-wasm",
   "crates/primitives",
   "crates/query",
   "crates/runtime",
@@ -39,6 +41,7 @@ members = [
   "crates/standalone",
   "crates/subscription",
   "crates/table",
+  "crates/test-datastore",
   "crates/testing",
   "crates/update",
   "modules/benchmarks",
@@ -159,6 +162,7 @@ spacetimedb-standalone = { path = "crates/standalone", version = "=2.7.0" }
 spacetimedb-subscription = { path = "crates/subscription", version = "=2.7.0" }
 spacetimedb-sql-parser = { path = "crates/sql-parser", version = "=2.7.0" }
 spacetimedb-table = { path = "crates/table", version = "=2.7.0" }
+spacetimedb-test-datastore = { path = "crates/test-datastore", version = "=2.7.0" }
 
 # Prevent `ahash` from pulling in `getrandom` by disabling default features.
 # Modules use `getrandom02` and we need to prevent an incompatible version

crates/bindings-macro/Cargo.toml

@@ -6,6 +6,9 @@ license-file = "LICENSE"
 description = "Easy support for interacting between SpacetimeDB and Rust."
 rust-version.workspace = true
 
+[features]
+test-utils = []
+
 [lib]
 proc-macro = true
 # Benching off, because of https://bheisler.github.io/criterion.rs/book/faq.html#cargo-bench-gives-unrecognized-option-errors-for-valid-command-line-options

crates/bindings-macro/src/lib.rs

@@ -109,7 +109,7 @@ use quote::quote;
 use std::time::Duration;
 use syn::{parse::ParseStream, Attribute};
 use syn::{ItemConst, ItemFn};
-use util::{cvt_attr, ok_or_compile_error};
+use util::{cvt_attr, native_test_utils_registration, ok_or_compile_error};
 
 mod sym {
     /// A symbol known at compile-time against
@@ -305,6 +305,9 @@ pub fn client_visibility_filter(args: StdTokenStream, item: StdTokenStream) -> S
         let item: ItemConst = syn::parse(item)?;
         let rls_ident = item.ident.clone();
         let register_rls_symbol = format!("__preinit__20_register_row_level_security_{rls_ident}");
+        let test_utils_registration = native_test_utils_registration(quote! {
+            spacetimedb::rt::register_row_level_security(#rls_ident.sql_text())
+        });
 
         Ok(quote! {
             #item
@@ -314,6 +317,8 @@ pub fn client_visibility_filter(args: StdTokenStream, item: StdTokenStream) -> S
                 extern "C" fn __register_client_visibility_filter() {
                     spacetimedb::rt::register_row_level_security(#rls_ident.sql_text())
                 }
+
+                #test_utils_registration
             };
         })
     })
@@ -357,6 +362,7 @@ pub fn settings(args: StdTokenStream, item: StdTokenStream) -> StdTokenStream {
             },
             _ => unreachable!("validated above"),
         };
+        let test_utils_registration = native_test_utils_registration(register_call.clone());
 
         Ok(quote! {
             #item
@@ -366,6 +372,8 @@ pub fn settings(args: StdTokenStream, item: StdTokenStream) -> StdTokenStream {
                 extern "C" fn __register_setting() {
                     #register_call
                 }
+
+                #test_utils_registration
             };
         })
     })

crates/bindings-macro/src/procedure.rs

@@ -1,6 +1,6 @@
 use crate::reducer::{assert_only_lifetime_generics, extract_typed_args, generate_explicit_names_impl};
 use crate::sym;
-use crate::util::{check_duplicate, ident_to_litstr, match_meta};
+use crate::util::{check_duplicate, ident_to_litstr, match_meta, native_test_utils_registration};
 use proc_macro2::TokenStream;
 use quote::quote;
 use syn::parse::Parser as _;
@@ -86,12 +86,16 @@ pub(crate) fn procedure_impl(_args: ProcedureArgs, original_function: &ItemFn) -
             spacetimedb::rt::register_procedure::<_, _, #func_name>(#func_name)
         }
     };
+    let test_utils_registration = native_test_utils_registration(quote! {
+        __register_describer()
+    });
 
     let generate_explicit_names = generate_explicit_names_impl(&procedure_name.value(), func_name, explicit_name);
 
     Ok(quote! {
         const _: () = {
             #generated_describe_function
+            #test_utils_registration
         };
         #[allow(non_camel_case_types)]
         #vis struct #func_name { _never: ::core::convert::Infallible }

crates/bindings-macro/src/reducer.rs

@@ -1,5 +1,5 @@
 use crate::sym;
-use crate::util::{check_duplicate, check_duplicate_msg, ident_to_litstr, match_meta};
+use crate::util::{check_duplicate, check_duplicate_msg, ident_to_litstr, match_meta, native_test_utils_registration};
 use proc_macro2::{Span, TokenStream};
 use quote::{quote, quote_spanned};
 use syn::parse::Parser as _;
@@ -140,10 +140,14 @@ pub(crate) fn reducer_impl(args: ReducerArgs, original_function: &ItemFn) -> syn
             spacetimedb::rt::register_reducer::<_, #func_name>(#func_name)
         }
     };
+    let test_utils_registration = native_test_utils_registration(quote! {
+        __register_describer()
+    });
 
     Ok(quote! {
         const _: () = {
             #generated_describe_function
+            #test_utils_registration
         };
         #[allow(non_camel_case_types)]
         #vis struct #func_name { _never: ::core::convert::Infallible }

crates/bindings-macro/src/table.rs

@@ -1,6 +1,6 @@
 use crate::sats;
 use crate::sym;
-use crate::util::{check_duplicate, check_duplicate_msg, match_meta};
+use crate::util::{check_duplicate, check_duplicate_msg, match_meta, native_test_utils_registration};
 use core::slice;
 use heck::ToSnakeCase;
 use proc_macro2::{Span, TokenStream};
@@ -589,7 +589,7 @@ impl ValidatedIndex<'_> {
         quote! {
             #[doc = #doc]
             #vis fn #column_ident(&self) -> #unique_ty<#tbl_token, #col_ty, __indices::#index_ident> {
-                #unique_ty::__NEW
+                #unique_ty::__new(self.__backend.clone())
             }
         }
     }
@@ -630,7 +630,7 @@ impl ValidatedIndex<'_> {
         quote! {
             #[doc = #doc]
             #vis fn #index_ident(&self) -> #handle_ty<#tbl_token, (#(#col_tys,)*), __indices::#index_ident> {
-                #handle_ty::__NEW
+                #handle_ty::__new(self.__backend.clone())
             }
         }
     }
@@ -677,6 +677,7 @@ impl ValidatedIndex<'_> {
             #vis struct #index_ident;
             impl spacetimedb::table::#index_kind_trait for #index_ident {}
             impl spacetimedb::table::Index for #index_ident {
+                const INDEX_NAME: &'static str = #index_name;
                 const NUM_COLS_INDEXED: usize = #num_cols;
                 fn index_id() -> spacetimedb::table::IndexId {
                     static INDEX_ID: std::sync::OnceLock<spacetimedb::table::IndexId> = std::sync::OnceLock::new();
@@ -1136,6 +1137,9 @@ pub(crate) fn table_impl(mut args: TableArgs, item: &syn::DeriveInput) -> syn::R
             #(const SCHEDULE: Option<spacetimedb::table::ScheduleDesc<'static>> = Some(#schedule);)*
 
             #table_id_from_name_func
+            fn __backend(&self) -> &spacetimedb::table::TableHandleBackend {
+                &self.__backend
+            }
             #default_fn
         }
     };
@@ -1157,12 +1161,14 @@ pub(crate) fn table_impl(mut args: TableArgs, item: &syn::DeriveInput) -> syn::R
         #[allow(non_camel_case_types, dead_code)]
         #vis trait #table_ident {
             #[allow(non_camel_case_types, dead_code)]
-            fn #table_ident(&self) -> &#tablehandle_ident;
+            fn #table_ident(&self) -> #tablehandle_ident;
         }
         impl #table_ident for spacetimedb::Local {
             #[allow(non_camel_case_types, dead_code)]
-            fn #table_ident(&self) -> &#tablehandle_ident {
-                &#tablehandle_ident {}
+            fn #table_ident(&self) -> #tablehandle_ident {
+                #tablehandle_ident {
+                    __backend: self.__table_backend(),
+                }
             }
         }
     };
@@ -1171,12 +1177,14 @@ pub(crate) fn table_impl(mut args: TableArgs, item: &syn::DeriveInput) -> syn::R
         #[allow(non_camel_case_types, dead_code)]
         #vis trait #view_trait_ident {
             #[allow(non_camel_case_types, dead_code)]
-            fn #table_ident(&self) -> &#viewhandle_ident;
+            fn #table_ident(&self) -> #viewhandle_ident;
         }
         impl #view_trait_ident for spacetimedb::LocalReadOnly {
             #[inline]
-            fn #table_ident(&self) -> &#viewhandle_ident {
-                &#viewhandle_ident {}
+            fn #table_ident(&self) -> #viewhandle_ident {
+                #viewhandle_ident {
+                    __backend: self.__table_backend(),
+                }
             }
         }
     };
@@ -1263,13 +1271,19 @@ pub(crate) fn table_impl(mut args: TableArgs, item: &syn::DeriveInput) -> syn::R
     let tablehandle_def = quote! {
         #[allow(non_camel_case_types)]
         #[non_exhaustive]
-        #vis struct #tablehandle_ident {}
+        #vis struct #tablehandle_ident {
+            #[doc(hidden)]
+            pub __backend: spacetimedb::table::TableHandleBackend,
+        }
     };
 
     let viewhandle_def = quote! {
         #[allow(non_camel_case_types)]
         #[non_exhaustive]
-        #vis struct #viewhandle_ident {}
+        #vis struct #viewhandle_ident {
+            #[doc(hidden)]
+            pub __backend: spacetimedb::table::TableHandleBackend,
+        }
     };
 
     let struct_name = original_struct_ident.to_string();
@@ -1281,6 +1295,13 @@ pub(crate) fn table_impl(mut args: TableArgs, item: &syn::DeriveInput) -> syn::R
         quote! {}
     };
 
+    let test_utils_registration = native_test_utils_registration(quote! {
+        spacetimedb::rt::register_table::<#tablehandle_ident>();
+        spacetimedb::test_utils::__register_table_name(
+            <#tablehandle_ident as spacetimedb::table::TableInternal>::TABLE_NAME,
+        );
+    });
+
     let emission = quote! {
         const _: () = {
             #(let _ = <#field_types as spacetimedb::rt::TableColumn>::_ITEM;)*
@@ -1304,7 +1325,13 @@ pub(crate) fn table_impl(mut args: TableArgs, item: &syn::DeriveInput) -> syn::R
             impl #viewhandle_ident {
                 #[inline]
                 pub fn count(&self) -> u64 {
-                    spacetimedb::table::count::<#tablehandle_ident>()
+                    let table_id = self
+                        .__backend
+                        .table_id(<#tablehandle_ident as spacetimedb::table::TableInternal>::TABLE_NAME)
+                        .expect("table_id_from_name() call failed");
+                    self.__backend
+                        .table_row_count(table_id)
+                        .expect("datastore_table_row_count() call failed")
                 }
 
                 #(#index_accessors_ro)*
@@ -1322,6 +1349,8 @@ pub(crate) fn table_impl(mut args: TableArgs, item: &syn::DeriveInput) -> syn::R
             }
 
             #describe_table_func
+
+            #test_utils_registration
         };
     };