Merge native query performance fixes into main

.github/actions/db-sync-server/action.yml

@@ -1,13 +1,15 @@
 name: Start db-sync server
 description: >-
-  Check out logseq@feature/native-mobile-app, build the db-sync node adapter,
+  Check out the native-compatible Logseq revision, build the db-sync node adapter,
   and start it on the given port (default 8787). The app e2e flows create sync
   graphs against it. logseq/logseq is public, so no credential is needed.
 inputs:
   ref:
     description: logseq/logseq ref to check out.
     required: false
-    default: feature/native-mobile-app
+    # This revision includes the snapshot transaction cursor required by the
+    # shared native bootstrap protocol. The old feature branch omitted it.
+    default: a63da5cb7993302f8c58ccf42eaf0083b77cf42b
   port:
     description: Port the server binds.
     required: false

android/app/src/androidTest/kotlin/com/logseq/app/AndroidNativeSurfacesTest.kt

@@ -28,6 +28,7 @@ import androidx.compose.ui.test.performTouchInput
 import androidx.compose.ui.test.swipeRight
 import androidx.compose.ui.unit.dp
 import androidx.test.ext.junit.runners.AndroidJUnit4
+import androidx.test.platform.app.InstrumentationRegistry
 import com.logseq.app.runtime.logseqIconResolver
 import dev.lui.LuiBackend
 import dev.lui.LuiEvent
@@ -135,6 +136,55 @@ class AndroidNativeSurfacesTest {
         ui.runOnIdle { assertEquals(listOf(LuiEvent.Dismiss(2)), events) }
     }
 
+    private fun assertModalControlsExposeResourceIds(kind: String) {
+        render(kind)
+        ui.runOnIdle {
+            val ops = JSONArray()
+            listOf("text-field", "switch", "button").forEachIndexed { index, control ->
+                val id = index + 3
+                ops.put(op("create-node", "id" to id, "kind" to control))
+                    .put(op("set-prop", "id" to id, "property" to "accessibility-identifier", "value" to "modal.$control"))
+                    .put(op("set-prop", "id" to id, "property" to "text", "value" to control))
+                    .put(op("insert-child", "parent" to 2, "child" to id, "index" to index))
+            }
+            apply(ops)
+        }
+        val automation = InstrumentationRegistry.getInstrumentation().uiAutomation
+        val originalInfo = automation.serviceInfo
+        automation.serviceInfo = android.accessibilityservice.AccessibilityServiceInfo().apply {
+            eventTypes = originalInfo.eventTypes
+            feedbackType = originalInfo.feedbackType
+            flags = originalInfo.flags or
+                android.accessibilityservice.AccessibilityServiceInfo.FLAG_REPORT_VIEW_IDS or
+                android.accessibilityservice.AccessibilityServiceInfo.FLAG_RETRIEVE_INTERACTIVE_WINDOWS
+        }
+        fun resourceIds(): Set<String> {
+            val pending = java.util.ArrayDeque<android.view.accessibility.AccessibilityNodeInfo>()
+            automation.windows.forEach { window -> window.root?.let { pending.add(it) } }
+            val ids = mutableSetOf<String>()
+            while (pending.isNotEmpty()) {
+                val node = pending.removeFirst()
+                node.viewIdResourceName?.let { ids.add(it) }
+                repeat(node.childCount) { index -> node.getChild(index)?.let { pending.add(it) } }
+            }
+            return ids
+        }
+        val expected = setOf("modal.text-field", "modal.switch", "modal.button")
+        try {
+            ui.waitUntil(5_000) { resourceIds().containsAll(expected) }
+        } finally {
+            automation.serviceInfo = originalInfo
+        }
+    }
+
+    @Test fun dialogControlsExposeResourceIdsInTheirOwnNativeWindow() {
+        assertModalControlsExposeResourceIds("dialog")
+    }
+
+    @Test fun sheetControlsExposeResourceIdsInTheirOwnNativeWindow() {
+        assertModalControlsExposeResourceIds("sheet")
+    }
+
     @Test fun selectedListItemsExposeNativeSingleSelection() {
         render("list-item", mapOf("text" to "Example", "selected" to true, "press-enabled" to true))
         ui.onNodeWithText("Example").assertIsSelected().performClick()

apple/Package.swift

@@ -44,7 +44,7 @@ let package = Package(
         .library(name: "LogseqModel", type: .dynamic, targets: ["LogseqModel"]),
     ],
     dependencies: [
-        .package(url: "ssh://git@github.com/logseq/lui.git", revision: "db5cca5056a76775b2ce12d085c9eadde44cc42d"),
+        .package(url: "ssh://git@github.com/logseq/lui.git", revision: "52aae8634c9add1e0c11fecfdda102b3f3644b08"),
         .package(url: "https://github.com/gonzalezreal/swiftui-math", from: "0.1.0"),
         .package(url: "https://github.com/appstefan/highlightswift.git", from: "1.1.0")
     ],

logseq.opam

@@ -25,16 +25,16 @@ depends: [
   "uunf"
 ]
 pin-depends: [
-  ["datascript_ocaml.dev" "git+https://github.com/logseq/datascript-ocaml.git#0561660e4894faee250d551ab2a32a6b5c25a5fb"]
-  ["datascript-ocaml-native.dev" "git+https://github.com/logseq/datascript-ocaml.git#0561660e4894faee250d551ab2a32a6b5c25a5fb"]
+  ["datascript_ocaml.dev" "git+https://github.com/logseq/datascript-ocaml.git#505c85564d62dc2864c43a0634ad65ee6f58550c"]
+  ["datascript-ocaml-native.dev" "git+https://github.com/logseq/datascript-ocaml.git#505c85564d62dc2864c43a0634ad65ee6f58550c"]
   ["persistent_sorted_set_ocaml.dev" "git+https://github.com/logseq/persistent-sorted-set-ocaml.git#695223e0a28cec1ff13a6ac7e870e4983ad3bb46"]
   ["melange-edn-core.0.5.0" "git+https://github.com/logseq/melange-edn.git#main"]
   ["melange-edn-native.0.5.0" "git+https://github.com/logseq/melange-edn.git#main"]
   ["melange-edn-melange.0.5.0" "git+https://github.com/logseq/melange-edn.git#main"]
   ["melange-transit-core.0.1.2" "git+https://github.com/logseq/melange-transit.git#main"]
   ["melange-transit-native.0.1.2" "git+https://github.com/logseq/melange-transit.git#main"]
   ["melange-transit-melange.0.1.2" "git+https://github.com/logseq/melange-transit.git#main"]
-  ["lui.0.1.0" "git+ssh://git@github.com/logseq/lui.git#db5cca5056a76775b2ce12d085c9eadde44cc42d"]
+  ["lui.0.1.0" "git+ssh://git@github.com/logseq/lui.git#52aae8634c9add1e0c11fecfdda102b3f3644b08"]
   ["drive.dev" "git+https://github.com/logseq/drive.git#dd735a65ed525d1c36db38ac8e8114aa81d314a1"]
   ["ocaml-signal.0.1.0" "git+https://github.com/logseq/ocaml-signal.git#02e18cdc5e0ad672aa223f638eaa3ba52fc9421e"]
   ["mldoc.dev" "git+https://github.com/logseq/mldoc.git#f11230c367964af2b575524d00779cfb1ffc7147"]

scripts/test-android-e2e-runner.sh

@@ -259,6 +259,8 @@ PATH="$mock_bin:$PATH" \
   "$runner" connect >/dev/null
 grep -Fxq -- '-s test-device reverse tcp:8787 tcp:8787' "$adb_args" \
   || die "Android E2E runner did not expose the host-local backend to the device"
+grep -Eq 'run-as com\.logseq\.app cp .* shared_prefs/logseq\.xml$' "$adb_args" \
+  || die "Android E2E server override did not use the app's SharedPreferences file"
 
 PATH="$mock_bin:$PATH" \
   ANDROID_SERIAL=test-device \

scripts/test-android-e2e.sh

@@ -380,7 +380,7 @@ if [[ -n ${LOGSEQ_E2E_BASE_URL:-} ]]; then
     '</map>' >"$preferences_file"
   adb -s "$device" push "$preferences_file" "$remote_preferences" >/dev/null
   adb -s "$device" shell run-as "$app_id" mkdir -p shared_prefs
-  adb -s "$device" shell run-as "$app_id" cp "$remote_preferences" shared_prefs/defaults.xml
+  adb -s "$device" shell run-as "$app_id" cp "$remote_preferences" shared_prefs/logseq.xml
   adb -s "$device" shell rm -f "$remote_preferences"
 fi
 

shared/src/logseq/core/graph_read.ml

@@ -1012,6 +1012,18 @@ let blocks_for_page decrypt_title db uuid =
        read_block ~with_breadcrumbs:false ~page_info decrypt_title db row.entity_id)
   |> List.of_seq
 
+let children_for_block decrypt_title db (parent : Model.block) =
+  match Ds.entid db "block/uuid" (Ds.Uuid parent.uuid) with
+  | None -> []
+  | Some eid ->
+    let reader = page_reader db parent.page_id in
+    let page_info = lazy (parent.page_id, parent.journal) in
+    ordered_page_children reader.tree eid
+    |> List.filter_map (fun child ->
+      if page_entity_visible reader.tree child then
+        read_block ~with_breadcrumbs:false ~page_info decrypt_title db child
+      else None)
+
 let node_destination decrypt_title db uuid =
   match Ds.entid db "block/uuid" (Ds.Uuid uuid) with
   | Some eid ->

shared/src/logseq/core/graph_read.mli

@@ -101,6 +101,12 @@ val blocks_for_page :
   Datascript.db ->
   string ->
   Cache_model.block list
+
+(** Hydrate direct outline children through the parent index, excluding metadata. *)
+val children_for_block :
+  (string -> (string, string) result) -> Datascript.db ->
+  Cache_model.block -> Cache_model.block list
+
 val references_for_node :
   (string -> (string, string) result) ->
   Datascript.db ->

shared/src/logseq/core/query_block.ml

@@ -138,7 +138,7 @@ let execute ?current_page db owner definition source =
               | Some rows -> rows | None -> invalid_arg "Invalid query DSL") in
         rows, field "result-transform" config, field "remove-block-children?" config <> Some (Ds.QueryFormBool false)
       | None ->
-        (match Dev.Db_query_dsl.execute_query db source opts with
+        (match Dev.Db_query_dsl.execute_query db source {opts with opt_block_attrs = Some "[:db/id]"} with
          | Some rows -> rows, None, true
          | None -> invalid_arg "Invalid query DSL") in
     let rows = filter_rows db owner definition remove_children query_rows in

shared/src/logseq/core/result_view.ml

@@ -229,9 +229,10 @@ let render_rows ?(input = "") db ~owner ~feature ~encode_block rows =
           | Ds.Keyword attr when public_attr attr -> Some attr | _ -> None) |> List.of_seq in
       let catalog = List.sort_uniq String.compare (known @ List.map (fun p -> p.attr) available)
         |> List.map (property db) in
+      let cell_properties = if typ = "list" then [] else properties in
       `Assoc (fields @ ["error", `Null; "columns", `List (List.map (column_json db original_rows) properties);
         "properties", `List (List.map (column_json db original_rows) catalog);
-        "rows", `List (List.map (row_json db encode_block properties) rows)])
+        "rows", `List (List.map (row_json db encode_block cell_properties) rows)])
     with error -> `Assoc (fields @ ["error", `String (match error with Invalid_argument message -> message | _ -> Printexc.to_string error);
       "rows", `List []; "columns", `List []]) in
   `Assoc ["ownerUuid", `String owner; "featureType", `String feature; "views", `List (List.map view_json views)]

shared/src/logseq/core/rpc_session.ml

@@ -101,30 +101,32 @@ let summary_json session summary = Rpc.summary_json (current_summary session sum
 
 let result_block_encoder db =
   let presentation = Node_presentation.decorate db in
-  let pages = Hashtbl.create 4 in
+  let children = Hashtbl.create 32 in
+  let encoded = Hashtbl.create 32 in
   let encode_tree block =
-    let children = match Hashtbl.find_opt pages block.Model.page_id with
+    let child_blocks (block : Model.block) = match Hashtbl.find_opt children block.uuid with
       | Some children -> children
       | None ->
-        let children = Hashtbl.create 16 in
-        List.iter (fun (block : Model.block) -> Option.iter (fun parent ->
-          let siblings = Option.value ~default:[] (Hashtbl.find_opt children parent) in
-          Hashtbl.replace children parent (block :: siblings)) block.parent_id)
-          (Graph_read.blocks_for_page (fun text -> Ok text) db block.page_id);
-        Hashtbl.iter (fun parent blocks -> Hashtbl.replace children parent
-          (List.sort Outliner.compare_blocks blocks)) children;
-        Hashtbl.replace pages block.page_id children;
-        children in
+        let blocks = Graph_read.children_for_block (fun text -> Ok text) db block
+          |> List.sort Outliner.compare_blocks in
+        Hashtbl.add children block.uuid blocks;
+        blocks in
     let rec encode ancestors depth (block : Model.block) =
       if depth > 128 then invalid_arg "Result block nesting is too deep";
-      let descendants = Option.value ~default:[] (Hashtbl.find_opt children block.uuid)
+      let descendants = child_blocks block
         |> List.filter (fun (child : Model.block) -> not (Outliner.Sset.mem child.uuid ancestors)) in
       let ancestors = Outliner.Sset.add block.uuid ancestors in
       match presentation block.uuid (Rpc.block_json block) with
       | `Assoc fields -> `Assoc (("children", `List (List.map (encode ancestors (depth + 1)) descendants)) :: fields)
       | value -> value in
     encode Outliner.Sset.empty 0 block in
-  encode_tree
+  fun (block : Model.block) ->
+    match Hashtbl.find_opt encoded block.uuid with
+    | Some value -> value
+    | None ->
+      let value = encode_tree block in
+      Hashtbl.add encoded block.uuid value;
+      value
 
 let query_for session db uuid =
   let current_page = match active_node_route session with

shared/src/logseq/view_query.ml

@@ -17,9 +17,9 @@ let rec decode_block depth json =
       | _ -> [] in
     {row = {row with Model.query_json = ""; has_children = children <> [];
       is_collapsed = children <> []}; children}
-let decode raw =
+let invalid_query = {rows = []; error = Some "Unable to display query results"; source = ""}
+let decode_json json =
   try
-    let json = Yojson.Basic.from_string raw in
     let rows = match member "rows" json with Some (`List rows) -> rows | _ -> [] in
     {source = string (member "source" json);
      error = (match member "error" json with Some (`String text) -> Some text | _ -> None);
@@ -31,7 +31,8 @@ let decode raw =
        | None -> let cells = match member "cells" json with Some (`List xs) -> xs | _ -> [] in
          {index; uuid = None; title = String.concat "  ·  " (List.map (fun value -> string (Some value)) cells);
           block = None; json}) rows}
-  with _ -> {rows = []; error = Some "Unable to display query results"; source = ""}
+  with _ -> invalid_query
+let decode raw = try decode_json (Yojson.Basic.from_string raw) with _ -> invalid_query
 
 let result_item owner render_block send source : t =
  fun context parent ->
@@ -112,8 +113,8 @@ let legacy_view ~render_block _model_source row_source send : t =
 let block_values ~render_block owner source send =
   let render_block source send =
     render_block (Signal.map (fun row -> {row with Model.row_breadcrumb = ""; row_breadcrumbs = []}) source) send in
-  let rows = Signal.map (fun json -> (decode (Yojson.Basic.to_string (`Assoc [
-    "rows", `List (match json with `List blocks -> List.map (fun block -> `Assoc ["block", block]) blocks | _ -> [])]))).rows) source in
+  let rows = Signal.map (fun json -> (decode_json (`Assoc [
+    "rows", `List (match json with `List blocks -> List.map (fun block -> `Assoc ["block", block]) blocks | _ -> [])])).rows) source in
   keyed ~source:rows ~key:(fun item -> Option.value ~default:"" item.uuid) ~cmp:String.compare
     ~mount:(result_item owner render_block send)
 
@@ -123,7 +124,7 @@ let jfield name json = Option.value ~default:`Null (member name json)
 let jtext json = string (Some json)
 let jlist = function `List values -> values | _ -> []
 let jbool = function `Bool value -> value | _ -> false
-let items json = (decode (Json.to_string json)).rows
+let items json = (decode_json json).rows
 let text_value = function
   | `String value -> value | `Int value -> string_of_int value | `Float value -> string_of_float value
   | `Bool value -> string_of_bool value | `Null -> "" | value -> Json.to_string value
@@ -138,12 +139,13 @@ let local_input set = function Lui_protocol.TextChanged (_, value) -> set value
 let list_take count values = List.filteri (fun index _ -> index < count) values
 
 type saved_view = {id : string; name : string; typ : string; data : Json.t}
-let saved_views raw =
-  try Json.from_string raw |> jfield "views" |> jlist |> List.mapi (fun index data ->
+let saved_views_json json =
+  try json |> jfield "views" |> jlist |> List.mapi (fun index data ->
     let id = jtext (jfield "uuid" data) in
     {id = (if id = "" then "invalid." ^ string_of_int index else id);
      name = jtext (jfield "name" data); typ = jtext (jfield "type" data); data})
   with _ -> []
+let saved_views raw = try saved_views_json (Json.from_string raw) with _ -> []
 let operator_title = function
   | "is" -> "Is" | "is-not" -> "Is not" | "text-contains" -> "Contains"
   | "text-not-contains" -> "Does not contain" | "number-gt" -> "Greater than"
@@ -554,31 +556,32 @@ let value_sheet owner feature view model_source send item property close =
 let collection ~render_block model_source raw_source send : t =
  fun context parent ->
   let own value = Signal.own_signal context.ui_scope value in
-  let initial = try Json.from_string (Signal.sample raw_source) with _ -> `Null in
+  let parsed = own (Signal.map (fun raw -> try Json.from_string raw with _ -> `Null) raw_source) in
+  let initial = Signal.sample parsed in
   let owner = jtext (jfield "ownerUuid" initial) and feature = jtext (jfield "featureType" initial) in
-  let views = own (Signal.map saved_views raw_source) in
+  let views = own (Signal.map saved_views_json parsed) in
   let active = Signal.state context.ui_scheduler "" and search = Signal.state context.ui_scheduler "" in
-  let is_query_expansion raw = try member "expanded" (Json.from_string raw) <> None with _ -> false in
-  let source_expanded raw = try match member "expanded" (Json.from_string raw) with Some (`Bool value) -> value | _ -> true with _ -> true in
-  let expanded = Signal.state context.ui_scheduler (source_expanded (Signal.sample raw_source)) in
-  ignore (Signal.own context.ui_scope (Signal.observe raw_source (fun raw ->
-    if is_query_expansion raw then Signal.set expanded (source_expanded raw))));
+  let is_query_expansion json = member "expanded" json <> None in
+  let source_expanded json = match member "expanded" json with Some (`Bool value) -> value | _ -> true in
+  let expanded = Signal.state context.ui_scheduler (source_expanded initial) in
+  ignore (Signal.own context.ui_scope (Signal.observe parsed (fun json ->
+    if is_query_expansion json then Signal.set expanded (source_expanded json))));
   let search_open = Signal.state context.ui_scheduler false in
   let chosen = own (Signal.map2 (fun views id ->
     match List.find_opt (fun view -> view.id = id) views with Some view -> [view]
     | None -> list_take 1 views) views (Signal.value active)) in
   let results_toggle () =
     let label = own (Signal.map2 (fun views expanded ->
       if not expanded then "Results" else
-      string_of_int (List.fold_left (fun count view -> max count (List.length (items view.data))) 0 views) ^ " results")
+      string_of_int (List.fold_left (fun count view -> max count (List.length (jlist (jfield "rows" view.data)))) 0 views) ^ " results")
       views (Signal.value expanded)) in
     View_base.with_label_signal label
       (button ~variant:`ghost ~text_alignment:`start ~grow:1.0
         ~accessibility_identifier:("query.toggle." ^ owner)
         ~disabled_signal:(waiting model_source owner)
         ~on_press:(fun _ ->
           let next = not (Signal.sample (Signal.value expanded)) in
-          if is_query_expansion (Signal.sample raw_source) then
+          if is_query_expansion (Signal.sample parsed) then
             ignore (send (Model.SubmitResultView (owner, payload owner feature "" "setExpanded" ["expanded", `Bool next])))
           else Signal.set expanded next)
         [results_disclosure context ("query.disclosure." ^ owner) (Signal.value expanded) label]) in