perf(data-inspector): lazy-load and inline jora on the node side (#250)

Co-authored-by: Anthony Fu <github@antfu.me>

Author: antfubot

.github/workflows/ci.yml

@@ -14,13 +14,9 @@ jobs:
     uses: sxzz/workflows/.github/workflows/unit-test.yml@main
     with:
       build: pnpm run ci:build
-      # The Build step above already produced a fresh dist/ for this exact
-      # checkout, so skip `test`'s own `build && vitest` - running plain
-      # vitest halves the number of full-monorepo `turbo run build` passes
-      # per job, which is where the flaky Windows native-toolchain crash
-      # (see scripts/ci-retry.ts) shows up.
       test: pnpm exec vitest
       lint: pnpm run lint && pnpm run knip
+      build-for-lint: true
 
   e2e:
     runs-on: ubuntu-latest

packages/devframe/src/rpc/wire-codec.ts

@@ -6,7 +6,6 @@ import { strictJsonStringify, STRUCTURED_CLONE_PREFIX } from './serialization'
  * The per-connection `serialize`/`deserialize` pair for a live RPC wire.
  *
  * @internal
- * implementations; not part of the stable public API.
  */
 export interface RpcWireCodec {
   serialize: (msg: any) => string
@@ -25,7 +24,6 @@ const EMPTY_WIRE_DEFS: ReadonlyMap<string, Pick<RpcFunctionDefinitionAny, 'jsonS
  * don't collide across connections.
  *
  * @internal
- * implementations; not part of the stable public API.
  */
 export function createRpcWireCodec(
   definitions: ReadonlyMap<string, Pick<RpcFunctionDefinitionAny, 'jsonSerializable'>> = EMPTY_WIRE_DEFS,
@@ -72,7 +70,6 @@ export function createRpcWireCodec(
  * handed to birpc proper.
  *
  * @internal
- * implementations; not part of the stable public API.
  */
 export function peekRpcWireFrame(raw: string): { t?: string, i?: string } {
   try {

plugins/data-inspector/package.json

@@ -62,8 +62,7 @@
     }
   },
   "dependencies": {
-    "cac": "catalog:deps",
-    "jora": "catalog:deps"
+    "cac": "catalog:deps"
   },
   "devDependencies": {
     "@antfu/design": "catalog:frontend",
@@ -79,6 +78,7 @@
     "devframe": "workspace:*",
     "dompurify": "catalog:frontend",
     "floating-vue": "catalog:frontend",
+    "jora": "catalog:inlined",
     "reka-ui": "catalog:frontend",
     "splitpanes": "catalog:frontend",
     "storybook": "catalog:storybook",

plugins/data-inspector/src/engine/query-engine.ts

@@ -9,10 +9,20 @@
  *     forms (`{ $type: 'Map', value }`), keeping queries portable;
  *   - suggestions come from jora's stat mode, flattened into plain
  *     RPC-safe completion items.
+ *
+ * jora itself loads lazily, on the first query: `import('jora')` only runs
+ * once `runQuery`/`runQueryAtPath`/`suggest` are actually called, so simply
+ * registering the data-inspector's RPC functions (which happens on every
+ * host that sets it up, whether or not anyone opens the panel) never pays
+ * for parsing jora. jora is a `devDependency` (`catalog:inlined` in the
+ * workspace catalog) rather than a regular `dependency`, so tsdown vendors
+ * it straight into this package's own `dist` on both the node and browser
+ * builds — the on-demand `import()` resolves a local chunk, and neither
+ * side needs consumers to install jora themselves.
  */
+import type { Jora } from 'jora'
 import type { NodePath, QueryOutcome, SuggestItem, SuggestOutcome } from './contract'
 import type { NormalizeOptions } from './normalize'
-import jora from 'jora'
 import { navigate, normalize } from './normalize'
 
 export type { SuggestItem, SuggestOutcome } from './contract'
@@ -45,51 +55,62 @@ function isSetLike(v: unknown): v is Set<unknown> {
     && typeof (v as Map<unknown, unknown>).get !== 'function'
 }
 
-const createQuery = jora.setup({
-  methods: {
-    /** Map(-like or normalized tag) -> plain object (string-coerced keys). */
-    fromMap: (v) => {
-      if (isMapLike(v))
-        return Object.fromEntries(v.entries())
-      if (isMapTag(v))
-        return v.value ?? Object.fromEntries((v.entries ?? []).map(e => [String(e.key), e.value]))
-      return v
-    },
-    /** Map(-like or normalized tag) -> [{ key, value }] preserving key identity. */
-    mapEntries: (v) => {
-      if (isMapLike(v))
-        return [...v.entries()].map(([key, value]) => ({ key, value }))
-      if (isMapTag(v)) {
-        if (v.entries)
-          return v.entries
-        return Object.entries(v.value ?? {}).map(([key, value]) => ({ key, value }))
-      }
-      return []
-    },
-    /** Set(-like or normalized tag) -> array. */
-    fromSet: (v) => {
-      if (isSetLike(v))
-        return [...v]
-      if (isSetTag(v))
-        return v.values ?? []
-      return v
-    },
-    /** Constructor name of any value. */
-    typeOf: (v) => {
-      if (v === null)
-        return 'null'
-      if (typeof v !== 'object')
-        return typeof v
-      return (v as object).constructor?.name ?? 'Object'
+type CreateQuery = ReturnType<Jora['setup']>
+
+/**
+ * jora loads on first use and is cached for the process lifetime — a single
+ * `import('jora')` + `setup()`, however many queries follow.
+ */
+let createQueryPromise: Promise<CreateQuery> | undefined
+
+function getCreateQuery(): Promise<CreateQuery> {
+  return createQueryPromise ??= import('jora').then(({ default: jora }) => jora.setup({
+    methods: {
+      /** Map(-like or normalized tag) -> plain object (string-coerced keys). */
+      fromMap: (v) => {
+        if (isMapLike(v))
+          return Object.fromEntries(v.entries())
+        if (isMapTag(v))
+          return v.value ?? Object.fromEntries((v.entries ?? []).map(e => [String(e.key), e.value]))
+        return v
+      },
+      /** Map(-like or normalized tag) -> [{ key, value }] preserving key identity. */
+      mapEntries: (v) => {
+        if (isMapLike(v))
+          return [...v.entries()].map(([key, value]) => ({ key, value }))
+        if (isMapTag(v)) {
+          if (v.entries)
+            return v.entries
+          return Object.entries(v.value ?? {}).map(([key, value]) => ({ key, value }))
+        }
+        return []
+      },
+      /** Set(-like or normalized tag) -> array. */
+      fromSet: (v) => {
+        if (isSetLike(v))
+          return [...v]
+        if (isSetTag(v))
+          return v.values ?? []
+        return v
+      },
+      /** Constructor name of any value. */
+      typeOf: (v) => {
+        if (v === null)
+          return 'null'
+        if (typeof v !== 'object')
+          return typeof v
+        return (v as object).constructor?.name ?? 'Object'
+      },
+      /** All own keys (incl. non-enumerable), as strings. */
+      ownKeys: v => (v && typeof v === 'object') ? Reflect.ownKeys(v).map(String) : [],
     },
-    /** All own keys (incl. non-enumerable), as strings. */
-    ownKeys: v => (v && typeof v === 'object') ? Reflect.ownKeys(v).map(String) : [],
-  },
-})
+  }))
+}
 
-export function runQuery(target: unknown, query: string, options?: NormalizeOptions): QueryOutcome {
+export async function runQuery(target: unknown, query: string, options?: NormalizeOptions): Promise<QueryOutcome> {
   try {
     const started = performance.now()
+    const createQuery = await getCreateQuery()
     const raw = createQuery(query)(target)
     const queryMs = Math.round((performance.now() - started) * 100) / 100
     const { data, stats } = normalize(raw, options)
@@ -110,9 +131,10 @@ export function runQuery(target: unknown, query: string, options?: NormalizeOpti
  * 'depth'` marker the client is expanding, so the same filter options must be
  * threaded through (they shift array indices and drop keys).
  */
-export function runQueryAtPath(target: unknown, query: string, path: NodePath, options?: NormalizeOptions): QueryOutcome {
+export async function runQueryAtPath(target: unknown, query: string, path: NodePath, options?: NormalizeOptions): Promise<QueryOutcome> {
   try {
     const started = performance.now()
+    const createQuery = await getCreateQuery()
     const raw = createQuery(query)(target)
     const node = navigate(raw, path, options)
     const queryMs = Math.round((performance.now() - started) * 100) / 100
@@ -140,9 +162,10 @@ interface JoraStatEntry {
  * its candidates in a nested `suggestions` array — flattened here into plain,
  * RPC-safe completion items.
  */
-export function suggest(target: unknown, query: string, pos: number, limit = 30): SuggestOutcome {
+export async function suggest(target: unknown, query: string, pos: number, limit = 30): Promise<SuggestOutcome> {
   try {
     const started = performance.now()
+    const createQuery = await getCreateQuery()
     const statApi = createQuery(query, { tolerant: true, stat: true })(target) as {
       suggestion: (pos: number, opts?: { limit?: number }) => JoraStatEntry[] | null
     }

plugins/data-inspector/test/engine.test.ts

@@ -101,54 +101,54 @@ describe('depth truncation + lazy expand', () => {
     expect(navigate(g, [['k', 'map'], ['mv', 0]])).toBe('v')
   })
 
-  it('runQueryAtPath re-runs and returns a fresh slice of the subtree', () => {
-    const out = runQueryAtPath(deep(), '$', [['k', 'level0'], ['k', 'level1']], { maxDepth: 3 })
+  it('runQueryAtPath re-runs and returns a fresh slice of the subtree', async () => {
+    const out = await runQueryAtPath(deep(), '$', [['k', 'level0'], ['k', 'level1']], { maxDepth: 3 })
     expect(out.ok).toBe(true)
     if (out.ok) {
       // The subtree normalizes from level2 with a fresh budget, reaching the leaf.
       expect(out.result).toMatchObject({ level2: { level3: { leaf: 'found' } } })
     }
   })
 
-  it('runQueryAtPath fails soft on a broken base query', () => {
-    expect(runQueryAtPath(deep(), 'nope.method()', []).ok).toBe(false)
+  it('runQueryAtPath fails soft on a broken base query', async () => {
+    expect((await runQueryAtPath(deep(), 'nope.method()', [])).ok).toBe(false)
   })
 })
 
 describe('runQuery (live)', () => {
-  it('queries live Maps and Sets through the bridge methods', () => {
-    const out = runQuery(liveGraph(), 'store.entries.mapEntries().key')
+  it('queries live Maps and Sets through the bridge methods', async () => {
+    const out = await runQuery(liveGraph(), 'store.entries.mapEntries().key')
     expect(out).toMatchObject({ ok: true, result: ['a', 'b'] })
-    const set = runQuery(liveGraph(), 'tags.fromSet()')
+    const set = await runQuery(liveGraph(), 'tags.fromSet()')
     expect(set).toMatchObject({ ok: true, result: ['alpha', 'beta'] })
   })
 
-  it('reports payload size and timings', () => {
-    const out = runQuery(liveGraph(), 'store.name')
+  it('reports payload size and timings', async () => {
+    const out = await runQuery(liveGraph(), 'store.name')
     expect(out.ok && out.stats.payloadBytes).toBeGreaterThan(0)
   })
 
-  it('fails soft with an error envelope', () => {
-    const out = runQuery(liveGraph(), 'nope.method()')
+  it('fails soft with an error envelope', async () => {
+    const out = await runQuery(liveGraph(), 'nope.method()')
     expect(out.ok).toBe(false)
   })
 })
 
 describe('runQuery (static portability)', () => {
-  it('the same query works against the NORMALIZED form of the data', () => {
+  it('the same query works against the NORMALIZED form of the data', async () => {
     const { data } = normalize(liveGraph())
     // `store.entries` is now a `{ $type: 'Map', value }` tag; the bridge
     // methods duck-type it so live-authored queries stay portable.
-    const out = runQuery(data, 'store.entries.mapEntries().key')
+    const out = await runQuery(data, 'store.entries.mapEntries().key')
     expect(out).toMatchObject({ ok: true, result: ['a', 'b'] })
-    const set = runQuery(data, 'tags.fromSet()')
+    const set = await runQuery(data, 'tags.fromSet()')
     expect(set).toMatchObject({ ok: true, result: ['alpha', 'beta'] })
   })
 })
 
 describe('suggest', () => {
-  it('returns flattened, prefix-ranged completion items', () => {
-    const out = suggest({ foo: { bar: 1, baz: 2 } }, 'foo.', 4)
+  it('returns flattened, prefix-ranged completion items', async () => {
+    const out = await suggest({ foo: { bar: 1, baz: 2 } }, 'foo.', 4)
     expect(out.ok).toBe(true)
     expect(out.suggestions.map(s => s.value)).toEqual(['bar', 'baz'])
     expect(out.suggestions[0]).toMatchObject({ from: 4, to: 4, current: '' })

plugins/data-inspector/test/registry.test.ts

@@ -158,7 +158,7 @@ describe('example source', () => {
     const data = await resolveSourceData(getDataSource(entry.id)!)
     const { runQuery } = await import('../src/engine/query-engine')
     for (const recipe of entry.queries ?? []) {
-      const out = runQuery(data, recipe.query.trim() || '$', recipe)
+      const out = await runQuery(data, recipe.query.trim() || '$', recipe)
       expect(out.ok, `suggested query "${recipe.title}" must run`).toBe(true)
     }
   })

pnpm-lock.yaml

@@ -89,7 +89,6 @@ catalogs:
     hono: ^4.13.1
     image-meta: ^0.2.2
     immer: ^11.1.16
-    jora: ^1.0.0-beta.16
     launch-editor: ^2.14.1
     mrmime: ^2.0.1
     nitro: ^3.0.260610-beta
@@ -152,6 +151,7 @@ catalogs:
     vue: ^3.5.41
   inlined:
     '@antfu/utils': ^9.3.0
+    jora: ^1.0.0-beta.16
     ua-parser-modern: ^0.1.1
   storybook:
     '@storybook/addon-a11y': *storybook

pnpm-workspace.yaml

@@ -148,12 +148,12 @@ export declare function normalize(_: unknown, _?: NormalizeOptions): {
   data: unknown;
   stats: NormalizeStats;
 };
-export declare function runQuery(_: unknown, _: string, _?: NormalizeOptions): QueryOutcome;
-export declare function runQueryAtPath(_: unknown, _: string, _: NodePath, _?: NormalizeOptions): QueryOutcome;
+export declare function runQuery(_: unknown, _: string, _?: NormalizeOptions): Promise<QueryOutcome>;
+export declare function runQueryAtPath(_: unknown, _: string, _: NodePath, _?: NormalizeOptions): Promise<QueryOutcome>;
 export declare function skeletonOf(_: unknown, _?: SkeletonOptions): {
   skeleton: unknown;
   nodes: number;
   ms: number;
 };
-export declare function suggest(_: unknown, _: string, _: number, _?: number): SuggestOutcome;
+export declare function suggest(_: unknown, _: string, _: number, _?: number): Promise<SuggestOutcome>;
 // #endregion

tests/__snapshots__/tsnapi/@devframes/plugin-data-inspector/engine.snapshot.d.ts

@@ -6,8 +6,8 @@ export function applyWrite(_, _, _) {}
 export function isExcludedKey(_, _) {}
 export function navigate(_, _, _) {}
 export function normalize(_, _) {}
-export function runQuery(_, _, _) {}
-export function runQueryAtPath(_, _, _, _) {}
+export async function runQuery(_, _, _) {}
+export async function runQueryAtPath(_, _, _, _) {}
 export function skeletonOf(_, _) {}
-export function suggest(_, _, _, _) {}
+export async function suggest(_, _, _, _) {}
 // #endregion