Derived Table support for @perspective-dev/viewer

Cargo.toml

@@ -44,6 +44,13 @@ codegen-units = 1
 lto = true
 strip = true
 
+[profile.fast]
+inherits = "release"
+opt-level = 1
+codegen-units = 256
+lto = "off"
+incremental = true
+
 # These are only respected when `cargo` is invoked from the project root
 [patch.crates-io]
 # simd-adler32 = { git = "https://github.com/mcountryman/simd-adler32.git", rev = "b279034d9eb554c3e5e0af523db044f08d8297ba" }

DEVELOPMENT.md

@@ -98,6 +98,7 @@ Other useful options:
 | Variable              | Effect                                                     |
 | --------------------- | ---------------------------------------------------------- |
 | `PSP_DEBUG=1`         | Debug build                                                |
+| `PSP_FAST=1`          | Release build with `-O1`, no LTO and no `wasm-opt`         |
 | `PSP_BUILD_VERBOSE=1` | Verbose C++ build output                                   |
 | `PSP_NUM_CPUS=<n>`    | Limit C++ build parallelism                                |
 | `PSP_WASM64=1`        | Also build the `wasm64` (Memory64) engine                  |

docs/md/explanation/derived_table.md

@@ -62,10 +62,10 @@ An unpivoted `View` yields a `Table` with the `View`'s own columns, including
 `View`'s `columns`, and its `limit` is inherited as well.
 
 A pivoted `View` yields the table you see on screen. Each `group_by` column
-becomes a typed key column of the same name, and each data column takes the
-name `View::column_paths` gives it. A column that is both a `group_by` key and
-one of the `View`'s `columns` would collide with itself, so that aggregate is
-qualified by its aggregate, as in `"Region (count)"` or
+becomes a typed key column of the same name, and each data column takes the name
+`View::column_paths` gives it. A column that is both a `group_by` key and one of
+the `View`'s `columns` would collide with itself, so that aggregate is qualified
+by its aggregate, as in `"Region (count)"` or
 `"Sales (weighted mean by Weight)"`; the key is always the bare column name:
 
 ```
@@ -115,7 +115,7 @@ Declare the columns up front to accept values that have not arrived yet:
 ```javascript
 const derived = await client.table(view, {
     schema: {
-        "Region": "string",
+        Region: "string",
         "Mon|Sales": "float",
         "Tue|Sales": "float",
         "Wed|Sales": "float",
@@ -290,6 +290,43 @@ On a [Virtual Server](./virtual_servers.md) a derived `Table` is a snapshot of
 the `View` taken when it is created, since a virtual `View` is itself a
 snapshot.
 
+## In the viewer
+
+`<perspective-viewer>` exposes derivation from the settings sidebar's query tab
+and from the panel's right-click context menu. **Derive table** (at the bottom
+of the settings panel) and the context menu's **Derive Table** each create a
+derived `Table` under a generated name — `"superstore GROUP BY Region"`-style —
+from whatever the panel currently displays, then rebind the panel to it. Both
+require a pivoted view.
+
+A derived `Table` created this way is an ordinary named table in the workspace:
+it appears in the table pickers and the "New" menu, other panels may bind to it,
+and it outlives the panel that created it. The "New" menu marks derived rows,
+shows how many panels and tables read each one, and offers a delete button once
+nothing does.
+
+The panel's derivation chain appears above the control, one row per table from
+the base up. The row for the panel's own table carries a close button:
+
+- **Click** rebinds the panel to the parent table, restoring the config the
+  derived table was created with. The derived table itself is untouched.
+- **Shift+click** rebinds _and_ deletes the derived table, which is offered only
+  when that panel is the only thing reading it.
+
+A backend which cannot promote a `View` to a `Table` reports so through the
+`view_derivations` capability, and the control is hidden for panels on that
+client. Where a backend can derive but does not stream — DuckDB and PostgreSQL —
+each level is badged as a snapshot, because its data is fixed at the moment it
+was derived.
+
+Derived tables persist in `saveWorkspace()` output under the top-level
+`derived_tables` key, each entry naming the `table` its source `View` reads and
+that view's config. Workspace panels refer to tables by name; the single-panel
+`save()` format instead inlines the definition recursively in its `table` field,
+so the token is self-contained, and `restore()` rebinds to a structurally
+identical derived table or creates one under a generated name. See
+[Save & restore](../how_to/javascript/save_restore.md).
+
 ## Changes to `client.table(view)`
 
 Previous releases copied the `View`'s data into an ordinary `Table` and

docs/md/explanation/view/config/windows.md

@@ -81,15 +81,20 @@ property does.</div>
 | `diff` | This row's value minus the value `offset` rows behind | `float` |
 | `rate` | Rate of change across the frame | `float` |
 | `ema` | Exponential moving average, smoothed by `alpha` | `float` |
+| `row_number` | 1-based position of the row in its partition's order | `integer` |
 
 `sum`, `avg`, `stddev`, `var`, `diff`, `rate` and `ema` require a numeric
 input column.
 
+`row_number` numbers rows, not values: it ignores `column` (any column
+satisfies it), never skips a row whose input is null, and breaks ties on
+`order_by` by primary key.
+
 ### Frame compatibility
 
 - `sum`, `avg`, `count`, `min`, `max`, `stddev` and `var` accept any frame.
-- `lag`, `lead`, `diff` and `ema` are frame-independent — they are computed
-  from row offsets rather than a frame.
+- `lag`, `lead`, `diff`, `ema` and `row_number` are frame-independent — they
+  are computed from row positions rather than a frame.
 - **`rate` requires a `range` frame**, and is invalid with `rows` or
   `cumulative`.
 

docs/md/how_to/javascript/save_restore.md

@@ -122,17 +122,63 @@ keys are ignored).
 
 Like `restore()`, `restoreWorkspace()` applies an _update_: a field that is
 absent leaves that part of the element unchanged, `null` resets it to its
-default, and a value replaces it. `panels` is the whole panel set — when
-present every existing panel is replaced (`{}` empties the element); when
-absent the panels are kept, and `layout`, `active` and `masters` name them by
-their existing ids. So a cross-filter can be applied to the current panels
-without re-creating them:
+default, and a value replaces it. `panels` is the whole panel set — when present
+every existing panel is replaced (`{}` empties the element); when absent the
+panels are kept, and `layout`, `active` and `masters` name them by their
+existing ids. So a cross-filter can be applied to the current panels without
+re-creating them:
 
 ```javascript
 await elem.restoreWorkspace({ global_filters: [["Region", "==", "West"]] });
 await elem.restoreWorkspace({ global_filters: null }); // clear
 ```
 
+### Derived tables
+
+A workspace token carries a top-level `derived_tables` map, one entry per
+derived `Table` the element owns, each naming the `table` its source `View`
+reads plus that view's config:
+
+```javascript
+await elem.restoreWorkspace({
+    panels: { one: { table: "Region Totals", plugin: "Datagrid" } },
+    derived_tables: {
+        "Region Totals": { table: "superstore", group_by: ["Region"] },
+    },
+});
+```
+
+Within a workspace token, panels refer to tables by name, so a panel entry never
+contains a derivation of its own — `one` above simply binds `"Region Totals"`.
+Entries may name another derived table as their `table`, and they are built
+parent-first regardless of the order they appear in.
+
+`derived_tables` follows the same update rule as `panels`: absent keeps the
+current set, a value replaces it entirely, and `{}` deletes every derived table.
+An entry whose definition has not changed is reused rather than rebuilt.
+
+A derived table whose `table` is not hosted yet does not fail the restore — it
+and anything under it wait, along with the panels bound to them, and are built
+when the parent table appears. A dependency cycle is rejected outright.
+
+The single-panel format inlines instead: `save()` of a panel bound to a derived
+table writes the whole definition recursively in the `table` field, so the token
+is self-contained:
+
+```javascript
+await elem.restore({
+    plugin: "Datagrid",
+    table: { table: "superstore", group_by: ["Region"] },
+});
+```
+
+`restore()` rebinds to an existing derived table when one was built from a
+structurally identical definition, and otherwise creates one under a generated
+name. Only the single-panel `restore()` accepts the inline form — in
+`restoreWorkspace()` every `table` field is a name.
+
+See [Derived tables](../../explanation/derived_table.md).
+
 ## Colors, palettes and gradients
 
 Per-column color styling lives in a panel's `columns_config`, keyed by column

packages/viewer-datagrid/src/ts/model/create.ts

@@ -57,7 +57,7 @@ function window_output_type(
     source_column: string,
     table_schema: Schema,
 ): ColumnType {
-    if (aggregate === "count") {
+    if (aggregate === "count" || aggregate === "row_number") {
         return "integer";
     }
 

rust/bundle/main.rs

@@ -26,18 +26,50 @@ struct BundleArgs {
     #[arg(short, long)]
     release: bool,
 
+    /// Compile with the `fast` profile?
+    #[arg(long, conflicts_with = "release")]
+    fast: bool,
+
     /// Extra features to build with
     #[arg(long)]
     features: Option<String>,
 }
 
 use wasm_bindgen_cli_support::{Bindgen, EncodeInto};
 
+/// The `cargo` profile to build and bundle.
+#[derive(Clone, Copy, PartialEq)]
+enum Profile {
+    Dev,
+    Release,
+    Fast,
+}
+
+impl Profile {
+    /// The `cargo build` arguments which select this profile.
+    fn cargo_args(self) -> &'static [&'static str] {
+        match self {
+            Profile::Dev => &[],
+            Profile::Release => &["--release"],
+            Profile::Fast => &["--profile", "fast"],
+        }
+    }
+
+    /// The name of this profile's directory under the `cargo` target dir.
+    fn dir(self) -> &'static str {
+        match self {
+            Profile::Dev => "debug",
+            Profile::Release => "release",
+            Profile::Fast => "fast",
+        }
+    }
+}
+
 /// Run the packages `build` task with the appropriate flags. These can't be
 /// defined in the `/.cargo/config.toml` because they would define this build
 /// script's parameters also, and there is no way to reset e.g. the `target`
 /// field to the host platform.
-fn build(pkg: Option<&str>, is_release: bool, features: Vec<String>) {
+fn build(pkg: Option<&str>, profile: Profile, features: Vec<String>) {
     let features = format!("tracing/release_max_level_warn,{}", features.join(","));
 
     // Build RUSTFLAGS including target-specific flags from config.toml and new
@@ -56,11 +88,8 @@ fn build(pkg: Option<&str>, is_release: bool, features: Vec<String>) {
         .args(["--lib"])
         .args(["--features", &features])
         .args(["--target", "wasm32-unknown-unknown"])
-        .args(["-Z", "build-std=std,panic_abort"]);
-
-    if is_release {
-        cmd.args(["--release"]);
-    }
+        .args(["-Z", "build-std=std,panic_abort"])
+        .args(profile.cargo_args());
 
     if let Some(pkg) = pkg {
         cmd.args(["-p", pkg]);
@@ -93,16 +122,16 @@ fn target_dir() -> PathBuf {
 }
 
 /// Generate the `wasm-bindgen` JavaScript and WASM bindings.
-fn bindgen(outdir: &Path, artifact: &str, is_release: bool) {
+fn bindgen(outdir: &Path, artifact: &str, profile: Profile) {
     let input = target_dir()
         .join("wasm32-unknown-unknown")
-        .join(if is_release { "release" } else { "debug" })
+        .join(profile.dir())
         .join(format!("{artifact}.wasm"));
 
     Bindgen::new()
         .web(true)
         .unwrap()
-        .keep_debug(!is_release)
+        .keep_debug(profile == Profile::Dev)
         .input_path(input)
         .encode_into(EncodeInto::Always)
         .typescript(true)
@@ -168,8 +197,8 @@ fn wasm_opt_version(bin: &Path) -> Option<u32> {
 }
 
 /// Run `wasm-opt` and output the new binary on top of the old one.
-fn opt(outpath: &Path, is_release: bool) {
-    if !is_release {
+fn opt(outpath: &Path, profile: Profile) {
+    if profile != Profile::Release {
         return;
     }
 
@@ -208,7 +237,12 @@ fn opt(outpath: &Path, is_release: bool) {
 fn main() {
     let args = BundleArgs::parse();
     let outdir = Path::new("dist/wasm");
-    let is_release = args.release;
+    let profile = match (args.release, args.fast) {
+        (_, true) => Profile::Fast,
+        (true, _) => Profile::Release,
+        _ => Profile::Dev,
+    };
+
     let package = args.artifact.clone().replace('_', "-");
     let outpath = &Path::new(outdir).join(format!(
         "{}.wasm",
@@ -222,9 +256,9 @@ fn main() {
         .map(|x| x.to_string())
         .collect();
 
-    build(Some(package.as_str()), is_release, features);
-    bindgen(outdir, args.artifact.as_str(), is_release);
-    opt(outpath, is_release);
+    build(Some(package.as_str()), profile, features);
+    bindgen(outdir, args.artifact.as_str(), profile);
+    opt(outpath, profile);
 }
 
 trait SimpleCommand {

rust/perspective-js/build.mjs

@@ -21,6 +21,8 @@ import { compress } from "pro_self_extracting_wasm";
 const IS_DEBUG =
     !!process.env.PSP_DEBUG || process.argv.indexOf("--debug") >= 0;
 
+const IS_FAST = !IS_DEBUG && !!process.env.PSP_FAST;
+
 const BUILD = [
     {
         entryPoints: ["src/ts/perspective-server.worker.ts"],
@@ -104,7 +106,7 @@ function get_host() {
 }
 
 async function build_rust() {
-    const release_flag = IS_DEBUG ? "" : "--release";
+    const release_flag = IS_DEBUG ? "" : IS_FAST ? "--fast" : "--release";
     execSync(
         `PSP_ROOT_DIR=../.. cargo bundle --target=${get_host()} -- perspective_js ${release_flag} --features="export-init,talc-allocator"`,
         INHERIT,

rust/perspective-js/src/ts/virtual_servers/clickhouse.ts

@@ -92,6 +92,7 @@ const WINDOW_AGGREGATES: WindowAggSpec[] = [
     { name: "lead", offset: true },
     { name: "diff", offset: true, result_type: "float" },
     { name: "rate", frames: ["range"], result_type: "float" },
+    { name: "row_number", result_type: "float" },
 ];
 
 const WINDOW_AGGREGATES_ANY: WindowAggSpec[] = [
@@ -100,6 +101,7 @@ const WINDOW_AGGREGATES_ANY: WindowAggSpec[] = [
     { name: "max", frames: FRAMES },
     { name: "lag", offset: true },
     { name: "lead", offset: true },
+    { name: "row_number", result_type: "float" },
 ];
 
 const FILTER_OPS = [

rust/perspective-js/test/js/duckdb/windows.spec.js

@@ -303,4 +303,31 @@ describeDuckDB("windows", (getClient) => {
         expect(result[first].length).toBe(rows.length);
         await view.delete();
     });
+
+    test("row_number partitioned by Region counts from one in Row ID order", async function () {
+        const rows = await raw_rows(getClient());
+        const running = {};
+        const expected = rows.map((r) => {
+            running[r.region] = (running[r.region] ?? 0) + 1;
+            return running[r.region];
+        });
+
+        const table = await getClient().open_table("memory.superstore");
+        const view = await table.view({
+            columns: ["Row ID", "rn"],
+            sort: [["Row ID", "asc"]],
+            windows: {
+                rn: {
+                    column: "Sales",
+                    aggregate: "row_number",
+                    order_by: ["Row ID", "asc"],
+                    partition_by: ["Region"],
+                },
+            },
+        });
+
+        const result = await view.to_columns();
+        expect(result["rn"].map(Number)).toEqual(expected);
+        await view.delete();
+    });
 });