diff --git a/.github/actions/install-deps/action.yaml b/.github/actions/install-deps/action.yaml index 1807dfb44d..73ebac2578 100644 --- a/.github/actions/install-deps/action.yaml +++ b/.github/actions/install-deps/action.yaml @@ -90,10 +90,17 @@ runs: shell: bash run: echo "CMAKE_GENERATOR=Ninja" >> "$GITHUB_ENV" + - name: Use Node.js ${{ matrix.node-version }} + uses: actions/setup-node@v4 + with: + node-version: ${{ matrix.node-version }} + - name: Install pnpm - uses: pnpm/action-setup@v3 + uses: pnpm/action-setup@v6 with: - version: 9 + version: 12 + cache: true + cache_dependency_path: pnpm-lock.yaml # emsdk installs to `/.emsdk` (`install_emsdk.mjs`), and # the emscripten sysroot libraries generated during the first build @@ -166,13 +173,6 @@ runs: # pip debug --verbose # shell: bash - - name: Use Node.js ${{ matrix.node-version }} - uses: actions/setup-node@v4 - with: - node-version: ${{ matrix.node-version }} - cache: "pnpm" - cache-dependency-path: pnpm-lock.yaml - - name: Install rust uses: dtolnay/rust-toolchain@nightly if: ${{ inputs.rust == 'true' && inputs.arch != 'aarch64' }} diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 7eaf3932d9..54aaa2dc47 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -559,7 +559,7 @@ jobs: path: rust/perspective-python/perspective/widget/static - name: Python Build Pyodide - run: pnpm install && pnpm run build + run: pnpm install && node tools/scripts/install_emsdk.mjs && pnpm run build env: PSP_PYODIDE: 1 PACKAGE: "python" diff --git a/Cargo.lock b/Cargo.lock index 86bbbe560e..d2349c1f26 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -187,17 +187,6 @@ dependencies = [ "event-listener", ] -[[package]] -name = "async-trait" -version = "0.1.89" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.111", -] - [[package]] name = "atomic-waker" version = "1.1.2" @@ -417,12 +406,6 @@ dependencies = [ "thiserror 2.0.17", ] -[[package]] -name = "cast" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" - [[package]] name = "cc" version = "1.2.48" @@ -1728,16 +1711,6 @@ dependencies = [ "unicase", ] -[[package]] -name = "minicov" -version = "0.3.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4869b6a491569605d66d3952bcdf03df789e5b536e5f0cf7758a7f08a55ae24d" -dependencies = [ - "cc", - "walkdir", -] - [[package]] name = "minimal-lexical" version = "0.2.1" @@ -1850,12 +1823,6 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" -[[package]] -name = "oorandom" -version = "11.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" - [[package]] name = "option-ext" version = "0.2.0" @@ -2034,7 +2001,6 @@ dependencies = [ "wasm-bindgen", "wasm-bindgen-derive", "wasm-bindgen-futures", - "wasm-bindgen-test", "web-sys", ] @@ -2124,7 +2090,6 @@ dependencies = [ "wasm-bindgen", "wasm-bindgen-derive", "wasm-bindgen-futures", - "wasm-bindgen-test", "web-sys", "yew", ] @@ -2738,15 +2703,6 @@ version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" -[[package]] -name = "same-file" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" -dependencies = [ - "winapi-util", -] - [[package]] name = "scopeguard" version = "1.2.0" @@ -3578,16 +3534,6 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "65dd7eed29412da847b0f78bcec0ac98588165988a8cfe41d4ea1d429f8ccfff" -[[package]] -name = "walkdir" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" -dependencies = [ - "same-file", - "winapi-util", -] - [[package]] name = "walrus" version = "0.24.4" @@ -3731,45 +3677,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "wasm-bindgen-test" -version = "0.3.58" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45649196a53b0b7a15101d845d44d2dda7374fc1b5b5e2bbf58b7577ff4b346d" -dependencies = [ - "async-trait", - "cast", - "js-sys", - "libm", - "minicov", - "nu-ansi-term 0.50.3", - "num-traits", - "oorandom", - "serde", - "serde_json", - "wasm-bindgen", - "wasm-bindgen-futures", - "wasm-bindgen-test-macro", - "wasm-bindgen-test-shared", -] - -[[package]] -name = "wasm-bindgen-test-macro" -version = "0.3.58" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f579cdd0123ac74b94e1a4a72bd963cf30ebac343f2df347da0b8df24cdebed2" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.111", -] - -[[package]] -name = "wasm-bindgen-test-shared" -version = "0.2.108" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8145dd1593bf0fb137dbfa85b8be79ec560a447298955877804640e40c2d6ea" - [[package]] name = "wasm-encoder" version = "0.240.0" diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 7e06a042b2..923fb61f7b 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -138,10 +138,10 @@ Deviating from this specific version of Emscripten specified in the project's `package.json` can introduce various errors that are extremely difficult to debug. -To install a specific version of Emscripten (e.g. `4.0.9`): +To install a specific version of Emscripten (e.g. `5.0.3`): ```bash -./emsdk install 4.0.9 +./emsdk install 5.0.3 ``` Set `PSP_SKIP_EMSDK_INSTALL=1` to prevent `pnpm install` from downloading the @@ -168,8 +168,7 @@ this within a virtual environment. To build for [Pyodide](https://pyodide.org/), select `perspective-python (pyodide)` in `pnpm run setup` and install the pinned -Pyodide distribution with `pnpm run install_pyodide`. Pyodide builds use the -separate `pyodide_emscripten` version pinned in `package.json`. +Pyodide distribution with `pnpm run install_pyodide`. ### `perspective-jupyterlab` @@ -288,9 +287,9 @@ tests are run with `PACKAGE=jupyterlab pnpm run test --jupyter`. Many UI tests compare against screenshot and DOM snapshots, which live in `tools/test/dist/snapshots` and are not checked in to this repository. CI -fetches them from a separate snapshots repository. To regenerate -snapshots locally after an intentional rendering change, or to generate them -for the first time from a known-passing (in CI) build you've checked out: +fetches them from a separate snapshots repository. To regenerate snapshots +locally after an intentional rendering change, or to generate them for the first +time from a known-passing (in CI) build you've checked out: ```bash pnpm run test --update-snapshots @@ -310,6 +309,10 @@ are forwarded to `pytest`. With `rust` selected, `pnpm run test` runs `cargo test` for the `perspective` and `perspective-client` crates. +`perspective-viewer` and `perspective-js` deliberately carry no Rust unit tests; +their behavior is tested through the Playwright suites, against the public +JavaScript API. + ## Lint ```bash diff --git a/docs/build.collage.mjs b/docs/build.collage.mjs index a1125e216a..cdf82caf6c 100644 --- a/docs/build.collage.mjs +++ b/docs/build.collage.mjs @@ -24,7 +24,7 @@ const COLLAGE_GAP = 2; const COLLAGE_BG = { light: "#ffffff", dark: "#242526" }; -const COLLAGE_SEED = 0x5eed; +const COLLAGE_SEED = 0xdeadbeef; function shuffled(items, seed) { let state = seed; @@ -55,27 +55,33 @@ function pngSize(file) { } } +/// Pick the column count whose natural tile aspect lands closest to filling the +/// frame, then stretch the rows so the grid covers it exactly. function bestFitGrid(count, aspect, width, height) { let best; for (let cols = 1; cols <= count; cols++) { const rows = Math.ceil(count / cols); const tileWidth = Math.floor((width - (cols - 1) * COLLAGE_GAP) / cols); + if (tileWidth <= 0) { + break; + } + + const natural = + rows * Math.round(tileWidth / aspect) + (rows - 1) * COLLAGE_GAP; - const tileHeight = Math.round(tileWidth / aspect); - const contentHeight = rows * tileHeight + (rows - 1) * COLLAGE_GAP; - if (contentHeight <= height && tileHeight > (best?.tileHeight ?? 0)) { - best = { cols, rows, tileWidth, tileHeight }; + const drift = Math.abs(natural - height); + if (best === undefined || drift < best.drift) { + best = { cols, rows, tileWidth, drift }; } } - return ( - best ?? { - cols: count, - rows: 1, - tileWidth: Math.floor(height * aspect), - tileHeight: height, - } - ); + const { cols, rows, tileWidth } = best; + return { + cols, + rows, + tileWidth, + tileHeight: Math.ceil((height - (rows - 1) * COLLAGE_GAP) / rows), + }; } export async function collage(page, ids, theme, { out, port }) { @@ -99,12 +105,12 @@ export async function collage(page, ids, theme, { out, port }) { height, ); - const tiles = present - .map( - (id) => - ``, - ) - .join(""); + const cells = cols * rows; + const tiles = Array.from( + { length: cells }, + (_, i) => + ``, + ).join(""); await page.setViewport({ width: COLLAGE_WIDTH, height }); await page.setContent( @@ -113,18 +119,17 @@ export async function collage(page, ids, theme, { out, port }) { .collage { width: ${COLLAGE_WIDTH}px; height: ${height}px; - display: flex; - flex-wrap: wrap; - align-content: center; - justify-content: center; + display: grid; + grid-template-columns: repeat(${cols}, 1fr); + grid-auto-rows: ${tileHeight}px; gap: ${COLLAGE_GAP}px; overflow: hidden; } .collage img { - width: ${tileWidth}px; + width: 100%; height: ${tileHeight}px; - /* The tile box shares the screenshots' aspect, so \`cover\` - only absorbs the sub-pixel rounding — no visible crop. */ + /* Rows are stretched to fill the frame exactly, so \`cover\` + absorbs the difference from the screenshots' own aspect. */ object-fit: cover; display: block; } @@ -134,9 +139,9 @@ export async function collage(page, ids, theme, { out, port }) { ); const tiles_found = await page.$$eval(".collage img", (x) => x.length); - if (tiles_found !== present.length) { + if (tiles_found !== cells) { throw new Error( - `collage page has ${tiles_found} tiles, expected ${present.length}`, + `collage page has ${tiles_found} tiles, expected ${cells}`, ); } @@ -159,8 +164,8 @@ export async function collage(page, ids, theme, { out, port }) { fs.writeFileSync(path.join(dir, COLLAGE), await page.screenshot()); console.log( - `Collage (${theme}): ${present.length} thumbnails, ${cols}×${rows} ` + - `grid of ${tileWidth}×${tileHeight} tiles, ` + + `Collage (${theme}): ${present.length} thumbnails in ${cells} cells, ` + + `${cols}×${rows} grid of ${tileWidth}×${tileHeight} tiles, ` + `${COLLAGE_WIDTH}×${height}.`, ); } diff --git a/docs/md/SUMMARY.md b/docs/md/SUMMARY.md index ecfb8fabac..6f6ab7c2b6 100644 --- a/docs/md/SUMMARY.md +++ b/docs/md/SUMMARY.md @@ -35,6 +35,7 @@ - [`expressions`](./explanation/view/config/expressions.md) - [Window Columns](./explanation/view/config/windows.md) - [Advanced View Operations](./explanation/view/advanced.md) +- [Derived `Table`s](./explanation/derived_table.md) - [`Join`](./explanation/join.md) - [Join Types](./explanation/join/join_types.md) - [Join Options](./explanation/join/options.md) diff --git a/docs/md/explanation/derived_table.md b/docs/md/explanation/derived_table.md new file mode 100644 index 0000000000..f4ac9f4545 --- /dev/null +++ b/docs/md/explanation/derived_table.md @@ -0,0 +1,310 @@ +# Derived `Table`s + +A `Table` can be constructed from a `View`, producing a read-only `Table` whose +rows are that `View`'s output. The engine maintains it in place: an update to +the source `Table` is reflected in the derived `Table`, and in any `View` of it, +within the same update cycle. + +This is how you query the result of a query — filter on an aggregate, group an +aggregate by a coarser key, or feed a pivot into a window function. It is also +the mechanism behind a +[Client/Server Replicated](./architecture/client_server.md) design, where it +handles `View` serialization and `on_update` forwarding for you. + +
+ +```javascript +const view = await table.view({ + group_by: ["Region"], + columns: ["Sales"], + aggregates: { Sales: "sum" }, + group_rollup_mode: "flat", +}); + +const totals = await client.table(view); +``` + +
+
+ +```python +view = table.view( + group_by=["Region"], + columns=["Sales"], + aggregates={"Sales": "sum"}, + group_rollup_mode="flat", +) + +totals = client.table(view) +``` + +
+
+ +```rust +let view = table.view(Some(ViewConfigUpdate { + group_by: Some(vec!["Region".into()]), + columns: Some(vec![Some("Sales".into())]), + group_rollup_mode: Some(GroupRollupMode::Flat), + ..ViewConfigUpdate::default() +})).await?; + +let totals = client.table(TableData::View(view), TableInitOptions::default()).await?; +``` + +
+ +## Columns + +An unpivoted `View` yields a `Table` with the `View`'s own columns, including +`expressions` and window columns, and one row for every row passing its +`filter`. The source's `index` is inherited when the index column is among the +`View`'s `columns`, and its `limit` is inherited as well. + +A pivoted `View` yields the table you see on screen. Each `group_by` column +becomes a typed key column of the same name, and each data column takes the +name `View::column_paths` gives it. A column that is both a `group_by` key and +one of the `View`'s `columns` would collide with itself, so that aggregate is +qualified by its aggregate, as in `"Region (count)"` or +`"Sales (weighted mean by Weight)"`; the key is always the bare column name: + +``` +Region,Central|Sales,East|Sales,South|Sales,West|Sales +Furniture,163797.16,208291.20,117298.68,252612.74 +Office Supplies,167026.41,205516.05,125651.31,220853.25 +``` + +Key columns are `null` below the depth of their row, which is how subtotal and +grand-total rows are represented. + +## Rollup modes are mirrored + +A derived `Table` has exactly the rows its source `View` displays. Under the +default `group_rollup_mode` of `"rollup"` that includes every subtotal row and +the grand total; under `"flat"` it is the leaves only; under `"total"` it is the +single total row. `split_rollup_mode` selects the data columns the same way. + +Set `group_rollup_mode: "flat"` on the source `View` +before grouping or filtering a derived `Table`, or its subtotal rows will be +counted alongside the rows they already summarize. + +`sort`, `expand`/`collapse` and `group_by_depth` describe how a `View` is +presented rather than what it contains, and do not affect the derived `Table`. + +## Read-only + +The engine owns a derived `Table`'s rows, so `update()`, `remove()`, `clear()` +and `replace()` all raise. Change the source `Table` instead. The `index` and +`limit` options are rejected for the same reason — identity comes from the +source. + +Deleting a `View` that a derived `Table` still reads raises as well; delete the +derived `Table` first. + +## The schema is fixed + +A derived `Table`'s columns are decided when it is created, which matters for +`split_by`, whose columns depend on the data. A split value that first appears +later has no column to land in and its cells are dropped; one that disappears +leaves its column in place, filled with `null`. + +Declare the columns up front to accept values that have not arrived yet: + +
+ +```javascript +const derived = await client.table(view, { + schema: { + "Region": "string", + "Mon|Sales": "float", + "Tue|Sales": "float", + "Wed|Sales": "float", + }, +}); +``` + +
+
+ +```python +derived = client.table( + view, + schema={ + "Region": "string", + "Mon|Sales": float, + "Tue|Sales": float, + "Wed|Sales": float, + }, +) +``` + +
+
+ +```rust +let derived = client.table(TableData::View(view), TableInitOptions { + schema: Some(IndexMap::from([ + ("Region".into(), ColumnType::String), + ("Mon|Sales".into(), ColumnType::Float), + ])), + ..TableInitOptions::default() +}).await?; +``` + +
+ +A declared column that the `View` does not have is `null` until its values +arrive; a column the `View` has but the schema omits is dropped. A declared type +which disagrees with the `View`'s is an error. + +## Filtering on an aggregate + +SQL's `HAVING` — show only the groups whose total passes a predicate — is a +`filter` on a derived `Table`: + +
+ +```javascript +const totals = await table.view({ + group_by: ["Category", "Sub-Category"], + columns: ["Sales"], + aggregates: { Sales: "sum" }, + group_rollup_mode: "flat", +}); + +const derived = await client.table(totals); +const large = await derived.view({ filter: [["Sales", ">", 100000]] }); +``` + +
+
+ +```python +totals = table.view( + group_by=["Category", "Sub-Category"], + columns=["Sales"], + aggregates={"Sales": "sum"}, + group_rollup_mode="flat", +) + +derived = client.table(totals) +large = derived.view(filter=[["Sales", ">", 100000]]) +``` + +
+
+ +```rust +let derived = client.table(TableData::View(totals), TableInitOptions::default()).await?; +let large = derived.view(Some(ViewConfigUpdate { + filter: Some(vec![Filter::new("Sales", ">", 100000)]), + ..ViewConfigUpdate::default() +})).await?; +``` + +
+ +## Change between pivot columns + +To compare an aggregate against the previous column of a pivot — day-over-day +change of an accruing balance, for example — group by the pivot key rather than +splitting on it, then apply a `diff` [window](./view/config/windows.md) to the +derived `Table` and split there: + +
+ +```javascript +const daily = await table.view({ + group_by: ["Portfolio", "Date"], + columns: ["PnL"], + aggregates: { PnL: "last" }, + group_rollup_mode: "flat", +}); + +const derived = await client.table(daily); +const change = await derived.view({ + group_by: ["Portfolio"], + split_by: ["Date"], + columns: ["change"], + aggregates: { change: "sum" }, + group_rollup_mode: "flat", + windows: { + change: { + column: "PnL", + aggregate: "diff", + partition_by: ["Portfolio"], + order_by: ["Date", "asc"], + }, + }, +}); +``` + +
+
+ +```python +daily = table.view( + group_by=["Portfolio", "Date"], + columns=["PnL"], + aggregates={"PnL": "last"}, + group_rollup_mode="flat", +) + +derived = client.table(daily) +change = derived.view( + group_by=["Portfolio"], + split_by=["Date"], + columns=["change"], + aggregates={"change": "sum"}, + group_rollup_mode="flat", + windows={ + "change": { + "column": "PnL", + "aggregate": "diff", + "partition_by": ["Portfolio"], + "order_by": ["Date", "asc"], + } + }, +) +``` + +
+ +A window runs down rows, so the axis it walks has to be a `group_by` of the +source. When each row of the source `Table` is already one observation per key — +one row per portfolio per day — no derived `Table` is needed, and the same +window applied directly to that `Table` gives the same result. + +## Limitations + +A derived `Table` re-aggregates its own rows, so an expression over aggregate +columns is only correct at the granularity those rows already have. A ratio of +sums, for instance, cannot be re-derived correctly for a coarser `group_by` of +the derived `Table`, because the ratios are averaged rather than recomputed. + +Constructing a `Table` from a `View` hosted by a _different_ `Client` copies the +`View`'s data across that boundary instead, which appends rather than updates in +place for a pivoted `View`. + +On a [Virtual Server](./virtual_servers.md) a derived `Table` is a snapshot of +the `View` taken when it is created, since a virtual `View` is itself a +snapshot. + +## Changes to `client.table(view)` + +Previous releases copied the `View`'s data into an ordinary `Table` and +forwarded updates to it. Derived `Table`s are maintained by the engine, which +changes several behaviors for code written against the old implementation: + +- A derived `Table` is read-only, where the copy accepted writes that the next + update would overwrite. +- Deleting the source `View` raises, where the copy became a frozen snapshot. +- Rows follow the source's insertion order; the source `View`'s `sort` no longer + orders the copy. +- A row updated out of the source `View`'s `filter` is removed, where the copy + retained it with stale values. +- Updates arrive in the same cycle as the source rather than eventually. +- A `View` which omits its source's index column is keyed by row identity, so + updates apply in place rather than appending. +- A pivoted `View` yields named key columns and only the rows the `View` + displays, where the copy appended its changed rows on every update. diff --git a/docs/md/explanation/view/advanced.md b/docs/md/explanation/view/advanced.md index a4063af1ec..fe7ac770fa 100644 --- a/docs/md/explanation/view/advanced.md +++ b/docs/md/explanation/view/advanced.md @@ -81,11 +81,11 @@ min_val, max_val = view.get_min_max("Sales") ## Describing a View Config -`Table::describe` validates a complete view config against a table and -reports the schema a `View` built from it would have - without creating one. -`describe` reports a `view_schema` if and only if `Table::view` with the same -config would succeed, and the two schemas are equal. It costs no engine -resources, so it is the right way to check a config before applying it. +`Table::describe` validates a complete view config against a table and reports +the schema a `View` built from it would have - without creating one. `describe` +reports a `view_schema` if and only if `Table::view` with the same config would +succeed, and the two schemas are equal. It costs no engine resources, so it is +the right way to check a config before applying it.
@@ -130,10 +130,10 @@ else: ## Expression Validation -`Table::validate_expressions` is a specialization of `Table::describe` over -a config that selects no columns, so only the expressions are checked. It -returns which expressions are valid and their inferred types, plus an -`expression_alias` map echoing the request: +`Table::validate_expressions` is a specialization of `Table::describe` over a +config that selects no columns, so only the expressions are checked. It returns +which expressions are valid and their inferred types, plus an `expression_alias` +map echoing the request:
@@ -265,22 +265,12 @@ view.remove_remove(callback)
-## Flattening a View into a Table +## Constructing a `Table` from a `View` -A [`Table`] can be constructed on a [`Table::view`] instance, which will return -a new [`Table`] based on the [`Table::view`]'s dataset, and all future updates -that affect the [`Table::view`] will be forwarded to the new [`Table`]. This is -particularly useful for implementing a -[Client/Server Replicated](../architecture/client_server.md) design, as it -handles the `View` serialization and `on_update` forwarding for you. This -pattern is available in JavaScript, Python and Rust. - -When the source `Table` has an `index`, and the `View` is unpivoted and includes -the index column, the new `Table` inherits that `index` and subscribes to the -source's `on_remove()`, so in-place updates and `remove()` calls on the source -are mirrored rather than appended. A pivoted `View`, or one which omits the -index column, produces an unindexed, append-only `Table`. A `limit` is inherited -the same way. `replace()` and `clear()` on the source are mirrored too. +A `Table` can be constructed on a `View`, producing a read-only `Table` of that +`View`'s output which the engine keeps current as the source is updated. See +[Derived `Table`s](../derived_table.md) for its columns, its constraints and +what it is good for.
@@ -310,7 +300,7 @@ let opts = TableInitOptions::default(); let data = TableData::Update(UpdateData::Csv("x,y\n1,2\n3,4".into())); let table = client.table(data, opts).await?; let view = table.view(None).await?; -let table2 = client.table(TableData::View(view)).await?; +let table2 = client.table(TableData::View(view), TableInitOptions::default()).await?; table.update(data).await?; ``` diff --git a/docs/md/explanation/view/config/grouping_and_pivots.md b/docs/md/explanation/view/config/grouping_and_pivots.md index bfc877131a..d967a41faf 100644 --- a/docs/md/explanation/view/config/grouping_and_pivots.md +++ b/docs/md/explanation/view/config/grouping_and_pivots.md @@ -228,15 +228,17 @@ a given column accepts depends on its type — see | Aggregate | Description | Result type | | --- | --- | --- | -| `sum` | Total of the group's values | `integer` or `float` | -| `sum not null` | As `sum`, but non-finite (`NaN`) values are skipped rather than poisoning the total | `integer` or `float` | -| `sum abs` | Sum of the absolute values — `Σ abs(v)` | `integer` or `float` | -| `abs sum` | Absolute value of the sum — `abs(Σ v)` | `integer` or `float` | +| `sum` | Total of the group's values, or `null` when the group has no non-null values | `integer` or `float` | +| `sum or zero` | As `sum`, but reports `0` rather than `null` for a group with no non-null values | `integer` or `float` | +| `sum abs` | Sum of the absolute values — `Σ abs(v)`; `null` when the group has no non-null values | `integer` or `float` | +| `abs sum` | Absolute value of the sum — `abs(Σ v)`; `null` when the group has no non-null values | `integer` or `float` | | `mul` | Product of the group's values | `integer` or `float` | | `gmv` | Gross market value — leaf rows are a plain `sum`, parent rows sum the _absolute_ subtotal of each immediate child group | `integer` or `float` | | `pct sum parent` | The group's `sum` as a percentage of its parent row's, `0`–`100`; `100` at the root, and `null` when the parent's sum is `0` | `float` | | `pct sum total` | The group's `sum` as a percentage of the grand total, `0`–`100` | `float` | +`sum not null` is accepted as a legacy alias for `sum`. + A numeric aggregate widens to the input's numeric class — `integer` columns accumulate as `integer`, `float` columns as `float`. @@ -311,7 +313,7 @@ aggregates differ: The aggregates a column accepts depend on its type: **Numeric columns** (`integer`, `float`): `sum`, `abs sum`, `sum abs`, -`sum not null`, `mul`, `gmv`, `any`, `avg`, `mean`, `count`, `distinct count`, +`sum or zero`, `mul`, `gmv`, `any`, `avg`, `mean`, `count`, `distinct count`, `distinct leaf`, `dominant`, `first`, `last`, `last by index`, `high`, `low`, `max`, `min`, `min by`, `max by`, `high minus low`, `last minus first`, `median`, `q1`, `q3`, `pct sum parent`, `pct sum total`, `stddev`, `var`, @@ -391,9 +393,10 @@ anywhere an aggregate is, and each group refers to one function: | `pct sum total` | `pct sum grand total`, `pct_sum_grand_total` | | `var` | `variance` | | `stddev` | `standard deviation` | +| `sum` | `sum not null`, `sum_not_null` (legacy name) | Most multi-word aggregates also answer to a snake_case spelling — -`sum_not_null`, `sum_abs`, `abs_sum`, `weighted_mean`, `distinct_leaf`, +`sum_or_zero`, `sum_abs`, `abs_sum`, `weighted_mean`, `distinct_leaf`, `pct_sum_parent`, `pct_sum_total`, `min_by`, `max_by`. Three do not, and are only accepted spelled with spaces: `high minus low`, `last minus first` and `last by index`. diff --git a/docs/src/data/features.js b/docs/src/data/features.js index 6772cec72e..15d6ccb3e4 100644 --- a/docs/src/data/features.js +++ b/docs/src/data/features.js @@ -193,16 +193,16 @@ export default [ ], columns_config: { Profit: { - pos_fg_color: "#32cd82", - neg_fg_color: "#f50fed", + fg_color: + "linear-gradient(to right, #f50fed 0%, #32cd82 100%)", }, Sales: { - pos_fg_color: "#780aff", - neg_fg_color: "#f5ac0f", + fg_color: + "linear-gradient(to right, #f5ac0f 0%, #780aff 100%)", }, Discount: { - pos_fg_color: "#f5ac0f", - neg_fg_color: "#780aff", + fg_color: + "linear-gradient(to right, #780aff 0%, #f5ac0f 100%)", }, }, sort: [["Sub-Category", "desc"]], @@ -217,22 +217,22 @@ export default [ plugin: "Datagrid", columns_config: { Discount: { - neg_bg_color: "#780aff", + bg_color: + "linear-gradient(to right, #780aff 0%, #f5ac0f 100%)", bg_mode: "color", fg_mode: "disabled", - pos_bg_color: "#f5ac0f", }, Profit: { - neg_bg_color: "#f50fed", + bg_color: + "linear-gradient(to right, #f50fed 0%, #32cd82 100%)", bg_mode: "color", fg_mode: "disabled", - pos_bg_color: "#32cd82", }, Sales: { - neg_bg_color: "#f5ac0f", + bg_color: + "linear-gradient(to right, #f5ac0f 0%, #780aff 100%)", bg_mode: "color", fg_mode: "disabled", - pos_bg_color: "#780aff", }, }, plugin_config: { @@ -268,20 +268,20 @@ export default [ Profit: { bg_mode: "gradient", bg_gradient: 600, - pos_bg_color: "#32cd82", - neg_bg_color: "#f50fed", + bg_color: + "linear-gradient(to right, #f50fed 0%, #ffffff 50%, #32cd82 100%)", }, Sales: { bg_mode: "gradient", bg_gradient: 2268, - pos_bg_color: "#780aff", - neg_bg_color: "#f5ac0f", + bg_color: + "linear-gradient(to right, #f5ac0f 0%, #ffffff 50%, #780aff 100%)", }, Discount: { bg_mode: "gradient", bg_gradient: 0.8, - pos_bg_color: "#f5ac0f", - neg_bg_color: "#780aff", + bg_color: + "linear-gradient(to right, #780aff 0%, #ffffff 50%, #f5ac0f 100%)", }, }, sort: [["Sub-Category", "desc"]], @@ -305,20 +305,20 @@ export default [ Profit: { fg_mode: "bar", fg_gradient: 600, - pos_fg_color: "#32cd82", - neg_fg_color: "#f50fed", + fg_color: + "linear-gradient(to right, #f50fed 0%, #32cd82 100%)", }, Sales: { fg_mode: "bar", fg_gradient: 2268, - pos_fg_color: "#780aff", - neg_fg_color: "#f5ac0f", + fg_color: + "linear-gradient(to right, #f5ac0f 0%, #780aff 100%)", }, Discount: { fg_mode: "bar", fg_gradient: 0.8, - pos_fg_color: "#f5ac0f", - neg_fg_color: "#780aff", + fg_color: + "linear-gradient(to right, #780aff 0%, #f5ac0f 100%)", }, }, sort: [["Sub-Category", "desc"]], @@ -373,20 +373,20 @@ export default [ Profit: { fg_mode: "label-bar", fg_gradient: 600, - pos_fg_color: "#32cd82", - neg_fg_color: "#f50fed", + fg_color: + "linear-gradient(to right, #f50fed 0%, #32cd82 100%)", }, Sales: { fg_mode: "label-bar", fg_gradient: 2268, - pos_fg_color: "#780aff", - neg_fg_color: "#f5ac0f", + fg_color: + "linear-gradient(to right, #f5ac0f 0%, #780aff 100%)", }, Discount: { fg_mode: "label-bar", fg_gradient: 0.8, - pos_fg_color: "#f5ac0f", - neg_fg_color: "#780aff", + fg_color: + "linear-gradient(to right, #780aff 0%, #f5ac0f 100%)", }, }, sort: [["Sub-Category", "desc"]], @@ -420,6 +420,113 @@ export default [ }, }, }, + { + name: "Zebra Stripes 2", + description: + "Data grid with 6 columns, banded with a zebra stripe every 2 rows.", + config: { + title: "Zebra Stripes 2", + plugin: "Datagrid", + columns: [ + "Category", + "Sales", + "Discount", + "Profit", + "Sub-Category", + "Order Date", + ], + plugin_config: { + zebra_rows: 2, + }, + sort: [["Sub-Category", "desc"]], + }, + }, + { + name: "Zebra Stripes 5", + description: + "Data grid with 6 columns, banded with a zebra stripe every 5 rows.", + config: { + title: "Zebra Stripes 5", + plugin: "Datagrid", + columns: [ + "Category", + "Sales", + "Discount", + "Profit", + "Sub-Category", + "Order Date", + ], + plugin_config: { + zebra_rows: 5, + }, + sort: [["Sub-Category", "desc"]], + }, + }, + { + name: "Large Rows", + description: + "Data grid with 6 columns whose rows are 48px tall, twice the default.", + config: { + title: "Large Row Height", + plugin: "Datagrid", + columns: [ + "Category", + "Sales", + "Discount", + "Profit", + "Sub-Category", + "Order Date", + ], + plugin_config: { + font_size: 20, + row_height: 48, + }, + sort: [["Sub-Category", "desc"]], + }, + }, + { + name: "Typewriter Ledger", + description: + "Data grid combining a monospace font, taller rows, 2-row zebra striping with a custom stripe color, and custom bar and gradient colors.", + config: { + title: "Typewriter Ledger", + plugin: "Datagrid", + columns: [ + "Category", + "Sales", + "Discount", + "Profit", + "Sub-Category", + "Order Date", + ], + plugin_config: { + font_family: "Courier New, monospace", + font_size: 15, + row_height: 32, + zebra_rows: 2, + zebra_color: "#666", + }, + columns_config: { + Sales: { + fg_mode: "bar", + fg_gradient: 2268, + fg_color: + "linear-gradient(to right, #b00020 0%, #0f9d58 100%)", + }, + Profit: { + bg_mode: "gradient", + bg_gradient: 600, + bg_color: + "linear-gradient(to right, #b00020 0%, #fdf1d6 50%, #0f9d58 100%)", + }, + Discount: { + fg_color: + "linear-gradient(to right, #7a5c00 0%, #1b5e20 100%)", + }, + }, + sort: [["Sub-Category", "desc"]], + }, + }, { name: "Y Bar", description: diff --git a/docs/src/data/projects/webcam_layouts.json b/docs/src/data/projects/webcam_layouts.json index 408dfb567c..5434466589 100644 --- a/docs/src/data/projects/webcam_layouts.json +++ b/docs/src/data/projects/webcam_layouts.json @@ -3,11 +3,10 @@ "plugin": "Datagrid", "columns_config": { "color": { + "bg_color": "linear-gradient(to right, #ff0000 0%, #ffff00 10%, #00ff00 20%, #00ffff 30%, #0000ff 40%, #000080 50%, #0000ff 60%, #00ffff 70%, #00ff00 80%, #ffff00 90%, #ff0000 100%)", "bg_gradient": 251.04, - "neg_bg_color": "#ffa38f", "bg_mode": "gradient", - "fg_mode": "disabled", - "pos_bg_color": "#346ead" + "fg_mode": "disabled" } }, "plugin_config": { @@ -40,6 +39,11 @@ }, "aggregates": { "New Column 1": "any" + }, + "columns_config": { + "color": { + "gradient": "linear-gradient(to right, #ff0000 0%, #ffff00 10%, #00ff00 20%, #00ffff 30%, #0000ff 40%, #000080 50%, #0000ff 60%, #00ffff 70%, #00ff00 80%, #ffff00 90%, #ff0000 100%)" + } } }, { @@ -56,7 +60,12 @@ "y": "bucket(-floor(\"index\" / 80), 3)", "x": "bucket(-\"index\" % 80, 3)" }, - "aggregates": {} + "aggregates": {}, + "columns_config": { + "color": { + "gradient": "linear-gradient(to right, #ff0000 0%, #ffff00 10%, #00ff00 20%, #00ffff 30%, #0000ff 40%, #000080 50%, #0000ff 60%, #00ffff 70%, #00ff00 80%, #ffff00 90%, #ff0000 100%)" + } + } }, { "plugin": "Y Bar", @@ -78,11 +87,10 @@ "plugin": "Datagrid", "columns_config": { "color": { + "bg_color": "linear-gradient(to right, #ff0000 0%, #ffff00 10%, #00ff00 20%, #00ffff 30%, #0000ff 40%, #000080 50%, #0000ff 60%, #00ffff 70%, #00ff00 80%, #ffff00 90%, #ff0000 100%)", "bg_gradient": 2463.68, - "neg_bg_color": "#ffa38f", "bg_mode": "gradient", - "fg_mode": "disabled", - "pos_bg_color": "#307bb0" + "fg_mode": "disabled" } }, "group_rollup_mode": "flat", @@ -117,7 +125,9 @@ "y": "floor(\"index\" / 80)", "New Column 2": "-floor(\"index\" / 80) * 20 - \"color\"" }, - "aggregates": { "New Column 2": "avg" } + "aggregates": { + "New Column 2": "avg" + } }, { "plugin": "X/Y Scatter", @@ -133,7 +143,15 @@ "x": "-\"index\" % 80", "y": "floor(\"index\" / 80)" }, - "aggregates": { "x": "avg", "New Column 2": "avg" } + "aggregates": { + "x": "avg", + "New Column 2": "avg" + }, + "columns_config": { + "color": { + "gradient": "linear-gradient(to right, #ff0000 0%, #ffff00 10%, #00ff00 20%, #00ffff 30%, #0000ff 40%, #000080 50%, #0000ff 60%, #00ffff 70%, #00ff00 80%, #ffff00 90%, #ff0000 100%)" + } + } }, { "plugin": "Datagrid", @@ -165,6 +183,11 @@ "aggregates": { "y": "any", "x": "any" + }, + "columns_config": { + "color": { + "gradient": "linear-gradient(to right, #00007f 0%, #0000ff 12.5%, #007fff 25%, #00ffff 37.5%, #7fff7f 50%, #ffff00 62.5%, #ff7f00 75%, #ff0000 87.5%, #7f0000 100%)" + } } } ] diff --git a/package.json b/package.json index e488c51875..a64512251d 100644 --- a/package.json +++ b/package.json @@ -13,8 +13,7 @@ } }, "type": "module", - "emscripten": "4.0.9", - "pyodide_emscripten": "5.0.3", + "emscripten": "5.0.3", "binaryen": "version_132", "llvm": "17.0.6", "pyodide": "314.0.7", @@ -80,7 +79,6 @@ "test_js": "node tools/scripts/test_js.mjs", "test_python": "node tools/scripts/test_python.mjs", "clean": "node tools/scripts/clean.mjs", - "start": "pnpm run start --workspace", "prepush": "node tools/scripts/prepush.mjs && pnpm run lint", "prepare": "husky install", "lint": "node tools/scripts/lint.mjs", diff --git a/packages/viewer-charts/src/ts/plugin/plugin.ts b/packages/viewer-charts/src/ts/plugin/plugin.ts index fc84e5af97..4889517bc9 100644 --- a/packages/viewer-charts/src/ts/plugin/plugin.ts +++ b/packages/viewer-charts/src/ts/plugin/plugin.ts @@ -255,6 +255,24 @@ const GLOBAL_STYLES = (() => { */ let BLIT_MODE: "direct" | "blit" = RENDER_BLIT_MODE; +/** + * A column's `date_format` / `number_format` controls, which are the only ones + * that apply to a value rendered as an axis tick or tooltip label. + */ +function format_fields( + column_type: string, +): Array & { kind: string }> { + if (column_type === "integer" || column_type === "float") { + return [{ kind: "NumberFormat", default: CHART_NUMBER_DEFAULTS }]; + } else if (column_type === "datetime") { + return [{ kind: "DatetimeFormat", default: CHART_DATETIME_DEFAULTS }]; + } else if (column_type === "date") { + return [{ kind: "DatetimeFormat", default: CHART_DATE_DEFAULTS }]; + } + + return []; +} + export class HTMLPerspectiveViewerWebGLPluginElement extends HTMLElement implements IPerspectiveViewerPlugin @@ -601,9 +619,15 @@ export class HTMLPerspectiveViewerWebGLPluginElement split_by?: string[]; group_rollup_mode?: string; }, + _plugin_config?: Record | null, + role?: "column" | "group_by" | "split_by", ) { const fields: Array & { kind: string }> = []; + if (role === "group_by" || role === "split_by") { + return { fields: format_fields(column_type) }; + } + if (group === "Color") { const numeric_gradient = this._chartType.category === "Hierarchical Charts" @@ -721,23 +745,7 @@ export class HTMLPerspectiveViewerWebGLPluginElement // Per-column formatter widgets. Surfaced for every chart type so // axes / tooltips / legends honor the user's format choice. - if (column_type === "integer" || column_type === "float") { - fields.push({ - kind: "NumberFormat", - default: CHART_NUMBER_DEFAULTS, - }); - } else if (column_type === "datetime") { - fields.push({ - kind: "DatetimeFormat", - default: CHART_DATETIME_DEFAULTS, - }); - } else if (column_type === "date") { - fields.push({ - kind: "DatetimeFormat", - default: CHART_DATE_DEFAULTS, - }); - } - + fields.push(...format_fields(column_type)); return { fields }; } diff --git a/packages/viewer-datagrid/src/ts/custom_elements/datagrid.ts b/packages/viewer-datagrid/src/ts/custom_elements/datagrid.ts index 3e2fe499d1..e0ac1b2d5e 100644 --- a/packages/viewer-datagrid/src/ts/custom_elements/datagrid.ts +++ b/packages/viewer-datagrid/src/ts/custom_elements/datagrid.ts @@ -26,9 +26,8 @@ import column_config_schema, { import plugin_config_schema from "../plugin/plugin_config_schema.js"; import datagridStyles from "../../../dist/css/perspective-viewer-datagrid.css"; import { format_raw } from "../data_listener/format_cell.js"; -import { sourceColumn } from "@perspective-dev/viewer/column-format"; -import type { View, ViewWindow } from "@perspective-dev/client"; +import type { ColumnWindow, View, ViewWindow } from "@perspective-dev/client"; import type { HTMLPerspectiveViewerElement, IPerspectiveViewerPlugin, @@ -42,8 +41,11 @@ import type { ColumnsConfig, ResolvedColumnsConfig, Align, + ColumnRole, } from "../types.js"; import { RegularTableElement } from "regular-table"; +import type { CellScalar } from "regular-table/dist/esm/types.js"; +import { isMetaColumn } from "../model/meta_columns.js"; type RenderTarget = "shadow" | "light"; @@ -237,6 +239,7 @@ export class HTMLPerspectiveViewerDatagridPluginElement group_rollup_mode?: string; }, plugin_config?: Record | null, + role?: ColumnRole, ): ColumnConfigSchema { return column_config_schema.call( this, @@ -246,6 +249,7 @@ export class HTMLPerspectiveViewerDatagridPluginElement current_value, viewer_config, plugin_config, + role, ); } @@ -267,8 +271,18 @@ export class HTMLPerspectiveViewerDatagridPluginElement } async render(view: View, viewport?: ViewWindow): Promise { - const json = await view.to_columns(viewport as any); - const cols = await view.column_paths(viewport as any); + const [json, area] = await Promise.all([ + view.to_columns(viewport as any), + view.column_paths(viewport as ColumnWindow) as Promise< + (CellScalar | null)[][] + >, + ]); + + const keys = Object.keys(json as Record).filter( + (x) => !isMetaColumn(x), + ); + + const names = area[area.length - 1] ?? []; const nrows = viewport?.end_row !== undefined && viewport?.end_row !== null && @@ -277,20 +291,23 @@ export class HTMLPerspectiveViewerDatagridPluginElement ? viewport.end_row - viewport.start_row : await view.num_rows(); + const pluginConfig = (this.regular_table as any)[ + PRIVATE_PLUGIN_SYMBOL + ] as ResolvedColumnsConfig | undefined; + + const formatters = keys.map((_, cidx) => { + const name = String(names[cidx] ?? ""); + return format_raw( + this.model!._schema[name], + pluginConfig?.[name] || {}, + ); + }); + let out = ""; for (let ridx = 0; ridx < nrows; ridx++) { - for (const col_name of cols) { - const col = (json as Record)[col_name]; - const type = this.model!._schema[col_name]; - const pluginConfig = (this.regular_table as any)[ - PRIVATE_PLUGIN_SYMBOL - ] as ResolvedColumnsConfig | undefined; - const columnName = sourceColumn(col_name); - const formatter = format_raw( - type, - pluginConfig?.[columnName] || {}, - ); - + for (let cidx = 0; cidx < keys.length; cidx++) { + const col = (json as Record)[keys[cidx]]; + const formatter = formatters[cidx]; if (formatter) { out += formatter.format(col[ridx]) + "\t"; } else { diff --git a/packages/viewer-datagrid/src/ts/data_listener/format_column_header.ts b/packages/viewer-datagrid/src/ts/data_listener/format_column_header.ts index 383bd4a570..52740efbe9 100644 --- a/packages/viewer-datagrid/src/ts/data_listener/format_column_header.ts +++ b/packages/viewer-datagrid/src/ts/data_listener/format_column_header.ts @@ -10,46 +10,15 @@ // ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ // ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ -import type { ColumnType } from "@perspective-dev/client"; import { format_cell } from "./format_cell.js"; import type { DatagridModel, ResolvedColumnsConfig } from "../types.js"; +import type { CellScalar } from "regular-table/dist/esm/types.js"; /** * A formatted `split_by` column header label. */ export type ColumnHeaderLabel = { toString(): string }; -const DATE_RE = - /^(\d{4})-(\d{2})-(\d{2})(?: (\d{2}):(\d{2}):(\d{2})(?:\.(\d{1,3}))?)?$/; - -/** - * Recover one `split_by` level's typed value from its column path text, or - * `undefined` when that text cannot be recovered exactly. - */ -export function parse_split_value( - type: ColumnType | undefined, - text: string, -): number | undefined { - if (type === "integer") { - const value = Number(text); - return /^-?\d+$/.test(text) && Number.isSafeInteger(value) - ? value - : undefined; - } - - if (type === "date" || type === "datetime") { - const match = DATE_RE.exec(text); - if (!match) { - return undefined; - } - - const [, y, mo, d, h = "0", mi = "0", s = "0", ms = "0"] = match; - return Date.UTC(+y, +mo - 1, +d, +h, +mi, +s, +ms.padEnd(3, "0")); - } - - return undefined; -} - /** * Formats `split_by` column path levels with their source column's * `date_format` / `number_format`, one cached label object per split group. @@ -60,21 +29,13 @@ export class ColumnHeaderLabels { format( model: DatagridModel, - path_parts: string[], + levels: (CellScalar | null)[], level: number, plugins: ResolvedColumnsConfig, ): string | ColumnHeaderLabel { const column = model._config.split_by[level]; - const text = path_parts[level]; - const type = (model._table_schema[column] || - model._schema[column] || - model._window_schema?.[column]) as ColumnType | undefined; - - const value = parse_split_value(type, text); - if (value === undefined) { - return text; - } - + const value = levels[level]; + const text = value === null || value === undefined ? "" : String(value); const formatted = format_cell.call(model, column, value, plugins, true); if (typeof formatted !== "string") { @@ -86,7 +47,7 @@ export class ColumnHeaderLabels { this._labels.clear(); } - const key = JSON.stringify([path_parts.slice(0, level + 1), formatted]); + const key = JSON.stringify([levels.slice(0, level + 1), formatted]); let label = this._labels.get(key); if (label === undefined) { label = { toString: () => formatted }; diff --git a/packages/viewer-datagrid/src/ts/data_listener/index.ts b/packages/viewer-datagrid/src/ts/data_listener/index.ts index f089e4c5c9..7c6d17703d 100644 --- a/packages/viewer-datagrid/src/ts/data_listener/index.ts +++ b/packages/viewer-datagrid/src/ts/data_listener/index.ts @@ -13,6 +13,12 @@ import { PRIVATE_PLUGIN_SYMBOL } from "../types.js"; import { isMetaColumn } from "../model/meta_columns.js"; import { reconcile_column_widths } from "../model/column_overrides.js"; +import { + column_name, + split_levels, + write_area, + write_flat_area, +} from "../model/column_path_area.js"; import { format_cell } from "./format_cell.js"; import { ColumnHeaderLabels, @@ -118,25 +124,35 @@ export function createDataListener( } if (changed_cols) { - const [a, b] = await Promise.all([ + const is_pivoted = this._config.split_by.length > 0; + const [a, b, area] = await Promise.all([ this._view.schema(), this._view.expression_schema(), + is_pivoted + ? this._view.column_paths({ + start_col: x0, + end_col: x1, + }) + : Promise.resolve(null), ]); + if (area === null) { + write_flat_area( + this._column_path_area, + new_col_paths, + new_window.start_col!, + ); + } else { + write_area( + this._column_path_area, + area as (CellScalar | null)[][], + new_window.start_col!, + ); + } + this._schema = { ...(a as Schema), ...(b as Schema) }; for (let i = 0; i < new_col_paths.length; i++) { - const column_path_parts = new_col_paths[i].split("|"); - - // Subtotal/total columns (`split_rollup_mode: "rollup"`) - // have fewer than `split_by.length` levels - the column - // name is always the last part. - const column = - column_path_parts[ - Math.min( - this._config.split_by.length, - column_path_parts.length - 1, - ) - ]; + const column = column_name(this, i + new_window.start_col!); this._is_editable[i + new_window.start_col!] = !!this._table_schema[column]; @@ -215,15 +231,10 @@ export function createDataListener( ++ipath ) { const path = this._column_paths[ipath]; - const path_parts = path.split("|"); - const n_split_levels = path_parts.length - 1; - - // Under `split_rollup_mode: "rollup"`, grand-total and subtotal - // columns have fewer than `split_by.length` levels. Pad between - // the split levels and the trailing column name so the name - // always lands at index `split_by.length` - every downstream - // `column_header` consumer indexes it there. - // + const name = column_name(this, ipath); + const levels = split_levels(this, ipath); + const n_split_levels = levels.length; + // Pads are zero-width spaces repeated by group depth, NOT `""`: // regular-table merges header cells on bare value equality // across the whole row, so a plain `""` pad would merge with an @@ -235,19 +246,20 @@ export function createDataListener( // inside a group and never across one. The grand-total group has // no real levels, so its entire header stack is pads (rendering // blank). - if (path_parts.length < this._config.split_by.length + 1) { - const column_name = path_parts.pop() as string; - const pad = "\u200b".repeat(path_parts.length + 1); + const path_parts: (string | ColumnHeaderLabel)[] = levels.slice( + 0, + ) as (string | ColumnHeaderLabel)[]; + + if (n_split_levels < this._config.split_by.length) { + const pad = "\u200b".repeat(n_split_levels + 1); while (path_parts.length < this._config.split_by.length) { path_parts.push(pad); } - - path_parts.push(column_name); } + path_parts.push(name); const column = columns[path] || new Array(y1 - y0).fill(null); - const column_config = - columns_config[path_parts[this._config.split_by.length]]; + const column_config = columns_config[name]; const wrap = (column_config?.word_wrap ?? this._word_wrap) && @@ -266,7 +278,7 @@ export function createDataListener( const cell = format_cell.call( this, - path_parts[this._config.split_by.length], + name, x, columns_config, ) as string | HTMLElement; @@ -280,7 +292,7 @@ export function createDataListener( for (let level = 0; level < n_split_levels; level++) { header[level] = header_labels.format( this, - path_parts, + levels, level, columns_config, ); diff --git a/packages/viewer-datagrid/src/ts/get_cell_config.ts b/packages/viewer-datagrid/src/ts/get_cell_config.ts index 0f36c35beb..78f6f91f6e 100644 --- a/packages/viewer-datagrid/src/ts/get_cell_config.ts +++ b/packages/viewer-datagrid/src/ts/get_cell_config.ts @@ -63,16 +63,21 @@ export default async function getCellConfig( }; let column_filters: Filter[] = []; - if (column_paths) { - const split_by_values = column_paths.split("|"); - result.column_names = [split_by_values[split_by.length]]; + if (column_paths && col_idx !== undefined) { + const area = (await _view.column_paths({ + start_col: col_idx, + end_col: col_idx + 1, + })) as (Scalar | undefined)[][]; + + result.column_names = [String(area[area.length - 1]?.[0] ?? "")]; column_filters = split_by .map((pivot, index): Filter | undefined => { - const pivot_value = split_by_values[index]; - return pivot_value ? [pivot, "==", pivot_value] : undefined; + const pivot_value = area[index]?.[0]; + return pivot_value === null || pivot_value === undefined + ? undefined + : [pivot, "==", pivot_value]; }) - .filter((x): x is Filter => x !== undefined) - .filter(([, , value]) => !isMetaColumn(value as string)); + .filter((x): x is Filter => x !== undefined); } const filter = _config.filter.concat(row_filters).concat(column_filters); diff --git a/packages/viewer-datagrid/src/ts/model/column_path_area.ts b/packages/viewer-datagrid/src/ts/model/column_path_area.ts new file mode 100644 index 0000000000..46691ea4ce --- /dev/null +++ b/packages/viewer-datagrid/src/ts/model/column_path_area.ts @@ -0,0 +1,112 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +import type { CellScalar } from "regular-table/dist/esm/types.js"; +import type { DatagridModel } from "../types.js"; + +/** + * A column header area, `[level][absolute column]`, sparse in the column axis. + */ +export type ColumnPathArea = (CellScalar | null)[][]; + +/** + * Write one window of `View.column_paths` into `area` at `start_col`, growing + * the level count if the view's `split_by` has since changed. + */ +export function write_area( + area: ColumnPathArea, + window_area: (CellScalar | null)[][], + start_col: number, +): void { + while (area.length < window_area.length) { + area.push([]); + } + + area.length = window_area.length; + for (let level = 0; level < window_area.length; level++) { + const row = window_area[level]; + for (let i = 0; i < row.length; i++) { + area[level][start_col + i] = row[i]; + } + } +} + +/** + * Synthesize the single-level area of an unpivoted view from the data keys it + * was already given, which saves the engine call entirely. + */ +export function write_flat_area( + area: ColumnPathArea, + paths: string[], + start_col: number, +): void { + area.length = 1; + area[0] ??= []; + for (let i = 0; i < paths.length; i++) { + area[0][start_col + i] = paths[i]; + } +} + +/** + * The name of column `x`, which always occupies the area's last level. + */ +export function column_name(model: DatagridModel, x: number): string { + const area = model._column_path_area; + const value = area[area.length - 1]?.[x]; + return value === undefined || value === null ? "" : String(value); +} + +/** + * How many levels column `x` pivots on, which is fewer than `split_by.length` + * for a subtotal or grand total under `split_rollup_mode: "rollup"`. + */ +export function split_depth(model: DatagridModel, x: number): number { + const area = model._column_path_area; + let depth = 0; + for (let level = 0; level + 1 < area.length; level++) { + if (area[level][x] !== null && area[level][x] !== undefined) { + depth++; + } + } + + return depth; +} + +/** + * The split values of column `x`, excluding its name and the levels it does + * not pivot on. + */ +export function split_levels( + model: DatagridModel, + x: number, +): (CellScalar | null)[] { + const area = model._column_path_area; + const levels: (CellScalar | null)[] = []; + for (let level = 0; level + 1 < area.length; level++) { + const value = area[level][x]; + if (value === null || value === undefined) { + break; + } + + levels.push(value); + } + + return levels; +} + +/** + * Whether column `x` has been loaded into the area at all. + */ +export function is_loaded(model: DatagridModel, x: number): boolean { + const area = model._column_path_area; + return area[area.length - 1]?.[x] !== undefined; +} diff --git a/packages/viewer-datagrid/src/ts/model/create.ts b/packages/viewer-datagrid/src/ts/model/create.ts index 41e62b8e30..6c43fbf529 100644 --- a/packages/viewer-datagrid/src/ts/model/create.ts +++ b/packages/viewer-datagrid/src/ts/model/create.ts @@ -32,6 +32,7 @@ import { type EditMode, } from "../types.js"; import type { HTMLPerspectiveViewerElement } from "@perspective-dev/viewer"; +import type { CellScalar } from "regular-table/dist/esm/types"; // Mirror of the engine's window-aggregate result types (the // `GetFeaturesResp.window_aggregates` table in `server.cpp`): these @@ -287,6 +288,11 @@ export async function createModel( ); const _column_paths: string[] = []; + const _column_path_area: (CellScalar | null)[][] = Array.from( + { length: config.split_by.length + 1 }, + () => [], + ); + const _is_editable: boolean[] = []; const _column_types: ColumnType[] = []; let _edit_mode: EditMode = this._edit_mode || "READ_ONLY"; @@ -319,6 +325,7 @@ export async function createModel( _ids: [], ...style, _column_paths, + _column_path_area, _column_types, _is_editable, _edit_mode, diff --git a/packages/viewer-datagrid/src/ts/plugin/column_config_schema.ts b/packages/viewer-datagrid/src/ts/plugin/column_config_schema.ts index 3fdfb9dbc9..b902715bfd 100644 --- a/packages/viewer-datagrid/src/ts/plugin/column_config_schema.ts +++ b/packages/viewer-datagrid/src/ts/plugin/column_config_schema.ts @@ -24,6 +24,7 @@ import { parse_fg_mode, type BgMode, type ColumnConfig, + type ColumnRole, type DatagridPluginElement, type FgMode, } from "../types.js"; @@ -44,6 +45,20 @@ export interface ColumnConfigSchema { * Plugin schema for the Datagrid column-settings sidebar. Returns the * controls the viewer should render in the Style tab for a given column. */ +/** + * A column's `date_format` / `number_format` controls, which are the only ones + * that apply to a value rendered as a pivot label rather than a cell. + */ +function format_fields(type: ColumnType): ControlSpec[] { + if (type === "integer" || type === "float") { + return [{ kind: "NumberFormat" }]; + } else if (type === "date" || type === "datetime") { + return [{ kind: "DatetimeFormat" }]; + } + + return []; +} + export default function column_config_schema( this: DatagridPluginElement, type: ColumnType, @@ -52,7 +67,12 @@ export default function column_config_schema( current_value: Record | null, viewer_config?: ViewerConfigLike, plugin_config?: Record | null, + role?: ColumnRole, ): ColumnConfigSchema { + if (role === "group_by" || role === "split_by") { + return { fields: format_fields(type) }; + } + const grid = plugin_config === undefined || plugin_config === null ? { @@ -160,11 +180,8 @@ export default function column_config_schema( fields.push({ kind: "Group", key: "color", fields: color_fields }); } - if (type === "integer" || type === "float") { - fields.push({ kind: "NumberFormat" }); - } else if (type === "date" || type === "datetime") { - fields.push({ kind: "DatetimeFormat" }); - } else if (type === "string") { + fields.push(...format_fields(type)); + if (type === "string") { fields.push({ kind: "Bool", key: "link" satisfies keyof ColumnConfig, diff --git a/packages/viewer-datagrid/src/ts/style_handlers/body.ts b/packages/viewer-datagrid/src/ts/style_handlers/body.ts index 33fb6fd853..c55aa246e3 100644 --- a/packages/viewer-datagrid/src/ts/style_handlers/body.ts +++ b/packages/viewer-datagrid/src/ts/style_handlers/body.ts @@ -37,6 +37,7 @@ import { type ColumnAlignment, } from "./column_alignment.js"; import { CollectedCell } from "./types.js"; +import { is_loaded, split_depth } from "../model/column_path_area.js"; const B_VALUE_NULL = 1; const B_USER_NULL = 2; @@ -162,9 +163,9 @@ export function applyBodyCellStyles( : undefined; const n_split_levels = - meta_x === undefined + meta_x === undefined || !is_loaded(model, meta_x) ? undefined - : model._column_paths[meta_x]?.split("|").length - 1; + : split_depth(model, meta_x); const is_rollup_col = n_split_by > 0 && diff --git a/packages/viewer-datagrid/src/ts/style_handlers/border_model.ts b/packages/viewer-datagrid/src/ts/style_handlers/border_model.ts index 1837edd49e..9ca8a72ed5 100644 --- a/packages/viewer-datagrid/src/ts/style_handlers/border_model.ts +++ b/packages/viewer-datagrid/src/ts/style_handlers/border_model.ts @@ -27,6 +27,9 @@ /// One edge of a cell. `miter-start` gaps the corner nearest the axis origin /// (top for vertical edges, left for horizontal edges). +import type { CellScalar } from "regular-table/dist/esm/types.js"; +import type { ColumnPathArea } from "../model/column_path_area.js"; + export type EdgeState = | "none" | "full" @@ -42,9 +45,9 @@ export interface CellBorders { } export interface HeaderCellInput { - /// Raw (unpadded) column paths, `model._column_paths`. May be sparse + /// The column path area, `model._column_path_area`. May be sparse /// where the virtual viewport has not loaded a column. - paths: (string | undefined)[]; + area: ColumnPathArea; /// `config.split_by.length`. split_by_len: number; @@ -86,14 +89,35 @@ const NONE: CellBorders = { left: "none", }; -/// The split levels of a raw column path - the path minus its trailing -/// column name. Total/subtotal columns (`split_rollup_mode: "rollup"`) have -/// fewer than `split_by_len` levels. NOTE a column NAME containing `"|"` -/// mis-splits here; this is a pre-existing ambiguity shared with the data -/// listener, not worsened. -export function split_levels(path: string, split_by_len: number): string[] { - const parts = path.split("|"); - return parts.slice(0, Math.min(split_by_len, parts.length - 1)); +/// The split levels of column `x` - its path minus the trailing column name. +/// Total/subtotal columns (`split_rollup_mode: "rollup"`) have fewer than +/// `split_by_len` levels. +export function split_levels( + area: ColumnPathArea, + split_by_len: number, + x: number, +): (CellScalar | null)[] { + const levels: (CellScalar | null)[] = []; + for (let level = 0; level < split_by_len && level < area.length; level++) { + const value = area[level][x]; + if (value === null || value === undefined) { + break; + } + + levels.push(value); + } + + return levels; +} + +/// The loaded extent of `area`, in data columns. +function area_width(area: ColumnPathArea): number { + return area[area.length - 1]?.length ?? 0; +} + +/// Whether column `x` has been loaded into `area`. +function is_loaded(area: ColumnPathArea, x: number): boolean { + return area[area.length - 1]?.[x] !== undefined; } /// Depth of the boundary between data columns `x` and `x + 1`: @@ -104,26 +128,24 @@ export function split_levels(path: string, split_by_len: number): string[] { /// levels: `0..split_by_len`, where `split_by_len` means "same group" /// (an aggregate-internal boundary). export function boundary_depth( - paths: (string | undefined)[], + area: ColumnPathArea, split_by_len: number, x: number, ): number | null { - const left = paths[x]; - if (left === undefined) { + if (!is_loaded(area, x)) { return null; } - if (x + 1 >= paths.length) { + if (x + 1 >= area_width(area)) { return 0; } - const right = paths[x + 1]; - if (right === undefined) { + if (!is_loaded(area, x + 1)) { return null; } - const a = split_levels(left, split_by_len); - const b = split_levels(right, split_by_len); + const a = split_levels(area, split_by_len, x); + const b = split_levels(area, split_by_len, x + 1); let depth = 0; while (depth < a.length && depth < b.length && a[depth] === b[depth]) { depth++; @@ -148,17 +170,16 @@ export function boundary_depth( /// `"Total"` label of a zero-level grand-total column counts as a pad too: /// the whole Total column header stack reads as one open block. function has_real_text( - paths: (string | undefined)[], + area: ColumnPathArea, split_by_len: number, x: number, y: number, ): boolean { - const path = paths[x]; - if (path === undefined) { + if (!is_loaded(area, x)) { return false; } - return y < split_levels(path, split_by_len).length; + return y < split_levels(area, split_by_len, x).length; } export function classify_header_cell(input: HeaderCellInput): CellBorders { @@ -190,8 +211,8 @@ export function classify_header_cell(input: HeaderCellInput): CellBorders { } const x_end = input.x + input.colspan - 1; - const depth = boundary_depth(input.paths, input.split_by_len, x_end); - const is_trailing = x_end >= input.paths.length - 1; + const depth = boundary_depth(input.area, input.split_by_len, x_end); + const is_trailing = x_end >= area_width(input.area) - 1; let right: EdgeState = "none"; if (depth !== null) { @@ -206,7 +227,7 @@ export function classify_header_cell(input: HeaderCellInput): CellBorders { const bottom: EdgeState = input.row_kind === "group" && - has_real_text(input.paths, input.split_by_len, input.x, input.y) + has_real_text(input.area, input.split_by_len, input.x, input.y) ? "miter-both" : "none"; diff --git a/packages/viewer-datagrid/src/ts/style_handlers/column_header.ts b/packages/viewer-datagrid/src/ts/style_handlers/column_header.ts index 397c68fabe..ca14cbcec7 100644 --- a/packages/viewer-datagrid/src/ts/style_handlers/column_header.ts +++ b/packages/viewer-datagrid/src/ts/style_handlers/column_header.ts @@ -16,6 +16,7 @@ import { CollectedHeaderRow } from "./types.js"; import type { HTMLPerspectiveViewerElement } from "@perspective-dev/viewer"; import { apply_borders, classify_header_cell } from "./border_model.js"; import { corner_boundary_x } from "./group_header.js"; +import { is_loaded, split_depth } from "../model/column_path_area.js"; /** * Apply selected column styling in response to column settings toggle events. @@ -117,7 +118,7 @@ export function styleColumnHeaderRow( const sort = model._config.sort.find((x) => x[0] === column_name); const is_corner = typeof metadata.x === "undefined"; const borders = classify_header_cell({ - paths: model._column_paths, + area: model._column_path_area, split_by_len, x: is_corner ? undefined : metadata.x, colspan: td.colSpan || 1, @@ -140,9 +141,10 @@ export function styleColumnHeaderRow( // Under `split_rollup_mode: "rollup"`, columns whose raw path has // fewer levels than `split_by` are subtotal (or, with zero levels, // grand-total) column groups. - const n_split_levels = is_corner - ? undefined - : model._column_paths[metadata.x!]?.split("|").length - 1; + const n_split_levels = + is_corner || !is_loaded(model, metadata.x!) + ? undefined + : split_depth(model, metadata.x!); const is_rollup_col = model._config.split_by.length > 0 && n_split_levels !== undefined && diff --git a/packages/viewer-datagrid/src/ts/style_handlers/group_header.ts b/packages/viewer-datagrid/src/ts/style_handlers/group_header.ts index a2f15d8eac..9b8d8206d7 100644 --- a/packages/viewer-datagrid/src/ts/style_handlers/group_header.ts +++ b/packages/viewer-datagrid/src/ts/style_handlers/group_header.ts @@ -89,7 +89,7 @@ export function applyGroupHeaderStyles( const is_data = metadata.type === "column_header" && metadata.x! >= 0; const borders = classify_header_cell({ - paths: model._column_paths, + area: model._column_path_area, split_by_len, x: is_data ? metadata.x : undefined, colspan: td.colSpan || 1, diff --git a/packages/viewer-datagrid/src/ts/types.ts b/packages/viewer-datagrid/src/ts/types.ts index 07a66fdb1f..08195e0d8f 100644 --- a/packages/viewer-datagrid/src/ts/types.ts +++ b/packages/viewer-datagrid/src/ts/types.ts @@ -28,7 +28,11 @@ import type { } from "@perspective-dev/viewer"; import type { RegularTableElement } from "regular-table"; -import type { CellMetadata, DataResponse } from "regular-table/dist/esm/types"; +import type { + CellMetadata, + CellScalar, + DataResponse, +} from "regular-table/dist/esm/types"; import type { GradientStopRgb } from "./color_utils.js"; // Re-export types from regular-table for use throughout the codebase @@ -51,6 +55,12 @@ export function get_psp_type( // Edit mode for the datagrid +/** + * Where in the `ViewConfig` a column being configured appears, which is a + * different axis from its plugin column slot. + */ +export type ColumnRole = "column" | "group_by" | "split_by"; + /** * Datagrid cell interaction mode (`plugin_config.edit_mode`): * `"READ_ONLY"` (default), `"EDIT"` (cells editable, writing back to the @@ -480,7 +490,15 @@ export interface DatagridModel { _default_bg_color_stops: GradientStopRgb[]; _series_palette: string[]; + + /// The joined identity key of each loaded column, read positionally out of + /// the data payload it keys - never parsed for structure, which is what + /// `_column_path_area` carries. _column_paths: string[]; + + /// `View.column_paths`' transposed area, `[level][absolute column]` and + /// sparse in the column axis exactly as `_column_paths` is. + _column_path_area: (CellScalar | null)[][]; _column_types: ColumnType[]; _is_editable: boolean[]; _edit_mode: EditMode; diff --git a/packages/viewer-datagrid/test/js/border_model.spec.ts b/packages/viewer-datagrid/test/js/border_model.spec.ts index 3fe932e3a9..7c7c8d5c64 100644 --- a/packages/viewer-datagrid/test/js/border_model.spec.ts +++ b/packages/viewer-datagrid/test/js/border_model.spec.ts @@ -22,27 +22,62 @@ import { split_levels, HeaderCellInput, } from "../../src/ts/style_handlers/border_model.js"; +import type { ColumnPathArea } from "../../src/ts/model/column_path_area.js"; + +function to_area( + paths: (string | undefined)[], + split_by_len: number, +): ColumnPathArea { + const depth = split_by_len + 1; + const area: ColumnPathArea = Array.from({ length: depth }, () => []); + for (let x = 0; x < paths.length; x++) { + const path = paths[x]; + if (path === undefined) { + continue; + } + + const parts = path.split("|"); + for (let level = 0; level + 1 < parts.length; level++) { + area[level][x] = parts[level]; + } + + area[depth - 1][x] = parts[parts.length - 1]; + } + + return area; +} -function cell(overrides: Partial): HeaderCellInput { - return { - paths: [], +function cell( + overrides: Partial> & { + paths?: (string | undefined)[]; + }, +): HeaderCellInput { + const { paths = [], ...rest } = overrides; + const base = { split_by_len: 0, colspan: 1, y: 0, - row_kind: "group", + row_kind: "group" as const, is_corner: false, is_last_header_row: false, single_header_row: false, - ...overrides, + ...rest, }; + + return { ...base, area: to_area(paths, base.split_by_len) }; } test.describe("split_levels", () => { test("strips the trailing column name", () => { - expect(split_levels("false|b|w", 2)).toStrictEqual(["false", "b"]); - expect(split_levels("false|w", 2)).toStrictEqual(["false"]); - expect(split_levels("w", 2)).toStrictEqual([]); - expect(split_levels("w", 0)).toStrictEqual([]); + expect(split_levels(to_area(["false|b|w"], 2), 2, 0)).toStrictEqual([ + "false", + "b", + ]); + expect(split_levels(to_area(["false|w"], 2), 2, 0)).toStrictEqual([ + "false", + ]); + expect(split_levels(to_area(["w"], 2), 2, 0)).toStrictEqual([]); + expect(split_levels(to_area(["w"], 0), 0, 0)).toStrictEqual([]); }); }); @@ -58,29 +93,29 @@ test.describe("boundary_depth", () => { ]; test("total-to-group boundary is depth 0", () => { - expect(boundary_depth(paths_2_level, 2, 0)).toEqual(0); + expect(boundary_depth(to_area(paths_2_level, 2), 2, 0)).toEqual(0); }); test("subtotal-to-first-child boundary is depth 1", () => { - expect(boundary_depth(paths_2_level, 2, 1)).toEqual(1); + expect(boundary_depth(to_area(paths_2_level, 2), 2, 1)).toEqual(1); }); test("sibling leaf boundary is depth 1", () => { - expect(boundary_depth(paths_2_level, 2, 2)).toEqual(1); + expect(boundary_depth(to_area(paths_2_level, 2), 2, 2)).toEqual(1); }); test("last-child-to-next-group boundary is depth 0", () => { - expect(boundary_depth(paths_2_level, 2, 3)).toEqual(0); + expect(boundary_depth(to_area(paths_2_level, 2), 2, 3)).toEqual(0); }); test("trailing edge is depth 0", () => { - expect(boundary_depth(paths_2_level, 2, 6)).toEqual(0); + expect(boundary_depth(to_area(paths_2_level, 2), 2, 6)).toEqual(0); }); test("aggregate-internal boundary is depth == split_by_len", () => { const paths = ["false|w", "false|x", "true|w", "true|x"]; - expect(boundary_depth(paths, 1, 0)).toEqual(1); - expect(boundary_depth(paths, 1, 1)).toEqual(0); + expect(boundary_depth(to_area(paths, 1), 1, 0)).toEqual(1); + expect(boundary_depth(to_area(paths, 1), 1, 1)).toEqual(0); }); test("total/subtotal aggregate-internal boundaries are never group boundaries", () => { @@ -95,17 +130,17 @@ test.describe("boundary_depth", () => { "false|b|w", "false|b|x", ]; - expect(boundary_depth(paths, 2, 0)).toEqual(2); - expect(boundary_depth(paths, 2, 1)).toEqual(0); - expect(boundary_depth(paths, 2, 2)).toEqual(2); - expect(boundary_depth(paths, 2, 3)).toEqual(1); - expect(boundary_depth(paths, 2, 4)).toEqual(2); + expect(boundary_depth(to_area(paths, 2), 2, 0)).toEqual(2); + expect(boundary_depth(to_area(paths, 2), 2, 1)).toEqual(0); + expect(boundary_depth(to_area(paths, 2), 2, 2)).toEqual(2); + expect(boundary_depth(to_area(paths, 2), 2, 3)).toEqual(1); + expect(boundary_depth(to_area(paths, 2), 2, 4)).toEqual(2); }); test("unloaded neighbor is null", () => { const paths: (string | undefined)[] = ["false|w", undefined, "true|w"]; - expect(boundary_depth(paths, 1, 0)).toBeNull(); - expect(boundary_depth(paths, 1, 1)).toBeNull(); + expect(boundary_depth(to_area(paths, 1), 1, 0)).toBeNull(); + expect(boundary_depth(to_area(paths, 1), 1, 1)).toBeNull(); }); }); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index beaa6ec69f..e5a88a32f7 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -56,7 +56,7 @@ catalogs: version: 5.6.2 commander: specifier: '>=14 <15' - version: 14.0.1 + version: 14.0.3 css-loader: specifier: '>=7 <8' version: 7.1.2 @@ -89,7 +89,7 @@ catalogs: version: 12.10.0 lightningcss: specifier: ^1.29.0 - version: 1.32.0 + version: 1.33.0 lodash: specifier: ^4.17.20 version: 4.17.21 @@ -169,6 +169,20 @@ catalogs: specifier: '>=8 <9' version: 8.8.5 +overrides: + '@perspective-dev/client@4.0.0>@perspective-dev/server': 4.0.0 + '@perspective-dev/client@4.1.0>@perspective-dev/server': 4.1.0 + '@perspective-dev/client@4.2.0>@perspective-dev/server': 4.2.0 + '@perspective-dev/client@4.3.0>@perspective-dev/server': 4.3.0 + '@perspective-dev/client@4.4.0>@perspective-dev/server': 4.4.0 + '@perspective-dev/client@4.5.0>@perspective-dev/server': 4.5.0 + '@perspective-dev/client@5.0.0>@perspective-dev/server': 5.0.0 + '@perspective-dev/client@5.1.0>@perspective-dev/server': 5.1.0 + '@perspective-dev/client@5.2.0>@perspective-dev/server': 5.2.0 + '@perspective-dev/client@5.3.0>@perspective-dev/server': 5.3.0 + '@perspective-dev/client@5.4.0>@perspective-dev/server': 5.4.0 + '@perspective-dev/client@5.5.0>@perspective-dev/server': 5.5.0 + importers: .: @@ -266,7 +280,7 @@ importers: version: 0.25.11 lightningcss: specifier: 'catalog:' - version: 1.32.0 + version: 1.33.0 typescript: specifier: 'catalog:' version: 5.9.3 @@ -659,7 +673,7 @@ importers: version: link:../../tools/esbuild-plugin lightningcss: specifier: 'catalog:' - version: 1.32.0 + version: 1.33.0 packages/cli: dependencies: @@ -677,7 +691,7 @@ importers: version: link:../viewer-datagrid commander: specifier: 'catalog:' - version: 14.0.1 + version: 14.0.3 puppeteer: specifier: 'catalog:' version: 24.26.1(supports-color@8.1.1)(typescript@5.9.3) @@ -724,7 +738,7 @@ importers: version: link:../../tools/test lightningcss: specifier: 'catalog:' - version: 1.32.0 + version: 1.33.0 packages/react: dependencies: @@ -779,7 +793,7 @@ importers: version: 24.9.1 lightningcss: specifier: 'catalog:' - version: 1.32.0 + version: 1.33.0 typescript: specifier: 'catalog:' version: 5.9.3 @@ -807,7 +821,7 @@ importers: version: link:../../tools/test lightningcss: specifier: 'catalog:' - version: 1.32.0 + version: 1.33.0 prettier: specifier: 'catalog:' version: 3.6.2 @@ -964,7 +978,7 @@ importers: version: 18.3.7(@types/react@19.2.2) lightningcss: specifier: 'catalog:' - version: 1.32.0 + version: 1.33.0 prettier: specifier: 'catalog:' version: 3.6.2 @@ -1193,6 +1207,7 @@ packages: '@clickhouse/client-common@1.17.0': resolution: {integrity: sha512-MiwwgXViFAQA2YZkN4ymF1ynzG0K49KeSX9/iOcmJetWkxqSekDdpyp1GjwATWa9R215uQ+hGzJtJujeQVZZIw==} + deprecated: 'Deprecated: import from @clickhouse/client or @clickhouse/client-web instead.' '@clickhouse/client-web@1.17.0': resolution: {integrity: sha512-rady6FKTSKfB4DL3dO1Y9R4XMykc0WxY8Yex4RWYgsNuYWkvP8WQqQjEgHQZqVUYvIA6jth6AMBSKAnZLo4p7A==} @@ -1924,6 +1939,7 @@ packages: '@octokit/webhooks-types@5.8.0': resolution: {integrity: sha512-8adktjIb76A7viIdayQSFuBEwOzwhDC+9yxZpKNHjfzrlostHCw0/N7JWpWMObfElwvJMk2fY2l1noENCk9wmw==} + deprecated: Package no longer supported. Contact Support at https://www.npmjs.com/support for more info. '@octokit/webhooks@9.26.3': resolution: {integrity: sha512-DLGk+gzeVq5oK89Bo601txYmyrelMQ7Fi5EnjHE0Xs8CWicy2xkmnJMKptKJrBJpstqbd/9oeDFi/Zj2pudBDQ==} @@ -1967,11 +1983,41 @@ packages: '@perspective-dev/client@5.5.0': resolution: {integrity: sha512-c6nN671nxKphyL2U9cjzTNnZm6+mwnTECXsCtaL+HWixPBeE5KMd9LTXsigiXMinC7gYrvpGZsvaJlRecv2jKQ==} + '@perspective-dev/server@4.0.0': + resolution: {integrity: sha512-S25Mr+cgRomU9aDiplXUj2gImuYZFXUUfytLdJC+f9iipFqX3MqFUCPI4jEzb1xTUnP3Fzg9xzs8p1SymHOF9g==} + + '@perspective-dev/server@4.1.0': + resolution: {integrity: sha512-ZYZ5dvyTLDy2TxF62SBS5BgJcP/WSuK367K5/Uts2aItS30I/4ElIl68Pt53Cr3qZBvsSU63YhDJGXlVBpMN4A==} + + '@perspective-dev/server@4.2.0': + resolution: {integrity: sha512-eK5YRPb53Ocnsqm15cnv4dMXyYK0168gk7SOxK2SpwD1w3ohLA6XpY1bpphmlmRBRNVCll5WISAThIZIIl0J4A==} + + '@perspective-dev/server@4.3.0': + resolution: {integrity: sha512-UH3ADscynozVx42RF07DTBmPE/0PUwH+SS0cgvMLuLfBjtvnPm6msfOU0tHlgFnNZOHJO0q6RZ9WI5fD6wF07A==} + + '@perspective-dev/server@4.4.0': + resolution: {integrity: sha512-hyFOmNRfZLyx96qS8AFq5EEjyNLQSEVlvp7zGrNabMHtkOYZCOfYrDgcdjKAvDhlvHrDVO0g7MavsepEuMDSww==} + + '@perspective-dev/server@4.5.0': + resolution: {integrity: sha512-1A3p1DOvS+U3Zc2eiRz0IkJYelLZ2VfmPh1w53qjI44lZAbQLYCMAq5iTSV1wgrL9P5sLliArjVCNMaIh0COBA==} + + '@perspective-dev/server@5.0.0': + resolution: {integrity: sha512-tuWAHeOusI+Jva34x5PPXBVnEaXB0Yeq/rfpioUygnLwmz4LJVk+SIjfoK4uEriScbV52dgatQjSbyWzUcKtqQ==} + '@perspective-dev/server@5.1.0': resolution: {integrity: sha512-lepN5Cvcx4MSCAv5FVYefA8HLqUhzVLgrO6h6FOtoLNoPR5C9N2MA7ZssSOyU3/6WSuidxs5iIqSFfldRYrRyw==} - '@perspective-dev/server@5.5.1': - resolution: {integrity: sha512-JBBdRLzg7yxm/WEhaif6dsAx3IFn1Qi172AswGtb0vb5ElBf7hv5ZOtCEP99Y/OBxz4xF9uNLkg1laNDKb8bqw==} + '@perspective-dev/server@5.2.0': + resolution: {integrity: sha512-WRBiokT2/BYM8Ipe7Dg1/2UYNb01Vsox79KBfFVI800VmwdId0GdqI95zufN5ZoFffeBfri1sr3S3AShBRFXKA==} + + '@perspective-dev/server@5.3.0': + resolution: {integrity: sha512-CcPb05fVGGQu90JZSL7Y+JVAqE6VEVatz4JR5T1Fb8qjbDnOAEleygx60ce5lUIHfOQfYXR/cDglSR7BtiFcFQ==} + + '@perspective-dev/server@5.4.0': + resolution: {integrity: sha512-HmUop3f0SfqbjCnojRy8gD/VIOeSgk91Oc5MafptBlJByaCSx8dbKZmCzLDc4p8MuClYe2PrYFMjJJchdl8ZRg==} + + '@perspective-dev/server@5.5.0': + resolution: {integrity: sha512-4wVCiabrrCpe7kvZu5Kx8BNJEjTGj2An7nJJiqZ9CvBZ8bObke6rqFNgTj3o1oGDmai+eTNFpg5LrFH4dZlPEw==} '@playwright/experimental-ct-core@1.62.0': resolution: {integrity: sha512-jmd7zmlTHrDf4coLZKxIQSmc8Dqr9NuNnadSQx1xeBlkoCUYO9eiiL5yIQ71LKHm/xcipRpe6HSIFEjoxXsAvQ==} @@ -2046,36 +2092,42 @@ packages: engines: {node: ^20.19.0 || >=22.12.0} cpu: [arm64] os: [linux] + libc: [glibc] '@rolldown/binding-linux-arm64-musl@1.2.8': resolution: {integrity: sha512-jFJTifHnNPY+yzOoNZQfSIysrVyXzEQPhPnOUjmD1bcQGHH6s7c8cViKWar8YplQImE5N9JRqMCLrM2CdxOrZA==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [arm64] os: [linux] + libc: [musl] '@rolldown/binding-linux-ppc64-gnu@1.2.8': resolution: {integrity: sha512-FhiOziBDWPBjbcmRzfLyIJnaP7AVMFXT7YCXPjXxj7wKU3vx24RjrCNN/zjvVa+N2vVoHJwCoUBvsrN/DG3zIA==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [ppc64] os: [linux] + libc: [glibc] '@rolldown/binding-linux-s390x-gnu@1.2.8': resolution: {integrity: sha512-WnHfADMzOV2Y55wlx1hzzQnar/wDt/VdvWSD99r18Mz9ylNieIGOkRx3UV21h7m/eJvjySYJkO26VvGNFkwsIQ==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [s390x] os: [linux] + libc: [glibc] '@rolldown/binding-linux-x64-gnu@1.2.8': resolution: {integrity: sha512-H9tRr5ibfXFVLxbPOseVewewFpl28zcEdjRDt2FTUZU7odxP0gEv1ki4/kGmcGOh78oRwZuuQllGLZ9zTJp84g==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [x64] os: [linux] + libc: [glibc] '@rolldown/binding-linux-x64-musl@1.2.8': resolution: {integrity: sha512-UefiqfM3D6IVNlZ8tSGs9+Ejjud2T+oxO0IHADU45Y+lyEjD2dVFyZHbkfX0LUb5Zugo/oIv1eCO/KVYhgYJYA==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [x64] os: [linux] + libc: [musl] '@rolldown/binding-openharmony-arm64@1.2.8': resolution: {integrity: sha512-637Ke4kWSy6rp9cxQ9gMOXlxPgIw/c1beASV4M//3+9I4uwBVOOl74G+e3zyU3u19U7RkRl/HuewixZ/Z6+Rjg==} @@ -2132,56 +2184,67 @@ packages: resolution: {integrity: sha512-PsNAbcyv9CcecAUagQefwX8fQn9LQ4nZkpDboBOttmyffnInRy8R8dSg6hxxl2Re5QhHBf6FYIDhIj5v982ATQ==} cpu: [arm] os: [linux] + libc: [glibc] '@rollup/rollup-linux-arm-musleabihf@4.52.5': resolution: {integrity: sha512-Fw4tysRutyQc/wwkmcyoqFtJhh0u31K+Q6jYjeicsGJJ7bbEq8LwPWV/w0cnzOqR2m694/Af6hpFayLJZkG2VQ==} cpu: [arm] os: [linux] + libc: [musl] '@rollup/rollup-linux-arm64-gnu@4.52.5': resolution: {integrity: sha512-a+3wVnAYdQClOTlyapKmyI6BLPAFYs0JM8HRpgYZQO02rMR09ZcV9LbQB+NL6sljzG38869YqThrRnfPMCDtZg==} cpu: [arm64] os: [linux] + libc: [glibc] '@rollup/rollup-linux-arm64-musl@4.52.5': resolution: {integrity: sha512-AvttBOMwO9Pcuuf7m9PkC1PUIKsfaAJ4AYhy944qeTJgQOqJYJ9oVl2nYgY7Rk0mkbsuOpCAYSs6wLYB2Xiw0Q==} cpu: [arm64] os: [linux] + libc: [musl] '@rollup/rollup-linux-loong64-gnu@4.52.5': resolution: {integrity: sha512-DkDk8pmXQV2wVrF6oq5tONK6UHLz/XcEVow4JTTerdeV1uqPeHxwcg7aFsfnSm9L+OO8WJsWotKM2JJPMWrQtA==} cpu: [loong64] os: [linux] + libc: [glibc] '@rollup/rollup-linux-ppc64-gnu@4.52.5': resolution: {integrity: sha512-W/b9ZN/U9+hPQVvlGwjzi+Wy4xdoH2I8EjaCkMvzpI7wJUs8sWJ03Rq96jRnHkSrcHTpQe8h5Tg3ZzUPGauvAw==} cpu: [ppc64] os: [linux] + libc: [glibc] '@rollup/rollup-linux-riscv64-gnu@4.52.5': resolution: {integrity: sha512-sjQLr9BW7R/ZiXnQiWPkErNfLMkkWIoCz7YMn27HldKsADEKa5WYdobaa1hmN6slu9oWQbB6/jFpJ+P2IkVrmw==} cpu: [riscv64] os: [linux] + libc: [glibc] '@rollup/rollup-linux-riscv64-musl@4.52.5': resolution: {integrity: sha512-hq3jU/kGyjXWTvAh2awn8oHroCbrPm8JqM7RUpKjalIRWWXE01CQOf/tUNWNHjmbMHg/hmNCwc/Pz3k1T/j/Lg==} cpu: [riscv64] os: [linux] + libc: [musl] '@rollup/rollup-linux-s390x-gnu@4.52.5': resolution: {integrity: sha512-gn8kHOrku8D4NGHMK1Y7NA7INQTRdVOntt1OCYypZPRt6skGbddska44K8iocdpxHTMMNui5oH4elPH4QOLrFQ==} cpu: [s390x] os: [linux] + libc: [glibc] '@rollup/rollup-linux-x64-gnu@4.52.5': resolution: {integrity: sha512-hXGLYpdhiNElzN770+H2nlx+jRog8TyynpTVzdlc6bndktjKWyZyiCsuDAlpd+j+W+WNqfcyAWz9HxxIGfZm1Q==} cpu: [x64] os: [linux] + libc: [glibc] '@rollup/rollup-linux-x64-musl@4.52.5': resolution: {integrity: sha512-arCGIcuNKjBoKAXD+y7XomR9gY6Mw7HnFBv5Rw7wQRvwYLR7gBAgV7Mb2QTyjXfTveBNFAtPt46/36vV9STLNg==} cpu: [x64] os: [linux] + libc: [musl] '@rollup/rollup-openharmony-arm64@4.52.5': resolution: {integrity: sha512-QoFqB6+/9Rly/RiPjaomPLmR/13cgkIGfA40LHly9zcH1S0bN2HVFYk3a1eAyHQyjs3ZJYlXvIGtcCs5tko9Cw==} @@ -2809,10 +2872,6 @@ packages: resolution: {integrity: sha512-y4Mg2tXshplEbSGzx7amzPwKKOCGuoSRP/CjEdwwk0FOGlUbq6lKuoyDZTNZkmxHdJtp54hdfY/JUrdL7Xfdug==} engines: {node: '>=14'} - commander@14.0.1: - resolution: {integrity: sha512-2JkV3gUZUVrbNA+1sjBOYLsMZ5cEEl8GTFP2a4AVz5hvasAMCQ1D2l2le/cX+pV4N6ZU17zjUahLpIXRrnWL8A==} - engines: {node: '>=20'} - commander@14.0.3: resolution: {integrity: sha512-H+y0Jo/T1RZ9qPP4Eh1pkcQcLRglraJaSLoyOtHxu6AapkjWVCy2Sit1QQ4x3Dng8qDlSsZEet7g5Pq06MvTgw==} engines: {node: '>=20'} @@ -3185,6 +3244,7 @@ packages: exenv-es6@1.1.1: resolution: {integrity: sha512-vlVu3N8d6yEMpMsEm+7sUBAI81aqYYuEvfK0jNqmdb/OPXzzH7QWDDnVjMvDSY47JdHEqx/dfC/q8WkfoTmpGQ==} + deprecated: Package no longer supported. Contact Support at https://www.npmjs.com/support for more info. extract-zip@2.0.1: resolution: {integrity: sha512-GDhU9ntwuKyGXdZBUgTIe+vXnWj0fppUEtMDL0+idd5Sta8TGpHssn/eusA9mrPr9qNDym6SxAYZjNvCn/9RBg==} @@ -3711,10 +3771,6 @@ packages: resolution: {integrity: sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==} hasBin: true - js-yaml@4.1.0: - resolution: {integrity: sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==} - hasBin: true - js-yaml@4.1.1: resolution: {integrity: sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA==} hasBin: true @@ -3806,119 +3862,63 @@ packages: webpack: optional: true - lightningcss-android-arm64@1.32.0: - resolution: {integrity: sha512-YK7/ClTt4kAK0vo6w3X+Pnm0D2cf2vPHbhOXdoNti1Ga0al1P4TBZhwjATvjNwLEBCnKvjJc2jQgHXH0NEwlAg==} - engines: {node: '>= 12.0.0'} - cpu: [arm64] - os: [android] - lightningcss-android-arm64@1.33.0: resolution: {integrity: sha512-gEpRTalKdosp4Bb8qWtc2iOgE5SeIHlpS1up9bFq2wAyYhl1UdTObYiHe98zEM9SQvSoqQZ1IQD0JNpg3Ml5pg==} engines: {node: '>= 12.0.0'} cpu: [arm64] os: [android] - lightningcss-darwin-arm64@1.32.0: - resolution: {integrity: sha512-RzeG9Ju5bag2Bv1/lwlVJvBE3q6TtXskdZLLCyfg5pt+HLz9BqlICO7LZM7VHNTTn/5PRhHFBSjk5lc4cmscPQ==} - engines: {node: '>= 12.0.0'} - cpu: [arm64] - os: [darwin] - lightningcss-darwin-arm64@1.33.0: resolution: {integrity: sha512-Sciaz8eenNTKn9b3t7+xr0ipTp9YxKQY4npwQ3mrRuL0BAVHBLyZxofhaKBAVtzmtRZ/zTyo0/to4B1uWG/Djg==} engines: {node: '>= 12.0.0'} cpu: [arm64] os: [darwin] - lightningcss-darwin-x64@1.32.0: - resolution: {integrity: sha512-U+QsBp2m/s2wqpUYT/6wnlagdZbtZdndSmut/NJqlCcMLTWp5muCrID+K5UJ6jqD2BFshejCYXniPDbNh73V8w==} - engines: {node: '>= 12.0.0'} - cpu: [x64] - os: [darwin] - lightningcss-darwin-x64@1.33.0: resolution: {integrity: sha512-Z5UPAxzrjlWNNyGy6i65cJzzvgJ5D3T6wMvs+gWpY9d7qRhANrxqAp6LhxIgZhWEw18RfJTGcRxjuLIBr+m8XQ==} engines: {node: '>= 12.0.0'} cpu: [x64] os: [darwin] - lightningcss-freebsd-x64@1.32.0: - resolution: {integrity: sha512-JCTigedEksZk3tHTTthnMdVfGf61Fky8Ji2E4YjUTEQX14xiy/lTzXnu1vwiZe3bYe0q+SpsSH/CTeDXK6WHig==} - engines: {node: '>= 12.0.0'} - cpu: [x64] - os: [freebsd] - lightningcss-freebsd-x64@1.33.0: resolution: {integrity: sha512-QQM/Ti/hQajJwCY+RiWuCZ9sdtI/XQk7nDK5vC8kkdwixezOlDgvDx7+RT+QjK6FcFT4MpsuoBnHIo/O3StRRg==} engines: {node: '>= 12.0.0'} cpu: [x64] os: [freebsd] - lightningcss-linux-arm-gnueabihf@1.32.0: - resolution: {integrity: sha512-x6rnnpRa2GL0zQOkt6rts3YDPzduLpWvwAF6EMhXFVZXD4tPrBkEFqzGowzCsIWsPjqSK+tyNEODUBXeeVHSkw==} - engines: {node: '>= 12.0.0'} - cpu: [arm] - os: [linux] - lightningcss-linux-arm-gnueabihf@1.33.0: resolution: {integrity: sha512-N7FVBe6iS24MlM6R/4RBTxGhQheZGs7tiQ9U32UtF75NzP5Q7xWPRqLBCKxlRQRk3rY1jCIPLzx7WzOhuUIRLQ==} engines: {node: '>= 12.0.0'} cpu: [arm] os: [linux] - lightningcss-linux-arm64-gnu@1.32.0: - resolution: {integrity: sha512-0nnMyoyOLRJXfbMOilaSRcLH3Jw5z9HDNGfT/gwCPgaDjnx0i8w7vBzFLFR1f6CMLKF8gVbebmkUN3fa/kQJpQ==} - engines: {node: '>= 12.0.0'} - cpu: [arm64] - os: [linux] - lightningcss-linux-arm64-gnu@1.33.0: resolution: {integrity: sha512-j2v/itmy4HlNxlc6voKXYgBqNi0Ng2LShg4z7GufpEgs05P+2suBVyi9I6YHq5uoVFx9ETin3eCEhLVyXGQnKg==} engines: {node: '>= 12.0.0'} cpu: [arm64] os: [linux] - - lightningcss-linux-arm64-musl@1.32.0: - resolution: {integrity: sha512-UpQkoenr4UJEzgVIYpI80lDFvRmPVg6oqboNHfoH4CQIfNA+HOrZ7Mo7KZP02dC6LjghPQJeBsvXhJod/wnIBg==} - engines: {node: '>= 12.0.0'} - cpu: [arm64] - os: [linux] + libc: [glibc] lightningcss-linux-arm64-musl@1.33.0: resolution: {integrity: sha512-yiO5ROMuYQgXbC60yjZU5CYSFZGKXL0HFATXt9mHJn1+zW55oCtMI9NfcVhYLMFDL7gV7oBPon/EmMMGg2OvtQ==} engines: {node: '>= 12.0.0'} cpu: [arm64] os: [linux] - - lightningcss-linux-x64-gnu@1.32.0: - resolution: {integrity: sha512-V7Qr52IhZmdKPVr+Vtw8o+WLsQJYCTd8loIfpDaMRWGUZfBOYEJeyJIkqGIDMZPwPx24pUMfwSxxI8phr/MbOA==} - engines: {node: '>= 12.0.0'} - cpu: [x64] - os: [linux] + libc: [musl] lightningcss-linux-x64-gnu@1.33.0: resolution: {integrity: sha512-ar+Ju7LmcN0Jo4FpL4hpFybwNG9/3A/Br5KW2n2jyODg3MEZXaDYADdemoNS+BDNfMgKvylJLj4S5tyRActuAg==} engines: {node: '>= 12.0.0'} cpu: [x64] os: [linux] - - lightningcss-linux-x64-musl@1.32.0: - resolution: {integrity: sha512-bYcLp+Vb0awsiXg/80uCRezCYHNg1/l3mt0gzHnWV9XP1W5sKa5/TCdGWaR/zBM2PeF/HbsQv/j2URNOiVuxWg==} - engines: {node: '>= 12.0.0'} - cpu: [x64] - os: [linux] + libc: [glibc] lightningcss-linux-x64-musl@1.33.0: resolution: {integrity: sha512-RYiYbkokw0trfKqqzfF55lginwEPrD3OJDfTuJzFs1MK6iFnDenaz1fqLLtX4ITG3OktJQXOeTaw1awrBAlZPw==} engines: {node: '>= 12.0.0'} cpu: [x64] os: [linux] - - lightningcss-win32-arm64-msvc@1.32.0: - resolution: {integrity: sha512-8SbC8BR40pS6baCM8sbtYDSwEVQd4JlFTOlaD3gWGHfThTcABnNDBda6eTZeqbofalIJhFx0qKzgHJmcPTnGdw==} - engines: {node: '>= 12.0.0'} - cpu: [arm64] - os: [win32] + libc: [musl] lightningcss-win32-arm64-msvc@1.33.0: resolution: {integrity: sha512-1K+MPfLSFVpphzpdbfkhlWk6wBrTObBzS2T6db10PNOZgR9GoVsAWzwNyuhUYYbTp23j+4RrncfujZ4uAzXvwA==} @@ -3926,22 +3926,12 @@ packages: cpu: [arm64] os: [win32] - lightningcss-win32-x64-msvc@1.32.0: - resolution: {integrity: sha512-Amq9B/SoZYdDi1kFrojnoqPLxYhQ4Wo5XiL8EVJrVsB8ARoC1PWW6VGtT0WKCemjy8aC+louJnjS7U18x3b06Q==} - engines: {node: '>= 12.0.0'} - cpu: [x64] - os: [win32] - lightningcss-win32-x64-msvc@1.33.0: resolution: {integrity: sha512-OlEICDx/Xl0FqSp4bry8zFnCvGpig3Gl4gCquvYwHuqJKEC1+n9NgDniFvqHGmMv1ZkqDJrDqKKSykTDX+ehuA==} engines: {node: '>= 12.0.0'} cpu: [x64] os: [win32] - lightningcss@1.32.0: - resolution: {integrity: sha512-NXYBzinNrblfraPGyrbPoD19C1h9lfI/1mzgWYvXUTe414Gz/X1FD2XBZSZM7rRTrMA8JL3OtAaGifrIKhQ5yQ==} - engines: {node: '>= 12.0.0'} - lightningcss@1.33.0: resolution: {integrity: sha512-WkUDrojuJs0xkgGf2udWxa3yGBRxPtxUkB79i6aCZLRgc7PM8fZe9TosfPDcvEpQZbuFASnHYmRLBLUbmLOIIA==} engines: {node: '>= 12.0.0'} @@ -5026,6 +5016,7 @@ packages: uuid@8.0.0: resolution: {integrity: sha512-jOXGuXZAWdsTH7eZLtyXMqUb9EcWMGZNbL9YcGBJl4MH4nrxHmZJhEHvyLFrkxo+28uLb/NYRcStH48fnD0Vzw==} + deprecated: uuid@10 and below is no longer supported. For ESM codebases, update to uuid@latest. For CommonJS codebases, use uuid@11 (but be aware this version will likely be deprecated in 2028). hasBin: true validate-npm-package-license@3.0.4: @@ -6518,7 +6509,7 @@ snapshots: '@perspective-dev/client@4.0.0(debug@4.4.3(supports-color@8.1.1))(supports-color@8.1.1)': dependencies: - '@perspective-dev/server': 5.1.0 + '@perspective-dev/server': 4.0.0 pro_self_extracting_wasm: 0.0.9(debug@4.4.3(supports-color@8.1.1))(supports-color@8.1.1) stoppable: 1.1.0 ws: 8.18.3 @@ -6530,7 +6521,7 @@ snapshots: '@perspective-dev/client@4.1.0(debug@4.4.3(supports-color@8.1.1))(supports-color@8.1.1)': dependencies: - '@perspective-dev/server': 5.1.0 + '@perspective-dev/server': 4.1.0 pro_self_extracting_wasm: 0.0.9(debug@4.4.3(supports-color@8.1.1))(supports-color@8.1.1) stoppable: 1.1.0 ws: 8.18.3 @@ -6542,7 +6533,7 @@ snapshots: '@perspective-dev/client@4.2.0(debug@4.4.3(supports-color@8.1.1))(supports-color@8.1.1)': dependencies: - '@perspective-dev/server': 5.1.0 + '@perspective-dev/server': 4.2.0 pro_self_extracting_wasm: 0.0.9(debug@4.4.3(supports-color@8.1.1))(supports-color@8.1.1) stoppable: 1.1.0 ws: 8.18.3 @@ -6554,7 +6545,7 @@ snapshots: '@perspective-dev/client@4.3.0(debug@4.4.3(supports-color@8.1.1))(supports-color@8.1.1)': dependencies: - '@perspective-dev/server': 5.1.0 + '@perspective-dev/server': 4.3.0 pro_self_extracting_wasm: 0.0.9(debug@4.4.3(supports-color@8.1.1))(supports-color@8.1.1) stoppable: 1.1.0 ws: 8.18.3 @@ -6566,7 +6557,7 @@ snapshots: '@perspective-dev/client@4.4.0(debug@4.4.3(supports-color@8.1.1))(supports-color@8.1.1)': dependencies: - '@perspective-dev/server': 5.1.0 + '@perspective-dev/server': 4.4.0 pro_self_extracting_wasm: 0.0.9(debug@4.4.3(supports-color@8.1.1))(supports-color@8.1.1) stoppable: 1.1.0 ws: 8.18.3 @@ -6578,7 +6569,7 @@ snapshots: '@perspective-dev/client@4.5.0(debug@4.4.3(supports-color@8.1.1))(supports-color@8.1.1)': dependencies: - '@perspective-dev/server': 5.1.0 + '@perspective-dev/server': 4.5.0 pro_self_extracting_wasm: 0.0.9(debug@4.4.3(supports-color@8.1.1))(supports-color@8.1.1) stoppable: 1.1.0 ws: 8.18.3 @@ -6590,7 +6581,7 @@ snapshots: '@perspective-dev/client@5.0.0(debug@4.4.3(supports-color@8.1.1))(supports-color@8.1.1)': dependencies: - '@perspective-dev/server': 5.1.0 + '@perspective-dev/server': 5.0.0 pro_self_extracting_wasm: 0.0.9(debug@4.4.3(supports-color@8.1.1))(supports-color@8.1.1) stoppable: 1.1.0 ws: 8.18.3 @@ -6614,7 +6605,7 @@ snapshots: '@perspective-dev/client@5.2.0(debug@4.4.3(supports-color@8.1.1))(supports-color@8.1.1)': dependencies: - '@perspective-dev/server': 5.1.0 + '@perspective-dev/server': 5.2.0 pro_self_extracting_wasm: 0.0.9(debug@4.4.3(supports-color@8.1.1))(supports-color@8.1.1) stoppable: 1.1.0 ws: 8.18.3 @@ -6626,7 +6617,7 @@ snapshots: '@perspective-dev/client@5.3.0(debug@4.4.3(supports-color@8.1.1))(supports-color@8.1.1)': dependencies: - '@perspective-dev/server': 5.1.0 + '@perspective-dev/server': 5.3.0 pro_self_extracting_wasm: 0.0.9(debug@4.4.3(supports-color@8.1.1))(supports-color@8.1.1) stoppable: 1.1.0 ws: 8.18.3 @@ -6638,7 +6629,7 @@ snapshots: '@perspective-dev/client@5.4.0(debug@4.4.3(supports-color@8.1.1))(supports-color@8.1.1)': dependencies: - '@perspective-dev/server': 5.5.1 + '@perspective-dev/server': 5.4.0 pro_self_extracting_wasm: 0.0.9(debug@4.4.3(supports-color@8.1.1))(supports-color@8.1.1) stoppable: 1.1.0 ws: 8.18.3 @@ -6650,7 +6641,7 @@ snapshots: '@perspective-dev/client@5.5.0(debug@4.4.3(supports-color@8.1.1))(supports-color@8.1.1)': dependencies: - '@perspective-dev/server': 5.5.1 + '@perspective-dev/server': 5.5.0 pro_self_extracting_wasm: 0.0.9(debug@4.4.3(supports-color@8.1.1))(supports-color@8.1.1) stoppable: 1.1.0 ws: 8.18.3 @@ -6660,9 +6651,29 @@ snapshots: - supports-color - utf-8-validate + '@perspective-dev/server@4.0.0': {} + + '@perspective-dev/server@4.1.0': {} + + '@perspective-dev/server@4.2.0': {} + + '@perspective-dev/server@4.3.0': {} + + '@perspective-dev/server@4.4.0': {} + + '@perspective-dev/server@4.5.0': {} + + '@perspective-dev/server@5.0.0': {} + '@perspective-dev/server@5.1.0': {} - '@perspective-dev/server@5.5.1': {} + '@perspective-dev/server@5.2.0': {} + + '@perspective-dev/server@5.3.0': {} + + '@perspective-dev/server@5.4.0': {} + + '@perspective-dev/server@5.5.0': {} '@playwright/experimental-ct-core@1.62.0(@types/node@24.9.1)(jiti@1.21.7)(less@4.4.2)(terser@5.44.0)(tsx@4.20.6)(yaml@2.8.1)': dependencies: @@ -7518,8 +7529,6 @@ snapshots: commander@10.0.1: {} - commander@14.0.1: {} - commander@14.0.3: {} commander@2.20.3: {} @@ -7561,7 +7570,7 @@ snapshots: dependencies: env-paths: 2.2.1 import-fresh: 3.3.1 - js-yaml: 4.1.0 + js-yaml: 4.1.1 parse-json: 5.2.0 optionalDependencies: typescript: 5.9.3 @@ -8554,10 +8563,6 @@ snapshots: esprima: 4.0.1 optional: true - js-yaml@4.1.0: - dependencies: - argparse: 2.0.1 - js-yaml@4.1.1: dependencies: argparse: 2.0.1 @@ -8666,88 +8671,39 @@ snapshots: optionalDependencies: webpack: 5.102.1(esbuild@0.25.11)(uglify-js@3.19.3)(webpack-cli@5.1.4) - lightningcss-android-arm64@1.32.0: - optional: true - lightningcss-android-arm64@1.33.0: optional: true - lightningcss-darwin-arm64@1.32.0: - optional: true - lightningcss-darwin-arm64@1.33.0: optional: true - lightningcss-darwin-x64@1.32.0: - optional: true - lightningcss-darwin-x64@1.33.0: optional: true - lightningcss-freebsd-x64@1.32.0: - optional: true - lightningcss-freebsd-x64@1.33.0: optional: true - lightningcss-linux-arm-gnueabihf@1.32.0: - optional: true - lightningcss-linux-arm-gnueabihf@1.33.0: optional: true - lightningcss-linux-arm64-gnu@1.32.0: - optional: true - lightningcss-linux-arm64-gnu@1.33.0: optional: true - lightningcss-linux-arm64-musl@1.32.0: - optional: true - lightningcss-linux-arm64-musl@1.33.0: optional: true - lightningcss-linux-x64-gnu@1.32.0: - optional: true - lightningcss-linux-x64-gnu@1.33.0: optional: true - lightningcss-linux-x64-musl@1.32.0: - optional: true - lightningcss-linux-x64-musl@1.33.0: optional: true - lightningcss-win32-arm64-msvc@1.32.0: - optional: true - lightningcss-win32-arm64-msvc@1.33.0: optional: true - lightningcss-win32-x64-msvc@1.32.0: - optional: true - lightningcss-win32-x64-msvc@1.33.0: optional: true - lightningcss@1.32.0: - dependencies: - detect-libc: 2.1.2 - optionalDependencies: - lightningcss-android-arm64: 1.32.0 - lightningcss-darwin-arm64: 1.32.0 - lightningcss-darwin-x64: 1.32.0 - lightningcss-freebsd-x64: 1.32.0 - lightningcss-linux-arm-gnueabihf: 1.32.0 - lightningcss-linux-arm64-gnu: 1.32.0 - lightningcss-linux-arm64-musl: 1.32.0 - lightningcss-linux-x64-gnu: 1.32.0 - lightningcss-linux-x64-musl: 1.32.0 - lightningcss-win32-arm64-msvc: 1.32.0 - lightningcss-win32-x64-msvc: 1.32.0 - lightningcss@1.33.0: dependencies: detect-libc: 2.1.2 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 28dcfed804..d0f24e56b8 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -20,6 +20,20 @@ packages: - "examples/*" - "docs" +overrides: + "@perspective-dev/client@4.0.0>@perspective-dev/server": "4.0.0" + "@perspective-dev/client@4.1.0>@perspective-dev/server": "4.1.0" + "@perspective-dev/client@4.2.0>@perspective-dev/server": "4.2.0" + "@perspective-dev/client@4.3.0>@perspective-dev/server": "4.3.0" + "@perspective-dev/client@4.4.0>@perspective-dev/server": "4.4.0" + "@perspective-dev/client@4.5.0>@perspective-dev/server": "4.5.0" + "@perspective-dev/client@5.0.0>@perspective-dev/server": "5.0.0" + "@perspective-dev/client@5.1.0>@perspective-dev/server": "5.1.0" + "@perspective-dev/client@5.2.0>@perspective-dev/server": "5.2.0" + "@perspective-dev/client@5.3.0>@perspective-dev/server": "5.3.0" + "@perspective-dev/client@5.4.0>@perspective-dev/server": "5.4.0" + "@perspective-dev/client@5.5.0>@perspective-dev/server": "5.5.0" + allowBuilds: "@fortawesome/fontawesome-free": true aws-sdk: true diff --git a/rust/perspective-client/perspective.proto b/rust/perspective-client/perspective.proto index 2d1d958432..2c13145c03 100644 --- a/rust/perspective-client/perspective.proto +++ b/rust/perspective-client/perspective.proto @@ -86,7 +86,7 @@ message ViewPort { optional uint32 start_col = 2; optional uint32 end_row = 3; optional uint32 end_col = 4; - optional bool emit_legacy_row_path_names = 5; + optional bool machine_column_names = 5; // optional bool id = 5; // optional bool index = 3; // optional bool formatted = 6; @@ -251,6 +251,9 @@ message GetFeaturesResp { bool unordered = 10; repeated SplitRollupMode split_rollup_mode = 11; + // Whether a `Table` can be derived from a `View` of this server. + bool view_derivations = 12; + message WindowAggregateOptions { repeated WindowAggregateArgs options = 1; } @@ -388,6 +391,10 @@ message MakeTableReq { optional bool page_to_disk = 3; optional ListFlatten list_flatten = 4; + + // The columns of a `Table` made from a `View`, in place of the ones + // inferred from it. Only valid with `MakeTableData.from_view`. + optional Schema view_schema = 5; } } @@ -534,16 +541,17 @@ message ViewColumnPathsReq { optional uint32 end_col = 2; } -// // TODO This is a better paths representations but its not compatible with -// // the legacy API. Let's do this when we can fix the API. - -// message ColumnPath { -// repeated string path = 1; -// } +// One row of a column path area, `width` values long. +message ColumnPathLevel { + repeated Scalar values = 1; +} message ViewColumnPathsResp { - repeated string paths = 1; - // repeated ColumnPath paths = 1; + reserved 1; + reserved "paths"; + + // Transposed, `split_by.len() + 1` rows of `width` values each. + repeated ColumnPathLevel area = 2; } message ViewDeleteReq {} diff --git a/rust/perspective-client/src/rust/client.rs b/rust/perspective-client/src/rust/client.rs index 1ff862081b..fad1620d2c 100644 --- a/rust/perspective-client/src/rust/client.rs +++ b/rust/perspective-client/src/rust/client.rs @@ -540,9 +540,14 @@ impl Client { /// [`Client`] connects to, where the data is stored and all calculation /// occurs. /// + /// When instantiated with a [`View`] of this [`Client`], the resulting + /// [`Table`] is _derived_: it is read-only, its rows are the [`View`]'s + /// output, and the engine keeps it current as the [`View`]'s source is + /// updated. + /// /// # Arguments /// - /// - `arg` - Either _schema_ or initialization _data_. + /// - `arg` - Either _schema_, initialization _data_, or a [`View`]. /// - `options` - Optional configuration which provides one of: /// - `limit` - The max number of rows the resulting [`Table`] can /// store. @@ -555,6 +560,8 @@ impl Client { /// `"json"`, `"columns"`, `"csv"` or `"arrow"`. This overrides /// language-specific type dispatch behavior, which allows stringified /// and byte array alternative inputs. + /// - `schema` - The columns of a [`Table`] derived from a [`View`], in + /// place of the ones inferred from it. /// /// # Examples /// @@ -575,6 +582,7 @@ impl Client { }; if let TableData::View(view) = &input { + let request_options: TableOptions = options.clone().into(); let mut options = options; let source_index = view.source.as_ref().and_then(|x| x.options.index.clone()); if let (None, Some(index)) = (&options.index, &source_index) { @@ -590,6 +598,18 @@ impl Client { options.limit = view.source.as_ref().and_then(|x| x.options.limit); } + if view.client() == self { + let data = TableData::View(view.clone()); + self.crate_table_inner(data, request_options, entity_id.clone()) + .await?; + + return Ok(Table::new(entity_id, self.clone(), options.into())); + } + + if options.schema.is_some() { + return Err(ClientError::BadTableOptions); + } + let window = ViewWindow::default(); let arrow = view.to_arrow(window).await?; let mut table = self @@ -697,6 +717,7 @@ impl Client { limit: None, page_to_disk: None, list_flatten: None, + schema: None, })), resp => Err(resp.into()), } @@ -743,6 +764,7 @@ impl Client { limit: info.limit, page_to_disk: None, list_flatten: None, + schema: None, }; let client = self.clone(); diff --git a/rust/perspective-client/src/rust/config/aggregates.rs b/rust/perspective-client/src/rust/config/aggregates.rs index e7f2e07af5..d04188acb7 100644 --- a/rust/perspective-client/src/rust/config/aggregates.rs +++ b/rust/perspective-client/src/rust/config/aggregates.rs @@ -15,7 +15,7 @@ use std::fmt::Display; use serde::{Deserialize, Serialize}; use ts_rs::TS; -use crate::proto::view_config; +use crate::proto::{ColumnType, view_config}; #[derive(Clone, Debug, Deserialize, Eq, Ord, PartialEq, PartialOrd, Serialize, TS)] #[serde(untagged)] @@ -24,6 +24,16 @@ pub enum Aggregate { MultiAggregate(String, Vec), } +impl Aggregate { + /// The aggregate a column of this type takes when `aggregates` names none. + pub fn default_for(column_type: ColumnType) -> Self { + match column_type { + ColumnType::Integer | ColumnType::Float => Aggregate::SingleAggregate("sum".into()), + _ => Aggregate::SingleAggregate("count".into()), + } + } +} + impl From<&'static str> for Aggregate { fn from(value: &'static str) -> Self { if value.contains(" by ") { diff --git a/rust/perspective-client/src/rust/config/view_config.rs b/rust/perspective-client/src/rust/config/view_config.rs index 5ca3034df3..c7d0875a2b 100644 --- a/rust/perspective-client/src/rust/config/view_config.rs +++ b/rust/perspective-client/src/rust/config/view_config.rs @@ -22,7 +22,7 @@ use super::filters::*; use super::sort::*; use super::windows::*; use crate::proto; -use crate::proto::columns_update; +use crate::proto::{ColumnType, columns_update}; #[derive(Clone, Copy, Debug, Default, Deserialize, Serialize, PartialEq, Eq, TS)] pub enum GroupRollupMode { @@ -567,6 +567,35 @@ impl ViewConfig { !self.group_by.is_empty() || self.group_rollup_mode == GroupRollupMode::Total } + /// The name a `View` column path takes in human-readable output, with its + /// leaf qualified by its aggregate when that column is also a `group_by` + /// key of this config. + pub fn readable_column_path( + &self, + path: &str, + column_type: impl FnOnce(&str) -> ColumnType, + ) -> String { + let (prefix, leaf) = match path.rsplit_once('|') { + Some((prefix, leaf)) if !self.split_by.is_empty() => (Some(prefix), leaf), + _ => (None, path), + }; + + if !self.group_by.iter().any(|x| x == leaf) { + return path.to_owned(); + } + + let aggregate = self + .aggregates + .get(leaf) + .cloned() + .unwrap_or_else(|| Aggregate::default_for(column_type(leaf))); + + match prefix { + Some(prefix) => format!("{prefix}|{leaf} ({aggregate})"), + None => format!("{leaf} ({aggregate})"), + } + } + pub fn is_column_expression_in_use(&self, name: &str) -> bool { let name = name.to_owned(); self.group_by.contains(&name) diff --git a/rust/perspective-client/src/rust/table.rs b/rust/perspective-client/src/rust/table.rs index e54c5163c2..2344dee64d 100644 --- a/rust/perspective-client/src/rust/table.rs +++ b/rust/perspective-client/src/rust/table.rs @@ -13,6 +13,7 @@ use std::collections::HashMap; use std::fmt::Display; +use indexmap::IndexMap; use serde::{Deserialize, Serialize}; use ts_rs::TS; @@ -108,6 +109,12 @@ pub struct TableInitOptions { #[serde(default)] #[ts(optional)] pub list_flatten: Option, + + /// The columns of a [`Table`] made from a [`View`], in place of the ones + /// inferred from it. + #[serde(default)] + #[ts(optional)] + pub schema: Option>, } impl TableInitOptions { @@ -122,9 +129,20 @@ impl TryFrom for MakeTableOptions { fn try_from(value: TableOptions) -> Result { let page_to_disk = value.page_to_disk; let list_flatten = value.list_flatten.map(|x| x as i32); + let view_schema = value.schema.clone().map(|schema| crate::proto::Schema { + schema: schema + .into_iter() + .map(|(name, r#type)| crate::proto::schema::KeyTypePair { + name, + r#type: r#type as i32, + }) + .collect(), + }); + Ok(MakeTableOptions { page_to_disk, list_flatten, + view_schema, make_table_type: match value { TableOptions { index: Some(_), @@ -149,6 +167,7 @@ pub(crate) struct TableOptions { pub limit: Option, pub page_to_disk: Option, pub list_flatten: Option, + pub schema: Option>, } /// The source [`View`] of a replica [`Table`] built by [`Client::table`], @@ -167,6 +186,7 @@ impl From for TableOptions { limit: value.limit, page_to_disk: value.page_to_disk, list_flatten: value.list_flatten, + schema: value.schema, } } } diff --git a/rust/perspective-client/src/rust/view.rs b/rust/perspective-client/src/rust/view.rs index 963764f90f..8a1c235472 100644 --- a/rust/perspective-client/src/rust/view.rs +++ b/rust/perspective-client/src/rust/view.rs @@ -112,12 +112,13 @@ pub struct ViewWindow { #[serde(skip_serializing_if = "Option::is_none")] pub compression: Option, - /// When `true`, group-by columns use legacy `"colname (Group by N)"` - /// naming. When `false`, they use `__ROW_PATH_N__` naming consistent - /// with the SQL backend. Defaults to `true` for backwards compatibility. + /// When `true`, [`View::to_arrow`] names `group_by` key columns + /// `__ROW_PATH_N__` and never qualifies an aggregate, instead of the + /// human-readable names a key column and its aggregate would otherwise + /// take. #[ts(optional)] #[serde(skip_serializing_if = "Option::is_none")] - pub emit_legacy_row_path_names: Option, + pub machine_column_names: Option, } impl From for ViewPort { @@ -127,7 +128,7 @@ impl From for ViewPort { start_col: window.start_col.map(|x| x.floor() as u32), end_row: window.end_row.map(|x| x.ceil() as u32), end_col: window.end_col.map(|x| x.ceil() as u32), - emit_legacy_row_path_names: window.emit_legacy_row_path_names, + machine_column_names: window.machine_column_names, } } } @@ -139,7 +140,7 @@ impl From for ViewWindow { start_col: window.start_col.map(|x| x as f64), end_row: window.end_row.map(|x| x as f64), end_col: window.end_col.map(|x| x as f64), - emit_legacy_row_path_names: window.emit_legacy_row_path_names, + machine_column_names: window.machine_column_names, ..ViewWindow::default() } } @@ -295,6 +296,10 @@ pub(crate) struct ViewSource { assert_view_api!(View); impl View { + pub(crate) fn client(&self) -> &Client { + &self.client + } + pub fn new(name: String, client: Client) -> Self { View { name, @@ -319,22 +324,40 @@ impl View { } } - /// Returns an array of strings containing the column paths of the [`View`] - /// without any of the source columns. + /// Returns this [`View`]'s column header area for `window`, transposed as + /// `area[level][column]`. /// /// A column path shows the columns that a given cell belongs to after - /// pivots are applied. - pub async fn column_paths(&self, window: ColumnWindow) -> ClientResult> { + /// pivots are applied. The area is rectangular with one level per + /// `split_by` plus one: a column's split values occupy the leading levels + /// and its name always occupies the last, so a subtotal or grand total + /// under `split_rollup_mode: "rollup"` reads [`Scalar::Null`] at the levels + /// it does not pivot on. Values keep their column's type - a `datetime` + /// split value is epoch milliseconds, not text - so formatting them is the + /// caller's choice. + /// + /// `window` slices the *column* axis; the number of levels does not depend + /// on it. + pub async fn column_paths( + &self, + window: ColumnWindow, + ) -> ClientResult>> { let msg = self.client_message(ClientReq::ViewColumnPathsReq(ViewColumnPathsReq { start_col: window.start_col.map(|x| x as u32), end_col: window.end_col.map(|x| x as u32), })); match self.client.oneshot(&msg).await? { - ClientResp::ViewColumnPathsResp(ViewColumnPathsResp { paths }) => { - // Ok(paths.into_iter().map(|x| x.path).collect()) - Ok(paths) - }, + ClientResp::ViewColumnPathsResp(ViewColumnPathsResp { area }) => Ok(area + .into_iter() + .map(|level| { + level + .values + .into_iter() + .map(crate::config::Scalar::from) + .collect() + }) + .collect()), resp => Err(resp.into()), } } diff --git a/rust/perspective-client/src/rust/virtual_server/data.rs b/rust/perspective-client/src/rust/virtual_server/data.rs index 900ba84205..f05fa8e5c4 100644 --- a/rust/perspective-client/src/rust/virtual_server/data.rs +++ b/rust/perspective-client/src/rust/virtual_server/data.rs @@ -33,7 +33,7 @@ use arrow_schema::{DataType, Field, Schema, TimeUnit}; use indexmap::IndexMap; use serde::Serialize; -use crate::config::{GroupRollupMode, Scalar, ViewConfig}; +use crate::config::{ColumnType, GroupRollupMode, Scalar, ViewConfig}; /// An Arrow column builder, used during the population phase of /// [`VirtualDataSlice`]. @@ -93,16 +93,15 @@ pub enum VirtualDataCell { #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub enum RowPathStyle { - /// Legacy: emit a single `__ROW_PATH__` sidecar (per-row nested - /// array in `render_to_rows`, array-of-arrays in - /// `render_to_columns_json`). `__ROW_PATH_N__` per-level columns - /// are filtered out. Matches the native engine's `to_json` / - /// `to_columns` shape. + /// Emit a single `__ROW_PATH__` sidecar (per-row nested array in + /// `render_to_rows`, array-of-arrays in `render_to_columns_json`) and + /// filter out the `__ROW_PATH_N__` per-level columns, matching the native + /// engine's `to_json`, `to_columns` and `to_ndjson` shapes. Sidecar, - /// Native: emit per-level `__ROW_PATH_0__`, `__ROW_PATH_1__`, … - /// columns directly. No `__ROW_PATH__` sidecar. Matches the native - /// engine's Arrow IPC, CSV, and NDJSON shapes. + /// Emit one column per `group_by` level named after its column, with an + /// aggregate of a `group_by` column qualified by its aggregate, matching + /// the native engine's `to_csv` shape. PerLevel, } @@ -317,6 +316,35 @@ fn cast_to_int64(array: &ArrayRef) -> Result, Box> { } /// Extracts a single cell from a *coerced* Arrow array as a [`Scalar`]. +fn column_type_of(data_type: &DataType) -> ColumnType { + match data_type { + DataType::Boolean => ColumnType::Boolean, + DataType::Int8 + | DataType::Int16 + | DataType::Int32 + | DataType::Int64 + | DataType::UInt8 + | DataType::UInt16 + | DataType::UInt32 + | DataType::UInt64 => ColumnType::Integer, + DataType::Float16 + | DataType::Float32 + | DataType::Float64 + | DataType::Decimal128(..) + | DataType::Decimal256(..) => ColumnType::Float, + DataType::Date32 | DataType::Date64 => ColumnType::Date, + DataType::Timestamp(..) => ColumnType::Datetime, + _ => ColumnType::String, + } +} + +fn row_path_level(name: &str) -> Option { + name.strip_prefix("__ROW_PATH_")? + .strip_suffix("__")? + .parse() + .ok() +} + fn extract_scalar(array: &ArrayRef, row_idx: usize) -> Scalar { if array.is_null(row_idx) { return Scalar::Null; @@ -641,7 +669,7 @@ impl VirtualDataSlice { /// viewer-charts to drive its categorical/numeric axis resolvers /// and tree-hierarchy walkers) see them inline — matching the /// native `perspective-server`'s `to_arrow` output when - /// `emit_legacy_row_path_names: false`. + /// `machine_column_names: true`. pub fn from_arrow_ipc(&mut self, ipc: &[u8]) -> Result<(), Box> { let cursor = std::io::Cursor::new(ipc); let (ipc_schema, batches) = if &ipc[0..6] == "ARROW1".as_bytes() { @@ -821,10 +849,9 @@ impl VirtualDataSlice { /// Converts the columnar data to a row-oriented representation for JSON /// serialization. /// - /// `style` selects between the legacy `__ROW_PATH__` sidecar - /// (`Sidecar`, used by `to_json`) and the native per-level - /// `__ROW_PATH_N__` columns (`PerLevel`, used by `to_csv` / - /// `to_ndjson`). See [`RowPathStyle`] for the deprecation plan. + /// `style` selects between the `__ROW_PATH__` sidecar (`Sidecar`, used + /// by `to_json` and `to_ndjson`) and one readable key column per + /// `group_by` level (`PerLevel`, used by `to_csv`), see [`RowPathStyle`]. pub(crate) fn render_to_rows( &mut self, style: RowPathStyle, @@ -840,6 +867,51 @@ impl VirtualDataSlice { .iter() .any(|x| x.name().starts_with("__ROW_PATH_")); + let key_types: IndexMap<&str, ColumnType> = self + .config + .group_by + .iter() + .enumerate() + .map(|(level, col)| { + let column_type = schema + .field_with_name(&format!("__ROW_PATH_{}__", level)) + .ok() + .map(|field| column_type_of(field.data_type())) + .or_else(|| { + self.row_path + .as_ref()? + .iter() + .find_map(|path| match path.get(level) { + Some(Scalar::Float(_)) => Some(ColumnType::Float), + Some(Scalar::Bool(_)) => Some(ColumnType::Boolean), + Some(Scalar::String(_)) => Some(ColumnType::String), + _ => None, + }) + }) + .unwrap_or(ColumnType::String); + + (col.as_str(), column_type) + }) + .collect(); + + let names: Vec = schema + .fields() + .iter() + .map(|field| { + let name = field.name(); + if style != RowPathStyle::PerLevel { + return name.clone(); + } + + match row_path_level(name).and_then(|level| self.config.group_by.get(level)) { + Some(key) => key.clone(), + None => self.config.readable_column_path(name, |leaf| { + key_types.get(leaf).copied().unwrap_or(ColumnType::String) + }), + } + }) + .collect(); + (0..num_rows) .map(|row_idx| { let mut row = IndexMap::new(); @@ -857,16 +929,13 @@ impl VirtualDataSlice { && let Some(ref rp) = self.row_path && row_idx < rp.len() { - for level in 0..self.config.group_by.len() { - row.insert( - format!("__ROW_PATH_{}__", level), - match rp[row_idx].get(level) { - Some(Scalar::String(x)) => VirtualDataCell::String(Some(x.clone())), - Some(Scalar::Float(x)) => VirtualDataCell::Float(Some(*x)), - Some(Scalar::Bool(x)) => VirtualDataCell::Boolean(Some(*x)), - Some(Scalar::Null) | None => VirtualDataCell::String(None), - }, - ); + for (level, key) in self.config.group_by.iter().enumerate() { + row.insert(key.clone(), match rp[row_idx].get(level) { + Some(Scalar::String(x)) => VirtualDataCell::String(Some(x.clone())), + Some(Scalar::Float(x)) => VirtualDataCell::Float(Some(*x)), + Some(Scalar::Bool(x)) => VirtualDataCell::Boolean(Some(*x)), + Some(Scalar::Null) | None => VirtualDataCell::String(None), + }); } } @@ -948,7 +1017,7 @@ impl VirtualDataSlice { }, } }; - row.insert(field.name().clone(), cell); + row.insert(names[col_idx].clone(), cell); } row diff --git a/rust/perspective-client/src/rust/virtual_server/features.rs b/rust/perspective-client/src/rust/virtual_server/features.rs index dfdd833651..f3e5ad3bf0 100644 --- a/rust/perspective-client/src/rust/virtual_server/features.rs +++ b/rust/perspective-client/src/rust/virtual_server/features.rs @@ -83,6 +83,12 @@ pub struct Features<'a> { #[serde(default)] #[ts(optional, as = "Option<_>")] pub unordered: bool, + + /// Whether a `Table` can be derived from a `View`, which requires + /// [`VirtualServerHandler::view_make_table`](super::VirtualServerHandler::view_make_table). + #[serde(default)] + #[ts(optional, as = "Option<_>")] + pub view_derivations: bool, } /// Specification for a window aggregate. @@ -159,6 +165,7 @@ impl<'a> From> for GetFeaturesResp { on_update: value.on_update, sort: value.sort, unordered: value.unordered, + view_derivations: value.view_derivations, window_aggregates: value .window_aggregates .iter() diff --git a/rust/perspective-client/src/rust/virtual_server/generic_sql_model.rs b/rust/perspective-client/src/rust/virtual_server/generic_sql_model.rs index b068e35f33..69488bb4a9 100644 --- a/rust/perspective-client/src/rust/virtual_server/generic_sql_model.rs +++ b/rust/perspective-client/src/rust/virtual_server/generic_sql_model.rs @@ -58,6 +58,17 @@ pub enum GenericSQLError { UnsupportedOperation(String), } +fn sql_type(ty: ColumnType) -> &'static str { + match ty { + ColumnType::String => "VARCHAR", + ColumnType::Integer => "INTEGER", + ColumnType::Float => "DOUBLE PRECISION", + ColumnType::Boolean => "BOOLEAN", + ColumnType::Date => "DATE", + ColumnType::Datetime => "TIMESTAMP", + } +} + impl fmt::Display for GenericSQLError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { @@ -349,6 +360,94 @@ impl GenericSQLVirtualServerModel { Ok(format!("CREATE {} {} AS ({})", template, view_id, query)) } + /// Returns the SQL query to create a table from a view, with the view's + /// row path unrolled into key columns named after their `group_by` + /// columns and aggregates of those columns qualified by their aggregate. + /// + /// # Arguments + /// * `view_id` - The identifier of the source view. + /// * `table_id` - The identifier for the new table. + /// * `config` - The configuration the view was created with. + /// * `view_schema` - The schema of the view's relation. + /// * `schema` - The columns of the new table, in place of the view's. + /// + /// # Returns + /// SQL: `CREATE TABLE {table_id} AS (SELECT ... FROM {view_id})` + pub fn view_make_table( + &self, + view_id: &str, + table_id: &str, + config: &ViewConfig, + view_schema: &IndexMap, + schema: Option<&IndexMap>, + ) -> GenericSQLResult { + let key_types: IndexMap<&str, ColumnType> = config + .group_by + .iter() + .enumerate() + .filter_map(|(i, col)| { + view_schema + .get(&format!("__ROW_PATH_{}__", i)) + .map(|ty| (col.as_str(), *ty)) + }) + .collect(); + + let mut selectable: IndexMap)> = config + .group_by + .iter() + .enumerate() + .map(|(i, col)| (col.clone(), (format!("\"__ROW_PATH_{}__\"", i), None))) + .collect(); + + for (name, ty) in view_schema + .iter() + .filter(|(name, _)| !name.starts_with("__")) + { + let readable = config.readable_column_path(name, |leaf| { + key_types.get(leaf).copied().unwrap_or(ColumnType::String) + }); + + selectable.insert(readable, (format!("\"{}\"", name), Some(*ty))); + } + + let select = |readable: &str, expr: &str| { + if expr == format!("\"{}\"", readable) { + expr.to_owned() + } else { + format!("{} AS \"{}\"", expr, readable) + } + }; + + let clauses: Vec = match schema { + None => selectable + .iter() + .map(|(readable, (expr, _))| select(readable, expr)) + .collect(), + Some(schema) => schema + .iter() + .map(|(name, ty)| match selectable.get(name) { + Some((_, Some(actual))) if actual != ty => { + Err(GenericSQLError::InvalidConfig(format!( + "Column \"{}\" does not have the type of the View's column", + name + ))) + }, + Some((expr, _)) => Ok(select(name, expr)), + None => Ok(format!("CAST(NULL AS {}) AS \"{}\"", sql_type(*ty), name)), + }) + .collect::>>()?, + }; + + let template = self.0.create_entity.as_deref().unwrap_or("TABLE"); + Ok(format!( + "CREATE {} {} AS (SELECT {} FROM {})", + template, + table_id, + clauses.join(", "), + view_id + )) + } + /// Returns the SQL query to fetch data from a view with the given viewport. /// /// # Arguments diff --git a/rust/perspective-client/src/rust/virtual_server/generic_sql_model/tests.rs b/rust/perspective-client/src/rust/virtual_server/generic_sql_model/tests.rs index 4f27d88a51..c67c314f95 100644 --- a/rust/perspective-client/src/rust/virtual_server/generic_sql_model/tests.rs +++ b/rust/perspective-client/src/rust/virtual_server/generic_sql_model/tests.rs @@ -1419,6 +1419,31 @@ fn test_view_get_min_max_escapes_double_quotes_in_column_name() { ); } +fn view_make_table_schema() -> IndexMap { + IndexMap::from([ + ("__GROUPING_ID__".to_string(), ColumnType::Integer), + ("__ROW_PATH_0__".to_string(), ColumnType::String), + ("Mon|x".to_string(), ColumnType::Float), + ("Tue|x".to_string(), ColumnType::Float), + ]) +} + +#[test] +fn test_view_make_table_flat() { + let builder = GenericSQLVirtualServerModel::new(GenericSQLVirtualServerModelArgs::default()); + let schema = IndexMap::from([ + ("a".to_string(), ColumnType::Integer), + ("b".to_string(), ColumnType::String), + ]); + + assert_eq!( + builder + .view_make_table("v", "t", &ViewConfig::default(), &schema, None) + .unwrap(), + "CREATE TABLE t AS (SELECT \"a\", \"b\" FROM v)" + ); +} + #[test] fn test_table_make_view_escapes_double_quotes_in_column_names() { let builder = GenericSQLVirtualServerModel::new(GenericSQLVirtualServerModelArgs::default()); @@ -1440,6 +1465,20 @@ fn test_table_make_view_escapes_double_quotes_in_column_names() { ); } +#[test] +fn test_view_make_table_unrolls_row_path() { + let builder = GenericSQLVirtualServerModel::new(GenericSQLVirtualServerModelArgs::default()); + let mut config = ViewConfig::default(); + config.group_by = vec!["g".to_string()]; + config.split_by = vec!["s".to_string()]; + assert_eq!( + builder + .view_make_table("v", "t", &config, &view_make_table_schema(), None) + .unwrap(), + "CREATE TABLE t AS (SELECT \"__ROW_PATH_0__\" AS \"g\", \"Mon|x\", \"Tue|x\" FROM v)" + ); +} + #[test] fn test_table_make_view_escapes_double_quotes_in_group_by() { let builder = GenericSQLVirtualServerModel::new(GenericSQLVirtualServerModelArgs::default()); @@ -1503,3 +1542,116 @@ fn test_view_get_data_orders_column_paths_containing_separator() { "SELECT \"a|b|amount\", \"a|b|qty\", \"b|c|amount\", \"b|c|qty\" FROM my_view" ); } + +#[test] +fn test_view_make_table_explicit_schema() { + let builder = GenericSQLVirtualServerModel::new(GenericSQLVirtualServerModelArgs::default()); + let mut config = ViewConfig::default(); + config.group_by = vec!["g".to_string()]; + config.split_by = vec!["s".to_string()]; + let declared = IndexMap::from([ + ("g".to_string(), ColumnType::String), + ("Tue|x".to_string(), ColumnType::Float), + ("Wed|x".to_string(), ColumnType::Float), + ]); + + assert_eq!( + builder + .view_make_table( + "v", + "t", + &config, + &view_make_table_schema(), + Some(&declared) + ) + .unwrap(), + "CREATE TABLE t AS (SELECT \"__ROW_PATH_0__\" AS \"g\", \"Tue|x\", CAST(NULL AS DOUBLE \ + PRECISION) AS \"Wed|x\" FROM v)" + ); +} + +#[test] +fn test_view_make_table_rejects_type_mismatch() { + let builder = GenericSQLVirtualServerModel::new(GenericSQLVirtualServerModelArgs::default()); + let declared = IndexMap::from([("Mon|x".to_string(), ColumnType::String)]); + assert!( + builder + .view_make_table( + "v", + "t", + &ViewConfig::default(), + &view_make_table_schema(), + Some(&declared) + ) + .is_err() + ); +} + +#[test] +fn test_view_make_table_qualifies_aggregate_of_key() { + let builder = GenericSQLVirtualServerModel::new(GenericSQLVirtualServerModelArgs::default()); + let mut config = ViewConfig::default(); + config.group_by = vec!["x".to_string()]; + config.split_by = vec!["s".to_string()]; + config.columns = vec![Some("x".to_string())]; + let view_schema = IndexMap::from([ + ("__GROUPING_ID__".to_string(), ColumnType::Integer), + ("__ROW_PATH_0__".to_string(), ColumnType::Float), + ("Mon|x".to_string(), ColumnType::Float), + ]); + + assert_eq!( + builder + .view_make_table("v", "t", &config, &view_schema, None) + .unwrap(), + "CREATE TABLE t AS (SELECT \"__ROW_PATH_0__\" AS \"x\", \"Mon|x\" AS \"Mon|x (sum)\" FROM \ + v)" + ); + + config.aggregates.insert( + "x".to_string(), + crate::config::Aggregate::from("weighted mean by w"), + ); + + assert_eq!( + builder + .view_make_table("v", "t", &config, &view_schema, None) + .unwrap(), + "CREATE TABLE t AS (SELECT \"__ROW_PATH_0__\" AS \"x\", \"Mon|x\" AS \"Mon|x (weighted \ + mean by w)\" FROM v)" + ); +} + +#[test] +fn test_view_make_table_explicit_schema_addresses_qualified_aggregate() { + let builder = GenericSQLVirtualServerModel::new(GenericSQLVirtualServerModelArgs::default()); + let mut config = ViewConfig::default(); + config.group_by = vec!["x".to_string()]; + config.split_by = vec!["s".to_string()]; + config.columns = vec![Some("x".to_string())]; + let view_schema = IndexMap::from([ + ("__GROUPING_ID__".to_string(), ColumnType::Integer), + ("__ROW_PATH_0__".to_string(), ColumnType::Float), + ("Mon|x".to_string(), ColumnType::Float), + ]); + + let declared = IndexMap::from([ + ("x".to_string(), ColumnType::Float), + ("Mon|x (sum)".to_string(), ColumnType::Float), + ]); + + assert_eq!( + builder + .view_make_table("v", "t", &config, &view_schema, Some(&declared)) + .unwrap(), + "CREATE TABLE t AS (SELECT \"__ROW_PATH_0__\" AS \"x\", \"Mon|x\" AS \"Mon|x (sum)\" FROM \ + v)" + ); + + let mismatched = IndexMap::from([("Mon|x (sum)".to_string(), ColumnType::String)]); + assert!( + builder + .view_make_table("v", "t", &config, &view_schema, Some(&mismatched)) + .is_err() + ); +} diff --git a/rust/perspective-client/src/rust/virtual_server/handler.rs b/rust/perspective-client/src/rust/virtual_server/handler.rs index edf286b914..e3022c21a8 100644 --- a/rust/perspective-client/src/rust/virtual_server/handler.rs +++ b/rust/perspective-client/src/rust/virtual_server/handler.rs @@ -159,6 +159,20 @@ pub trait VirtualServerHandler { // Unused + /// Creates a table from a view, with `schema` in place of the view's + /// columns when given. + /// + /// Default implementation panics with "not implemented". + fn view_make_table( + &mut self, + _view_id: &str, + _table_id: &str, + _config: &ViewConfig, + _schema: Option<&IndexMap>, + ) -> VirtualServerFuture<'_, Result<(), Self::Error>> { + Box::pin(async { unimplemented!("view_make_table not implemented") }) + } + /// Creates a new table with the given data. /// /// Default implementation panics with "not implemented". diff --git a/rust/perspective-client/src/rust/virtual_server/server.rs b/rust/perspective-client/src/rust/virtual_server/server.rs index 43d898ed36..e25b4c6959 100644 --- a/rust/perspective-client/src/rust/virtual_server/server.rs +++ b/rust/perspective-client/src/rust/virtual_server/server.rs @@ -12,7 +12,7 @@ use std::collections::HashMap; -use indexmap::IndexMap; +use indexmap::{IndexMap, IndexSet}; use prost::Message as ProstMessage; use prost::bytes::{Bytes, BytesMut}; @@ -23,14 +23,14 @@ use super::handler::VirtualServerHandler; use crate::config::{ViewConfig, ViewConfigUpdate}; use crate::proto::response::ClientResp; use crate::proto::{ - ColumnType, GetFeaturesResp, GetHostedTablesResp, MakeTableResp, Request, Response, - ServerError, TableDescribeResp, TableMakePortResp, TableMakeViewResp, TableOnDeleteResp, - TableRemoveDeleteResp, TableSchemaResp, TableSizeResp, ViewColumnPathsResp, ViewDeleteResp, - ViewDescription, ViewDimensionsResp, ViewExpressionSchemaResp, ViewGetConfigResp, - ViewGetMinMaxResp, ViewOnDeleteResp, ViewOnRemoveResp, ViewOnUpdateResp, ViewRemoveDeleteResp, - ViewRemoveOnRemoveResp, ViewRemoveOnUpdateResp, ViewSchemaResp, ViewToArrowResp, - ViewToColumnsStringResp, ViewToCsvResp, ViewToNdjsonStringResp, ViewToRowsStringResp, - table_describe_resp, + ColumnPathLevel, ColumnType, GetFeaturesResp, GetHostedTablesResp, MakeTableResp, Request, + Response, Scalar, ServerError, TableDeleteResp, TableDescribeResp, TableMakePortResp, + TableMakeViewResp, TableOnDeleteResp, TableRemoveDeleteResp, TableSchemaResp, TableSizeResp, + ViewColumnPathsResp, ViewDeleteResp, ViewDescription, ViewDimensionsResp, + ViewExpressionSchemaResp, ViewGetConfigResp, ViewGetMinMaxResp, ViewOnDeleteResp, + ViewOnRemoveResp, ViewOnUpdateResp, ViewRemoveDeleteResp, ViewRemoveOnRemoveResp, + ViewRemoveOnUpdateResp, ViewSchemaResp, ViewToArrowResp, ViewToColumnsStringResp, + ViewToCsvResp, ViewToNdjsonStringResp, ViewToRowsStringResp, scalar, table_describe_resp, }; use crate::table::{DescribeError, Description}; @@ -62,6 +62,9 @@ pub struct VirtualServer { view_configs: IndexMap, view_schemas: IndexMap>, + /// The tables made from views, which this server created and may drop. + view_tables: IndexSet, + /// Per-view `table_describe` answers, computed LAZILY on the first /// `ViewExpressionSchemaReq` for that view — never on view creation. view_descriptions: IndexMap, @@ -75,6 +78,7 @@ impl VirtualServer { view_configs: IndexMap::default(), view_to_table: IndexMap::default(), view_schemas: IndexMap::default(), + view_tables: IndexSet::default(), view_descriptions: IndexMap::default(), } } @@ -184,6 +188,17 @@ impl VirtualServer { .insert(req.view_id.clone(), msg.entity_id.clone()); let mut config: ViewConfigUpdate = req.config.clone().unwrap_or_default().into(); + if let Some(group_by) = &config.group_by + && let Some(duplicate) = group_by + .iter() + .enumerate() + .find(|(idx, col)| group_by[..*idx].contains(*col)) + { + return Err(VirtualServerError::Other(format!( + "Duplicate column `{}` in `group_by`", + duplicate.1 + ))); + } // An UNORDERED store has no natural row order to fall back // on, so every window must carry an explicit `order_by`. @@ -338,9 +353,48 @@ impl VirtualServer { .end_col .map_or(paths.len(), |x| x as usize); - let paths = paths.into_iter().take(end).skip(start).collect::>(); + let paths = paths + .into_iter() + .skip(start) + .take(end.saturating_sub(start)) + .collect::>(); + + let depth = config.split_by.len() + 1; + let mut area = vec![ + ColumnPathLevel { + values: Vec::with_capacity(paths.len()) + }; + depth + ]; + + for path in paths { + let mut split = path.splitn(depth, '|'); + let mut levels = Vec::with_capacity(depth); + for _ in 0..depth { + match split.next() { + Some(x) => levels.push(Some(x.to_owned())), + None => levels.push(None), + } + } + + let name = levels + .iter() + .rposition(|x| x.is_some()) + .map(|i| levels[i].take().unwrap()) + .unwrap_or_default(); + + for (level, value) in area.iter_mut().zip(levels).take(depth - 1) { + level.values.push(Scalar { + scalar: value.map(scalar::Scalar::String), + }); + } + + area[depth - 1].values.push(Scalar { + scalar: Some(scalar::Scalar::String(name)), + }); + } - respond!(msg, ViewColumnPathsResp { paths }) + respond!(msg, ViewColumnPathsResp { area }) }, ViewToArrowReq(view_to_arrow_req) => { let viewport = view_to_arrow_req.viewport.unwrap(); @@ -394,7 +448,7 @@ impl VirtualServer { .view_get_data(msg.entity_id.as_str(), config, &schema, &viewport) .await?; - let rows = cols.render_to_rows(RowPathStyle::PerLevel); + let rows = cols.render_to_rows(RowPathStyle::Sidecar); let ndjson_string = rows .iter() .map(serde_json::to_string) @@ -445,9 +499,47 @@ impl VirtualServer { respond!(msg, ViewDeleteResp {}) }, MakeTableReq(req) => { - self.handler - .make_table(&msg.entity_id, req.data.as_ref().unwrap()) - .await?; + let data = req.data.as_ref().unwrap(); + if let Some(crate::proto::make_table_data::Data::FromView(view_id)) = &data.data { + if !self.handler.get_features().await?.view_derivations { + return Err(VirtualServerError::Other( + "This data source cannot derive a Table from a View".to_string(), + )); + } + + let options = req.options.clone().unwrap_or_default(); + if options.make_table_type.is_some() { + return Err(VirtualServerError::Other( + "`index` and `limit` cannot be set on a Table made from a View" + .to_string(), + )); + } + + let config = self.view_configs.get(view_id).ok_or_else(|| { + VirtualServerError::Other(format!("View \"{}\" does not exist", view_id)) + })?; + + let schema = options + .view_schema + .map(|schema| { + schema + .schema + .into_iter() + .map(|x| Ok((x.name, ColumnType::try_from(x.r#type)?))) + .collect::, prost::DecodeError>>() + }) + .transpose() + .map_err(|e| VirtualServerError::Other(e.to_string()))?; + + self.handler + .view_make_table(view_id, &msg.entity_id, config, schema.as_ref()) + .await?; + + self.view_tables.insert(msg.entity_id.clone()); + } else { + self.handler.make_table(&msg.entity_id, data).await?; + } + respond!(msg, MakeTableResp {}) }, ViewGetMinMaxReq(req) => { @@ -462,6 +554,12 @@ impl VirtualServer { }) }, + TableDeleteReq(_) if self.view_tables.contains(&msg.entity_id) => { + self.handler.view_delete(msg.entity_id.as_str()).await?; + self.view_tables.shift_remove(&msg.entity_id); + respond!(msg, TableDeleteResp {}) + }, + // Stub implementations for callback/update requests that VirtualServer doesn't support TableOnDeleteReq(_) => { respond!(msg, TableOnDeleteResp {}) diff --git a/rust/perspective-js/Cargo.toml b/rust/perspective-js/Cargo.toml index 31af11cc3f..5c0760a5eb 100644 --- a/rust/perspective-js/Cargo.toml +++ b/rust/perspective-js/Cargo.toml @@ -61,9 +61,6 @@ trace-allocator = [] serde_json = { version = "1.0.107", features = ["raw_value"] } anyhow = "1.0.66" -[dev-dependencies] -wasm-bindgen-test = "0.3.13" - [dependencies] perspective-client = { version = "5.5.1", features = ["sendable"] } arrow-array = { version = "57.3.0", default-features = false } diff --git a/rust/perspective-js/src/rust/generic_sql_model.rs b/rust/perspective-js/src/rust/generic_sql_model.rs index ce8ce57dd2..27c9e5848f 100644 --- a/rust/perspective-js/src/rust/generic_sql_model.rs +++ b/rust/perspective-js/src/rust/generic_sql_model.rs @@ -166,6 +166,31 @@ impl GenericSQLVirtualServerModel { .map_err(|e| JsValue::from_str(&e.to_string())) } + /// Returns the SQL query to create a table from a view. + #[wasm_bindgen(js_name = "viewMakeTable")] + pub fn view_make_table( + &self, + view_id: &str, + table_id: &str, + config: JsValue, + view_schema: JsValue, + schema: JsValue, + ) -> Result { + let config: ViewConfig = serde_wasm_bindgen::from_value(config) + .map_err(|e| JsValue::from_str(&e.to_string()))?; + + let view_schema = self.parse_schema(view_schema)?; + let schema = if schema.is_undefined() || schema.is_null() { + None + } else { + Some(self.parse_schema(schema)?) + }; + + self.inner + .view_make_table(view_id, table_id, &config, &view_schema, schema.as_ref()) + .map_err(|e| JsValue::from_str(&e.to_string())) + } + /// Returns the SQL query to fetch data from a view with the given viewport. #[wasm_bindgen(js_name = "viewGetData")] pub fn view_get_data( diff --git a/rust/perspective-js/src/rust/view.rs b/rust/perspective-js/src/rust/view.rs index 1031971993..7572690eaa 100644 --- a/rust/perspective-js/src/rust/view.rs +++ b/rust/perspective-js/src/rust/view.rs @@ -94,12 +94,21 @@ impl View { self.0.name.clone() } - /// Returns an array of strings containing the column paths of the [`View`] - /// without any of the source columns. + /// Returns this [`View`]'s column header area for `window`, transposed as + /// `area[level][column]`. /// /// A column path shows the columns that a given cell belongs to after - /// pivots are applied. - #[wasm_bindgen] + /// pivots are applied. The area is rectangular with one level per + /// `split_by` plus one: a column's split values occupy the leading levels + /// and its name always occupies the last, so a subtotal or grand total + /// under `split_rollup_mode: "rollup"` reads `null` at the levels it does + /// not pivot on. Values keep their column's type - a `datetime` split value + /// is epoch milliseconds, not text - so formatting them is the caller's + /// choice. + /// + /// `window` slices the *column* axis; the number of levels does not depend + /// on it. + #[wasm_bindgen(unchecked_return_type = "(string | number | boolean | null)[][]")] pub async fn column_paths(&self, window: Option) -> ApiResult { let window = window.into_serde_ext::>()?; let columns = self.0.column_paths(window.unwrap_or_default()).await?; @@ -290,7 +299,7 @@ impl View { let float32 = opts.float32; let mut view_window: ViewWindow = opts.into(); - view_window.emit_legacy_row_path_names = Some(false); + view_window.machine_column_names = Some(true); let arrow = self.0.to_arrow(view_window).await?; crate::typed_array::decode_and_call(&arrow, float32, &callback).await?; Ok(()) diff --git a/rust/perspective-js/src/rust/virtual_server.rs b/rust/perspective-js/src/rust/virtual_server.rs index 8aeb60acfb..3ed3fdd3b4 100644 --- a/rust/perspective-js/src/rust/virtual_server.rs +++ b/rust/perspective-js/src/rust/virtual_server.rs @@ -183,6 +183,12 @@ export interface VirtualServerHandler { tableId: string, data: string | Uint8Array, ): void | Promise; + viewMakeTable?( + viewId: string, + tableId: string, + config: ViewConfig, + schema?: Record, + ): void | Promise; } "#; @@ -544,6 +550,55 @@ impl VirtualServerHandler for JsServerHandler { }) } + fn view_make_table( + &mut self, + view_id: &str, + table_id: &str, + config: &perspective_client::config::ViewConfig, + schema: Option<&IndexMap>, + ) -> HandlerFuture> { + let has_method = Reflect::get(&self.0, &JsValue::from_str("viewMakeTable")) + .map(|val| !val.is_undefined()) + .unwrap_or(false); + + if !has_method { + return Box::pin(async { + Err(JsError(JsValue::from_str("viewMakeTable not implemented"))) + }); + } + + let handler = self.0.clone(); + let view_id = view_id.to_string(); + let table_id = table_id.to_string(); + let config_value = JsValue::from_serde_ext(config); + let schema_value = schema.map(|schema| { + let object = Object::new(); + for (name, ty) in schema { + let _ = Reflect::set( + &object, + &JsValue::from_str(name), + &JsValue::from_str(&ty.to_string()), + ); + } + + JsValue::from(object) + }); + + Box::pin(async move { + let this = JsServerHandler(handler); + let args = Array::new(); + args.push(&JsValue::from_str(&view_id)); + args.push(&JsValue::from_str(&table_id)); + args.push(&config_value?); + if let Some(schema_value) = schema_value { + args.push(&schema_value); + } + + this.call_method_js_async("viewMakeTable", &args).await?; + Ok(()) + }) + } + fn make_table( &mut self, table_id: &str, diff --git a/rust/perspective-js/src/ts/virtual_servers/duckdb.ts b/rust/perspective-js/src/ts/virtual_servers/duckdb.ts index 68a5bbe3fa..7b427d2305 100644 --- a/rust/perspective-js/src/ts/virtual_servers/duckdb.ts +++ b/rust/perspective-js/src/ts/virtual_servers/duckdb.ts @@ -299,6 +299,7 @@ export class DuckDBHandler implements perspective.VirtualServerHandler { split_by: true, sort: true, expressions: true, + view_derivations: true, window_aggregates: { // `ema` is recursive and has no SQL window translation. integer: WINDOW_AGGREGATES, @@ -485,6 +486,23 @@ export class DuckDBHandler implements perspective.VirtualServerHandler { return { expression_schema, expression_errors }; } + async viewMakeTable( + viewId: string, + tableId: string, + config: ViewConfig, + schema?: Record, + ) { + const query = this.sqlBuilder.viewMakeTable( + viewId, + tableId, + config, + await this.tableSchema(viewId), + schema, + ); + + await runQuery(this.db, query); + } + async viewDelete(viewId: string) { const query = this.sqlBuilder.viewDelete(viewId); await runQuery(this.db, query); diff --git a/rust/perspective-js/src/ts/wasm/engine.ts b/rust/perspective-js/src/ts/wasm/engine.ts index cee59e972f..49dad4b5e7 100644 --- a/rust/perspective-js/src/ts/wasm/engine.ts +++ b/rust/perspective-js/src/ts/wasm/engine.ts @@ -247,7 +247,8 @@ async function convert_typed_array_to_pointer( * * @param callback A callback to which is passed the responses. THe responses * must be fully processed or copied before the callback returns, as it - * references memory on the wasm stack. + * references memory on the wasm stack, which a `callback` that allocates may + * also detach - hence the message descriptors are read before dispatching. */ async function decode_api_responses( core: MainModule, @@ -255,28 +256,32 @@ async function decode_api_responses( callback: (_: ApiResponse) => Promise, ) { const is_64 = core._psp_is_memory64(); + const response_ptr = is_64 ? Number(ptr) : Number(ptr) >>> 0; const response = new DataView( core.HEAPU8.buffer, - is_64 ? Number(ptr) : Number(ptr) >>> 0, + response_ptr, is_64 ? 12 : 8, ); const num_msgs = response.getUint32(0, true); - const msgs_ptr = is_64 - ? response.getBigInt64(4, true) - : response.getUint32(4, true); + const msgs_ptr = Number( + is_64 ? response.getBigInt64(4, true) : response.getUint32(4, true), + ); const messages = new DataView( core.HEAPU8.buffer, - Number(msgs_ptr), + msgs_ptr, num_msgs * (is_64 ? 16 : 12), ); - try { - for (let i = 0; i < num_msgs; i++) { - const [data_ptr, data_len, client_id] = is_64 + const free = (x: number) => core._psp_free((is_64 ? BigInt(x) : x) as any); + + const descriptors: [number, number, number][] = []; + for (let i = 0; i < num_msgs; i++) { + descriptors.push( + is_64 ? [ - messages.getBigInt64(i * 16, true), + Number(messages.getBigInt64(i * 16, true)), messages.getInt32(i * 16 + 8, true), messages.getInt32(i * 16 + 12, true), ] @@ -284,35 +289,21 @@ async function decode_api_responses( messages.getUint32(i * 12, true), messages.getUint32(i * 12 + 4, true), messages.getInt32(i * 12 + 8, true), - ]; - - const data = new Uint8Array( - core.HEAPU8.buffer, - Number(data_ptr), - data_len, - ); + ], + ); + } - const resp = { client_id, data }; - await callback(resp); + try { + for (const [data_ptr, data_len, client_id] of descriptors) { + const data = new Uint8Array(core.HEAPU8.buffer, data_ptr, data_len); + await callback({ client_id, data }); } } finally { - for (let i = 0; i < num_msgs; i++) { - const data_ptr = is_64 - ? messages.getBigInt64(i * 16, true) - : messages.getInt32(i * 12, true); - - core._psp_free(data_ptr as any); + for (const [data_ptr] of descriptors) { + free(data_ptr); } - core._psp_free( - is_64 - ? (BigInt(messages.byteOffset) as any as number) - : (messages.byteOffset as any), - ); - core._psp_free( - is_64 - ? (BigInt(response.byteOffset) as any as number) - : (response.byteOffset as any), - ); + free(msgs_ptr); + free(response_ptr); } } diff --git a/rust/perspective-js/test/js/aggregates.spec.js b/rust/perspective-js/test/js/aggregates.spec.js index 4c565a73b2..b5cabacb25 100644 --- a/rust/perspective-js/test/js/aggregates.spec.js +++ b/rust/perspective-js/test/js/aggregates.spec.js @@ -12,6 +12,7 @@ import { test, expect } from "@perspective-dev/test"; import perspective from "./perspective_client"; +import { joined_column_paths } from "./column_paths.js"; var data = [ { x: 1, y: "a", z: true }, @@ -151,7 +152,7 @@ const std = (nums) => { y: "last", }, }); - const paths = await view.column_paths(); + const paths = await joined_column_paths(view); expect(paths).toEqual(["y", "z"]); const answer = [ { __ROW_PATH__: [], y: "c", z: true }, @@ -219,7 +220,7 @@ const std = (nums) => { x: "count", }, }); - const paths = await view.column_paths(); + const paths = await joined_column_paths(view); expect(paths).toEqual(["y", "z"]); const answer = [ { __ROW_PATH__: [], y: 4, z: 4 }, @@ -1544,7 +1545,7 @@ const std = (nums) => { __ROW_PATH__: [[]], x: [""], y: [0], - index: [0], + index: [null], }); await table.update({ diff --git a/rust/perspective-js/test/js/column_paths.ts b/rust/perspective-js/test/js/column_paths.ts new file mode 100644 index 0000000000..f02831c6cf --- /dev/null +++ b/rust/perspective-js/test/js/column_paths.ts @@ -0,0 +1,46 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +/** + * A `View`'s column paths as `"|"`-joined strings, for the many tests whose + * subject is which columns exist in what order rather than the typed values + * `column_paths` now returns. + * + * Tests of `column_paths` itself assert the area directly - this helper is + * deliberately lossy in the same way the old wire format was, so it must not + * be used to pin a split value's type or rendering. + */ +export async function joined_column_paths( + view: { + column_paths( + window?: unknown, + ): Promise<(string | number | boolean | null)[][]>; + }, + window?: unknown, +): Promise { + const area = await view.column_paths(window ?? {}); + const width = area[area.length - 1]?.length ?? 0; + const out: string[] = []; + for (let x = 0; x < width; x++) { + const parts: string[] = []; + for (let level = 0; level < area.length; level++) { + const value = area[level][x]; + if (value !== null && value !== undefined) { + parts.push(String(value)); + } + } + + out.push(parts.join("|")); + } + + return out; +} diff --git a/rust/perspective-js/test/js/constructors.spec.js b/rust/perspective-js/test/js/constructors.spec.js index 8a1053d1ae..7388036918 100644 --- a/rust/perspective-js/test/js/constructors.spec.js +++ b/rust/perspective-js/test/js/constructors.spec.js @@ -842,6 +842,17 @@ function validate_typed_array(typed_array, column_data) { await table.delete(); }); + test("View constructor should reject a duplicate group_by column", async function () { + const table = await perspective.table(int_float_string_data); + await expect( + table.view({ group_by: ["string", "string"] }), + ).rejects.toThrow( + "Abort(): Duplicate column 'string' found in View group_by.\n", + ); + + await table.delete(); + }); + test("Table constructor should throw an exception on await", async function () { let error; try { @@ -961,7 +972,7 @@ function validate_typed_array(typed_array, column_data) { group_by: ["z"], columns: ["x"], }); - var answer = `"z (Group by 1)","x"\n,10\nfalse,6\ntrue,4\n`; + var answer = `"z","x"\n,10\nfalse,6\ntrue,4\n`; let result = await view.to_csv(); expect(result).toEqual(answer); view.delete(); @@ -975,7 +986,7 @@ function validate_typed_array(typed_array, column_data) { split_by: ["y"], columns: ["x"], }); - var answer = `"z (Group by 1)","a|x","b|x","c|x","d|x"\n,1,2,3,4\nfalse,,2,,4\ntrue,1,,3,\n`; + var answer = `"z","a|x","b|x","c|x","d|x"\n,1,2,3,4\nfalse,,2,,4\ntrue,1,,3,\n`; let result = await view.to_csv(); expect(result).toEqual(answer); view.delete(); diff --git a/rust/perspective-js/test/js/delta.spec.js b/rust/perspective-js/test/js/delta.spec.js index 435d0872b3..b2da4f17e8 100644 --- a/rust/perspective-js/test/js/delta.spec.js +++ b/rust/perspective-js/test/js/delta.spec.js @@ -864,8 +864,10 @@ function it_old_behavior(name, capture) { }); view.on_update( async function (updated) { - // underlying data changes, but only total aggregate row is affected - const expected = [{ x: 10, y: 3, z: 2 }]; + const expected = [ + { x: 10, y: 3, z: 2 }, + { x: 5, y: 1, z: 2 }, + ]; await match_delta(perspective, updated.delta, expected); view.delete(); table.delete(); @@ -1253,9 +1255,11 @@ function it_old_behavior(name, capture) { }); view.on_update( async function (updated) { - // underlying data changes, but only total aggregate row is affected - const expected = await view.to_json(); - expected.splice(3, 1); + const expected = (await view.to_json()).filter( + (d) => + d["__ROW_PATH__"].length === 0 || + d["__ROW_PATH__"][0] !== "c", + ); expected.map((d) => { delete d["__ROW_PATH__"]; }); @@ -1285,9 +1289,11 @@ function it_old_behavior(name, capture) { }); view.on_update( async function (updated) { - // underlying data changes, but only total aggregate row is affected - const expected = await view.to_json(); - expected.splice(3, 1); + const expected = (await view.to_json()).filter( + (d) => + d["__ROW_PATH__"].length === 0 || + d["__ROW_PATH__"][0] !== "c", + ); expected.map((d) => { delete d["__ROW_PATH__"]; }); diff --git a/rust/perspective-js/test/js/derived/backing.spec.ts b/rust/perspective-js/test/js/derived/backing.spec.ts new file mode 100644 index 0000000000..5827eebe8e --- /dev/null +++ b/rust/perspective-js/test/js/derived/backing.spec.ts @@ -0,0 +1,65 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +import { test } from "@perspective-dev/test"; +import perspective from "../perspective_client.ts"; +import { expect_flat_parity, expect_tree_parity } from "./oracle.ts"; +import { SCHEMA, rows } from "./fixtures.ts"; + +const TREE = { + group_by: ["g", "h"], + columns: ["x", "y", "h"], + aggregates: { x: "sum", y: "mean", h: "last" }, +}; + +test.describe("Derived tables over a disk-backed source", function () { + test("group_by parent", async function () { + const source = await perspective.table(SCHEMA, { + index: "id", + page_to_disk: true, + }); + + await source.update(rows(24)); + const view = await source.view(TREE); + const derived = await perspective.table(view); + await expect_tree_parity(source, TREE, derived, view); + await source.update(rows(12, 18)); + await source.remove([0, 1, 2]); + await expect_tree_parity(source, TREE, derived, view); + await derived.delete(); + await view.delete(); + await source.delete(); + }); + + test("flat parent", async function () { + const source = await perspective.table(SCHEMA, { + index: "id", + page_to_disk: true, + }); + + await source.update(rows(24)); + const view = await source.view({ + expressions: { double: '"x" * 2' }, + filter: [["y", ">", 3]], + }); + + const derived = await perspective.table(view); + await expect_flat_parity(view, derived); + await source.update(rows(12, 18)); + await source.update([{ id: 0, y: 11 }]); + await source.remove([5]); + await expect_flat_parity(view, derived); + await derived.delete(); + await view.delete(); + await source.delete(); + }); +}); diff --git a/rust/perspective-js/test/js/derived/chain.spec.ts b/rust/perspective-js/test/js/derived/chain.spec.ts new file mode 100644 index 0000000000..204544f199 --- /dev/null +++ b/rust/perspective-js/test/js/derived/chain.spec.ts @@ -0,0 +1,165 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +import { test, expect } from "@perspective-dev/test"; +import perspective from "../perspective_client.ts"; +import { expect_flat_parity, expect_tree_parity } from "./oracle.ts"; +import { make_source, rows } from "./fixtures.ts"; + +const TREE = { + group_by: ["g", "h"], + columns: ["x", "y"], + aggregates: { x: "sum", y: "sum" }, + group_rollup_mode: "flat", +}; + +test.describe("Chained derived tables", function () { + test("tree then tree", async function () { + const source = await make_source(); + const v1 = await source.view(TREE); + const d1 = await perspective.table(v1); + const config2 = { + group_by: ["g"], + columns: ["x"], + aggregates: { x: "sum" }, + }; + + const v2 = await d1.view(config2); + const d2 = await perspective.table(v2); + await source.update(rows(9, 20)); + await source.remove([0, 1]); + await expect_tree_parity(d1, config2, d2); + const direct = await source.view({ + group_by: ["g"], + columns: ["x"], + aggregates: { x: "sum" }, + group_rollup_mode: "flat", + }); + + const c2 = await d2.view({ + filter: [["g", "is not null"]], + columns: ["x"], + }); + + expect((await c2.to_columns()).x.sort()).toEqual( + (await direct.to_columns()).x.sort(), + ); + + await c2.delete(); + await direct.delete(); + await d2.delete(); + await v2.delete(); + await d1.delete(); + await v1.delete(); + await source.delete(); + }); + + test("tree then flat", async function () { + const source = await make_source(); + const v1 = await source.view(TREE); + const d1 = await perspective.table(v1); + const v2 = await d1.view({ filter: [["x", ">", 50]] }); + const d2 = await perspective.table(v2); + await source.update(rows(9, 20)); + await source.update([{ id: 0, x: -500 }]); + await expect_flat_parity(v2, d2); + await d2.delete(); + await v2.delete(); + await d1.delete(); + await v1.delete(); + await source.delete(); + }); + + test("flat then tree", async function () { + const source = await make_source(); + const v1 = await source.view({ filter: [["y", ">", 3]] }); + const d1 = await perspective.table(v1); + const v2 = await d1.view(TREE); + const d2 = await perspective.table(v2); + await source.update(rows(9, 20)); + await source.update([{ id: 0, y: 10 }]); + await expect_tree_parity(d1, TREE, d2); + await expect_tree_parity( + source, + { ...TREE, filter: [["y", ">", 3]] }, + d2, + ); + await d2.delete(); + await v2.delete(); + await d1.delete(); + await v1.delete(); + await source.delete(); + }); + + test("flat then flat", async function () { + const source = await make_source(); + const v1 = await source.view({ filter: [["y", ">", 3]] }); + const d1 = await perspective.table(v1); + const v2 = await d1.view({ filter: [["x", ">", 10]] }); + const d2 = await perspective.table(v2); + await source.update(rows(9, 20)); + await source.update([{ id: 0, y: 10, x: 100 }]); + await expect_flat_parity(v2, d2); + const direct = await source.view({ + filter: [ + ["y", ">", 3], + ["x", ">", 10], + ], + }); + + await expect_flat_parity(direct, d2); + await direct.delete(); + await d2.delete(); + await v2.delete(); + await d1.delete(); + await v1.delete(); + await source.delete(); + }); + + test("a join over a derived table updates with its source", async function () { + const source = await make_source(); + const view = await source.view({ + group_by: ["g"], + columns: ["x"], + aggregates: { x: "sum" }, + group_rollup_mode: "flat", + }); + + const derived = await perspective.table(view); + const names = await perspective.table( + [ + { g: "a", label: "Alpha" }, + { g: "b", label: "Beta" }, + { g: "c", label: "Gamma" }, + ], + { index: "g" }, + ); + + const joined = await perspective.join(derived, names, "g"); + + const jv = await joined.view(); + const dv = await derived.view(); + await source.update([{ id: 0, x: 12345 }]); + const want = (await dv.to_json()).map((r: any) => r.x).sort(); + await expect + .poll(async () => (await jv.to_json()).map((r: any) => r.x).sort()) + .toEqual(want); + + await dv.delete(); + await jv.delete(); + await joined.delete(); + await names.delete(); + await derived.delete(); + await view.delete(); + await source.delete(); + }); +}); diff --git a/rust/perspective-js/test/js/derived/child_views.spec.ts b/rust/perspective-js/test/js/derived/child_views.spec.ts new file mode 100644 index 0000000000..9652567956 --- /dev/null +++ b/rust/perspective-js/test/js/derived/child_views.spec.ts @@ -0,0 +1,276 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +import { test, expect } from "@perspective-dev/test"; +import perspective from "../perspective_client.ts"; +import { expect_child_parity } from "./oracle.ts"; +import { make_source, rows, ROLLUPS } from "./fixtures.ts"; + +test.describe("Views on a derived table", function () { + test("the default view shows members only", async function () { + const source = await make_source(); + const view = await source.view({ + group_by: ["g", "h"], + columns: ["x"], + group_rollup_mode: "flat", + }); + + const derived = await perspective.table(view); + const child = await derived.view(); + expect(await child.num_rows()).toEqual(6); + expect((await child.to_json()).length).toEqual(6); + await child.delete(); + await derived.delete(); + await view.delete(); + await source.delete(); + }); + + for (const group_rollup_mode of ROLLUPS) { + test(`re-aggregating a ${group_rollup_mode} parent matches a snapshot`, async function () { + const source = await make_source(); + const config = { + group_by: ["g", "h"], + columns: ["x", "y"], + aggregates: { x: "sum", y: "mean" }, + group_rollup_mode, + }; + + const child_config = { + group_by: ["g"], + columns: ["x", "y"], + aggregates: { x: "sum", y: "mean" }, + }; + + const view = await source.view(config); + const derived = await perspective.table(view); + const live = await derived.view(child_config); + await expect_child_parity( + source, + config, + derived, + child_config, + view, + ); + for (let i = 0; i < 10; i++) { + await source.update(rows(3, i * 5)); + await source.update([{ id: i, x: i * 11, g: "b" }]); + await source.remove([i + 12]); + await expect_child_parity( + source, + config, + derived, + child_config, + view, + live, + ); + } + + await live.delete(); + await derived.delete(); + await view.delete(); + await source.delete(); + }); + } + + test("a flat parent re-aggregates to the direct aggregation", async function () { + const source = await make_source(); + const view = await source.view({ + group_by: ["g", "h"], + columns: ["x"], + aggregates: { x: "sum" }, + group_rollup_mode: "flat", + }); + + const derived = await perspective.table(view); + const child = await derived.view({ + group_by: ["g"], + columns: ["x"], + aggregates: { x: "sum" }, + }); + + const direct = await source.view({ + group_by: ["g"], + columns: ["x"], + aggregates: { x: "sum" }, + }); + + await source.update(rows(10, 20)); + await source.remove([1, 2]); + const a = await child.to_columns(); + const b = await direct.to_columns(); + expect(a.x).toEqual(b.x); + expect(a.__ROW_PATH__).toEqual(b.__ROW_PATH__); + await direct.delete(); + await child.delete(); + await derived.delete(); + await view.delete(); + await source.delete(); + }); + + test("filtering on an aggregate", async function () { + const source = await make_source(); + const view = await source.view({ + group_by: ["g", "h"], + columns: ["x"], + aggregates: { x: "sum" }, + group_rollup_mode: "flat", + }); + + const derived = await perspective.table(view); + const child = await derived.view({ filter: [["x", ">", 60]] }); + const all = await derived.view(); + const expected = (await all.to_json()).filter((r) => r.x > 60); + expect(await child.to_json()).toEqual(expected); + await source.update([{ id: 0, x: 1000 }]); + const expected2 = (await all.to_json()).filter((r) => r.x > 60); + expect(await child.to_json()).toEqual(expected2); + await all.delete(); + await child.delete(); + await derived.delete(); + await view.delete(); + await source.delete(); + }); + + test("an expression over two split columns", async function () { + const source = await make_source(); + const view = await source.view({ + group_by: ["g"], + split_by: ["s"], + columns: ["x"], + aggregates: { x: "sum" }, + group_rollup_mode: "flat", + }); + + const derived = await perspective.table(view); + const child = await derived.view({ + columns: ["Mon|x", "Tue|x", "chg"], + expressions: { chg: '"Tue|x" - "Mon|x"' }, + }); + + await source.update([{ id: 0, x: 500 }]); + const json = await child.to_json(); + for (const row of json) { + expect(row.chg).toBeCloseTo(row["Tue|x"] - row["Mon|x"]); + } + + await child.delete(); + await derived.delete(); + await view.delete(); + await source.delete(); + }); + + test("a diff window across days on a group_by promotion", async function () { + const source = await perspective.table( + [ + { id: 1, p: "A", d: 1, pnl: 110 }, + { id: 2, p: "A", d: 2, pnl: 130 }, + { id: 3, p: "A", d: 3, pnl: 150 }, + { id: 4, p: "B", d: 1, pnl: 50 }, + { id: 5, p: "B", d: 2, pnl: 70 }, + { id: 6, p: "B", d: 3, pnl: 65 }, + ], + { index: "id" }, + ); + + const view = await source.view({ + group_by: ["p", "d"], + columns: ["pnl"], + aggregates: { pnl: "avg" }, + group_rollup_mode: "flat", + }); + + const derived = await perspective.table(view); + const child = await derived.view({ + group_by: ["p"], + split_by: ["d"], + columns: ["chg"], + aggregates: { chg: "sum" }, + group_rollup_mode: "flat", + windows: { + chg: { + column: "pnl", + aggregate: "diff", + partition_by: ["p"], + order_by: ["d", "asc"], + }, + }, + }); + + const before = await child.to_columns(); + expect(before["2|chg"]).toEqual([20, 20]); + expect(before["3|chg"]).toEqual([20, -5]); + await source.update([{ id: 5, pnl: 90 }]); + const after = await child.to_columns(); + expect(after["2|chg"]).toEqual([20, 40]); + expect(after["3|chg"]).toEqual([20, -25]); + await child.delete(); + await derived.delete(); + await view.delete(); + await source.delete(); + }); + + test("on_update fires once per parent update with a row delta", async function () { + const source = await make_source(); + const view = await source.view({ + group_by: ["g"], + columns: ["x"], + aggregates: { x: "sum" }, + group_rollup_mode: "flat", + }); + + const derived = await perspective.table(view); + const child = await derived.view(); + const deltas: any[] = []; + await child.on_update( + async (updated: any) => { + const t = await perspective.table(updated.delta); + const v = await t.view(); + deltas.push(await v.to_json()); + await v.delete(); + await t.delete(); + }, + { mode: "row" }, + ); + + await source.update([{ id: 0, x: 1000 }]); + await expect.poll(() => deltas.length).toEqual(1); + expect(deltas[0]).toHaveLength(1); + expect(deltas[0][0]["g"]).toEqual("a"); + await source.update([{ id: 1, x: 2000 }]); + await expect.poll(() => deltas.length).toEqual(2); + await child.delete(); + await derived.delete(); + await view.delete(); + await source.delete(); + }); + + test("a read straight after a source update is fresh", async function () { + const source = await make_source(); + const view = await source.view({ + group_by: ["g"], + columns: ["x"], + aggregates: { x: "sum" }, + group_rollup_mode: "total", + }); + + const derived = await perspective.table(view); + const child = await derived.view(); + const before = (await child.to_json())[0].x; + await source.update([{ id: 1000, g: "a", x: 5 }]); + expect((await child.to_json())[0].x).toBeCloseTo(before + 5); + expect(await derived.size()).toEqual(1); + await child.delete(); + await derived.delete(); + await view.delete(); + await source.delete(); + }); +}); diff --git a/rust/perspective-js/test/js/derived/explicit_schema.spec.ts b/rust/perspective-js/test/js/derived/explicit_schema.spec.ts new file mode 100644 index 0000000000..ba017c32e0 --- /dev/null +++ b/rust/perspective-js/test/js/derived/explicit_schema.spec.ts @@ -0,0 +1,152 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +import { test, expect } from "@perspective-dev/test"; +import perspective from "../perspective_client.ts"; +import { expect_rejects } from "./oracle.ts"; +import { make_source } from "./fixtures.ts"; + +const CONFIG = { + group_by: ["g"], + split_by: ["s"], + columns: ["x"], + group_rollup_mode: "flat", +}; + +const SCHEMA = { + g: "string", + "Mon|x": "integer", + "Tue|x": "integer", + "Wed|x": "integer", +}; + +async function make() { + return await perspective.table( + [ + { id: 1, g: "a", s: "Mon", x: 1 }, + { id: 2, g: "a", s: "Tue", x: 2 }, + ], + { index: "id" }, + ); +} + +test.describe("Derived table with an explicit schema", function () { + test("a declared column fills in when its split value arrives", async function () { + const source = await make(); + const view = await source.view(CONFIG); + const derived = await perspective.table(view, { schema: SCHEMA }); + expect(await derived.schema()).toEqual(SCHEMA); + const child = await derived.view(); + expect(await child.to_json()).toEqual([ + { g: "a", "Mon|x": 1, "Tue|x": 2, "Wed|x": null }, + ]); + + await source.update([{ id: 3, g: "a", s: "Wed", x: 3 }]); + expect(await child.to_json()).toEqual([ + { g: "a", "Mon|x": 1, "Tue|x": 2, "Wed|x": 3 }, + ]); + + await child.delete(); + await derived.delete(); + await view.delete(); + await source.delete(); + }); + + test("an undeclared view column is dropped", async function () { + const source = await make(); + const view = await source.view(CONFIG); + const derived = await perspective.table(view, { + schema: { g: "string", "Tue|x": "integer" }, + }); + + const child = await derived.view(); + expect(await child.to_json()).toEqual([{ g: "a", "Tue|x": 2 }]); + + await child.delete(); + await derived.delete(); + await view.delete(); + await source.delete(); + }); + + test("key columns may be omitted", async function () { + const source = await make(); + const view = await source.view(CONFIG); + const derived = await perspective.table(view, { + schema: { "Mon|x": "integer" }, + }); + + const child = await derived.view(); + expect(await child.to_json()).toEqual([{ "Mon|x": 1 }]); + await child.delete(); + await derived.delete(); + await view.delete(); + await source.delete(); + }); + + test("a dtype mismatch is rejected", async function () { + const source = await make(); + const view = await source.view(CONFIG); + await expect_rejects( + perspective.table(view, { schema: { "Mon|x": "string" } }), + ); + + await view.delete(); + await source.delete(); + }); + + test("schema without a view source is rejected", async function () { + await expect_rejects( + perspective.table([{ x: 1 }], { schema: { x: "integer" } }), + ); + }); + + test("inferred and explicit schemas are equivalent", async function () { + const source = await make_source(); + const config = { group_by: ["g"], columns: ["x", "y"] }; + const view = await source.view(config); + const inferred = await perspective.table(view); + const explicit = await perspective.table(view, { + schema: await inferred.schema(), + }); + + const a = await inferred.view(); + const b = await explicit.view(); + expect(await b.to_json()).toEqual(await a.to_json()); + await source.update([{ id: 0, x: 500 }]); + expect(await b.to_json()).toEqual(await a.to_json()); + await a.delete(); + await b.delete(); + await explicit.delete(); + await inferred.delete(); + await view.delete(); + await source.delete(); + }); + + test("a flat parent", async function () { + const source = await make_source(); + const view = await source.view({ columns: ["id", "x", "g"] }); + const derived = await perspective.table(view, { + schema: { id: "integer", x: "float", missing: "string" }, + }); + + const child = await derived.view({ columns: ["id", "x"] }); + const parent = await source.view({ columns: ["id", "x"] }); + expect(await child.to_json()).toEqual(await parent.to_json()); + await source.update([{ id: 2, x: 77 }]); + expect(await child.to_json()).toEqual(await parent.to_json()); + await parent.delete(); + await child.delete(); + await derived.delete(); + await view.delete(); + await source.delete(); + }); +}); diff --git a/rust/perspective-viewer/src/rust/utils/browser/tests/mod.rs b/rust/perspective-js/test/js/derived/fixtures.ts similarity index 66% rename from rust/perspective-viewer/src/rust/utils/browser/tests/mod.rs rename to rust/perspective-js/test/js/derived/fixtures.ts index 6edfb458d7..bce8795ff0 100644 --- a/rust/perspective-viewer/src/rust/utils/browser/tests/mod.rs +++ b/rust/perspective-js/test/js/derived/fixtures.ts @@ -10,4 +10,43 @@ // ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ // ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ -mod debounce; +import perspective from "../perspective_client.ts"; + +export const SCHEMA = { + id: "integer", + g: "string", + h: "string", + s: "string", + x: "float", + y: "integer", + w: "float", +}; + +export function rows(n: number, offset = 0) { + const G = ["a", "b", "c"]; + const H = ["p", "q"]; + const S = ["Mon", "Tue", "Wed"]; + const out = []; + for (let i = offset; i < offset + n; i++) { + out.push({ + id: i, + g: G[i % G.length], + h: H[i % H.length], + s: S[Math.floor(i / 3) % S.length], + x: i * 1.5 + 1, + y: (i * 7) % 11, + w: (i % 4) + 1, + }); + } + + return out; +} + +export async function make_source(n = 24) { + const table = await perspective.table(SCHEMA, { index: "id" }); + await table.update(rows(n)); + return table; +} + +export const ROLLUPS = ["rollup", "flat", "total"]; +export const SPLIT_ROLLUPS = ["flat", "rollup"]; diff --git a/rust/perspective-js/test/js/derived/flat_expressions_windows.spec.ts b/rust/perspective-js/test/js/derived/flat_expressions_windows.spec.ts new file mode 100644 index 0000000000..fd6da3d52b --- /dev/null +++ b/rust/perspective-js/test/js/derived/flat_expressions_windows.spec.ts @@ -0,0 +1,67 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +import { test } from "@perspective-dev/test"; +import perspective from "../perspective_client.ts"; +import { expect_flat_parity } from "./oracle.ts"; +import { make_source, rows } from "./fixtures.ts"; + +test.describe("Derived table carries expression and window columns", function () { + test("expression columns update", async function () { + const source = await make_source(); + const view = await source.view({ + columns: ["id", "x", "double", "label"], + expressions: { + double: '"x" * 2', + label: 'concat("g", \'-\', "h")', + }, + }); + + const derived = await perspective.table(view); + await expect_flat_parity(view, derived); + await source.update([ + { id: 0, x: 42, g: "zz" }, + { id: 99, x: 1, g: "a", h: "p" }, + ]); + + await expect_flat_parity(view, derived); + await derived.delete(); + await view.delete(); + await source.delete(); + }); + + test("window columns update, including rows outside the batch", async function () { + const source = await make_source(); + const view = await source.view({ + columns: ["id", "g", "x", "running"], + windows: { + running: { + column: "x", + aggregate: "sum", + partition_by: ["g"], + order_by: ["id", "asc"], + }, + }, + }); + + const derived = await perspective.table(view); + await expect_flat_parity(view, derived); + await source.update([{ id: 0, x: 1000 }]); + await expect_flat_parity(view, derived); + await source.update(rows(6, 24)); + await source.remove([3]); + await expect_flat_parity(view, derived); + await derived.delete(); + await view.delete(); + await source.delete(); + }); +}); diff --git a/rust/perspective-js/test/js/derived/flat_filter_membership.spec.ts b/rust/perspective-js/test/js/derived/flat_filter_membership.spec.ts new file mode 100644 index 0000000000..79f59c4180 --- /dev/null +++ b/rust/perspective-js/test/js/derived/flat_filter_membership.spec.ts @@ -0,0 +1,90 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +import { test, expect } from "@perspective-dev/test"; +import perspective from "../perspective_client.ts"; +import { expect_flat_parity } from "./oracle.ts"; +import { make_source } from "./fixtures.ts"; + +test.describe("Derived table membership follows the parent's filter", function () { + test("a row updated into the filter appears", async function () { + const source = await make_source(); + const view = await source.view({ filter: [["y", ">", 8]] }); + const derived = await perspective.table(view); + const size = await derived.size(); + await source.update([{ id: 0, y: 10 }]); + expect(await derived.size()).toEqual(size + 1); + await expect_flat_parity(view, derived); + await derived.delete(); + await view.delete(); + await source.delete(); + }); + + test("a row updated out of the filter disappears", async function () { + const source = await make_source(); + const view = await source.view({ filter: [["y", ">", 8]] }); + const derived = await perspective.table(view); + const size = await derived.size(); + const inside = (await view.to_json())[0].id; + await source.update([{ id: inside, y: 0 }]); + expect(await derived.size()).toEqual(size - 1); + await expect_flat_parity(view, derived); + await derived.delete(); + await view.delete(); + await source.delete(); + }); + + test("removing a row outside the filter is a no-op", async function () { + const source = await make_source(); + const view = await source.view({ filter: [["y", ">", 8]] }); + const derived = await perspective.table(view); + const size = await derived.size(); + await source.remove([0]); + expect(await derived.size()).toEqual(size); + await expect_flat_parity(view, derived); + await derived.delete(); + await view.delete(); + await source.delete(); + }); + + test("removing a row inside the filter removes it", async function () { + const source = await make_source(); + const view = await source.view({ filter: [["y", ">", 8]] }); + const derived = await perspective.table(view); + const inside = (await view.to_json())[0].id; + await source.remove([inside]); + await expect_flat_parity(view, derived); + await derived.delete(); + await view.delete(); + await source.delete(); + }); + + test("a filter on an expression column", async function () { + const source = await make_source(); + const view = await source.view({ + expressions: { big: '"x" * 2' }, + filter: [["big", ">", 30]], + }); + + const derived = await perspective.table(view); + await expect_flat_parity(view, derived); + await source.update([ + { id: 0, x: 100 }, + { id: 23, x: 0 }, + ]); + + await expect_flat_parity(view, derived); + await derived.delete(); + await view.delete(); + await source.delete(); + }); +}); diff --git a/rust/perspective-js/test/js/derived/flat_keyed_updates.spec.ts b/rust/perspective-js/test/js/derived/flat_keyed_updates.spec.ts new file mode 100644 index 0000000000..0900485e57 --- /dev/null +++ b/rust/perspective-js/test/js/derived/flat_keyed_updates.spec.ts @@ -0,0 +1,70 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +import { test, expect } from "@perspective-dev/test"; +import perspective from "../perspective_client.ts"; +import { expect_flat_parity } from "./oracle.ts"; +import { make_source, SCHEMA, rows } from "./fixtures.ts"; + +test.describe("Derived table updates from a flat view", function () { + test("a view omitting the index column updates in place", async function () { + const source = await make_source(); + const view = await source.view({ columns: ["x", "g"] }); + const derived = await perspective.table(view); + for (let i = 0; i < 10; i++) { + await source.update([{ id: 3, x: i }]); + } + + expect(await derived.size()).toEqual(24); + await expect_flat_parity(view, derived); + await derived.delete(); + await view.delete(); + await source.delete(); + }); + + test("an unindexed source appends", async function () { + const source = await perspective.table(SCHEMA); + const view = await source.view(); + const derived = await perspective.table(view); + await source.update(rows(5)); + await source.update(rows(5)); + expect(await derived.size()).toEqual(10); + await expect_flat_parity(view, derived); + await derived.delete(); + await view.delete(); + await source.delete(); + }); + + test("partial updates and nulls", async function () { + const source = await make_source(); + const view = await source.view(); + const derived = await perspective.table(view); + await source.update([{ id: 1, x: null }]); + await source.update([{ id: 2, g: "new" }]); + await expect_flat_parity(view, derived); + await derived.delete(); + await view.delete(); + await source.delete(); + }); + + test("removes mirror", async function () { + const source = await make_source(); + const view = await source.view(); + const derived = await perspective.table(view); + await source.remove([1, 2, 3]); + expect(await derived.size()).toEqual(21); + await expect_flat_parity(view, derived); + await derived.delete(); + await view.delete(); + await source.delete(); + }); +}); diff --git a/rust/perspective-viewer/src/rust/utils/tests/request_animation_frame.rs b/rust/perspective-js/test/js/derived/flat_limit.spec.ts similarity index 69% rename from rust/perspective-viewer/src/rust/utils/tests/request_animation_frame.rs rename to rust/perspective-js/test/js/derived/flat_limit.spec.ts index 7a51b502dc..439e31769e 100644 --- a/rust/perspective-viewer/src/rust/utils/tests/request_animation_frame.rs +++ b/rust/perspective-js/test/js/derived/flat_limit.spec.ts @@ -10,33 +10,23 @@ // ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ // ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ -use std::cell::*; -use std::rc::*; +import { test, expect } from "@perspective-dev/test"; +import perspective from "../perspective_client.ts"; +import { expect_flat_parity } from "./oracle.ts"; +import { SCHEMA, rows } from "./fixtures.ts"; -use wasm_bindgen_futures::spawn_local; -use wasm_bindgen_test::*; - -use crate::utils::*; - -#[wasm_bindgen_test] -async fn test_request_animation_frame_async() { - // Merely test that this Promise resolves at all .. - request_animation_frame().await; -} - -#[wasm_bindgen_test] -async fn test_async_in_correct_order() { - let cell = Rc::new(RefCell::new(vec![])); - spawn_local({ - clone!(cell); - async move { - request_animation_frame().await; - cell.borrow_mut().push("1"); - } +test.describe("Derived table from a limit table's view", function () { + test("wrap-around overwrites in place", async function () { + const source = await perspective.table(SCHEMA, { limit: 5 }); + const view = await source.view(); + const derived = await perspective.table(view); + await source.update(rows(4)); + await expect_flat_parity(view, derived); + await source.update(rows(4, 4)); + expect(await derived.size()).toEqual(5); + await expect_flat_parity(view, derived); + await derived.delete(); + await view.delete(); + await source.delete(); }); - - request_animation_frame().await; - cell.borrow_mut().push("2"); - request_animation_frame().await; - assert_eq!(vec!["2", "1"], *cell.borrow()); -} +}); diff --git a/rust/perspective-js/test/js/derived/flat_shape.spec.ts b/rust/perspective-js/test/js/derived/flat_shape.spec.ts new file mode 100644 index 0000000000..68b2bc177f --- /dev/null +++ b/rust/perspective-js/test/js/derived/flat_shape.spec.ts @@ -0,0 +1,79 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +import { test, expect } from "@perspective-dev/test"; +import perspective from "../perspective_client.ts"; +import { expect_rejects, expect_flat_parity } from "./oracle.ts"; +import { make_source, SCHEMA } from "./fixtures.ts"; + +test.describe("Derived table shape from a flat view", function () { + test("schema is the view's columns in order", async function () { + const source = await make_source(); + const view = await source.view({ + columns: ["x", "double", "g"], + expressions: { double: '"x" * 2' }, + }); + + const derived = await perspective.table(view); + expect(await derived.columns()).toEqual(["x", "double", "g"]); + expect(await derived.schema()).toEqual({ + x: "float", + double: "float", + g: "string", + }); + + await expect_flat_parity(view, derived); + await derived.delete(); + await view.delete(); + await source.delete(); + }); + + test("inherits the index when its column is in the view", async function () { + const source = await make_source(); + const view = await source.view(); + const derived = await perspective.table(view); + expect(await derived.get_index()).toEqual("id"); + expect(await derived.size()).toEqual(24); + await derived.delete(); + await view.delete(); + await source.delete(); + }); + + test("has no index when its column is not in the view", async function () { + const source = await make_source(); + const view = await source.view({ columns: ["x", "g"] }); + const derived = await perspective.table(view); + expect(await derived.get_index()).toBeUndefined(); + await derived.delete(); + await view.delete(); + await source.delete(); + }); + + test("inherits the limit", async function () { + const source = await perspective.table(SCHEMA, { limit: 5 }); + const view = await source.view(); + const derived = await perspective.table(view); + expect(await derived.get_limit()).toEqual(5); + await derived.delete(); + await view.delete(); + await source.delete(); + }); + + test("rejects index and limit options", async function () { + const source = await make_source(); + const view = await source.view(); + await expect_rejects(perspective.table(view, { index: "x" })); + await expect_rejects(perspective.table(view, { limit: 10 })); + await view.delete(); + await source.delete(); + }); +}); diff --git a/rust/perspective-viewer/src/rust/utils/tests/mod.rs b/rust/perspective-js/test/js/derived/flat_sort_ignored.spec.ts similarity index 65% rename from rust/perspective-viewer/src/rust/utils/tests/mod.rs rename to rust/perspective-js/test/js/derived/flat_sort_ignored.spec.ts index 41a91ea4b9..0a53c5b5a0 100644 --- a/rust/perspective-viewer/src/rust/utils/tests/mod.rs +++ b/rust/perspective-js/test/js/derived/flat_sort_ignored.spec.ts @@ -10,5 +10,26 @@ // ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ // ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ -mod pubsub; -mod request_animation_frame; +import { test, expect } from "@perspective-dev/test"; +import perspective from "../perspective_client.ts"; +import { expect_flat_parity } from "./oracle.ts"; +import { make_source, rows } from "./fixtures.ts"; + +test.describe("Derived table ignores a flat parent's sort", function () { + test("order is the source's insertion order", async function () { + const source = await make_source(); + const view = await source.view({ sort: [["x", "desc"]] }); + const derived = await perspective.table(view); + const unsorted = await source.view(); + const child = await derived.view(); + expect(await child.to_json()).toEqual(await unsorted.to_json()); + await source.update(rows(4, 40)); + expect(await child.to_json()).toEqual(await unsorted.to_json()); + await expect_flat_parity(view, derived); + await child.delete(); + await unsorted.delete(); + await derived.delete(); + await view.delete(); + await source.delete(); + }); +}); diff --git a/rust/perspective-js/test/js/derived/flat_unit_parent.spec.ts b/rust/perspective-js/test/js/derived/flat_unit_parent.spec.ts new file mode 100644 index 0000000000..0411c47d46 --- /dev/null +++ b/rust/perspective-js/test/js/derived/flat_unit_parent.spec.ts @@ -0,0 +1,45 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +import { test, expect } from "@perspective-dev/test"; +import perspective from "../perspective_client.ts"; +import { expect_flat_parity } from "./oracle.ts"; +import { SCHEMA, rows } from "./fixtures.ts"; + +test.describe("Derived table from an unconfigured view", function () { + test("mirrors an unindexed table", async function () { + const source = await perspective.table(SCHEMA); + await source.update(rows(10)); + const view = await source.view(); + const derived = await perspective.table(view); + await expect_flat_parity(view, derived); + await source.update(rows(10, 10)); + expect(await derived.size()).toEqual(20); + await expect_flat_parity(view, derived); + await derived.delete(); + await view.delete(); + await source.delete(); + }); + + test("an empty table fills in", async function () { + const source = await perspective.table(SCHEMA); + const view = await source.view(); + const derived = await perspective.table(view); + expect(await derived.size()).toEqual(0); + await source.update(rows(7)); + expect(await derived.size()).toEqual(7); + await expect_flat_parity(view, derived); + await derived.delete(); + await view.delete(); + await source.delete(); + }); +}); diff --git a/rust/perspective-js/test/js/derived/lifecycle.spec.ts b/rust/perspective-js/test/js/derived/lifecycle.spec.ts new file mode 100644 index 0000000000..d94ba3976a --- /dev/null +++ b/rust/perspective-js/test/js/derived/lifecycle.spec.ts @@ -0,0 +1,122 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +import { test, expect } from "@perspective-dev/test"; +import perspective from "../perspective_client.ts"; +import { + expect_rejects, + expect_flat_parity, + expect_tree_parity, +} from "./oracle.ts"; +import { make_source, SCHEMA, rows } from "./fixtures.ts"; + +const PARENTS: Record = { + flat: { filter: [["y", ">", 2]] }, + group_by: { group_by: ["g", "h"], columns: ["x", "y"] }, + split_by: { group_by: ["g"], split_by: ["s"], columns: ["x"] }, +}; + +async function expect_parity( + source: any, + view: any, + config: any, + derived: any, +) { + if (config.group_by || config.split_by) { + await expect_tree_parity(source, config, derived, view); + } else { + await expect_flat_parity(view, derived); + } +} + +test.describe("Derived table lifecycle", function () { + for (const [name, config] of Object.entries(PARENTS)) { + test(`${name}: parent clear then update`, async function () { + const source = await make_source(); + const view = await source.view(config); + const derived = await perspective.table(view); + await source.clear(); + await expect_parity(source, view, config, derived); + await source.update(rows(8, 3)); + await expect_parity(source, view, config, derived); + await source.update([{ id: 4, x: 99, y: 9 }]); + await expect_parity(source, view, config, derived); + await derived.delete(); + await view.delete(); + await source.delete(); + }); + + test(`${name}: parent clear with no further data`, async function () { + const source = await make_source(); + const view = await source.view(config); + const derived = await perspective.table(view); + const child = await derived.view(); + await source.clear(); + await expect_parity(source, view, config, derived); + await child.delete(); + await derived.delete(); + await view.delete(); + await source.delete(); + }); + + test(`${name}: parent replace`, async function () { + const source = await make_source(); + const view = await source.view(config); + const derived = await perspective.table(view); + await source.replace(rows(5, 50)); + await expect_parity(source, view, config, derived); + await source.update([{ id: 51, x: 3, y: 10 }]); + await expect_parity(source, view, config, derived); + await derived.delete(); + await view.delete(); + await source.delete(); + }); + + test(`${name}: first update into an empty table`, async function () { + const source = await perspective.table(SCHEMA, { index: "id" }); + const view = await source.view(config); + const derived = await perspective.table(view); + expect(await derived.size()).toBeLessThanOrEqual(1); + await source.update(rows(12)); + await expect_parity(source, view, config, derived); + await source.update(rows(12, 6)); + await expect_parity(source, view, config, derived); + await derived.delete(); + await view.delete(); + await source.delete(); + }); + + test(`${name}: deleting a view with a dependent table is rejected`, async function () { + const source = await make_source(); + const view = await source.view(config); + const derived = await perspective.table(view); + await expect_rejects(view.delete()); + await derived.delete(); + }); + + test(`${name}: two derived tables on one view`, async function () { + const source = await make_source(); + const view = await source.view(config); + const a = await perspective.table(view); + const b = await perspective.table(view); + await source.update(rows(6, 20)); + await expect_parity(source, view, config, a); + await expect_parity(source, view, config, b); + await a.delete(); + await source.update([{ id: 1, x: 5 }]); + await expect_parity(source, view, config, b); + await b.delete(); + await view.delete(); + await source.delete(); + }); + } +}); diff --git a/rust/perspective-js/test/js/derived/memory.spec.ts b/rust/perspective-js/test/js/derived/memory.spec.ts new file mode 100644 index 0000000000..7ef94b248a --- /dev/null +++ b/rust/perspective-js/test/js/derived/memory.spec.ts @@ -0,0 +1,122 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +import { test, expect } from "@perspective-dev/test"; +import perspective from "../perspective_client.ts"; + +const GROUPS = 200_000; + +async function make_wide() { + const table = await perspective.table({ + id: "integer", + a: "float", + b: "float", + c: "float", + d: "float", + e: "float", + f: "float", + }); + + const size = GROUPS; + const column = () => Array.from({ length: size }, (_, i) => i * 1.5); + await table.update({ + id: Array.from({ length: size }, (_, i) => i), + a: column(), + b: column(), + c: column(), + d: column(), + e: column(), + f: column(), + }); + + return table; +} + +async function used() { + return Number((await perspective.system_info()).used_size); +} + +const EXTRA = ["b", "c", "d", "e", "f"]; +const EXTRA_BYTES = GROUPS * EXTRA.length * 8; + +async function growth(view: any, schema: Record) { + const before = await used(); + const derived = await perspective.table(view, { schema }); + const after = await used(); + expect(await derived.size()).toEqual(GROUPS); + await derived.delete(); + return after - before; +} + +function floats(names: string[]) { + return Object.fromEntries(names.map((n) => [n, "float"])); +} + +test.describe("Derived table memory", function () { + test("a group_by promotion does not copy its aggregates", async function () { + const source = await make_wide(); + const view = await source.view({ + group_by: ["id"], + columns: ["a", ...EXTRA], + group_rollup_mode: "flat", + }); + + await growth(view, floats(["a"])); + const narrow = await growth(view, floats(["a"])); + const wide = await growth(view, floats(["a", ...EXTRA])); + expect(wide - narrow).toBeLessThan(EXTRA_BYTES * 0.25); + await view.delete(); + await source.delete(); + }); + + test("a flat promotion does not copy its columns", async function () { + const source = await make_wide(); + const view = await source.view({ + columns: ["a", ...EXTRA], + filter: [["id", ">=", 0]], + }); + + await growth(view, floats(["a"])); + const narrow = await growth(view, floats(["a"])); + const wide = await growth(view, floats(["a", ...EXTRA])); + expect(wide - narrow).toBeLessThan(EXTRA_BYTES * 0.25); + await view.delete(); + await source.delete(); + }); + + test("creating and deleting derived tables does not leak", async function () { + const source = await perspective.table( + Array.from({ length: 500 }, (_, i) => ({ id: i, g: i % 7, x: i })), + ); + + const view = await source.view({ group_by: ["g"], columns: ["x"] }); + const cycle = async () => { + const derived = await perspective.table(view); + const child = await derived.view(); + await child.to_columns(); + await child.delete(); + await derived.delete(); + }; + + await cycle(); + await cycle(); + const before = await used(); + for (let i = 0; i < 200; i++) { + await cycle(); + } + + const after = await used(); + expect(after / before).toBeLessThan(1.2); + await view.delete(); + await source.delete(); + }); +}); diff --git a/rust/perspective-js/test/js/derived/oracle.ts b/rust/perspective-js/test/js/derived/oracle.ts new file mode 100644 index 0000000000..a7ac42e5a4 --- /dev/null +++ b/rust/perspective-js/test/js/derived/oracle.ts @@ -0,0 +1,241 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +import { expect } from "@perspective-dev/test"; +import perspective from "../perspective_client.ts"; + +type Columns = Record; +type Row = Record; + +export function key_name(column: string): string { + return column; +} + +function aggregate_name(aggregate: any): string { + return Array.isArray(aggregate) + ? `${aggregate[0]} by ${aggregate[1].join(", ")}` + : aggregate; +} + +export function readable_name( + name: string, + config: any, + schema: Record, +): string { + const split = + (config.split_by ?? []).length > 0 ? name.lastIndexOf("|") : -1; + const prefix = split < 0 ? "" : name.slice(0, split + 1); + const leaf = split < 0 ? name : name.slice(split + 1); + if (!(config.group_by ?? []).includes(leaf)) { + return name; + } + + const aggregate = + config.aggregates?.[leaf] ?? + (["integer", "float"].includes(schema[leaf]) ? "sum" : "count"); + + return `${prefix}${leaf} (${aggregate_name(aggregate)})`; +} + +export function to_rows(columns: Columns): Row[] { + const names = Object.keys(columns); + const size = names.length === 0 ? 0 : columns[names[0]].length; + const rows: Row[] = []; + for (let i = 0; i < size; i++) { + const row: Row = {}; + for (const name of names) { + row[name] = columns[name][i] ?? null; + } + + rows.push(row); + } + + return rows; +} + +function normalize(value: any): any { + if (typeof value === "number" && !Number.isInteger(value)) { + return Number(value.toPrecision(10)); + } + + return value ?? null; +} + +export function sorted(rows: Row[], names: string[]): string[] { + return rows + .map((row) => JSON.stringify(names.map((n) => normalize(row[n])))) + .sort(); +} + +async function unroll( + view: any, + config: any, + schema: Record, +): Promise { + const columns: Columns = await view.to_columns(); + const paths: any[][] = columns.__ROW_PATH__ ?? []; + delete columns.__ROW_PATH__; + for (const name of Object.keys(columns)) { + const readable = readable_name(name, config, schema); + if (readable !== name) { + columns[readable] = columns[name]; + delete columns[name]; + } + } + + const rows = + Object.keys(columns).length === 0 + ? paths.map(() => ({})) + : to_rows(columns); + const group_by: string[] = config.group_by ?? []; + if (group_by.length === 0) { + return rows; + } + + return rows.map((row, i) => { + const out: Row = {}; + group_by.forEach((g, d) => { + out[key_name(g)] = paths[i][d] ?? null; + }); + + return { ...out, ...row }; + }); +} + +export async function tree_oracle( + source: any, + config: any, + live?: any, +): Promise { + const schema = await source.schema(); + if (live) { + return await unroll(live, config, schema); + } + + const { sort, group_by_depth, ...rest } = config; + const view = await source.view(rest); + const rows = await unroll(view, config, schema); + await view.delete(); + return rows; +} + +export async function derived_rows(derived: any): Promise { + const view = await derived.view(); + const columns = await view.to_columns(); + await view.delete(); + return to_rows(columns); +} + +export async function expect_tree_parity( + source: any, + config: any, + derived: any, + live?: any, +) { + const expected = await tree_oracle(source, config, live); + const actual = await derived_rows(derived); + const names = Object.keys(await derived.schema()); + const known = new Set(expected.length > 0 ? Object.keys(expected[0]) : []); + const shared = names.filter((n) => known.has(n)); + for (const name of names.filter((n) => !known.has(n))) { + if (expected.length > 0) { + expect(actual.every((row) => row[name] === null)).toBe(true); + } + } + + expect(sorted(actual, shared)).toEqual(sorted(expected, shared)); +} + +export async function expect_flat_parity(view: any, derived: any) { + const expected = to_rows(await view.to_columns()); + const actual = await derived_rows(derived); + const names = Object.keys(await derived.schema()); + expect(sorted(actual, names)).toEqual(sorted(expected, names)); +} + +async function expect_snapshot_parity( + derived: any, + rows: Row[], + child_config: any, + child?: any, +) { + if (rows.length === 0) { + return; + } + + const snapshot = await perspective.table(await derived.schema()); + await snapshot.update(rows); + const expected_view = await snapshot.view(child_config); + const actual_view = child ?? (await derived.view(child_config)); + const expected = await expected_view.to_columns(); + const actual = await actual_view.to_columns(); + await expected_view.delete(); + if (!child) { + await actual_view.delete(); + } + + await snapshot.delete(); + const names = Object.keys(expected).filter((n) => n !== "__ROW_PATH__"); + const with_path = (columns: Columns) => + to_rows(columns).map((row, i) => ({ + ...row, + __path: JSON.stringify(columns.__ROW_PATH__?.[i] ?? null), + })); + + expect(sorted(with_path(actual), ["__path", ...names])).toEqual( + sorted(with_path(expected), ["__path", ...names]), + ); +} + +export async function expect_child_parity( + source: any, + config: any, + derived: any, + child_config: any, + live?: any, + child?: any, +) { + const rows = await tree_oracle(source, config, live); + await expect_snapshot_parity(derived, rows, child_config, child); +} + +export async function expect_flat_child_parity( + view: any, + derived: any, + child_config: any, + child?: any, +) { + const rows = to_rows(await view.to_columns()); + await expect_snapshot_parity(derived, rows, child_config, child); +} + +export async function expect_rejects(promise: Promise) { + let error; + try { + await promise; + } catch (e) { + error = e; + } + + expect(error).toBeDefined(); +} + +export function mulberry32(seed: number): () => number { + let a = seed; + return function () { + a |= 0; + a = (a + 0x6d2b79f5) | 0; + let t = Math.imul(a ^ (a >>> 15), 1 | a); + t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t; + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; + }; +} diff --git a/rust/perspective-js/test/js/derived/readonly.spec.ts b/rust/perspective-js/test/js/derived/readonly.spec.ts new file mode 100644 index 0000000000..4e24fdf36e --- /dev/null +++ b/rust/perspective-js/test/js/derived/readonly.spec.ts @@ -0,0 +1,40 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +import { test, expect } from "@perspective-dev/test"; +import perspective from "../perspective_client.ts"; +import { expect_rejects } from "./oracle.ts"; +import { make_source } from "./fixtures.ts"; + +const PARENTS = { + flat: {}, + group_by: { group_by: ["g"], columns: ["x"] }, + split_by: { group_by: ["g"], split_by: ["s"], columns: ["x"] }, +}; + +test.describe("Derived tables are read-only", function () { + for (const [name, config] of Object.entries(PARENTS)) { + test(`${name} parent`, async function () { + const source = await make_source(); + const view = await source.view(config); + const derived = await perspective.table(view); + await expect_rejects(derived.update([{ x: 1 }])); + await expect_rejects(derived.remove([1])); + await expect_rejects(derived.clear()); + await expect_rejects(derived.replace([{ x: 1 }])); + expect(typeof (await derived.make_port())).toEqual("number"); + await derived.delete(); + await view.delete(); + await source.delete(); + }); + } +}); diff --git a/rust/perspective-js/test/js/derived/soak.spec.ts b/rust/perspective-js/test/js/derived/soak.spec.ts new file mode 100644 index 0000000000..6007c0d188 --- /dev/null +++ b/rust/perspective-js/test/js/derived/soak.spec.ts @@ -0,0 +1,178 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +import { test } from "@perspective-dev/test"; +import perspective from "../perspective_client.ts"; +import { + expect_child_parity, + expect_flat_child_parity, + expect_flat_parity, + expect_tree_parity, + mulberry32, +} from "./oracle.ts"; +import { SCHEMA, ROLLUPS, SPLIT_ROLLUPS } from "./fixtures.ts"; + +const G = ["a", "b", "c", "d"]; +const H = ["p", "q"]; +const S = ["Mon", "Tue", "Wed"]; +const KEYS = 30; +const TICKS = 300; + +function pick(rand: () => number, xs: T[]): T { + return xs[Math.floor(rand() * xs.length)]; +} + +async function tick( + rand: () => number, + source: any, + live: Set, + null_keys = true, +) { + const op = rand(); + if (op < 0.2 && live.size > 0) { + const ids = [...live].filter(() => rand() < 0.15); + ids.forEach((id) => live.delete(id)); + if (ids.length > 0) { + await source.remove(ids); + } + + return; + } + + const n = 1 + Math.floor(rand() * 4); + const batch = []; + for (let i = 0; i < n; i++) { + const id = Math.floor(rand() * KEYS); + const row: any = { id }; + if (!live.has(id) || rand() < 0.3) { + const is_null = rand() < 0.05 && null_keys; + row.g = is_null ? null : pick(rand, G); + row.h = pick(rand, H); + row.s = pick(rand, S); + } + + if (!live.has(id) || rand() < 0.8) { + row.x = Math.round(rand() * 1000) / 4; + row.y = Math.floor(rand() * 12); + row.w = 1 + Math.floor(rand() * 4); + } + + live.add(id); + batch.push(row); + } + + await source.update(batch); +} + +async function soak_tree(config: any, child_config: any) { + const rand = mulberry32(1234); + const live = new Set(); + const source = await perspective.table(SCHEMA, { index: "id" }); + await source.update( + S.map((s, i) => ({ id: 1000 + i, g: "a", h: "p", s, x: i, y: i })), + ); + + const view = await source.view(config); + const derived = await perspective.table(view); + const child = await derived.view(child_config); + for (let i = 0; i < TICKS; i++) { + await tick(rand, source, live); + await expect_tree_parity(source, config, derived, view); + await expect_child_parity( + source, + config, + derived, + child_config, + view, + child, + ); + } + + await child.delete(); + await derived.delete(); + await view.delete(); + await source.delete(); +} + +const AGGS = { x: "sum", y: "mean" }; + +test.describe("Derived table soak", function () { + for (const group_rollup_mode of ROLLUPS) { + test(`group_by parent, ${group_rollup_mode}`, async function () { + await soak_tree( + { + group_by: ["g", "h"], + columns: ["x", "y"], + aggregates: AGGS, + group_rollup_mode, + }, + { + group_by: ["h"], + columns: ["x", "y"], + aggregates: AGGS, + }, + ); + }); + + for (const split_rollup_mode of SPLIT_ROLLUPS) { + test(`split_by parent, ${group_rollup_mode} rows, ${split_rollup_mode} columns`, async function () { + await soak_tree( + { + group_by: ["g", "h"], + split_by: ["s"], + columns: ["x"], + aggregates: { x: "sum" }, + group_rollup_mode, + split_rollup_mode, + }, + { + group_by: ["g"], + columns: ["Mon|x", "Wed|x"], + aggregates: { "Mon|x": "sum", "Wed|x": "mean" }, + }, + ); + }); + } + } + + test("filtered flat parent", async function () { + const rand = mulberry32(99); + const live = new Set(); + const source = await perspective.table(SCHEMA, { index: "id" }); + const view = await source.view({ + expressions: { big: '"x" * 2' }, + filter: [ + ["big", ">", 100], + ["y", "<", 9], + ], + }); + + const derived = await perspective.table(view); + const child_config = { + group_by: ["g"], + columns: ["x", "y"], + aggregates: AGGS, + }; + + const child = await derived.view(child_config); + for (let i = 0; i < TICKS; i++) { + await tick(rand, source, live, false); + await expect_flat_parity(view, derived); + await expect_flat_child_parity(view, derived, child_config, child); + } + + await child.delete(); + await derived.delete(); + await view.delete(); + await source.delete(); + }); +}); diff --git a/rust/perspective-js/test/js/derived/tree_aggregates.spec.ts b/rust/perspective-js/test/js/derived/tree_aggregates.spec.ts new file mode 100644 index 0000000000..42154fdc2b --- /dev/null +++ b/rust/perspective-js/test/js/derived/tree_aggregates.spec.ts @@ -0,0 +1,108 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +import { test } from "@perspective-dev/test"; +import perspective from "../perspective_client.ts"; +import { expect_tree_parity } from "./oracle.ts"; +import { make_source, rows } from "./fixtures.ts"; + +const NUMERIC = [ + "sum", + "mul", + "avg", + "mean", + "count", + "distinct count", + "unique", + "any", + "q1", + "q3", + "median", + "dominant", + "first", + "last", + "first by index", + "last by index", + "last minus first", + "max", + "min", + "high", + "low", + "high minus low", + "sum abs", + "abs sum", + "sum or zero", + "pct sum parent", + "pct sum grand total", + "var", + "stddev", + ["weighted mean", ["w"]], + ["max by", ["y"]], + ["min by", ["y"]], +]; + +const STRING = [ + "count", + "distinct count", + "unique", + "any", + "dominant", + "first", + "last", + "join", +]; + +async function check(column: string, aggregate: any, split: boolean) { + const source = await make_source(); + const config: any = { + group_by: ["g", "h"], + columns: [column], + aggregates: { [column]: aggregate }, + }; + + if (split) { + config.split_by = ["s"]; + } + + const view = await source.view(config); + const derived = await perspective.table(view); + await expect_tree_parity(source, config, derived, view); + await source.update(rows(9, 20)); + await expect_tree_parity(source, config, derived, view); + await source.remove([0, 1, 2, 3, 25]); + await expect_tree_parity(source, config, derived, view); + await derived.delete(); + await view.delete(); + await source.delete(); +} + +test.describe("Derived table aggregate parity", function () { + for (const aggregate of NUMERIC) { + test(`numeric ${JSON.stringify(aggregate)}`, async function () { + await check("x", aggregate, false); + }); + + test(`numeric ${JSON.stringify(aggregate)} with split_by`, async function () { + await check("x", aggregate, true); + }); + } + + for (const aggregate of STRING) { + test(`string ${JSON.stringify(aggregate)}`, async function () { + await check("h", aggregate, false); + }); + + test(`string ${JSON.stringify(aggregate)} with split_by`, async function () { + await check("h", aggregate, true); + }); + } +}); diff --git a/rust/perspective-js/test/js/derived/tree_removes.spec.ts b/rust/perspective-js/test/js/derived/tree_removes.spec.ts new file mode 100644 index 0000000000..45587debc5 --- /dev/null +++ b/rust/perspective-js/test/js/derived/tree_removes.spec.ts @@ -0,0 +1,90 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +import { test, expect } from "@perspective-dev/test"; +import perspective from "../perspective_client.ts"; +import { expect_tree_parity } from "./oracle.ts"; +import { make_source } from "./fixtures.ts"; + +const CONFIG = { + group_by: ["g"], + columns: ["x", "y"], + aggregates: { x: "sum", y: "last" }, +}; + +test.describe("Derived table removes from a group_by view", function () { + test("emptying a group removes its row", async function () { + const source = await make_source(6); + const view = await source.view(CONFIG); + const derived = await perspective.table(view); + expect(await derived.size()).toEqual(4); + await source.remove([0, 3]); + expect(await derived.size()).toEqual(3); + await expect_tree_parity(source, CONFIG, derived, view); + await derived.delete(); + await view.delete(); + await source.delete(); + }); + + test("a freed slot is reused by a different group", async function () { + const source = await make_source(6); + const view = await source.view(CONFIG); + const derived = await perspective.table(view); + await source.remove([0, 3]); + await expect_tree_parity(source, CONFIG, derived, view); + await source.update([{ id: 50, g: "zz", x: 7, y: 8 }]); + await expect_tree_parity(source, CONFIG, derived, view); + await derived.delete(); + await view.delete(); + await source.delete(); + }); + + test("dropping and re-adding the same key", async function () { + const source = await make_source(6); + const view = await source.view(CONFIG); + const derived = await perspective.table(view); + await source.remove([0, 3]); + await source.update([{ id: 60, g: "a", x: 1, y: 1 }]); + await expect_tree_parity(source, CONFIG, derived, view); + await derived.delete(); + await view.delete(); + await source.delete(); + }); + + test("removing and adding a group in one update", async function () { + const source = await make_source(6); + const view = await source.view(CONFIG); + const derived = await perspective.table(view); + await source.update([ + { id: 0, g: "n" }, + { id: 3, g: "n" }, + ]); + + await expect_tree_parity(source, CONFIG, derived, view); + await derived.delete(); + await view.delete(); + await source.delete(); + }); + + test("removing every row", async function () { + const source = await make_source(6); + const view = await source.view(CONFIG); + const derived = await perspective.table(view); + await source.remove([0, 1, 2, 3, 4, 5]); + await expect_tree_parity(source, CONFIG, derived, view); + await source.update([{ id: 1, g: "a", x: 1, y: 1 }]); + await expect_tree_parity(source, CONFIG, derived, view); + await derived.delete(); + await view.delete(); + await source.delete(); + }); +}); diff --git a/rust/perspective-js/test/js/derived/tree_schema_frozen.spec.ts b/rust/perspective-js/test/js/derived/tree_schema_frozen.spec.ts new file mode 100644 index 0000000000..8fb8fb8d42 --- /dev/null +++ b/rust/perspective-js/test/js/derived/tree_schema_frozen.spec.ts @@ -0,0 +1,107 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +import { test, expect } from "@perspective-dev/test"; +import perspective from "../perspective_client.ts"; +import { expect_tree_parity } from "./oracle.ts"; + +const CONFIG = { + group_by: ["g"], + split_by: ["s"], + columns: ["x"], + group_rollup_mode: "flat", +}; + +async function make() { + return await perspective.table( + [ + { id: 1, g: "a", s: "Mon", x: 1 }, + { id: 2, g: "a", s: "Tue", x: 2 }, + { id: 3, g: "b", s: "Mon", x: 3 }, + ], + { index: "id" }, + ); +} + +test.describe("Derived table schema is frozen", function () { + test("a new split value is dropped", async function () { + const source = await make(); + const view = await source.view(CONFIG); + const derived = await perspective.table(view); + const schema = await derived.schema(); + await source.update([ + { id: 4, g: "a", s: "Wed", x: 10 }, + { id: 5, g: "b", s: "Tue", x: 20 }, + ]); + + expect(await derived.schema()).toEqual(schema); + await expect_tree_parity(source, CONFIG, derived, view); + await derived.delete(); + await view.delete(); + await source.delete(); + }); + + test("a row that exists only under a new split value still appears", async function () { + const source = await make(); + const view = await source.view(CONFIG); + const derived = await perspective.table(view); + await source.update([{ id: 6, g: "c", s: "Wed", x: 10 }]); + const child = await derived.view(); + expect(await child.to_json()).toEqual([ + { g: "a", "Mon|x": 1, "Tue|x": 2 }, + { g: "b", "Mon|x": 3, "Tue|x": null }, + { g: "c", "Mon|x": null, "Tue|x": null }, + ]); + + await child.delete(); + await derived.delete(); + await view.delete(); + await source.delete(); + }); + + test("a vanished split value leaves a null column", async function () { + const source = await make(); + const view = await source.view(CONFIG); + const derived = await perspective.table(view); + const schema = await derived.schema(); + await source.remove([2]); + expect(await derived.schema()).toEqual(schema); + const child = await derived.view(); + expect(await child.to_json()).toEqual([ + { g: "a", "Mon|x": 1, "Tue|x": null }, + { g: "b", "Mon|x": 3, "Tue|x": null }, + ]); + + await child.delete(); + await derived.delete(); + await view.delete(); + await source.delete(); + }); + + test("the schema survives a parent clear", async function () { + const source = await make(); + const view = await source.view(CONFIG); + const derived = await perspective.table(view); + const schema = await derived.schema(); + await source.clear(); + await source.update([ + { id: 1, g: "a", s: "Tue", x: 5 }, + { id: 2, g: "a", s: "Wed", x: 6 }, + ]); + + expect(await derived.schema()).toEqual(schema); + await expect_tree_parity(source, CONFIG, derived, view); + await derived.delete(); + await view.delete(); + await source.delete(); + }); +}); diff --git a/rust/perspective-js/test/js/derived/tree_shape.spec.ts b/rust/perspective-js/test/js/derived/tree_shape.spec.ts new file mode 100644 index 0000000000..7a7327920a --- /dev/null +++ b/rust/perspective-js/test/js/derived/tree_shape.spec.ts @@ -0,0 +1,131 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +import { test, expect } from "@perspective-dev/test"; +import perspective from "../perspective_client.ts"; +import { expect_rejects, expect_tree_parity } from "./oracle.ts"; +import { make_source, ROLLUPS } from "./fixtures.ts"; + +test.describe("Derived table shape from a group_by view", function () { + test("schema is keys plus aggregates", async function () { + const source = await make_source(); + const view = await source.view({ + group_by: ["g", "h"], + columns: ["x", "y"], + aggregates: { x: "sum", y: "mean" }, + }); + + const derived = await perspective.table(view); + expect(await derived.schema()).toEqual({ + g: "string", + h: "string", + x: "float", + y: "float", + }); + + await derived.delete(); + await view.delete(); + await source.delete(); + }); + + test("an aggregate of a key is qualified by its aggregate", async function () { + const source = await make_source(); + const view = await source.view({ + group_by: ["g"], + columns: ["g", "x"], + aggregates: { g: "count", x: "sum" }, + }); + + const derived = await perspective.table(view); + expect(await derived.schema()).toEqual({ + g: "string", + "g (count)": "integer", + x: "float", + }); + + const child = await derived.view({ columns: ["g", "g (count)"] }); + expect(await child.schema()).toEqual({ + g: "string", + "g (count)": "integer", + }); + + expect(await child.num_rows()).toEqual(await view.num_rows()); + await child.delete(); + await derived.delete(); + await view.delete(); + await source.delete(); + }); + + test("has no index", async function () { + const source = await make_source(); + const view = await source.view({ group_by: ["g"], columns: ["x"] }); + const derived = await perspective.table(view); + expect(await derived.get_index()).toBeUndefined(); + await derived.delete(); + await view.delete(); + await source.delete(); + }); + + test("rejects index and limit options", async function () { + const source = await make_source(); + const view = await source.view({ group_by: ["g"], columns: ["x"] }); + await expect_rejects(perspective.table(view, { index: "x" })); + await expect_rejects(perspective.table(view, { limit: 10 })); + await view.delete(); + await source.delete(); + }); + + for (const group_rollup_mode of ROLLUPS) { + test(`mirrors group_rollup_mode ${group_rollup_mode}`, async function () { + const source = await make_source(); + const config = { + group_by: ["g", "h"], + columns: ["x", "y"], + aggregates: { x: "sum", y: "mean" }, + group_rollup_mode, + }; + + const view = await source.view(config); + const derived = await perspective.table(view); + const expected = { rollup: 10, flat: 6, total: 1 }[ + group_rollup_mode + ]; + + expect(await derived.size()).toEqual(expected); + await expect_tree_parity(source, config, derived); + await derived.delete(); + await view.delete(); + await source.delete(); + }); + } + + test("rollup rows have null keys beyond their depth", async function () { + const source = await make_source(); + const view = await source.view({ + group_by: ["g", "h"], + columns: ["x"], + }); + + const derived = await perspective.table(view); + const child = await derived.view(); + const json = await child.to_json(); + const totals = json.filter((r) => r.g === null); + const mids = json.filter((r) => r.g !== null && r.h === null); + + expect(totals).toHaveLength(1); + expect(mids).toHaveLength(3); + await child.delete(); + await derived.delete(); + await view.delete(); + await source.delete(); + }); +}); diff --git a/rust/perspective-js/test/js/derived/tree_split_by.spec.ts b/rust/perspective-js/test/js/derived/tree_split_by.spec.ts new file mode 100644 index 0000000000..85a409d41f --- /dev/null +++ b/rust/perspective-js/test/js/derived/tree_split_by.spec.ts @@ -0,0 +1,180 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +import { test, expect } from "@perspective-dev/test"; +import perspective from "../perspective_client.ts"; +import { expect_tree_parity, key_name } from "./oracle.ts"; +import { make_source, rows, ROLLUPS, SPLIT_ROLLUPS } from "./fixtures.ts"; +import { joined_column_paths } from "../column_paths.js"; + +const CONFIG = { + group_by: ["g"], + split_by: ["s"], + columns: ["x", "y"], + aggregates: { x: "sum", y: "mean" }, +}; + +test.describe("Derived table from a split_by view", function () { + test("columns are the view's column paths", async function () { + const source = await make_source(); + const view = await source.view(CONFIG); + const derived = await perspective.table(view); + const paths = await joined_column_paths(view); + expect(await derived.columns()).toEqual([ + key_name("g"), + ...paths.filter((p: string) => p !== "__ROW_PATH__"), + ]); + + await derived.delete(); + await view.delete(); + await source.delete(); + }); + + for (const group_rollup_mode of ROLLUPS) { + for (const split_rollup_mode of SPLIT_ROLLUPS) { + test(`parity with ${group_rollup_mode} rows and ${split_rollup_mode} columns`, async function () { + const source = await make_source(); + const config = { + ...CONFIG, + group_by: ["g", "h"], + group_rollup_mode, + split_rollup_mode, + }; + + const view = await source.view(config); + const derived = await perspective.table(view); + await expect_tree_parity(source, config, derived, view); + await source.update(rows(9, 20)); + await expect_tree_parity(source, config, derived, view); + await source.remove([0, 1, 2, 3, 4, 5]); + await expect_tree_parity(source, config, derived, view); + await derived.delete(); + await view.delete(); + await source.delete(); + }); + } + } + + test("a sparse cell is null", async function () { + const source = await perspective.table( + [ + { id: 1, g: "a", s: "Mon", x: 1 }, + { id: 2, g: "a", s: "Tue", x: 2 }, + { id: 3, g: "b", s: "Mon", x: 3 }, + ], + { index: "id" }, + ); + + const config = { + group_by: ["g"], + split_by: ["s"], + columns: ["x"], + group_rollup_mode: "flat", + }; + + const view = await source.view(config); + const derived = await perspective.table(view); + const child = await derived.view(); + expect(await child.to_json()).toEqual([ + { g: "a", "Mon|x": 1, "Tue|x": 2 }, + { g: "b", "Mon|x": 3, "Tue|x": null }, + ]); + + await child.delete(); + await derived.delete(); + await view.delete(); + await source.delete(); + }); + + test("a dropped cell becomes null while its row survives", async function () { + const source = await perspective.table( + [ + { id: 1, g: "a", s: "Mon", x: 1 }, + { id: 2, g: "a", s: "Tue", x: 2 }, + { id: 3, g: "b", s: "Tue", x: 3 }, + ], + { index: "id" }, + ); + + const config = { + group_by: ["g"], + split_by: ["s"], + columns: ["x"], + group_rollup_mode: "flat", + }; + + const view = await source.view(config); + const derived = await perspective.table(view); + await source.remove([2]); + const child = await derived.view(); + expect(await child.to_json()).toEqual([ + { g: "a", "Mon|x": 1, "Tue|x": null }, + { g: "b", "Mon|x": null, "Tue|x": 3 }, + ]); + + await source.remove([1]); + expect(await child.to_json()).toEqual([ + { g: "b", "Mon|x": null, "Tue|x": 3 }, + ]); + + await child.delete(); + await derived.delete(); + await view.delete(); + await source.delete(); + }); + + test("a column-only parent has one row per source row", async function () { + const source = await make_source(); + const config = { split_by: ["s"], columns: ["x"] }; + const view = await source.view(config); + const derived = await perspective.table(view); + expect(await derived.size()).toEqual(24); + await expect_tree_parity(source, config, derived, view); + await source.update(rows(5, 30)); + await expect_tree_parity(source, config, derived, view); + await derived.delete(); + await view.delete(); + await source.delete(); + }); + + test("multi-level split_by", async function () { + const source = await make_source(); + const config = { + group_by: ["g"], + split_by: ["s", "h"], + columns: ["x"], + split_rollup_mode: "rollup", + }; + + const view = await source.view(config); + const derived = await perspective.table(view); + await expect_tree_parity(source, config, derived, view); + await source.update(rows(9, 20)); + await expect_tree_parity(source, config, derived, view); + await derived.delete(); + await view.delete(); + await source.delete(); + }); + + test("a hidden sort aggregate is not exposed", async function () { + const source = await make_source(); + const config = { ...CONFIG, columns: ["x"], sort: [["y", "desc"]] }; + const view = await source.view(config); + const derived = await perspective.table(view); + const names = await derived.columns(); + expect(names.some((n: string) => n.endsWith("|y"))).toBe(false); + await expect_tree_parity(source, config, derived, view); + await derived.delete(); + await view.delete(); + await source.delete(); + }); +}); diff --git a/rust/perspective-js/test/js/derived/tree_traversal_independence.spec.ts b/rust/perspective-js/test/js/derived/tree_traversal_independence.spec.ts new file mode 100644 index 0000000000..e7b1a7c3a2 --- /dev/null +++ b/rust/perspective-js/test/js/derived/tree_traversal_independence.spec.ts @@ -0,0 +1,80 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +import { test } from "@perspective-dev/test"; +import perspective from "../perspective_client.ts"; +import { expect_tree_parity } from "./oracle.ts"; +import { make_source, rows } from "./fixtures.ts"; + +test.describe("Derived table ignores presentation state", function () { + test("parent sort is ignored", async function () { + const source = await make_source(); + const config = { + group_by: ["g", "h"], + columns: ["x"], + sort: [["x", "desc"]], + }; + + const view = await source.view(config); + const derived = await perspective.table(view); + await expect_tree_parity(source, config, derived); + await source.update(rows(6, 24)); + await expect_tree_parity(source, config, derived); + await derived.delete(); + await view.delete(); + await source.delete(); + }); + + test("parent sort on a hidden column is not exposed", async function () { + const source = await make_source(); + const config = { + group_by: ["g"], + columns: ["x"], + sort: [["y", "desc"]], + }; + + const view = await source.view(config); + const derived = await perspective.table(view); + await expect_tree_parity(source, config, derived); + await derived.delete(); + await view.delete(); + await source.delete(); + }); + + test("collapsed parent rows are still present and updated", async function () { + const source = await make_source(); + const config = { group_by: ["g", "h"], columns: ["x"] }; + const view = await source.view(config); + await view.collapse(1); + await view.collapse(0); + const derived = await perspective.table(view); + await expect_tree_parity(source, config, derived); + await source.update(rows(6, 24)); + await expect_tree_parity(source, config, derived); + await derived.delete(); + await view.delete(); + await source.delete(); + }); + + test("set_depth on the parent changes nothing", async function () { + const source = await make_source(); + const config = { group_by: ["g", "h"], columns: ["x"] }; + const view = await source.view(config); + const derived = await perspective.table(view); + await view.set_depth(0); + await source.update(rows(6, 24)); + await expect_tree_parity(source, config, derived); + await derived.delete(); + await view.delete(); + await source.delete(); + }); +}); diff --git a/rust/perspective-js/test/js/derived/tree_updates.spec.ts b/rust/perspective-js/test/js/derived/tree_updates.spec.ts new file mode 100644 index 0000000000..d64932e1fe --- /dev/null +++ b/rust/perspective-js/test/js/derived/tree_updates.spec.ts @@ -0,0 +1,140 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +import { test, expect } from "@perspective-dev/test"; +import perspective from "../perspective_client.ts"; +import { expect_tree_parity } from "./oracle.ts"; +import { make_source, rows } from "./fixtures.ts"; + +const CONFIG = { + group_by: ["g", "h"], + columns: ["x", "y"], + aggregates: { x: "sum", y: "mean" }, +}; + +test.describe("Derived table updates from a group_by view", function () { + test("repeated updates to the same groups do not append", async function () { + const source = await make_source(); + const view = await source.view(CONFIG); + const derived = await perspective.table(view); + const size = await derived.size(); + for (let i = 0; i < 20; i++) { + await source.update([{ id: i % 6, x: i * 3.5 }]); + expect(await derived.size()).toEqual(size); + } + + await expect_tree_parity(source, CONFIG, derived, view); + await derived.delete(); + await view.delete(); + await source.delete(); + }); + + test("a new group adds rows", async function () { + const source = await make_source(); + const view = await source.view(CONFIG); + const derived = await perspective.table(view); + const size = await derived.size(); + await source.update([{ id: 100, g: "z", h: "p", x: 1, y: 2 }]); + expect(await derived.size()).toEqual(size + 2); + await expect_tree_parity(source, CONFIG, derived, view); + await derived.delete(); + await view.delete(); + await source.delete(); + }); + + test("partial updates", async function () { + const source = await make_source(); + const view = await source.view(CONFIG); + const derived = await perspective.table(view); + await source.update([{ id: 3, y: 99 }]); + await expect_tree_parity(source, CONFIG, derived, view); + await source.update([{ id: 3, x: null }]); + await expect_tree_parity(source, CONFIG, derived, view); + await derived.delete(); + await view.delete(); + await source.delete(); + }); + + test("null group keys", async function () { + const source = await make_source(); + const view = await source.view(CONFIG); + const derived = await perspective.table(view); + await source.update([ + { id: 200, g: null, h: "p", x: 5, y: 1 }, + { id: 201, g: "a", h: null, x: 6, y: 2 }, + ]); + + await expect_tree_parity(source, CONFIG, derived, view); + await derived.delete(); + await view.delete(); + await source.delete(); + }); + + test("a row moving between groups", async function () { + const source = await make_source(); + const view = await source.view(CONFIG); + const derived = await perspective.table(view); + await source.update([{ id: 0, g: "c", h: "q" }]); + await expect_tree_parity(source, CONFIG, derived, view); + await derived.delete(); + await view.delete(); + await source.delete(); + }); + + test("updates through a port", async function () { + const source = await make_source(); + const view = await source.view(CONFIG); + const derived = await perspective.table(view); + const port = await source.make_port(); + await source.update(rows(4, 40), { port_id: port }); + await source.update([{ id: 1, x: 1000 }]); + await expect_tree_parity(source, CONFIG, derived, view); + await derived.delete(); + await view.delete(); + await source.delete(); + }); + + test("a filtered parent", async function () { + const source = await make_source(); + const config = { ...CONFIG, filter: [["y", ">", 4]] }; + const view = await source.view(config); + const derived = await perspective.table(view); + await expect_tree_parity(source, config, derived, view); + await source.update([ + { id: 0, y: 10 }, + { id: 1, y: 0 }, + ]); + + await expect_tree_parity(source, config, derived, view); + await derived.delete(); + await view.delete(); + await source.delete(); + }); + + test("a parent grouped by an expression", async function () { + const source = await make_source(); + const config = { + group_by: ["bucket"], + columns: ["x"], + expressions: { bucket: 'bucket("y", 5)' }, + }; + + const view = await source.view(config); + const derived = await perspective.table(view); + await expect_tree_parity(source, config, derived, view); + await source.update(rows(10, 30)); + await expect_tree_parity(source, config, derived, view); + await derived.delete(); + await view.delete(); + await source.delete(); + }); +}); diff --git a/rust/perspective-js/test/js/duckdb/coerce_types.spec.js b/rust/perspective-js/test/js/duckdb/coerce_types.spec.js index 4c4125df17..d44d88274c 100644 --- a/rust/perspective-js/test/js/duckdb/coerce_types.spec.js +++ b/rust/perspective-js/test/js/duckdb/coerce_types.spec.js @@ -139,7 +139,7 @@ describeDuckDB("coerce_types", (getClient) => { const view = await table.view({ group_by: ["enum"], columns: [] }); const csv = await view.to_csv(); expect(csv.split("\n").filter((x) => x.length > 0)).toEqual([ - "__ROW_PATH_0__", + "enum", "null", '"happy"', '"sad"', diff --git a/rust/perspective-js/test/js/duckdb/combined.spec.js b/rust/perspective-js/test/js/duckdb/combined.spec.js index 1373afe3bb..bd13a91fa6 100644 --- a/rust/perspective-js/test/js/duckdb/combined.spec.js +++ b/rust/perspective-js/test/js/duckdb/combined.spec.js @@ -12,6 +12,7 @@ import { test, expect } from "@perspective-dev/test"; import { describeDuckDB } from "./setup.js"; +import { joined_column_paths } from "../column_paths.js"; describeDuckDB("combined operations", (getClient) => { test("group_by + filter + sort", async function () { @@ -52,7 +53,7 @@ describeDuckDB("combined operations", (getClient) => { aggregates: { Sales: "sum" }, }); - const paths = await view.column_paths(); + const paths = await joined_column_paths(view); expect(paths).toEqual([ "Central|Sales", "East|Sales", @@ -106,7 +107,7 @@ describeDuckDB("combined operations", (getClient) => { filter: [["Quantity", ">", 3]], }); - const paths = await view.column_paths(); + const paths = await joined_column_paths(view); expect(paths).toEqual([ "Central|Sales", "East|Sales", @@ -138,7 +139,7 @@ describeDuckDB("combined operations", (getClient) => { filter: [["Quantity", ">", 3]], }); - const paths = await view.column_paths(); + const paths = await joined_column_paths(view); expect(paths).toEqual([ "Central|Sales", "East|Sales", diff --git a/rust/perspective-js/test/js/duckdb/split_by.spec.js b/rust/perspective-js/test/js/duckdb/split_by.spec.js index 58ffa51e44..db9531a45d 100644 --- a/rust/perspective-js/test/js/duckdb/split_by.spec.js +++ b/rust/perspective-js/test/js/duckdb/split_by.spec.js @@ -12,6 +12,7 @@ import { test, expect } from "@perspective-dev/test"; import { describeDuckDB } from "./setup.js"; +import { joined_column_paths } from "../column_paths.js"; describeDuckDB("split_by", (getClient) => { test("single split_by", async function () { @@ -23,7 +24,7 @@ describeDuckDB("split_by", (getClient) => { aggregates: { Sales: "sum" }, }); - const columns = await view.column_paths(); + const columns = await joined_column_paths(view); expect(columns).toEqual([ "Central|Sales", "East|Sales", @@ -115,7 +116,7 @@ describeDuckDB("split_by", (getClient) => { columns: ["Sales"], split_by: ["Category"], }); - const paths = await view.column_paths(); + const paths = await joined_column_paths(view); expect(paths.some((c) => c.includes("Furniture"))).toBe(true); expect(paths.some((c) => c.includes("Office Supplies"))).toBe(true); expect(paths.some((c) => c.includes("Technology"))).toBe(true); @@ -133,9 +134,8 @@ describeDuckDB("split_by", (getClient) => { }); expect(await view.column_paths()).toEqual([ - "a_b|amount", - "plain|amount", - 'say "hi"|amount', + ["a_b", "plain", 'say "hi"'], + ["amount", "amount", "amount"], ]); expect(await view.to_json()).toEqual([ @@ -171,7 +171,10 @@ describeDuckDB("split_by", (getClient) => { aggregates: { amount: "sum" }, }); - expect(await view.column_paths()).toEqual(["g1|amount", "g2|amount"]); + expect(await view.column_paths()).toEqual([ + ["g1", "g2"], + ["amount", "amount"], + ]); await view.delete(); }); }); diff --git a/rust/perspective-js/test/js/duckdb/split_rollup_mode.spec.js b/rust/perspective-js/test/js/duckdb/split_rollup_mode.spec.js index 6d4fca9bbb..49d6f724ff 100644 --- a/rust/perspective-js/test/js/duckdb/split_rollup_mode.spec.js +++ b/rust/perspective-js/test/js/duckdb/split_rollup_mode.spec.js @@ -17,6 +17,7 @@ import { test, expect } from "@perspective-dev/test"; import { describeDuckDB } from "./setup.js"; +import { joined_column_paths } from "../column_paths.js"; describeDuckDB("split_rollup_mode", (getClient) => { test("rollup emits grand-total column group in totals-before order", async function () { @@ -28,7 +29,7 @@ describeDuckDB("split_rollup_mode", (getClient) => { aggregates: { total_sales: "sum" }, split_rollup_mode: "rollup", }); - const paths = await view.column_paths(); + const paths = await joined_column_paths(view); expect(paths).toEqual([ "total_sales", "bay_area|total_sales", @@ -58,7 +59,7 @@ describeDuckDB("split_rollup_mode", (getClient) => { aggregates: { total_sales: "sum" }, split_rollup_mode: "rollup", }); - const paths = await view.column_paths(); + const paths = await joined_column_paths(view); expect(paths).toEqual([ "total_sales", "east_coast|total_sales", diff --git a/rust/perspective-js/test/js/duckdb/table_from_view.spec.js b/rust/perspective-js/test/js/duckdb/table_from_view.spec.js new file mode 100644 index 0000000000..d5cdc0ba62 --- /dev/null +++ b/rust/perspective-js/test/js/duckdb/table_from_view.spec.js @@ -0,0 +1,206 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +import { test, expect } from "@perspective-dev/test"; +import { describeDuckDB } from "./setup.js"; + +async function expect_rejects(promise) { + let error; + try { + await promise; + } catch (e) { + error = e; + } + + expect(error).toBeDefined(); +} + +describeDuckDB("table from view", (getClient) => { + test("a flat view", async function () { + const client = getClient(); + const table = await client.open_table("memory.superstore"); + const view = await table.view({ + columns: ["Region", "Sales"], + filter: [["Sales", ">", 5000]], + }); + + const derived = await client.table(view, { name: "derived_flat" }); + expect(await derived.schema()).toEqual({ + Region: "string", + Sales: "float", + }); + + const child = await derived.view({ columns: ["Region", "Sales"] }); + expect(await child.to_json()).toEqual(await view.to_json()); + await child.delete(); + await derived.delete(); + await view.delete(); + }); + + test("a group_by view unrolls its row path", async function () { + const client = getClient(); + const table = await client.open_table("memory.superstore"); + const view = await table.view({ + columns: ["Sales"], + group_by: ["Region"], + aggregates: { Sales: "sum" }, + }); + + const derived = await client.table(view, { name: "derived_group" }); + expect(await derived.schema()).toEqual({ + Region: "string", + Sales: "float", + }); + + const child = await derived.view({ + columns: ["Region", "Sales"], + }); + + const expected = (await view.to_json()).map((row) => ({ + Region: row.__ROW_PATH__[0] ?? null, + Sales: row.Sales, + })); + + const key = (row) => String(row["Region"]); + const by_key = (a, b) => key(a).localeCompare(key(b)); + expect((await child.to_json()).sort(by_key)).toEqual( + expected.sort(by_key), + ); + + await child.delete(); + await derived.delete(); + await view.delete(); + }); + + test("a flat-rollup split_by view", async function () { + const client = getClient(); + const table = await client.open_table("memory.superstore"); + const view = await table.view({ + columns: ["Sales"], + group_by: ["Category"], + split_by: ["Region"], + aggregates: { Sales: "sum" }, + group_rollup_mode: "flat", + }); + + const derived = await client.table(view, { name: "derived_split" }); + expect(await derived.columns()).toEqual([ + "Category", + "Central|Sales", + "East|Sales", + "South|Sales", + "West|Sales", + ]); + + const child = await derived.view({ + columns: ["Category", "West|Sales"], + filter: [["West|Sales", ">", 250000]], + }); + + const expected = (await view.to_json()) + .filter((row) => row["West|Sales"] > 250000) + .map((row) => row.__ROW_PATH__[0]) + .sort(); + + const actual = (await child.to_json()) + .map((row) => row["Category"]) + .sort(); + + expect(actual).toEqual(expected); + await child.delete(); + await derived.delete(); + await view.delete(); + }); + + test("an explicit schema", async function () { + const client = getClient(); + const table = await client.open_table("memory.superstore"); + const view = await table.view({ + columns: ["Sales"], + group_by: ["Category"], + split_by: ["Region"], + aggregates: { Sales: "sum" }, + group_rollup_mode: "flat", + }); + + const derived = await client.table(view, { + name: "derived_schema", + schema: { + Category: "string", + "West|Sales": "float", + "North|Sales": "float", + }, + }); + + expect(await derived.schema()).toEqual({ + Category: "string", + "West|Sales": "float", + "North|Sales": "float", + }); + + const child = await derived.view({ + columns: ["Category", "West|Sales", "North|Sales"], + }); + + const json = await child.to_json(); + expect(json).toHaveLength(3); + expect(json.every((row) => row["North|Sales"] === null)).toBe(true); + expect(json.every((row) => row["West|Sales"] > 0)).toBe(true); + await child.delete(); + await derived.delete(); + await view.delete(); + }); + + test("an aggregate of a group_by column is qualified", async function () { + const client = getClient(); + const table = await client.open_table("memory.superstore"); + const view = await table.view({ + columns: ["Region", "Sales"], + group_by: ["Region"], + aggregates: { Region: "count", Sales: "sum" }, + }); + + const derived = await client.table(view, { name: "derived_qualified" }); + expect(await derived.columns()).toEqual([ + "Region", + "Region (count)", + "Sales", + ]); + + const child = await derived.view({ + columns: ["Region", "Region (count)"], + }); + + expect(await child.num_rows()).toEqual(await view.num_rows()); + await child.delete(); + await derived.delete(); + await view.delete(); + }); + + test("rejects a mismatched type, index and limit", async function () { + const client = getClient(); + const table = await client.open_table("memory.superstore"); + const view = await table.view({ + columns: ["Sales"], + group_by: ["Region"], + aggregates: { Sales: "sum" }, + }); + + await expect_rejects( + client.table(view, { schema: { Sales: "string" } }), + ); + + await expect_rejects(client.table(view, { index: "Sales" })); + await expect_rejects(client.table(view, { limit: 10 })); + await view.delete(); + }); +}); diff --git a/rust/perspective-js/test/js/duckdb/underscore_columns.spec.js b/rust/perspective-js/test/js/duckdb/underscore_columns.spec.js index 53c412254b..28ec5519a0 100644 --- a/rust/perspective-js/test/js/duckdb/underscore_columns.spec.js +++ b/rust/perspective-js/test/js/duckdb/underscore_columns.spec.js @@ -16,6 +16,7 @@ import { test, expect } from "@perspective-dev/test"; import { describeDuckDB } from "./setup.js"; +import { joined_column_paths } from "../column_paths.js"; describeDuckDB("underscore columns", (getClient) => { test("table schema() preserves underscore names", async function () { @@ -35,7 +36,7 @@ describeDuckDB("underscore columns", (getClient) => { columns: ["account_number", "total_sales"], }); - const paths = await view.column_paths(); + const paths = await joined_column_paths(view); expect(paths).toEqual(["account_number", "total_sales"]); const schema = await view.schema(); @@ -108,7 +109,7 @@ describeDuckDB("underscore columns", (getClient) => { split_by: ["region_name"], }); - const paths = await view.column_paths(); + const paths = await joined_column_paths(view); expect(paths).toEqual([ "east_coast|total_sales", "west_coast|total_sales", @@ -140,7 +141,7 @@ describeDuckDB("underscore columns", (getClient) => { aggregates: { account_number: "sum", total_sales: "sum" }, }); - const paths = await view.column_paths(); + const paths = await joined_column_paths(view); expect(paths).toEqual([ "east_coast|account_number", "east_coast|total_sales", @@ -247,7 +248,7 @@ describeDuckDB("underscore columns", (getClient) => { split_by: ["region_name", "sub_region"], }); - const paths = await view.column_paths(); + const paths = await joined_column_paths(view); expect(paths).toEqual([ "east_coast|new_jersey|total_sales", "east_coast|new_york|total_sales", @@ -292,7 +293,7 @@ describeDuckDB("underscore columns", (getClient) => { expressions: { double_sales: '"total_sales" * 2' }, }); - const paths = await view.column_paths(); + const paths = await joined_column_paths(view); expect(paths).toEqual([ "east_coast|double_sales", "west_coast|double_sales", diff --git a/rust/perspective-js/test/js/duckdb/view.spec.js b/rust/perspective-js/test/js/duckdb/view.spec.js index aa21716c73..a4acf4a53a 100644 --- a/rust/perspective-js/test/js/duckdb/view.spec.js +++ b/rust/perspective-js/test/js/duckdb/view.spec.js @@ -12,6 +12,7 @@ import { test, expect } from "@perspective-dev/test"; import { describeDuckDB } from "./setup.js"; +import { joined_column_paths } from "../column_paths.js"; describeDuckDB("view", (getClient) => { test("num_rows()", async function () { @@ -84,7 +85,7 @@ describeDuckDB("view", (getClient) => { const view = await table.view({ columns: ["Sales", "Profit", "State"], }); - const paths = await view.column_paths(); + const paths = await joined_column_paths(view); expect(paths).toEqual(["Sales", "Profit", "State"]); await view.delete(); }); diff --git a/rust/perspective-js/test/js/expressions/functionality.spec.js b/rust/perspective-js/test/js/expressions/functionality.spec.js index 756923d737..74fdfb0ab9 100644 --- a/rust/perspective-js/test/js/expressions/functionality.spec.js +++ b/rust/perspective-js/test/js/expressions/functionality.spec.js @@ -14,6 +14,7 @@ import * as expressions_common from "./common.js"; import { test, expect } from "@perspective-dev/test"; import perspective from "../perspective_client"; +import { joined_column_paths } from "../column_paths.js"; /** * Tests the functionality of `View`-based expressions, specifically that @@ -2353,7 +2354,7 @@ import perspective from "../perspective_client"; expressions: { column: '"w" + "x"' }, }); - let paths = await view.column_paths(); + let paths = await joined_column_paths(view); expect(paths).toEqual(["w", "x", "y", "z", "column"]); await view.delete(); @@ -2371,7 +2372,7 @@ import perspective from "../perspective_client"; expressions: { column: '"w" + "x"' }, columns: expected, }); - paths = await view.column_paths(); + paths = await joined_column_paths(view); expect(paths).toEqual(output); view.delete(); } @@ -2383,7 +2384,7 @@ import perspective from "../perspective_client"; expressions: { column: '"w" + "x"' }, columns: expected, }); - paths = await view.column_paths(); + paths = await joined_column_paths(view); expect(paths).toEqual(output); view.delete(); } @@ -2407,7 +2408,7 @@ import perspective from "../perspective_client"; // default order let view = await table.view(config); - let paths = await view.column_paths(); + let paths = await joined_column_paths(view); expect(paths).toEqual(["w", "x", "y", "z", "1234"]); await view.delete(); @@ -2424,7 +2425,7 @@ import perspective from "../perspective_client"; ...config, columns: expected, }); - paths = await view.column_paths(); + paths = await joined_column_paths(view); expect(paths).toEqual(output); view.delete(); } @@ -2435,7 +2436,7 @@ import perspective from "../perspective_client"; ...config, columns: expected, }); - paths = await view.column_paths(); + paths = await joined_column_paths(view); expect(paths).toEqual(output); view.delete(); } diff --git a/rust/perspective-js/test/js/expressions/numeric.spec.js b/rust/perspective-js/test/js/expressions/numeric.spec.js index d00e74aba1..9f4952bd0d 100644 --- a/rust/perspective-js/test/js/expressions/numeric.spec.js +++ b/rust/perspective-js/test/js/expressions/numeric.spec.js @@ -13,6 +13,7 @@ import { test, expect } from "@perspective-dev/test"; import perspective from "../perspective_client"; import * as common from "./common.js"; +import { joined_column_paths } from "../column_paths.js"; const NUMERIC_TYPES = [ "i8", @@ -224,7 +225,7 @@ function validate_binary_operations(output, expressions, operator) { expect(await view.to_columns()).toEqual({ __ROW_PATH__: [[], [null], [1], [2], [3], [4]], a: [295, 220, 10, 15, 20, 30], - b: [10, 0, 1, 2, 3, 4], + b: [10, null, 1, 2, 3, 4], '"a"': [295, 220, 10, 15, 20, 30], }); await view.delete(); @@ -243,7 +244,7 @@ function validate_binary_operations(output, expressions, operator) { expressions, }); - const col_names = await view.column_paths(); + const col_names = await joined_column_paths(view); for (const expr of expressions) { expect(col_names.includes(expr)).toBe(true); @@ -266,7 +267,7 @@ function validate_binary_operations(output, expressions, operator) { expressions, }); - const col_names = await view.column_paths(); + const col_names = await joined_column_paths(view); for (const expr of expressions) { expect(col_names.includes(expr)).toBe(true); @@ -287,7 +288,7 @@ function validate_binary_operations(output, expressions, operator) { expressions, }); - const col_names = await view.column_paths(); + const col_names = await joined_column_paths(view); for (const expr of expressions) { expect(col_names.includes(expr)).toBe(true); @@ -308,7 +309,7 @@ function validate_binary_operations(output, expressions, operator) { expressions, }); - const col_names = await view.column_paths(); + const col_names = await joined_column_paths(view); for (const expr of expressions) { expect(col_names.includes(expr)).toBe(true); @@ -330,7 +331,7 @@ function validate_binary_operations(output, expressions, operator) { expressions, }); - const col_names = await view.column_paths(); + const col_names = await joined_column_paths(view); for (const expr of expressions) { expect(col_names.includes(expr)).toBe(true); @@ -352,7 +353,7 @@ function validate_binary_operations(output, expressions, operator) { expressions, }); - const col_names = await view.column_paths(); + const col_names = await joined_column_paths(view); for (const expr of expressions) { expect(col_names.includes(expr)).toBe(true); @@ -371,7 +372,7 @@ function validate_binary_operations(output, expressions, operator) { expressions, }); - const col_names = await view.column_paths(); + const col_names = await joined_column_paths(view); for (const expr of expressions) { expect(col_names.includes(expr)).toBe(true); @@ -392,7 +393,7 @@ function validate_binary_operations(output, expressions, operator) { expressions, }); - const col_names = await view.column_paths(); + const col_names = await joined_column_paths(view); for (const expr of expressions) { expect(col_names.includes(expr)).toBe(true); diff --git a/rust/perspective-js/test/js/expressions/updates.spec.js b/rust/perspective-js/test/js/expressions/updates.spec.js index f5495336ea..1193f30f6c 100644 --- a/rust/perspective-js/test/js/expressions/updates.spec.js +++ b/rust/perspective-js/test/js/expressions/updates.spec.js @@ -508,7 +508,7 @@ const pivot_data = [ expect(after).toEqual({ __ROW_PATH__: [[], ["a"], ["b"], ["c"], ["hello"], ["world"]], 'lower("y")': [6, 1, 1, 2, 1, 1], - x: [10, 1, 2, 7, 0, 0], + x: [10, 1, 2, 7, null, null], y: [6, 1, 1, 2, 1, 1], }); view.delete(); diff --git a/rust/perspective-js/test/js/group_by.spec.js b/rust/perspective-js/test/js/group_by.spec.js index dad637ec00..0458c56812 100644 --- a/rust/perspective-js/test/js/group_by.spec.js +++ b/rust/perspective-js/test/js/group_by.spec.js @@ -12,6 +12,7 @@ import { test, expect } from "@perspective-dev/test"; import perspective from "./perspective_client"; +import { joined_column_paths } from "./column_paths.js"; var data = [ { x: 1, y: "a", z: true }, @@ -991,7 +992,7 @@ const std = (nums) => { test("Should return all columns, 0-sided view from schema", async function () { const table = await perspective.table(meta); const view = await table.view(); - const paths = await view.column_paths(); + const paths = await joined_column_paths(view); expect(paths).toEqual(["x", "y", "z"]); view.delete(); table.delete(); @@ -1002,7 +1003,7 @@ const std = (nums) => { const view = await table.view({ columns: ["z", "y", "x"], }); - const paths = await view.column_paths(); + const paths = await joined_column_paths(view); expect(paths).toEqual(["z", "y", "x"]); view.delete(); table.delete(); @@ -1013,7 +1014,7 @@ const std = (nums) => { const view = await table.view({ columns: ["x"], }); - const paths = await view.column_paths(); + const paths = await joined_column_paths(view); expect(paths).toEqual(["x"]); view.delete(); table.delete(); @@ -1022,7 +1023,7 @@ const std = (nums) => { test("Should return all columns, 0-sided view", async function () { const table = await perspective.table(data); const view = await table.view(); - const paths = await view.column_paths(); + const paths = await joined_column_paths(view); expect(paths).toEqual(["x", "y", "z"]); view.delete(); table.delete(); @@ -1033,7 +1034,7 @@ const std = (nums) => { const view = await table.view({ columns: ["z", "y", "x"], }); - const paths = await view.column_paths(); + const paths = await joined_column_paths(view); expect(paths).toEqual(["z", "y", "x"]); view.delete(); table.delete(); @@ -1044,7 +1045,7 @@ const std = (nums) => { const view = await table.view({ columns: ["x"], }); - const paths = await view.column_paths(); + const paths = await joined_column_paths(view); expect(paths).toEqual(["x"]); view.delete(); table.delete(); @@ -1055,7 +1056,7 @@ const std = (nums) => { const view = await table.view({ group_by: ["x"], }); - const paths = await view.column_paths(); + const paths = await joined_column_paths(view); expect(paths).toEqual(["x", "y", "z"]); view.delete(); table.delete(); @@ -1082,7 +1083,7 @@ const std = (nums) => { 1234: "sum", }, }); - const paths = await view.column_paths(); + const paths = await joined_column_paths(view); expect(paths).toEqual(["2345", "1234", "x", "1.23456789"]); view.delete(); table.delete(); @@ -1094,7 +1095,7 @@ const std = (nums) => { group_by: ["x"], columns: ["z", "y", "x"], }); - const paths = await view.column_paths(); + const paths = await joined_column_paths(view); expect(paths).toEqual(["z", "y", "x"]); view.delete(); table.delete(); @@ -1106,7 +1107,7 @@ const std = (nums) => { columns: ["x"], group_by: ["x"], }); - const paths = await view.column_paths(); + const paths = await joined_column_paths(view); expect(paths).toEqual(["x"]); view.delete(); table.delete(); @@ -1118,7 +1119,7 @@ const std = (nums) => { group_by: ["x"], split_by: ["y"], }); - const paths = await view.column_paths(); + const paths = await joined_column_paths(view); expect(paths).toEqual([ "a|x", "a|y", @@ -1144,7 +1145,7 @@ const std = (nums) => { group_by: ["x"], split_by: ["y"], }); - const paths = await view.column_paths(); + const paths = await joined_column_paths(view); expect(paths).toEqual([ "a|z", "a|y", @@ -1170,13 +1171,13 @@ const std = (nums) => { group_by: ["x"], split_by: ["y"], }); - const paths = await view.column_paths(); + const paths = await joined_column_paths(view); expect(paths).toEqual(["a|x", "b|x", "c|x", "d|x"]); view.delete(); table.delete(); }); - test("Should format date columns in split_by", async function () { + test("Should return date split_by values as epoch milliseconds", async function () { const table = await perspective.table({ w: "float", x: "integer", @@ -1186,16 +1187,12 @@ const std = (nums) => { await table.update(data_8); const view = await table.view({ group_by: ["y"], split_by: ["z"] }); - const paths = await view.column_paths(); - expect(paths).toEqual([ - "2019-04-11|w", - "2019-04-11|x", - "2019-04-11|y", - "2019-04-11|z", - "2019-04-13|w", - "2019-04-13|x", - "2019-04-13|y", - "2019-04-13|z", + const area = await view.column_paths(); + const d11 = Date.UTC(2019, 3, 11); + const d13 = Date.UTC(2019, 3, 13); + expect(area).toEqual([ + [d11, d11, d11, d11, d13, d13, d13, d13], + ["w", "x", "y", "z", "w", "x", "y", "z"], ]); view.delete(); diff --git a/rust/perspective-js/test/js/pivot_nulls.spec.js b/rust/perspective-js/test/js/pivot_nulls.spec.js index 17d85c6552..8fb78a56f7 100644 --- a/rust/perspective-js/test/js/pivot_nulls.spec.js +++ b/rust/perspective-js/test/js/pivot_nulls.spec.js @@ -332,6 +332,229 @@ import perspective from "./perspective_client"; }, ]); }); + test.describe("an all-null group", function () { + const SOURCE = { g: "string", a: "float" }; + const ROWS = { g: ["x", "x", "y", "y"], a: [null, null, 1, 3] }; + + async function grouped(aggregates, expressions, column) { + const table = await perspective.table(SOURCE); + await table.update(ROWS); + const view = await table.view({ + group_by: ["g"], + columns: [column], + aggregates, + expressions, + }); + + const result = await view.to_columns(); + await view.delete(); + await table.delete(); + return result[column]; + } + + test("sum reports null for an all-null group", async function () { + expect(await grouped({}, {}, "a")).toEqual([4, null, 4]); + }); + + test("sum or zero reports zero, keeping its incremental fast path", async function () { + expect(await grouped({ a: "sum or zero" }, {}, "a")).toEqual([ + 4, 0, 4, + ]); + }); + + test("sum reports null for an expression column too", async function () { + const column = await grouped({}, { e: '"a"' }, "e"); + expect(column).toEqual([4, null, 4]); + }); + + test("sum not null is a legacy alias for sum", async function () { + const column = await grouped({ a: "sum not null" }, {}, "a"); + expect(column).toEqual([4, null, 4]); + }); + + test("sum or zero still matches sum when a value is present", async function () { + const plain = await grouped({}, {}, "a"); + const or_zero = await grouped({ a: "sum or zero" }, {}, "a"); + expect([plain[0], plain[2]]).toEqual([or_zero[0], or_zero[2]]); + }); + + test("sum abs reports null for an all-null group", async function () { + const column = await grouped({ a: "sum abs" }, {}, "a"); + expect(column).toEqual([4, null, 4]); + }); + + test("abs sum reports null for an all-null group", async function () { + const column = await grouped({ a: "abs sum" }, {}, "a"); + expect(column).toEqual([4, null, 4]); + }); + + test("pct sum parent reports null for an all-null group", async function () { + const column = await grouped({ a: "pct sum parent" }, {}, "a"); + expect(column).toEqual([100, null, 100]); + }); + + test("pct sum grand total reports null for an all-null group", async function () { + const column = await grouped( + { a: "pct sum grand total" }, + {}, + "a", + ); + expect(column).toEqual([100, null, 100]); + }); + + test("a group with one non-null value still sums to it", async function () { + const table = await perspective.table(SOURCE); + await table.update({ g: ["x", "x"], a: [null, 5] }); + const view = await table.view({ + group_by: ["g"], + columns: ["a"], + }); + + expect((await view.to_columns()).a).toEqual([5, 5]); + await view.delete(); + await table.delete(); + }); + + test("min, max and mean already report null for an all-null group", async function () { + for (const agg of ["min", "max", "mean"]) { + const column = await grouped({ a: agg }, {}, "a"); + expect([agg, column[1]]).toEqual([agg, null]); + } + }); + }); + test.describe("sum across accumulator dtypes", function () { + for (const dtype of ["integer", "float"]) { + async function grouped(rows, aggregates) { + const table = await perspective.table({ + g: "string", + a: dtype, + }); + + await table.update(rows); + const view = await table.view({ + group_by: ["g"], + columns: ["a"], + aggregates: aggregates ?? { a: "sum" }, + }); + + const out = (await view.to_columns()).a; + await view.delete(); + await table.delete(); + return out; + } + + test(`${dtype} > an all-null group is null`, async function () { + const out = await grouped({ + g: ["x", "x", "y", "y"], + a: [null, null, 1, 3], + }); + + expect(out).toEqual([4, null, 4]); + }); + + test(`${dtype} > a populated group sums normally`, async function () { + const out = await grouped({ + g: ["x", "x", "y"], + a: [2, 3, 7], + }); + + expect(out).toEqual([12, 5, 7]); + }); + + test(`${dtype} > a group netting to zero is zero, not null`, async function () { + const out = await grouped({ + g: ["x", "x", "y"], + a: [5, -5, 1], + }); + + expect(out).toEqual([1, 0, 1]); + }); + + test(`${dtype} > sum or zero still reports zero for an all-null group`, async function () { + const out = await grouped( + { g: ["x", "x", "y", "y"], a: [null, null, 1, 3] }, + { a: "sum or zero" }, + ); + + expect(out).toEqual([4, 0, 4]); + }); + + test(`${dtype} > nulling a group's only value reports null`, async function () { + const table = await perspective.table( + { ticker: "string", pnl: dtype }, + { index: "ticker" }, + ); + + await table.update([ + { ticker: "IBM", pnl: 100 }, + { ticker: "AAPL", pnl: 100 }, + ]); + + const view = await table.view({ + group_by: ["ticker"], + columns: ["pnl"], + aggregates: { pnl: "sum" }, + }); + + await table.update([{ ticker: "AAPL", pnl: null }]); + expect(await view.to_json()).toEqual([ + { __ROW_PATH__: [], pnl: 100 }, + { __ROW_PATH__: ["AAPL"], pnl: null }, + { __ROW_PATH__: ["IBM"], pnl: 100 }, + ]); + + await view.delete(); + await table.delete(); + }); + } + + test("float > draining a group across batches reports null, not residue", async function () { + const table = await perspective.table( + { id: "integer", g: "string", pnl: "float" }, + { index: "id" }, + ); + + await table.update([ + { id: 0, g: "x", pnl: 0.1 }, + { id: 1, g: "x", pnl: 0.2 }, + { id: 2, g: "y", pnl: 1 }, + ]); + + const view = await table.view({ + group_by: ["g"], + columns: ["pnl"], + }); + + await view.to_columns(); + await table.update([{ id: 0, pnl: null }]); + await table.update([{ id: 1, pnl: null }]); + expect((await view.to_columns()).pnl).toEqual([1, null, 1]); + await view.delete(); + await table.delete(); + }); + + test("float > a NaN is skipped rather than poisoning the total", async function () { + const table = await perspective.table({ + g: "string", + a: "float", + }); + + await table.update({ + g: ["x", "x", "x"], + a: [1, Number.NaN, 4], + }); + + const view = await table.view({ + group_by: ["g"], + columns: ["a"], + aggregates: { a: "sum" }, + }); + + expect((await view.to_columns()).a).toEqual([5, 5]); + await view.delete(); + await table.delete(); + }); + }); test.describe("sum aggregate with null updates (#1256)", function () { test("sum does not accumulate when an indexed row flips between null and a value", async function () { const table = await perspective.table( @@ -352,7 +575,7 @@ import perspective from "./perspective_client"; const nulled = [ { __ROW_PATH__: [], pnl: 100 }, - { __ROW_PATH__: ["AAPL"], pnl: 0 }, + { __ROW_PATH__: ["AAPL"], pnl: null }, { __ROW_PATH__: ["IBM"], pnl: 100 }, ]; @@ -393,7 +616,7 @@ import perspective from "./perspective_client"; const nulled = [ { __ROW_PATH__: [], pnl: 100.5 }, - { __ROW_PATH__: ["AAPL"], pnl: 0 }, + { __ROW_PATH__: ["AAPL"], pnl: null }, { __ROW_PATH__: ["IBM"], pnl: 100.5 }, ]; diff --git a/rust/perspective-js/test/js/sort.spec.js b/rust/perspective-js/test/js/sort.spec.js index 7010a6ad93..9e491021fc 100644 --- a/rust/perspective-js/test/js/sort.spec.js +++ b/rust/perspective-js/test/js/sort.spec.js @@ -12,6 +12,7 @@ import { test, expect } from "@perspective-dev/test"; import perspective from "./perspective_client"; +import { joined_column_paths } from "./column_paths.js"; const data = { w: [1.5, 2.5, 3.5, 4.5, 5.5, 6.5, 7.5, 8.5], @@ -177,7 +178,7 @@ const data3 = { ["a"], ["b"], ], - w: [12, 0, 0, 0, 0, 1.5, 2.5, 3.5, 4.5], + w: [12, null, null, null, null, 1.5, 2.5, 3.5, 4.5], x: [null, 3, 4, 4, 3, 2, 1, 1, 2], }); @@ -214,7 +215,17 @@ const data3 = { ["a"], ["b"], ], - w: [11.92, 0, 0, 0, 0, 1.57, 2.59, 3.25, 4.51], + w: [ + 11.92, + null, + null, + null, + null, + 1.57, + 2.59, + 3.25, + 4.51, + ], x: [null, 3, 4, 4, 3, 2, 1, 1, 2], }); @@ -411,7 +422,7 @@ const data3 = { columns: ["w", "y"], sort: [["x", "desc"]], }); - const paths = await view.column_paths(); + const paths = await joined_column_paths(view); expect(paths).toEqual(["w", "y"]); view.delete(); table.delete(); @@ -425,7 +436,7 @@ const data3 = { split_by: ["z"], sort: [["x", "col desc"]], }); - const paths = await view.column_paths(); + const paths = await joined_column_paths(view); expect(paths).toEqual(["false|w", "true|w"]); view.delete(); table.delete(); @@ -442,7 +453,7 @@ const data3 = { ["y", "desc"], ], }); - const paths = await view.column_paths(); + const paths = await joined_column_paths(view); expect(paths).toEqual(["false|w", "true|w"]); view.delete(); table.delete(); @@ -601,7 +612,7 @@ const data3 = { split_by: ["y"], sort: [["x", "desc"]], }); - const paths = await view.column_paths(); + const paths = await joined_column_paths(view); expect(paths).toEqual(["a|w", "b|w", "c|w", "d|w"]); const answer = [ { "a|w": null, "b|w": null, "c|w": null, "d|w": 4.5 }, @@ -626,7 +637,7 @@ const data3 = { split_by: ["y"], sort: [["x", "col desc"]], }); - const paths = await view.column_paths(); + const paths = await joined_column_paths(view); expect(paths).toEqual(["d|w", "c|w", "b|w", "a|w"]); const answer = { "d|w": [null, null, null, 4.5, null, null, null, 8.5], @@ -651,7 +662,7 @@ const data3 = { sort: [["y", "desc"]], }); - const paths = await view.column_paths(); + const paths = await joined_column_paths(view); // regular non-col sort should not change order of column paths expect(paths).toEqual(["a|x", "b|x"]); @@ -675,7 +686,7 @@ const data3 = { sort: [["y", "col desc"]], }); - const paths = await view.column_paths(); + const paths = await joined_column_paths(view); expect(paths).toEqual(["b|x", "a|x"]); const result = await view.to_columns(); @@ -704,7 +715,7 @@ const data3 = { }, }); - const paths = await view.column_paths(); + const paths = await joined_column_paths(view); // regular non-col sort should not change order of column paths expect(paths).toEqual(["a|x", "b|x"]); @@ -734,7 +745,7 @@ const data3 = { }, }); - const paths = await view.column_paths(); + const paths = await joined_column_paths(view); expect(paths).toEqual(["b|x", "a|x"]); const result = await view.to_columns(); @@ -758,7 +769,7 @@ const data3 = { sort: [["y", "col desc"]], }); - const paths = await view.column_paths(); + const paths = await joined_column_paths(view); // Col sort will override aggregate expect(paths).toEqual(["b|x", "a|x"]); @@ -811,7 +822,7 @@ const data3 = { ["y", "col asc"], ], }); - const paths = await view.column_paths(); + const paths = await joined_column_paths(view); expect(paths).toEqual(["x|z", "y|z"]); const expected = { __ROW_PATH__: [[], ["a"], ["b"], ["c"]], @@ -839,7 +850,7 @@ const data3 = { ["y", "col desc"], ], }); - const paths = await view.column_paths(); + const paths = await joined_column_paths(view); expect(paths).toEqual(["y|z", "x|z"]); const expected = { __ROW_PATH__: [[], ["c"], ["b"], ["a"]], @@ -864,7 +875,7 @@ const data3 = { split_by: ["x"], sort: [["x", "desc"]], }); - const paths = await view.column_paths(); + const paths = await joined_column_paths(view); expect(paths).toEqual(["a|z", "b|z", "c|z"]); const expected = { __ROW_PATH__: [[], ["x"], ["y"]], diff --git a/rust/perspective-js/test/js/split_rollup_mode.spec.js b/rust/perspective-js/test/js/split_rollup_mode.spec.js index b98ffd82fe..5904a60df7 100644 --- a/rust/perspective-js/test/js/split_rollup_mode.spec.js +++ b/rust/perspective-js/test/js/split_rollup_mode.spec.js @@ -12,6 +12,7 @@ import { test, expect } from "@perspective-dev/test"; import perspective from "./perspective_client"; +import { joined_column_paths } from "./column_paths.js"; const data = { w: [1.5, 2.5, 3.5, 4.5, 5.5, 6.5, 7.5, 8.5], @@ -31,7 +32,7 @@ const data = { split_by: ["z"], split_rollup_mode: "rollup", }); - const paths = await view.column_paths(); + const paths = await joined_column_paths(view); expect(paths).toStrictEqual(["w", "false|w", "true|w"]); expect(await view.num_columns()).toEqual(3); view.delete(); @@ -65,7 +66,7 @@ const data = { split_by: ["z", "y"], split_rollup_mode: "rollup", }); - const paths = await view.column_paths(); + const paths = await joined_column_paths(view); expect(paths).toStrictEqual([ "w", "false|w", @@ -120,7 +121,7 @@ const data = { split_rollup_mode: "rollup", sort: [["x", "asc"]], }); - const paths = await view.column_paths(); + const paths = await joined_column_paths(view); expect(paths).toStrictEqual(["w", "false|w", "true|w"]); expect(await view.num_columns()).toEqual(3); view.delete(); @@ -154,7 +155,7 @@ const data = { split_by: ["z"], split_rollup_mode: "rollup", }); - const paths = await view.column_paths(); + const paths = await joined_column_paths(view); expect(paths).toStrictEqual(["w", "false|w", "true|w"]); // Each column-only row is a single-row group, so the @@ -178,7 +179,7 @@ const data = { split_by: ["z"], split_rollup_mode: "rollup", }); - const paths = await view.column_paths(); + const paths = await joined_column_paths(view); expect(paths).toStrictEqual(["w", "false|w", "true|w"]); const cols = await view.to_columns(); @@ -199,7 +200,7 @@ const data = { split_by: ["z"], split_rollup_mode: "rollup", }); - const paths = await view.column_paths(); + const paths = await joined_column_paths(view); expect(paths).toStrictEqual([ "w", "x", @@ -353,7 +354,7 @@ const data = { group_by: ["y"], split_by: ["z"], }); - const paths = await view.column_paths(); + const paths = await joined_column_paths(view); expect(paths).toStrictEqual(["false|w", "true|w"]); const config = await view.get_config(); expect(config.split_rollup_mode).toEqual("flat"); diff --git a/rust/perspective-js/test/js/to_format/to_arrow.spec.js b/rust/perspective-js/test/js/to_format/to_arrow.spec.js index 85c6b73777..49ec6142f7 100644 --- a/rust/perspective-js/test/js/to_format/to_arrow.spec.js +++ b/rust/perspective-js/test/js/to_format/to_arrow.spec.js @@ -14,6 +14,20 @@ import { test, expect } from "@perspective-dev/test"; import perspective from "../perspective_client"; import { STD_DATE, int_float_string_data, pivoted_output } from "./_shared"; +const unpivot = (rows) => + rows.map((x) => { + x.__ROW_PATH__ = [x.string].filter((x) => x); + delete x.string; + for (const name of Object.keys(x)) { + if (name.endsWith("string (count)")) { + x[name.slice(0, -" (count)".length)] = x[name]; + delete x[name]; + } + } + + return x; + }); + ((perspective) => { test.describe("to_arrow()", function () { test("serializes boolean arrays correctly", async function () { @@ -182,15 +196,7 @@ import { STD_DATE, int_float_string_data, pivoted_output } from "./_shared"; let view2 = await table2.view(); let json2 = await view2.to_json(); - expect(json).toEqual( - json2.map((x) => { - x.__ROW_PATH__ = [x["string (Group by 1)"]].filter( - (x) => x, - ); - delete x["string (Group by 1)"]; - return x; - }), - ); + expect(json).toEqual(unpivot(json2)); view2.delete(); table2.delete(); @@ -213,9 +219,9 @@ import { STD_DATE, int_float_string_data, pivoted_output } from "./_shared"; let result = await view2.to_columns(); expect(result).toEqual({ - "string (Group by 1)": [null, "a", "b"], + string: [null, "a", "b"], float: [2.75, 1.75, 3.75], - string: [4, 2, 2], + "string (count)": [4, 2, 2], }); await view2.delete(); @@ -224,6 +230,56 @@ import { STD_DATE, int_float_string_data, pivoted_output } from "./_shared"; await table.delete(); }); + test("Arrow output qualifies an aggregate of a group_by column", async function () { + let table = await perspective.table(int_float_string_data); + let view = await table.view({ + group_by: ["string"], + columns: ["string", "float"], + aggregates: { string: "distinct count", float: "sum" }, + }); + + let readable = await perspective.table(await view.to_arrow()); + expect(await readable.columns()).toEqual([ + "string", + "string (distinct count)", + "float", + ]); + + let machine = await perspective.table( + await view.to_arrow({ machine_column_names: true }), + ); + + expect(await machine.columns()).toEqual([ + "__ROW_PATH_0__", + "string", + "float", + ]); + + await machine.delete(); + await readable.delete(); + await view.delete(); + await table.delete(); + }); + + test("Arrow output names a multi-aggregate of a group_by column", async function () { + let table = await perspective.table(int_float_string_data); + let view = await table.view({ + group_by: ["float"], + columns: ["float"], + aggregates: { float: ["weighted mean", ["int"]] }, + }); + + let table2 = await perspective.table(await view.to_arrow()); + expect(await table2.columns()).toEqual([ + "float", + "float (weighted mean by int)", + ]); + + await table2.delete(); + await view.delete(); + await table.delete(); + }); + test("Transitive arrow output 1-sided mean", async function () { let table = await perspective.table(int_float_string_data); let view = await table.view({ @@ -236,15 +292,7 @@ import { STD_DATE, int_float_string_data, pivoted_output } from "./_shared"; let view2 = await table2.view(); let json2 = await view2.to_json(); - expect(json).toEqual( - json2.map((x) => { - x.__ROW_PATH__ = [x["string (Group by 1)"]].filter( - (x) => x, - ); - delete x["string (Group by 1)"]; - return x; - }), - ); + expect(json).toEqual(unpivot(json2)); view2.delete(); table2.delete(); @@ -265,15 +313,7 @@ import { STD_DATE, int_float_string_data, pivoted_output } from "./_shared"; let view2 = await table2.view(); let json2 = await view2.to_json(); - expect(json).toEqual( - json2.map((x) => { - x.__ROW_PATH__ = [x["string (Group by 1)"]].filter( - (x) => x, - ); - delete x["string (Group by 1)"]; - return x; - }), - ); + expect(json).toEqual(unpivot(json2)); view2.delete(); table2.delete(); @@ -294,15 +334,7 @@ import { STD_DATE, int_float_string_data, pivoted_output } from "./_shared"; let view2 = await table2.view(); let json2 = await view2.to_json(); - expect(json).toEqual( - json2.map((x) => { - x.__ROW_PATH__ = [x["string (Group by 1)"]].filter( - (x) => x, - ); - delete x["string (Group by 1)"]; - return x; - }), - ); + expect(json).toEqual(unpivot(json2)); view2.delete(); table2.delete(); @@ -319,15 +351,7 @@ import { STD_DATE, int_float_string_data, pivoted_output } from "./_shared"; let view2 = await table2.view(); let json2 = await view2.to_json(); - expect(json).toEqual( - json2.map((x) => { - x.__ROW_PATH__ = [x["string (Group by 1)"]].filter( - (x) => x, - ); - delete x["string (Group by 1)"]; - return x; - }), - ); + expect(json).toEqual(unpivot(json2)); view2.delete(); table2.delete(); @@ -344,15 +368,7 @@ import { STD_DATE, int_float_string_data, pivoted_output } from "./_shared"; let view2 = await table2.view(); let json2 = await view2.to_json(); - expect(json).toEqual( - json2.map((x) => { - x.__ROW_PATH__ = [x["string (Group by 1)"]].filter( - (x) => x, - ); - delete x["string (Group by 1)"]; - return x; - }), - ); + expect(json).toEqual(unpivot(json2)); view2.delete(); table2.delete(); @@ -372,15 +388,7 @@ import { STD_DATE, int_float_string_data, pivoted_output } from "./_shared"; let view2 = await table2.view(); let json2 = await view2.to_json(); - expect(json).toEqual( - json2.map((x) => { - x.__ROW_PATH__ = [x["string (Group by 1)"]].filter( - (x) => x, - ); - delete x["string (Group by 1)"]; - return x; - }), - ); + expect(json).toEqual(unpivot(json2)); view2.delete(); table2.delete(); @@ -401,15 +409,7 @@ import { STD_DATE, int_float_string_data, pivoted_output } from "./_shared"; let view2 = await table2.view(); let json2 = await view2.to_json(); - expect(json).toEqual( - json2.map((x) => { - x.__ROW_PATH__ = [x["string (Group by 1)"]].filter( - (x) => x, - ); - delete x["string (Group by 1)"]; - return x; - }), - ); + expect(json).toEqual(unpivot(json2)); view2.delete(); table2.delete(); @@ -429,15 +429,7 @@ import { STD_DATE, int_float_string_data, pivoted_output } from "./_shared"; let view2 = await table2.view(); let json2 = await view2.to_json(); - expect(json).toEqual( - json2.map((x) => { - x.__ROW_PATH__ = [x["string (Group by 1)"]].filter( - (x) => x, - ); - delete x["string (Group by 1)"]; - return x; - }), - ); + expect(json).toEqual(unpivot(json2)); view2.delete(); table2.delete(); @@ -457,15 +449,7 @@ import { STD_DATE, int_float_string_data, pivoted_output } from "./_shared"; let view2 = await table2.view(); let json2 = await view2.to_json(); - expect(json).toEqual( - json2.map((x) => { - x.__ROW_PATH__ = [x["string (Group by 1)"]].filter( - (x) => x, - ); - delete x["string (Group by 1)"]; - return x; - }), - ); + expect(json).toEqual(unpivot(json2)); view2.delete(); table2.delete(); diff --git a/rust/perspective-js/test/js/to_format/to_csv.spec.ts b/rust/perspective-js/test/js/to_format/to_csv.spec.ts index a2bcef1401..9943931abe 100644 --- a/rust/perspective-js/test/js/to_format/to_csv.spec.ts +++ b/rust/perspective-js/test/js/to_format/to_csv.spec.ts @@ -47,8 +47,25 @@ test.describe("to_csv", () => { ]); const view = await table.view({ group_by: ["y"] }); const csv = await view.to_csv(); + expect(csv).toEqual('"y","x","y (count)"\n,6,3\n"a",3,2\n"b",3,1\n'); + view.delete(); + table.delete(); + }); + + test("1-sided view qualifies an aggregate of its key", async () => { + const table = await perspective.table([ + { x: 1, y: "a" }, + { x: 2, y: "a" }, + { x: 3, y: "b" }, + ]); + const view = await table.view({ + group_by: ["y"], + columns: ["y", "x"], + aggregates: { y: "distinct count" }, + }); + const csv = await view.to_csv(); expect(csv).toEqual( - '"y (Group by 1)","x","y"\n,6,3\n"a",3,2\n"b",3,1\n', + '"y","y (distinct count)","x"\n,2,6\n"a",1,3\n"b",1,3\n', ); view.delete(); table.delete(); diff --git a/rust/perspective-js/test/js/to_format/to_format_viewport.spec.js b/rust/perspective-js/test/js/to_format/to_format_viewport.spec.js index 52ca8eeba3..85e993103f 100644 --- a/rust/perspective-js/test/js/to_format/to_format_viewport.spec.js +++ b/rust/perspective-js/test/js/to_format/to_format_viewport.spec.js @@ -12,6 +12,7 @@ import { test, expect } from "@perspective-dev/test"; import perspective from "../perspective_client"; +import { joined_column_paths } from "../column_paths.js"; const data = { w: [ @@ -298,7 +299,7 @@ test.describe("to_format viewport", function () { test("0 sided", async function () { const table = await perspective.table(data); const view = await table.view({}); - const paths = await view.column_paths({ + const paths = await joined_column_paths(view, { start_col: 1, end_col: 3, }); @@ -311,10 +312,10 @@ test.describe("to_format viewport", function () { const table = await perspective.table(data); const view = await table.view({ group_by: ["y"] }); expect( - await view.column_paths({ start_col: 0, end_col: 1 }), + await joined_column_paths(view, { start_col: 0, end_col: 1 }), ).toEqual(["w"]); expect( - await view.column_paths({ start_col: 1, end_col: 3 }), + await joined_column_paths(view, { start_col: 1, end_col: 3 }), ).toEqual(["x", "y"]); view.delete(); table.delete(); @@ -327,13 +328,13 @@ test.describe("to_format viewport", function () { split_by: ["z"], }); expect( - await view.column_paths({ start_col: 0, end_col: 1 }), + await joined_column_paths(view, { start_col: 0, end_col: 1 }), ).toEqual(["false|w"]); expect( - await view.column_paths({ start_col: 1, end_col: 2 }), + await joined_column_paths(view, { start_col: 1, end_col: 2 }), ).toEqual(["false|x"]); expect( - await view.column_paths({ start_col: 3, end_col: 5 }), + await joined_column_paths(view, { start_col: 3, end_col: 5 }), ).toEqual(["false|z", "true|w"]); view.delete(); table.delete(); @@ -343,7 +344,7 @@ test.describe("to_format viewport", function () { const table = await perspective.table(data); const view = await table.view({ split_by: ["z"] }); expect( - await view.column_paths({ start_col: 0, end_col: 1 }), + await joined_column_paths(view, { start_col: 0, end_col: 1 }), ).toEqual(["false|w"]); view.delete(); table.delete(); @@ -365,7 +366,7 @@ test.describe("to_format viewport", function () { }; const cols = await view.to_columns(viewport); - const paths = await view.column_paths({ + const paths = await joined_column_paths(view, { start_col: viewport.start_col, end_col: viewport.end_col, }); diff --git a/rust/perspective-js/test/js/to_format/with_column_typed_array.spec.ts b/rust/perspective-js/test/js/to_format/with_column_typed_array.spec.ts index 8ef6f291f1..98510d1531 100644 --- a/rust/perspective-js/test/js/to_format/with_column_typed_array.spec.ts +++ b/rust/perspective-js/test/js/to_format/with_column_typed_array.spec.ts @@ -485,8 +485,7 @@ test.describe("with_typed_arrays()", () => { }); expect(names).toContain("__ROW_PATH_0__"); - // Should NOT contain the legacy "category (Group by 1)" naming - expect(names).not.toContain("category (Group by 1)"); + expect(names).not.toContain("category (count)"); await view.delete(); await table.delete(); }); @@ -547,8 +546,8 @@ test.describe("with_typed_arrays()", () => { expect(names).toContain("__ROW_PATH_0__"); expect(names).toContain("__ROW_PATH_1__"); - expect(names).not.toContain("region (Group by 1)"); - expect(names).not.toContain("country (Group by 2)"); + expect(names).not.toContain("region (count)"); + expect(names).not.toContain("country (count)"); await view.delete(); await table.delete(); }); @@ -586,10 +585,7 @@ test.describe("with_typed_arrays()", () => { await table.delete(); }); - test("regular to_arrow still uses legacy naming by default", async () => { - // Sanity check: `to_arrow` (not with_typed_arrays) should - // still use legacy "colname (Group by N)" naming for backwards - // compatibility. `with_typed_arrays` forces the new naming. + test("regular to_arrow uses readable names by default", async () => { const table = await perspective.table({ category: ["a", "b"], value: [1, 2], @@ -600,10 +596,13 @@ test.describe("with_typed_arrays()", () => { }); const arrow = await view.to_arrow(); - // The Arrow IPC bytes should contain the legacy name. - const bytes = new Uint8Array(arrow); - const text = new TextDecoder().decode(bytes); - expect(text.includes("(Group by 1)")).toBe(true); + const table2 = await perspective.table(arrow); + expect(await table2.columns()).toEqual([ + "category", + "category (count)", + "value", + ]); + await table2.delete(); await view.delete(); await table.delete(); }); diff --git a/rust/perspective-js/test/js/updates.spec.js b/rust/perspective-js/test/js/updates.spec.js index ed94b56b49..6d668ce372 100644 --- a/rust/perspective-js/test/js/updates.spec.js +++ b/rust/perspective-js/test/js/updates.spec.js @@ -552,7 +552,7 @@ async function match_delta(perspective, delta, expected) { __ROW_PATH__: [[]], x: [""], y: [0], - idx: [0], + idx: [null], }); await view.delete(); @@ -2124,7 +2124,7 @@ async function match_delta(perspective, delta, expected) { expect(json).toEqual([ { __ROW_PATH__: [], y: 1 }, { __ROW_PATH__: [1], y: 1 }, - { __ROW_PATH__: [2], y: 0 }, + { __ROW_PATH__: [2], y: null }, ]); view.delete(); table.delete(); diff --git a/rust/perspective-python/build.mjs b/rust/perspective-python/build.mjs index 60e30a8383..da04eb8d0a 100644 --- a/rust/perspective-python/build.mjs +++ b/rust/perspective-python/build.mjs @@ -45,10 +45,11 @@ const env = { ...process.env }; let emsdk_prefix = ""; if (is_pyodide) { const emsdkdir = path.resolve(__dirname, "../../.emsdk"); - const { pyodide_emscripten } = JSON.parse( + const { emscripten } = JSON.parse( fs.readFileSync(path.resolve(__dirname, "../../package.json")), ); - emsdk_prefix = `cd ${emsdkdir} && . ./emsdk_env.sh && ./emsdk activate ${pyodide_emscripten} && cd ${cwd} && `; + + emsdk_prefix = `cd ${emsdkdir} && . ./emsdk_env.sh && ./emsdk activate ${emscripten} && cd ${cwd} && `; env.MATURIN_PYEMSCRIPTEN_PLATFORM_VERSION = process.env.MATURIN_PYEMSCRIPTEN_PLATFORM_VERSION || "2026_0"; } @@ -132,9 +133,7 @@ if (build_sdist) { ); } - if ( - !fs.existsSync("perspective/widget/static/perspective-anywidget.js") - ) { + if (!fs.existsSync("perspective/widget/static/perspective-anywidget.js")) { throw new Error( "anywidget bundle is not present in perspective/widget/static, please build `@perspective-dev/anywidget`", ); diff --git a/rust/perspective-python/perspective/tests/column_paths.py b/rust/perspective-python/perspective/tests/column_paths.py new file mode 100644 index 0000000000..e54f38ef41 --- /dev/null +++ b/rust/perspective-python/perspective/tests/column_paths.py @@ -0,0 +1,33 @@ +# ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +# ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +# ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +# ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +# ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +# ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +# ┃ Copyright (c) 2017, the Perspective Authors. ┃ +# ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +# ┃ This file is part of the Perspective library, distributed under the terms ┃ +# ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +# ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + + +def joined_column_paths(view, **window): + """A View's column paths as "|"-joined strings. + + For the many tests whose subject is which columns exist in what order + rather than the typed values `column_paths` now returns. Tests of + `column_paths` itself assert the area directly - this helper is + deliberately lossy in the same way the old wire format was, so it must + not be used to pin a split value's type or rendering. + """ + area = view.column_paths(**window) + width = len(area[-1]) if area else 0 + out = [] + for x in range(width): + parts = [ + str(area[level][x]) + for level in range(len(area)) + if area[level][x] is not None + ] + out.append("|".join(parts)) + return out diff --git a/rust/perspective-python/perspective/tests/multi_threaded/test_derived_threaded.py b/rust/perspective-python/perspective/tests/multi_threaded/test_derived_threaded.py new file mode 100644 index 0000000000..5c4917d4eb --- /dev/null +++ b/rust/perspective-python/perspective/tests/multi_threaded/test_derived_threaded.py @@ -0,0 +1,103 @@ +# ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +# ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +# ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +# ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +# ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +# ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +# ┃ Copyright (c) 2017, the Perspective Authors. ┃ +# ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +# ┃ This file is part of the Perspective library, distributed under the terms ┃ +# ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +# ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +from perspective import Server + +import random +import threading + + +PARENTS = { + "flat": dict(filter=[["y", ">", 2]]), + "group_by": dict( + group_by=["g"], + columns=["x", "y"], + aggregates={"x": "sum", "y": "mean"}, + group_rollup_mode="flat", + ), + "split_by": dict( + group_by=["g"], + split_by=["s"], + columns=["x"], + group_rollup_mode="flat", + ), +} + + +def _row(rand, i): + return { + "id": i, + "g": rand.choice(["a", "b", "c", "d"]), + "s": rand.choice(["Mon", "Tue", "Wed"]), + "x": rand.randint(0, 4000) / 4, + "y": rand.randint(0, 9), + } + + +def _key(row): + return sorted((k, str(v)) for k, v in row.items()) + + +def _run(config): + rand = random.Random(7) + client = Server().new_local_client() + table = client.table( + {"id": "integer", "g": "string", "s": "string", "x": "float", "y": "integer"}, + index="id", + ) + + table.update([_row(rand, i) for i in range(60)]) + view = table.view(**config) + derived = client.table(view) + child = derived.view() + errors = [] + done = threading.Event() + + def read(): + try: + while not done.is_set(): + child.to_columns() + derived.size() + except Exception as e: + errors.append(e) + + readers = [threading.Thread(target=read) for _ in range(4)] + for reader in readers: + reader.start() + + try: + for _ in range(500): + if rand.random() < 0.2: + table.remove([rand.randint(0, 59) for _ in range(3)]) + else: + table.update([_row(rand, rand.randint(0, 59)) for _ in range(4)]) + finally: + done.set() + for reader in readers: + reader.join() + + assert errors == [] + fresh = client.table(view) + expected = fresh.view().to_records() + actual = child.to_records() + assert sorted(map(_key, actual)) == sorted(map(_key, expected)) + + +class TestDerivedTableThreaded(object): + def test_flat_parent(self): + _run(PARENTS["flat"]) + + def test_group_by_parent(self): + _run(PARENTS["group_by"]) + + def test_split_by_parent(self): + _run(PARENTS["split_by"]) diff --git a/rust/perspective-python/perspective/tests/multi_threaded/test_multi_threaded.py b/rust/perspective-python/perspective/tests/multi_threaded/test_multi_threaded.py index 4b43bb5277..8f2c21a8db 100644 --- a/rust/perspective-python/perspective/tests/multi_threaded/test_multi_threaded.py +++ b/rust/perspective-python/perspective/tests/multi_threaded/test_multi_threaded.py @@ -253,3 +253,41 @@ def feed(): executor.shutdown() assert checked > 0 + + def test_concurrent_flat_view_reads_are_threadsafe(self): + client = Server().new_local_client() + table = client.table( + {"a": "integer", "b": "string", "index": "integer"}, index="index" + ) + table.update( + [{"a": i, "b": ascii_letters[i % 52], "index": i} for i in range(60)] + ) + for config in ({}, {"filter": [["a", ">", 2]]}): + view = table.view(**config) + errors = [] + stop = threading.Event() + + def read(): + try: + while not stop.is_set(): + view.column_paths() + view.to_columns() + view.to_arrow() + except Exception as e: + errors.append(e) + + readers = [threading.Thread(target=read) for _ in range(8)] + for reader in readers: + reader.start() + + try: + for i in range(300): + table.update([{"a": i, "b": "z", "index": i % 60}]) + finally: + stop.set() + for reader in readers: + reader.join() + + assert errors == [] + assert view.num_rows() == len(view.to_columns()["a"]) + view.delete() diff --git a/rust/perspective-python/perspective/tests/table/test_column_paths.py b/rust/perspective-python/perspective/tests/table/test_column_paths.py index 4ea63e7fa7..d46c7602bd 100644 --- a/rust/perspective-python/perspective/tests/table/test_column_paths.py +++ b/rust/perspective-python/perspective/tests/table/test_column_paths.py @@ -20,7 +20,7 @@ def test_column_paths(self, superstore): tbl = client.table(superstore) view = tbl.view() paths = view.column_paths() - assert paths == [ + assert paths == [[ "index", "Row ID", "Order ID", @@ -41,7 +41,7 @@ def test_column_paths(self, superstore): "Quantity", "Discount", "Profit", - ] + ]] view.delete() tbl.delete() @@ -51,9 +51,8 @@ def test_column_paths_split_by(self, superstore): view = tbl.view(group_by=["State"], columns=["Sales"], split_by=["Ship Mode"]) paths = view.column_paths() assert paths == [ - "First Class|Sales", - "Second Class|Sales", - "Standard Class|Sales", + ["First Class", "Second Class", "Standard Class"], + ["Sales", "Sales", "Sales"], ] view.delete() @@ -64,15 +63,16 @@ def test_column_paths_split_by_range(self, superstore): view = tbl.view(group_by=["State"], columns=["Sales"], split_by=["Ship Mode"]) paths = view.column_paths(start_col=1) assert paths == [ - "Second Class|Sales", - "Standard Class|Sales", + ["Second Class", "Standard Class"], + ["Sales", "Sales"], ] view.delete() view = tbl.view(group_by=["State"], columns=["Sales"], split_by=["Ship Mode"]) paths = view.column_paths(start_col=1, end_col=2) assert paths == [ - "Second Class|Sales", + ["Second Class"], + ["Sales"], ] view.delete() @@ -80,7 +80,8 @@ def test_column_paths_split_by_range(self, superstore): view = tbl.view(group_by=["State"], columns=["Sales"], split_by=["Ship Mode"]) paths = view.column_paths(end_col=1) assert paths == [ - "First Class|Sales", + ["First Class"], + ["Sales"], ] view.delete() diff --git a/rust/perspective-python/perspective/tests/table/test_derived.py b/rust/perspective-python/perspective/tests/table/test_derived.py new file mode 100644 index 0000000000..e54fbcc1cf --- /dev/null +++ b/rust/perspective-python/perspective/tests/table/test_derived.py @@ -0,0 +1,145 @@ +# ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +# ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +# ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +# ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +# ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +# ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +# ┃ Copyright (c) 2017, the Perspective Authors. ┃ +# ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +# ┃ This file is part of the Perspective library, distributed under the terms ┃ +# ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +# ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +from perspective import Server + +import pytest + + +def _rows(n, offset=0): + return [ + { + "id": i, + "g": ["a", "b", "c"][i % 3], + "s": ["Mon", "Tue"][(i // 3) % 2], + "x": i * 1.5, + "y": i % 7, + } + for i in range(offset, offset + n) + ] + + +def _unrolled(view, group_by): + columns = view.to_columns() + paths = columns.pop("__ROW_PATH__", []) + names = list(columns.keys()) + out = [] + for i, path in enumerate(paths): + row = { + g: (path[d] if d < len(path) else None) + for d, g in enumerate(group_by) + } + + row.update({n: columns[n][i] for n in names}) + out.append(row) + + return out + + +def _key(row): + return sorted((k, str(v)) for k, v in row.items()) + + +def _same(a, b): + return sorted(map(_key, a)) == sorted(map(_key, b)) + + +class TestDerivedTable(object): + def _source(self): + client = Server().new_local_client() + table = client.table( + {"id": "integer", "g": "string", "s": "string", "x": "float", "y": "integer"}, + index="id", + ) + + table.update(_rows(24)) + return client, table + + def test_group_by_parent_tracks_updates_and_removes(self): + client, table = self._source() + view = table.view( + group_by=["g"], columns=["x", "y"], aggregates={"x": "sum", "y": "mean"} + ) + + derived = client.table(view) + assert derived.schema() == { + "g": "string", + "x": "float", + "y": "float", + } + + child = derived.view() + assert _same(child.to_records(), _unrolled(view, ["g"])) + for i in range(10): + table.update([{"id": i, "x": i * 100.0}]) + + assert derived.size() == 4 + table.remove([0, 3, 6, 9, 12, 15, 18, 21]) + assert derived.size() == 3 + assert _same(child.to_records(), _unrolled(view, ["g"])) + + def test_split_by_parent(self): + client, table = self._source() + view = table.view( + group_by=["g"], split_by=["s"], columns=["x"], group_rollup_mode="flat" + ) + + derived = client.table(view) + child = derived.view() + table.update(_rows(12, 18)) + assert _same(child.to_records(), _unrolled(view, ["g"])) + + def test_flat_parent_follows_filter(self): + client, table = self._source() + view = table.view(filter=[["y", ">", 3]]) + derived = client.table(view) + assert derived.get_index() == "id" + child = derived.view() + assert _same(child.to_records(), view.to_records()) + table.update([{"id": 0, "y": 6}, {"id": 4, "y": 0}]) + table.remove([5]) + assert _same(child.to_records(), view.to_records()) + + def test_explicit_schema(self): + client, table = self._source() + view = table.view( + group_by=["g"], split_by=["s"], columns=["x"], group_rollup_mode="flat" + ) + + derived = client.table( + view, schema={"g": "string", "Tue|x": "float", "Wed|x": "float"} + ) + + assert derived.columns() == ["g", "Tue|x", "Wed|x"] + child = derived.view() + assert all(v is None for v in child.to_columns()["Wed|x"]) + table.update([{"id": 100, "g": "a", "s": "Wed", "x": 7.0, "y": 1}]) + assert child.to_columns()["Wed|x"].count(7.0) == 1 + + def test_read_only(self): + client, table = self._source() + view = table.view(group_by=["g"], columns=["x"]) + derived = client.table(view) + with pytest.raises(Exception): + derived.update([{"x": 1.0}]) + + with pytest.raises(Exception): + derived.clear() + + def test_index_and_limit_rejected(self): + client, table = self._source() + view = table.view(group_by=["g"], columns=["x"]) + with pytest.raises(Exception): + client.table(view, index="x") + + with pytest.raises(Exception): + client.table(view, limit=10) diff --git a/rust/perspective-python/perspective/tests/table/test_exception.py b/rust/perspective-python/perspective/tests/table/test_exception.py index 9613bb9fde..6c44f4b6a7 100644 --- a/rust/perspective-python/perspective/tests/table/test_exception.py +++ b/rust/perspective-python/perspective/tests/table/test_exception.py @@ -41,6 +41,14 @@ def test_exception_from_core(self): assert str(ex.value) == "Abort(): Invalid column 'b' found in View group_by.\n" + def test_exception_duplicate_group_by(self): + tbl = Table({"a": [1, 2, 3]}) + + with raises(PerspectiveError) as ex: + tbl.view(group_by=["a", "a"]) + + assert str(ex.value) == "Abort(): Duplicate column 'a' found in View group_by.\n" + def test_exception_from_core_catch_generic(self): tbl = Table({"a": [1, 2, 3]}) # `PerspectiveCppError` should inherit from `Exception` diff --git a/rust/perspective-python/perspective/tests/table/test_table_view_table.py b/rust/perspective-python/perspective/tests/table/test_table_view_table.py index e6f1c9b608..f21f407acc 100644 --- a/rust/perspective-python/perspective/tests/table/test_table_view_table.py +++ b/rust/perspective-python/perspective/tests/table/test_table_view_table.py @@ -56,15 +56,15 @@ def test_view_one(self): assert dimms["num_view_rows"] == 3 assert dimms["num_view_columns"] == 3 assert view2.schema() == { - "a (Group by 1)": "integer", "a": "integer", + "a (sum)": "integer", "b": "integer", } assert view2.to_records() == [ - {"a (Group by 1)": None, "a": 4, "b": 6}, - {"a (Group by 1)": 1, "a": 1, "b": 2}, - {"a (Group by 1)": 3, "a": 3, "b": 4}, + {"a": None, "a (sum)": 4, "b": 6}, + {"a": 1, "a (sum)": 1, "b": 2}, + {"a": 3, "a (sum)": 3, "b": 4}, ] def test_view_two(self): @@ -77,33 +77,33 @@ def test_view_two(self): assert dimms["num_view_rows"] == 3 assert dimms["num_view_columns"] == 5 assert view2.schema() == { - "a (Group by 1)": "integer", - "2|a": "integer", + "a": "integer", + "2|a (sum)": "integer", "2|b": "integer", - "4|a": "integer", + "4|a (sum)": "integer", "4|b": "integer", } assert view2.to_records() == [ { - "a (Group by 1)": None, - "2|a": 1, + "a": None, + "2|a (sum)": 1, "2|b": 2, - "4|a": 3, + "4|a (sum)": 3, "4|b": 4, }, { - "a (Group by 1)": 1, - "2|a": 1, + "a": 1, + "2|a (sum)": 1, "2|b": 2, - "4|a": None, + "4|a (sum)": None, "4|b": None, }, { - "a (Group by 1)": 3, - "2|a": None, + "a": 3, + "2|a (sum)": None, "2|b": None, - "4|a": 3, + "4|a (sum)": 3, "4|b": 4, }, ] diff --git a/rust/perspective-python/perspective/tests/table/test_to_arrow.py b/rust/perspective-python/perspective/tests/table/test_to_arrow.py index 22748ccc97..61a249103e 100644 --- a/rust/perspective-python/perspective/tests/table/test_to_arrow.py +++ b/rust/perspective-python/perspective/tests/table/test_to_arrow.py @@ -164,15 +164,14 @@ def test_to_arrow_one_symmetric(self): arrow = view.to_arrow() tbl2 = Table(arrow) assert tbl2.schema() == { - "a (Group by 1)": "integer", "a": "integer", + "a (sum)": "integer", "b": "integer", "c": "integer", } d = view.to_columns() - d["a (Group by 1)"] = [ - x[0] if len(x) > 0 else None for x in d.pop("__ROW_PATH__") - ] + d["a (sum)"] = d.pop("a") + d["a"] = [x[0] if len(x) > 0 else None for x in d.pop("__ROW_PATH__")] assert tbl2.view().to_columns() == d def test_to_arrow_two_symmetric(self): @@ -186,24 +185,24 @@ def test_to_arrow_two_symmetric(self): arrow = view.to_arrow() tbl2 = Table(arrow) assert tbl2.schema() == { - "a (Group by 1)": "integer", - "hello|a": "integer", + "a": "integer", + "hello|a (sum)": "integer", "hello|b": "integer", "hello|c": "integer", - "world|a": "integer", + "world|a (sum)": "integer", "world|b": "integer", "world|c": "integer", - "hello2|a": "integer", + "hello2|a (sum)": "integer", "hello2|b": "integer", "hello2|c": "integer", - "world2|a": "integer", + "world2|a (sum)": "integer", "world2|b": "integer", "world2|c": "integer", } d = view.to_columns() - d["a (Group by 1)"] = [ - x[0] if len(x) > 0 else None for x in d.pop("__ROW_PATH__") - ] + for name in [name for name in d if name.endswith("|a")]: + d[name + " (sum)"] = d.pop(name) + d["a"] = [x[0] if len(x) > 0 else None for x in d.pop("__ROW_PATH__")] assert tbl2.view().to_columns() == d def test_to_arrow_column_only_symmetric(self): @@ -414,4 +413,4 @@ def test_to_arrow_one_mean(self): view2 = table2.view() result = view2.to_columns() - assert result == {"b (Group by 1)": [None, "a", "b"], "a": [2.5, 1.5, 3.5]} + assert result == {"b": [None, "a", "b"], "a": [2.5, 1.5, 3.5]} diff --git a/rust/perspective-python/perspective/tests/table/test_to_format.py b/rust/perspective-python/perspective/tests/table/test_to_format.py index 7bca866663..505f5b6bc5 100644 --- a/rust/perspective-python/perspective/tests/table/test_to_format.py +++ b/rust/perspective-python/perspective/tests/table/test_to_format.py @@ -852,13 +852,19 @@ def test_to_csv_one(self): data = [{"a": 1, "b": 2}, {"a": 1, "b": 2}] tbl = Table(data) view = tbl.view(group_by=["a"]) - assert view.to_csv() == '"a (Group by 1)","a","b"\n,2,4\n1,2,4\n' + assert view.to_csv() == '"a","a (sum)","b"\n,2,4\n1,2,4\n' + + def test_to_csv_one_qualifies_key_aggregate(self): + data = [{"a": 1, "b": 2}, {"a": 1, "b": 2}] + tbl = Table(data) + view = tbl.view(group_by=["a"], columns=["a", "b"], aggregates={"a": "count"}) + assert view.to_csv() == '"a","a (count)","b"\n,2,4\n1,2,4\n' def test_to_csv_two(self): data = [{"a": 1, "b": 2}, {"a": 1, "b": 2}] tbl = Table(data) view = tbl.view(group_by=["a"], split_by=["b"]) - assert view.to_csv() == '"a (Group by 1)","2|a","2|b"\n,2,4\n1,2,4\n' + assert view.to_csv() == '"a","2|a (sum)","2|b"\n,2,4\n1,2,4\n' def test_to_csv_column_only(self): data = [{"a": 1, "b": 2}, {"a": 1, "b": 2}] @@ -870,13 +876,13 @@ def test_to_csv_one_no_columns(self): data = [{"a": 1, "b": 2}, {"a": 1, "b": 2}] tbl = Table(data) view = tbl.view(group_by=["a"], columns=[]) - assert view.to_csv() == '"a (Group by 1)"\n\n1\n' + assert view.to_csv() == '"a"\n\n1\n' def test_to_csv_two_no_columns(self): data = [{"a": 1, "b": 2}, {"a": 1, "b": 2}] tbl = Table(data) view = tbl.view(group_by=["a"], split_by=["b"], columns=[]) - assert view.to_csv() == '"a (Group by 1)"\n\n1\n' + assert view.to_csv() == '"a"\n\n1\n' def test_to_csv_column_only_no_columns(self): data = [{"a": 1, "b": 2}, {"a": 1, "b": 2}] diff --git a/rust/perspective-python/perspective/tests/table/test_view.py b/rust/perspective-python/perspective/tests/table/test_view.py index e59d13dca2..7532b7e412 100644 --- a/rust/perspective-python/perspective/tests/table/test_view.py +++ b/rust/perspective-python/perspective/tests/table/test_view.py @@ -17,6 +17,7 @@ from pytest import approx, mark, raises import perspective as psp +from perspective.tests.column_paths import joined_column_paths client = psp.Server().new_local_client() Table = client.table @@ -90,42 +91,42 @@ def test_view_column_path_zero(self): data = {"a": [1, 2, 3], "b": [1.5, 2.5, 3.5]} tbl = Table(data) view = tbl.view() - paths = view.column_paths() + paths = joined_column_paths(view) assert paths == ["a", "b"] def test_view_column_path_zero_schema(self): data = {"a": "integer", "b": "float"} tbl = Table(data) view = tbl.view() - paths = view.column_paths() + paths = joined_column_paths(view) assert paths == ["a", "b"] def test_view_column_path_zero_hidden(self): data = {"a": [1, 2, 3], "b": [1.5, 2.5, 3.5]} tbl = Table(data) view = tbl.view(columns=["b"]) - paths = view.column_paths() + paths = joined_column_paths(view) assert paths == ["b"] def test_view_column_path_zero_respects_order(self): data = {"a": [1, 2, 3], "b": [1.5, 2.5, 3.5]} tbl = Table(data) view = tbl.view(columns=["b", "a"]) - paths = view.column_paths() + paths = joined_column_paths(view) assert paths == ["b", "a"] def test_view_column_path_one(self): data = {"a": [1, 2, 3], "b": [1.5, 2.5, 3.5]} tbl = Table(data) view = tbl.view(group_by=["a"]) - paths = view.column_paths() + paths = joined_column_paths(view) assert paths == ["a", "b"] def test_view_column_path_one_numeric_names(self): data = {"a": [1, 2, 3], "b": [1.5, 2.5, 3.5], "1234": [5, 6, 7]} tbl = Table(data) view = tbl.view(group_by=["a"], columns=["b", "1234", "a"]) - paths = view.column_paths() + paths = joined_column_paths(view) assert paths == ["b", "1234", "a"] def test_view_column_path_two(self): @@ -134,12 +135,8 @@ def test_view_column_path_two(self): view = tbl.view(group_by=["a"], split_by=["b"]) paths = view.column_paths() assert paths == [ - "1.5|a", - "1.5|b", - "2.5|a", - "2.5|b", - "3.5|a", - "3.5|b", + [1.5, 1.5, 2.5, 2.5, 3.5, 3.5], + ["a", "b", "a", "b", "a", "b"], ] def test_view_column_path_two_column_only(self): @@ -147,13 +144,16 @@ def test_view_column_path_two_column_only(self): tbl = Table(data) view = tbl.view(split_by=["b"]) paths = view.column_paths() - assert paths == ["1.5|a", "1.5|b", "2.5|a", "2.5|b", "3.5|a", "3.5|b"] + assert paths == [ + [1.5, 1.5, 2.5, 2.5, 3.5, 3.5], + ["a", "b", "a", "b", "a", "b"], + ] def test_view_column_path_hidden_sort(self): data = {"a": [1, 2, 3], "b": [1.5, 2.5, 3.5], "c": [3, 2, 1]} tbl = Table(data) view = tbl.view(columns=["a", "b"], sort=[["c", "desc"]]) - paths = view.column_paths() + paths = joined_column_paths(view) assert paths == ["a", "b"] def test_view_column_path_hidden_col_sort(self): @@ -161,14 +161,20 @@ def test_view_column_path_hidden_col_sort(self): tbl = Table(data) view = tbl.view(split_by=["a"], columns=["a", "b"], sort=[["c", "col desc"]]) paths = view.column_paths() - assert paths == ["1|a", "1|b", "2|a", "2|b", "3|a", "3|b"] + assert paths == [ + [1, 1, 2, 2, 3, 3], + ["a", "b", "a", "b", "a", "b"], + ] def test_view_column_path_pivot_by_bool(self): data = {"a": [1, 2, 3], "b": [True, False, True], "c": [3, 2, 1]} tbl = Table(data) view = tbl.view(split_by=["b"], columns=["a", "b", "c"]) paths = view.column_paths() - assert paths == ["false|a", "false|b", "false|c", "true|a", "true|b", "true|c"] + assert paths == [ + [False, False, False, True, True, True], + ["a", "b", "c", "a", "b", "c"], + ] # schema correctness @@ -244,7 +250,7 @@ def test_view_dataframe_column_order(self): ) ) view = table.view(columns=["-0.1", "-0.05", "0.0", "0.1"], group_by=["str"]) - assert view.column_paths() == ["-0.1", "-0.05", "0.0", "0.1"] + assert joined_column_paths(view) == ["-0.1", "-0.05", "0.0", "0.1"] def test_view_aggregate_order_with_columns(self): """If `columns` is provided, order is always guaranteed.""" @@ -257,7 +263,7 @@ def test_view_aggregate_order_with_columns(self): ) order = ["a", "b", "c", "d"] - assert view.column_paths() == order + assert joined_column_paths(view) == order def test_view_df_aggregate_order_with_columns(self): """If `columns` is provided, order is always guaranteed.""" @@ -272,7 +278,7 @@ def test_view_df_aggregate_order_with_columns(self): ) order = ["index", "d", "a", "c", "b"] - assert view.column_paths() == order + assert joined_column_paths(view) == order def test_view_aggregates_with_no_columns(self): data = [{"a": 1, "b": 2, "c": 3, "d": 4}, {"a": 3, "b": 4, "c": 5, "d": 6}] @@ -280,7 +286,7 @@ def test_view_aggregates_with_no_columns(self): view = tbl.view( group_by=["a"], aggregates={"c": "avg", "a": "last"}, columns=[] ) - assert view.column_paths() == [] + assert joined_column_paths(view) == [] assert view.to_records() == [ {"__ROW_PATH__": []}, {"__ROW_PATH__": [1]}, @@ -296,7 +302,7 @@ def test_view_aggregates_default_column_order(self): cols = tbl.columns() view = tbl.view(group_by=["a"], aggregates={"c": "avg", "a": "last"}) - assert view.column_paths() == cols + assert joined_column_paths(view) == cols # check that default aggregates have been applied result = view.to_columns() @@ -325,14 +331,15 @@ def test_view_group_by_datetime_row_paths_are_same_as_data(self, util): "b": [1], } - def test_view_split_by_datetime_names_utc(self): + def test_view_split_by_datetime_names_utc(self, util): """Tests column paths for datetimes in UTC. Timezone-related tests are in the `test_table_datetime` file.""" data = {"a": [datetime(2019, 7, 11, 12, 30)], "b": [1]} tbl = Table(data) view = tbl.view(split_by=["a"]) cols = view.column_paths() - assert cols == ["2019-07-11 12:30:00.000|a", "2019-07-11 12:30:00.000|b"] + ts = util.to_timestamp(datetime(2019, 7, 11, 12, 30)) + assert cols == [[ts, ts], ["a", "b"]] # TODO: time slightly off! thinks its NYE 1969 @mark.skip # We do not support python datetimes. @@ -347,7 +354,7 @@ def test_view_split_by_datetime_names_min(self): tbl.update(data) view = tbl.view(split_by=["a"]) cols = view.column_paths() - assert cols == ["1970-01-01 00:00:00.000|a", "1970-01-01 00:00:00.000|b"] + assert cols == [[0, 0], ["a", "b"]] @mark.skip # We dont support python datetimes. def test_view_split_by_datetime_names_max(self): @@ -357,7 +364,7 @@ def test_view_split_by_datetime_names_max(self): tbl = Table(data) view = tbl.view(split_by=["a"]) cols = view.column_paths() - assert cols == ["10000-01-01 00:00:00.000|a", "10000-01-01 00:00:00.000|b"] + assert cols == [[253402300800000, 253402300800000], ["a", "b"]] # aggregate @@ -539,18 +546,70 @@ def test_view_aggregate_gmv(self): def test_view_aggregate_gmv_split_by(self): data = { "division": [ - "D1", "D2", "D1", "D2", "D1", "D2", "D1", "D2", "D1", "D2", - "D1", "D2", "D1", "D2", "D1", "D2", "D1", "D2", "D1", "D2", + "D1", + "D2", + "D1", + "D2", + "D1", + "D2", + "D1", + "D2", + "D1", + "D2", + "D1", + "D2", + "D1", + "D2", + "D1", + "D2", + "D1", + "D2", + "D1", + "D2", ], "symbol": [ - "AAPL", "GOOG", "MSFT", "AAPL", "GOOG", "MSFT", "AAPL", - "GOOG", "MSFT", "AAPL", "GOOG", "MSFT", "AAPL", "GOOG", - "MSFT", "AAPL", "GOOG", "MSFT", "AAPL", "GOOG", + "AAPL", + "GOOG", + "MSFT", + "AAPL", + "GOOG", + "MSFT", + "AAPL", + "GOOG", + "MSFT", + "AAPL", + "GOOG", + "MSFT", + "AAPL", + "GOOG", + "MSFT", + "AAPL", + "GOOG", + "MSFT", + "AAPL", + "GOOG", ], "MV": [ - 1500, 1200, 1300, 1400, 1600, 1100, 1700, 1800, 1900, 2000, - -2100, -2200, -2300, -2400, -2500, -2600, -2700, -2800, - -2900, -3000, + 1500, + 1200, + 1300, + 1400, + 1600, + 1100, + 1700, + 1800, + 1900, + 2000, + -2100, + -2200, + -2300, + -2400, + -2500, + -2600, + -2700, + -2800, + -2900, + -3000, ], } @@ -1476,7 +1535,7 @@ def test_view_sort_sum_nan(self): ["a"], ["b"], ], - "w": [12, 0, 0, 0, 0, 1.5, 2.5, 3.5, 4.5], + "w": [12, None, None, None, None, 1.5, 2.5, 3.5, 4.5], "x": [None, 3, 4, 4, 3, 2, 1, 1, 2], } diff --git a/rust/perspective-python/perspective/tests/virtual_servers/test_duckdb.py b/rust/perspective-python/perspective/tests/virtual_servers/test_duckdb.py index dd8dbb99f5..10f59fe7b6 100644 --- a/rust/perspective-python/perspective/tests/virtual_servers/test_duckdb.py +++ b/rust/perspective-python/perspective/tests/virtual_servers/test_duckdb.py @@ -19,6 +19,7 @@ from perspective import Client from perspective.virtual_servers.duckdb import DuckDBVirtualServer +from perspective.tests.column_paths import joined_column_paths _SUPERSTORE_LOCAL = os.path.join( os.path.dirname(__file__), @@ -227,7 +228,7 @@ def test_to_columns(self, client): def test_column_paths(self, client): table = client.open_table("memory.superstore") view = table.view(columns=["Sales", "Profit", "State"]) - paths = view.column_paths() + paths = joined_column_paths(view) assert paths == ["Sales", "Profit", "State"] view.delete() @@ -362,7 +363,7 @@ def test_single_split_by(self, client): aggregates={"Sales": "sum"}, ) - column_paths = view.column_paths() + column_paths = joined_column_paths(view) assert column_paths == [ "Central|Sales", "East|Sales", @@ -409,7 +410,7 @@ def test_split_by_without_group_by(self, client): columns=["Sales"], split_by=["Category"], ) - paths = view.column_paths() + paths = joined_column_paths(view) assert any("Furniture" in c for c in paths) assert any("Office Supplies" in c for c in paths) assert any("Technology" in c for c in paths) @@ -905,7 +906,7 @@ def test_split_by_group_by_filter(self, client): aggregates={"Sales": "sum"}, ) - paths = view.column_paths() + paths = joined_column_paths(view) assert paths == [ "Central|Sales", "East|Sales", @@ -957,7 +958,7 @@ def test_split_by_only(self, client): filter=[["Quantity", ">", 3]], ) - paths = view.column_paths() + paths = joined_column_paths(view) assert paths == [ "Central|Sales", "East|Sales", @@ -1165,7 +1166,7 @@ def test_column_values_view(self, client): view = table.view(group_by=["enum"], columns=[]) csv = view.to_csv() assert [line for line in csv.splitlines() if line] == [ - "__ROW_PATH_0__", + "enum", "null", '"happy"', '"sad"', @@ -1184,3 +1185,96 @@ def test_filter_matching_nothing(self, client): assert view.to_json() == [] assert view.to_columns() == {"tiny": []} view.delete() + + +class TestDuckDBTableFromView: + def test_group_by_view_unrolls_row_path(self, client): + table = client.open_table("memory.superstore") + view = table.view( + columns=["Sales"], group_by=["Region"], aggregates={"Sales": "sum"} + ) + + derived = client.table(view, name="py_derived_group") + assert derived.schema() == { + "Region": "string", + "Sales": "float", + } + + child = derived.view(columns=["Region", "Sales"]) + expected = sorted( + (str(row["__ROW_PATH__"][0]) if row["__ROW_PATH__"] else "None", row["Sales"]) + for row in view.to_json() + ) + + actual = sorted( + (str(row["Region"]), row["Sales"]) for row in child.to_json() + ) + + assert actual == expected + child.delete() + derived.delete() + view.delete() + + def test_group_by_view_qualifies_key_aggregate(self, client): + table = client.open_table("memory.superstore") + view = table.view( + columns=["Region", "Sales"], + group_by=["Region"], + aggregates={"Region": "count", "Sales": "sum"}, + ) + + derived = client.table(view, name="py_derived_qualified") + assert derived.columns() == ["Region", "Region (count)", "Sales"] + child = derived.view(columns=["Region", "Region (count)"]) + assert child.num_rows() == view.num_rows() + child.delete() + derived.delete() + view.delete() + + def test_explicit_schema(self, client): + table = client.open_table("memory.superstore") + view = table.view( + columns=["Sales"], + group_by=["Category"], + split_by=["Region"], + aggregates={"Sales": "sum"}, + group_rollup_mode="flat", + ) + + derived = client.table( + view, + name="py_derived_schema", + schema={ + "Category": "string", + "West|Sales": "float", + "North|Sales": "float", + }, + ) + + assert derived.columns() == [ + "Category", + "West|Sales", + "North|Sales", + ] + + child = derived.view( + columns=["Category", "West|Sales", "North|Sales"] + ) + + rows = child.to_json() + assert len(rows) == 3 + assert all(row["North|Sales"] is None for row in rows) + child.delete() + derived.delete() + view.delete() + + def test_rejects_mismatched_type(self, client): + table = client.open_table("memory.superstore") + view = table.view( + columns=["Sales"], group_by=["Region"], aggregates={"Sales": "sum"} + ) + + with pytest.raises(Exception): + client.table(view, schema={"Sales": "string"}) + + view.delete() diff --git a/rust/perspective-python/perspective/tests/virtual_servers/test_polars.py b/rust/perspective-python/perspective/tests/virtual_servers/test_polars.py index 9d20747b0b..9d857baae8 100644 --- a/rust/perspective-python/perspective/tests/virtual_servers/test_polars.py +++ b/rust/perspective-python/perspective/tests/virtual_servers/test_polars.py @@ -18,6 +18,7 @@ from perspective import Client from perspective.virtual_servers.polars import PolarsVirtualServer import urllib +from perspective.tests.column_paths import joined_column_paths def approx_json(expected): @@ -190,7 +191,7 @@ def test_to_columns(self, client): def test_column_paths(self, client): table = client.open_table("superstore") view = table.view(columns=["Sales", "Profit", "State"]) - paths = view.column_paths() + paths = joined_column_paths(view) assert paths == ["Sales", "Profit", "State"] view.delete() @@ -478,7 +479,7 @@ def test_single_split_by(self, client): aggregates={"Sales": "sum"}, ) - column_paths = view.column_paths() + column_paths = joined_column_paths(view) assert column_paths == [ "Central_Sales", "East_Sales", @@ -527,7 +528,7 @@ def test_split_by_without_group_by(self, client): columns=["Sales"], split_by=["Category"], ) - paths = view.column_paths() + paths = joined_column_paths(view) assert any("Furniture" in c for c in paths) assert any("Office Supplies" in c for c in paths) assert any("Technology" in c for c in paths) @@ -934,7 +935,7 @@ def test_split_by_group_by_filter(self, client): aggregates={"Sales": "sum"}, ) - paths = view.column_paths() + paths = joined_column_paths(view) assert paths == [ "Central_Sales", "East_Sales", @@ -988,7 +989,7 @@ def test_split_by_only(self, client): filter=[["Quantity", ">", 3]], ) - paths = view.column_paths() + paths = joined_column_paths(view) assert paths == [ "Central_Sales", "East_Sales", @@ -1064,7 +1065,7 @@ def test_column_values_view(self, client): view = table.view(group_by=["Region"], columns=[]) csv = view.to_csv() assert [line for line in csv.splitlines() if line] == [ - "__ROW_PATH_0__", + "Region", "null", '"Central"', '"East"', diff --git a/rust/perspective-python/perspective/tests/virtual_servers/test_postgres.py b/rust/perspective-python/perspective/tests/virtual_servers/test_postgres.py index 9e78136bbc..cbbb6b1468 100644 --- a/rust/perspective-python/perspective/tests/virtual_servers/test_postgres.py +++ b/rust/perspective-python/perspective/tests/virtual_servers/test_postgres.py @@ -22,6 +22,7 @@ from perspective import Client from perspective.virtual_servers.postgres import PostgresVirtualServer +from perspective.tests.column_paths import joined_column_paths # Set `PSP_TEST_POSTGRES_DSN` to point these tests at a server; without a # reachable PostgreSQL >= 16 they skip rather than fail. @@ -266,7 +267,7 @@ def test_to_columns(self, client): def test_column_paths(self, client): table = client.open_table("psp_test.superstore") view = table.view(columns=["Sales", "Profit", "State"]) - paths = view.column_paths() + paths = joined_column_paths(view) assert paths == ["Sales", "Profit", "State"] view.delete() @@ -945,7 +946,7 @@ def test_column_values_view(self, client): view = table.view(group_by=["enum"], columns=[]) csv = view.to_csv() assert [line for line in csv.splitlines() if line] == [ - "__ROW_PATH_0__", + "enum", "null", '"happy"', '"sad"', diff --git a/rust/perspective-python/perspective/virtual_servers/duckdb.py b/rust/perspective-python/perspective/virtual_servers/duckdb.py index 613d59a3b3..84430e949c 100644 --- a/rust/perspective-python/perspective/virtual_servers/duckdb.py +++ b/rust/perspective-python/perspective/virtual_servers/duckdb.py @@ -178,6 +178,7 @@ def get_features(self): "split_by": True, "sort": True, "expressions": True, + "view_derivations": True, "group_rollup_mode": ["rollup", "flat", "total"], "split_rollup_mode": ["flat", "rollup"], "filter_ops": { @@ -254,6 +255,13 @@ def _describe_query(self, query): if not row[0].startswith("__") } + def view_make_table(self, view_name, table_name, config, schema=None): + query = self.sql_builder.view_make_table( + view_name, table_name, config, self.table_schema(view_name), schema + ) + + run_query(self.db, query, execute=True) + def view_delete(self, view_name): query = self.sql_builder.view_delete(view_name) run_query(self.db, query, execute=True) diff --git a/rust/perspective-python/perspective/virtual_servers/postgres.py b/rust/perspective-python/perspective/virtual_servers/postgres.py index 849e3c5519..70b1a90269 100644 --- a/rust/perspective-python/perspective/virtual_servers/postgres.py +++ b/rust/perspective-python/perspective/virtual_servers/postgres.py @@ -193,6 +193,7 @@ def get_features(self): "split_by": False, "sort": True, "expressions": True, + "view_derivations": True, "group_rollup_mode": ["rollup", "flat", "total"], # `ctid` is not a stable row identity, so natural-order windows # are unsupported. @@ -294,6 +295,13 @@ def _describe_query(self, query): if not d.name.startswith("__") } + def view_make_table(self, view_name, table_name, config, schema=None): + query = self.sql_builder.view_make_table( + view_name, table_name, config, self.table_schema(view_name), schema + ) + + run_query(self.db, query, execute=True) + def view_delete(self, view_name): query = self.sql_builder.view_delete(view_name) run_query(self.db, query, execute=True) diff --git a/rust/perspective-python/src/client/client_async.rs b/rust/perspective-python/src/client/client_async.rs index 2cd9b1e190..d7ab194f6d 100644 --- a/rust/perspective-python/src/client/client_async.rs +++ b/rust/perspective-python/src/client/client_async.rs @@ -16,7 +16,8 @@ use std::str::FromStr; use std::sync::Arc; use futures::FutureExt; -use perspective_client::config::ViewConfigUpdate; +use indexmap::IndexMap; +use perspective_client::config::{ColumnType, ViewConfigUpdate}; use perspective_client::proto::ListFlatten; use perspective_client::{ Client, ColumnWindow, DeleteOptions, DescribeVerdict, OnRemoveData, OnUpdateData, OnUpdateMode, @@ -30,7 +31,7 @@ use pythonize::depythonize; use super::pandas::arrow_to_pandas; use super::polars::arrow_to_polars; -use super::table_data::TableDataExt; +use super::table_data::{TableDataExt, psp_type_from_py_type}; use super::update_data::UpdateDataExt; use super::{pandas, polars, pyarrow}; use crate::py_async::{self, AllowThreads}; @@ -55,6 +56,16 @@ fn py_to_table_ref_from_owned(py: Python<'_>, val: &Py) -> PyResult, + schema: &Bound<'_, PyDict>, +) -> PyResult> { + schema + .iter() + .map(|(name, ty)| Ok((name.extract::()?, psp_type_from_py_type(py, ty)?))) + .collect() +} + fn parse_list_flatten(value: Option) -> PyResult> { match value.as_deref() { None => Ok(None), @@ -182,6 +193,8 @@ impl AsyncClient { /// and byte array alternative inputs. /// - `page_to_disk` - Back this [`Table`]'s canonical data with the /// on-disk (memory-mapped) storage backend instead of memory. + /// - `schema` - The columns of a [`Table`] derived from a `View`, in + /// place of the ones inferred from it. /// /// # Python Examples /// @@ -191,7 +204,7 @@ impl AsyncClient { /// table = await client.table("x,y\n1,2\n3,4") /// ``` #[allow(clippy::too_many_arguments)] - #[pyo3(signature=(input, limit=None, index=None, name=None, format=None, page_to_disk=None, list_flatten=None))] + #[pyo3(signature=(input, limit=None, index=None, name=None, format=None, page_to_disk=None, list_flatten=None, schema=None))] pub async fn table( &self, input: Py, @@ -201,6 +214,7 @@ impl AsyncClient { format: Option>, page_to_disk: Option, list_flatten: Option>, + schema: Option>, ) -> PyResult { let client = self.client.clone(); let py_client = Python::attach(|_| self.clone()); @@ -209,6 +223,7 @@ impl AsyncClient { name: name.map(|x| x.extract::(py)).transpose()?, page_to_disk, list_flatten: parse_list_flatten(list_flatten.map(|x| x.to_string()))?, + schema: schema.map(|x| parse_schema(py, x.bind(py))).transpose()?, ..TableInitOptions::default() }; @@ -739,17 +754,27 @@ assert_view_api!(AsyncView); #[pymethods] impl AsyncView { - /// Returns an array of strings containing the column paths of the [`View`] - /// without any of the source columns. + /// Returns this [`View`]'s column header area for `window`, transposed as + /// `area[level][column]`. /// /// A column path shows the columns that a given cell belongs to after - /// pivots are applied. - pub async fn column_paths(&self, window: Option>) -> PyResult> { + /// pivots are applied. The area is rectangular with one level per + /// `split_by` plus one: a column's split values occupy the leading levels + /// and its name always occupies the last, so a subtotal or grand total + /// under `split_rollup_mode: "rollup"` reads `None` at the levels it does + /// not pivot on. Values keep their column's type - a `datetime` split value + /// is epoch milliseconds, not text - so formatting them is the caller's + /// choice. + /// + /// `window` slices the *column* axis; the number of levels does not depend + /// on it. + pub async fn column_paths(&self, window: Option>) -> PyResult> { let window: ColumnWindow = Python::attach(|py| window.map(|x| depythonize(x.bind(py)))) .transpose()? .unwrap_or_default(); - self.view.column_paths(window).await.into_pyerr() + let area = self.view.column_paths(window).await.into_pyerr()?; + Python::attach(|py| Ok(pythonize::pythonize(py, &area)?.unbind())) } /// Delete this [`View`] and clean up all resources associated with it. diff --git a/rust/perspective-python/src/client/client_sync.rs b/rust/perspective-python/src/client/client_sync.rs index f6f9e97e8c..c770583ed3 100644 --- a/rust/perspective-python/src/client/client_sync.rs +++ b/rust/perspective-python/src/client/client_sync.rs @@ -157,6 +157,8 @@ impl Client { /// `"json"`, `"columns"`, `"csv"` or `"arrow"`. This overrides /// language-specific type dispatch behavior, which allows stringified /// and byte array alternative inputs. + /// - `schema` - The columns of a [`Table`] derived from a `View`, in + /// place of the ones inferred from it. /// /// # Python Examples /// @@ -166,7 +168,7 @@ impl Client { /// table = client.table("x,y\n1,2\n3,4") /// ``` #[allow(clippy::too_many_arguments)] - #[pyo3(signature = (input, limit=None, index=None, name=None, format=None, page_to_disk=None, list_flatten=None))] + #[pyo3(signature = (input, limit=None, index=None, name=None, format=None, page_to_disk=None, list_flatten=None, schema=None))] pub fn table( &self, py: Python<'_>, @@ -177,6 +179,7 @@ impl Client { format: Option>, page_to_disk: Option, list_flatten: Option>, + schema: Option>, ) -> PyResult
{ Ok(Table( self.0 @@ -188,6 +191,7 @@ impl Client { format, page_to_disk, list_flatten, + schema, ) .py_block_on(py)?, )) @@ -579,17 +583,22 @@ impl View { )) } - /// Returns an array of strings containing the column paths of the [`View`] - /// without any of the source columns. + /// Returns this [`View`]'s column header area for `window`, transposed as + /// `area[level][column]`. /// /// A column path shows the columns that a given cell belongs to after - /// pivots are applied. + /// pivots are applied. The area is rectangular with one level per + /// `split_by` plus one: a column's split values occupy the leading levels + /// and its name always occupies the last, so a subtotal or grand total + /// under `split_rollup_mode: "rollup"` reads `None` at the levels it does + /// not pivot on. Values keep their column's type - a `datetime` split value + /// is epoch milliseconds, not text - so formatting them is the caller's + /// choice. + /// + /// `window` slices the *column* axis; the number of levels does not depend + /// on it. #[pyo3(signature = (**window))] - pub fn column_paths( - &self, - py: Python<'_>, - window: Option>, - ) -> PyResult> { + pub fn column_paths(&self, py: Python<'_>, window: Option>) -> PyResult> { self.0.column_paths(window).py_block_on(py) } diff --git a/rust/perspective-python/src/client/table_data.rs b/rust/perspective-python/src/client/table_data.rs index b7fafd1112..a1be4e8bf7 100644 --- a/rust/perspective-python/src/client/table_data.rs +++ b/rust/perspective-python/src/client/table_data.rs @@ -19,7 +19,10 @@ use pyo3::types::{PyAny, PyAnyMethods, PyDict, PyList, PyString, PyType}; use super::update_data::UpdateDataExt; use crate::py_err::ResultTClientErrorExt; -fn psp_type_from_py_type(_py: Python<'_>, val: Bound<'_, PyAny>) -> PyResult { +pub(crate) fn psp_type_from_py_type( + _py: Python<'_>, + val: Bound<'_, PyAny>, +) -> PyResult { if let Ok(pystr) = val.cast::() { ColumnType::try_from(pystr.to_string_lossy().as_ref()).into_pyerr() } else if let Ok(val) = val.cast::() { diff --git a/rust/perspective-python/src/server/generic_sql_model.rs b/rust/perspective-python/src/server/generic_sql_model.rs index 2f1e9bb23f..70aca5593a 100644 --- a/rust/perspective-python/src/server/generic_sql_model.rs +++ b/rust/perspective-python/src/server/generic_sql_model.rs @@ -150,6 +150,33 @@ impl PyGenericSQLVirtualServerModel { }) } + #[pyo3(signature = (view_id, table_id, config, view_schema, schema=None))] + pub fn view_make_table( + &self, + view_id: &str, + table_id: &str, + config: Py, + view_schema: Py, + schema: Option>, + ) -> PyResult { + Python::attach(|py| { + let config: ViewConfig = pythonize::depythonize(config.bind(py)) + .map_err(|e| PyValueError::new_err(e.to_string()))?; + + let as_dict = |value: &Py| { + self.parse_schema(value.cast_bound::(py).map_err(|_| { + PyValueError::new_err("Schema must be a dict mapping column names to types") + })?) + }; + + let view_schema = as_dict(&view_schema)?; + let schema = schema.as_ref().map(as_dict).transpose()?; + self.inner + .view_make_table(view_id, table_id, &config, &view_schema, schema.as_ref()) + .map_err(|e| PyValueError::new_err(e.to_string())) + }) + } + pub fn view_get_data( &self, view_id: &str, @@ -237,7 +264,7 @@ impl From for ViewPort { start_col: value.start_col, end_row: value.end_row, end_col: value.end_col, - emit_legacy_row_path_names: None, + machine_column_names: None, } } } diff --git a/rust/perspective-python/src/server/virtual_server_sync.rs b/rust/perspective-python/src/server/virtual_server_sync.rs index 8381dc0513..0507c0fff7 100644 --- a/rust/perspective-python/src/server/virtual_server_sync.rs +++ b/rust/perspective-python/src/server/virtual_server_sync.rs @@ -284,6 +284,47 @@ impl VirtualServerHandler for PyServerHandler { }) } + fn view_make_table( + &mut self, + view_id: &str, + table_id: &str, + config: &perspective_client::config::ViewConfig, + schema: Option<&IndexMap>, + ) -> VirtualServerFuture<'_, Result<(), Self::Error>> { + let handler = Python::attach(|py| self.0.clone_ref(py)); + let view_id = view_id.to_string(); + let table_id = table_id.to_string(); + let config = config.clone(); + let schema = schema.cloned(); + Box::pin(async move { + Python::attach(|py| { + let schema = schema + .map(|schema| { + let dict = PyDict::new(py); + for (name, ty) in schema { + dict.set_item(name, ty.to_string())?; + } + + PyResult::Ok(dict) + }) + .transpose()?; + + handler.call_method1( + py, + pyo3::intern!(py, "view_make_table"), + ( + &view_id, + &table_id, + pythonize::pythonize(py, &config)?, + schema, + ), + )?; + + Ok(()) + }) + }) + } + fn view_delete(&self, view_id: &str) -> VirtualServerFuture<'_, Result<(), Self::Error>> { let handler = Python::attach(|py| self.0.clone_ref(py)); let view_id = view_id.to_string(); diff --git a/rust/perspective-python/test.mjs b/rust/perspective-python/test.mjs index d4172faf84..7be395a22d 100644 --- a/rust/perspective-python/test.mjs +++ b/rust/perspective-python/test.mjs @@ -59,6 +59,7 @@ if (process.env.PSP_PYODIDE) { "perspective/tests", "-W error", "--timeout=300", + "--capture=sys", ...process.argv.slice(2), ], execOpts, diff --git a/rust/perspective-server/cpp/perspective/CMakeLists.txt b/rust/perspective-server/cpp/perspective/CMakeLists.txt index a65c04f8e5..0a996c327b 100644 --- a/rust/perspective-server/cpp/perspective/CMakeLists.txt +++ b/rust/perspective-server/cpp/perspective/CMakeLists.txt @@ -539,6 +539,7 @@ set(SOURCE_FILES ${PSP_CPP_SRC}/src/cpp/vocab.cpp ${PSP_CPP_SRC}/src/cpp/arrow_csv.cpp ${PSP_CPP_SRC}/src/cpp/join_engine.cpp + ${PSP_CPP_SRC}/src/cpp/derived_source.cpp ${PSP_CPP_SRC}/src/cpp/server.cpp ${PSP_CPP_SRC}/src/cpp/binding_api.cpp ) @@ -598,7 +599,6 @@ else() # -s MEMORY_GROWTH_GEOMETRIC_CAP=536870912 \ set(PSP_FILESYSTEM_FLAGS "-s NO_FILESYSTEM=1 ") - set(PSP_CLOSURE_FLAGS "--closure=1 ") # On-disk tables in the browser are backed by OPFS via emscripten's WasmFS. if(PSP_WASM_OPFS) @@ -607,7 +607,6 @@ else() set(PSP_WASM_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} \ --no-entry \ - ${PSP_CLOSURE_FLAGS} \ ${PSP_FILESYSTEM_FLAGS} \ -s ALLOW_MEMORY_GROWTH=1 \ -s MODULARIZE=1 \ diff --git a/rust/perspective-server/cpp/perspective/src/cpp/aggregate.cpp b/rust/perspective-server/cpp/perspective/src/cpp/aggregate.cpp index 3abb77ae08..22febba88e 100644 --- a/rust/perspective-server/cpp/perspective/src/cpp/aggregate.cpp +++ b/rust/perspective-server/cpp/perspective/src/cpp/aggregate.cpp @@ -32,6 +32,10 @@ void t_aggregate::init() { switch (m_aggtype) { case AGGTYPE_SUM: + case AGGTYPE_SUM_OR_ZERO: + case AGGTYPE_SUM_ABS: + case AGGTYPE_ABS_SUM: + case AGGTYPE_VALID_COUNT: case AGGTYPE_PCT_SUM_PARENT: case AGGTYPE_PCT_SUM_GRAND_TOTAL: { switch (m_icolumns[0]->get_dtype()) { diff --git a/rust/perspective-server/cpp/perspective/src/cpp/aggspec.cpp b/rust/perspective-server/cpp/perspective/src/cpp/aggspec.cpp index 3668855095..cdc04fe0f4 100644 --- a/rust/perspective-server/cpp/perspective/src/cpp/aggspec.cpp +++ b/rust/perspective-server/cpp/perspective/src/cpp/aggspec.cpp @@ -220,8 +220,11 @@ t_aggspec::agg_str() const { ss << "udf_reducer_" << disp_name(); return ss.str(); } - case AGGTYPE_SUM_NOT_NULL: { - return "sum not null"; + case AGGTYPE_SUM_OR_ZERO: { + return "sum or zero"; + } + case AGGTYPE_VALID_COUNT: { + return "valid count"; } case AGGTYPE_MEAN_BY_COUNT: { return "mean by count"; @@ -269,15 +272,13 @@ get_simple_accumulator_type(t_dtype coltype) { case DTYPE_INT64: case DTYPE_INT32: case DTYPE_INT16: - case DTYPE_INT8: { - return DTYPE_INT64; - } break; + case DTYPE_INT8: case DTYPE_UINT64: case DTYPE_UINT32: case DTYPE_UINT16: case DTYPE_UINT8: { - return DTYPE_UINT64; - } + return DTYPE_INT64; + } break; case DTYPE_FLOAT64: case DTYPE_FLOAT32: { return DTYPE_FLOAT64; @@ -350,12 +351,15 @@ t_aggspec::get_output_specs(const t_schema& schema) const { case AGGTYPE_PCT_SUM_PARENT: case AGGTYPE_PCT_SUM_GRAND_TOTAL: case AGGTYPE_MUL: - case AGGTYPE_SUM_NOT_NULL: { + case AGGTYPE_SUM_OR_ZERO: { t_dtype coltype = schema.get_dtype(m_dependencies[0].name()); return mk_col_name_type_vec( name(), get_simple_accumulator_type(coltype) ); } + case AGGTYPE_VALID_COUNT: { + return mk_col_name_type_vec(name(), DTYPE_INT64); + } case AGGTYPE_ANY: case AGGTYPE_UNIQUE: case AGGTYPE_DOMINANT: diff --git a/rust/perspective-server/cpp/perspective/src/cpp/base.cpp b/rust/perspective-server/cpp/perspective/src/cpp/base.cpp index 8452ef55d4..3922fc9065 100644 --- a/rust/perspective-server/cpp/perspective/src/cpp/base.cpp +++ b/rust/perspective-server/cpp/perspective/src/cpp/base.cpp @@ -588,8 +588,11 @@ maybe_str_to_aggtype(const std::string& str) { if (str == "abs sum" || str == "abs_sum") { return t_aggtype::AGGTYPE_ABS_SUM; } + if (str == "sum or zero" || str == "sum_or_zero") { + return t_aggtype::AGGTYPE_SUM_OR_ZERO; + } if (str == "sum not null" || str == "sum_not_null") { - return t_aggtype::AGGTYPE_SUM_NOT_NULL; + return t_aggtype::AGGTYPE_SUM; } if (str == "mean by count" || str == "mean_by_count") { return t_aggtype::AGGTYPE_MEAN_BY_COUNT; diff --git a/rust/perspective-server/cpp/perspective/src/cpp/config.cpp b/rust/perspective-server/cpp/perspective/src/cpp/config.cpp index b8292a5dd5..5f4f6be231 100644 --- a/rust/perspective-server/cpp/perspective/src/cpp/config.cpp +++ b/rust/perspective-server/cpp/perspective/src/cpp/config.cpp @@ -229,7 +229,7 @@ t_config::setup( case AGGTYPE_MAX: case AGGTYPE_MAX_BY: case AGGTYPE_MIN_BY: - case AGGTYPE_SUM_NOT_NULL: + case AGGTYPE_SUM_OR_ZERO: case AGGTYPE_SUM_ABS: case AGGTYPE_ABS_SUM: case AGGTYPE_GMV: @@ -302,10 +302,11 @@ t_config::get_num_columns() const { return m_detail_columns.size(); } -std::string +const std::string& t_config::col_at(t_uindex idx) const { + static const std::string empty; if (idx >= m_detail_columns.size()) { - return ""; + return empty; } return m_detail_columns[idx]; } diff --git a/rust/perspective-server/cpp/perspective/src/cpp/context_grouped_pkey.cpp b/rust/perspective-server/cpp/perspective/src/cpp/context_grouped_pkey.cpp index 78e6d4a802..95c1d7dd27 100644 --- a/rust/perspective-server/cpp/perspective/src/cpp/context_grouped_pkey.cpp +++ b/rust/perspective-server/cpp/perspective/src/cpp/context_grouped_pkey.cpp @@ -210,7 +210,6 @@ t_ctx_grouped_pkey::get_data( void t_ctx_grouped_pkey::notify( const t_data_table& flattened, - const t_data_table& delta, const t_data_table& prev, const t_data_table& current, const t_data_table& transitions, @@ -761,7 +760,6 @@ t_ctx_grouped_pkey::compute_expressions( const std::shared_ptr& master, const t_gstate::t_mapping& pkey_map, const std::shared_ptr& flattened, - const std::shared_ptr& delta, const std::shared_ptr& prev, const std::shared_ptr& current, const std::shared_ptr& transitions, @@ -802,16 +800,6 @@ t_ctx_grouped_pkey::compute_expressions( regex_mapping ); - // delta: for each numerical column, the numerical delta between the - // previous value and the current value in the row. - expr->compute( - delta, - pkey_map, - m_expression_tables->m_delta, - expression_vocab, - regex_mapping - ); - // prev: the values of the updated rows before this update was applied expr->compute( prev, @@ -841,7 +829,6 @@ t_ctx_grouped_pkey::compute_expressions( m_expression_tables->m_flattened, m_expression_tables->m_prev, m_expression_tables->m_current, - m_expression_tables->m_delta, flattened, existed ); diff --git a/rust/perspective-server/cpp/perspective/src/cpp/context_one.cpp b/rust/perspective-server/cpp/perspective/src/cpp/context_one.cpp index 55b61b1cc8..f19488264b 100644 --- a/rust/perspective-server/cpp/perspective/src/cpp/context_one.cpp +++ b/rust/perspective-server/cpp/perspective/src/cpp/context_one.cpp @@ -303,7 +303,6 @@ t_ctx1::notify(const t_data_table& flattened, bool /* is_registration */) { void t_ctx1::notify( const t_data_table& flattened, - const t_data_table& delta, const t_data_table& prev, const t_data_table& current, const t_data_table& transitions, @@ -320,7 +319,6 @@ t_ctx1::notify( m_config.get_sortby_pairs(), m_sortby, flattened, - delta, prev, current, transitions, @@ -567,6 +565,7 @@ t_ctx1::get_cell_delta(t_index bidx, t_index eidx) const { void t_ctx1::reset(bool reset_expressions) { + ++m_storage_generation; auto pivots = m_config.get_row_pivots(); m_tree = std::make_shared( pivots, m_config.get_aggregates(), m_schema, m_config @@ -725,7 +724,6 @@ t_ctx1::compute_expressions( const std::shared_ptr& master, const t_gstate::t_mapping& pkey_map, const std::shared_ptr& flattened, - const std::shared_ptr& delta, const std::shared_ptr& prev, const std::shared_ptr& current, const std::shared_ptr& transitions, @@ -766,16 +764,6 @@ t_ctx1::compute_expressions( regex_mapping ); - // delta: for each numerical column, the numerical delta between the - // previous value and the current value in the row. - expr->compute( - delta, - pkey_map, - m_expression_tables->m_delta, - expression_vocab, - regex_mapping - ); - // prev: the values of the updated rows before this update was applied expr->compute( prev, @@ -805,7 +793,6 @@ t_ctx1::compute_expressions( m_expression_tables->m_flattened, m_expression_tables->m_prev, m_expression_tables->m_current, - m_expression_tables->m_delta, flattened, existed ); diff --git a/rust/perspective-server/cpp/perspective/src/cpp/context_two.cpp b/rust/perspective-server/cpp/perspective/src/cpp/context_two.cpp index cab874e870..574f28587e 100644 --- a/rust/perspective-server/cpp/perspective/src/cpp/context_two.cpp +++ b/rust/perspective-server/cpp/perspective/src/cpp/context_two.cpp @@ -592,7 +592,6 @@ t_ctx2::notify(const t_data_table& flattened, bool /* is_registration */) { void t_ctx2::notify( const t_data_table& flattened, - const t_data_table& delta, const t_data_table& prev, const t_data_table& current, const t_data_table& transitions, @@ -610,7 +609,6 @@ t_ctx2::notify( m_config.get_sortby_pairs(), m_sortby, flattened, - delta, prev, current, transitions, @@ -628,7 +626,6 @@ t_ctx2::notify( m_config.get_sortby_pairs(), m_column_sortby, flattened, - delta, prev, current, transitions, @@ -646,7 +643,6 @@ t_ctx2::notify( m_config.get_sortby_pairs(), std::vector(), flattened, - delta, prev, current, transitions, @@ -1181,6 +1177,7 @@ t_ctx2::get_rows_changed() { void t_ctx2::reset(bool reset_expressions) { + ++m_storage_generation; for (t_uindex treeidx = 0, tree_loop_end = m_trees.size(); treeidx < tree_loop_end; ++treeidx) { @@ -1343,7 +1340,6 @@ t_ctx2::compute_expressions( const std::shared_ptr& master, const t_gstate::t_mapping& pkey_map, const std::shared_ptr& flattened, - const std::shared_ptr& delta, const std::shared_ptr& prev, const std::shared_ptr& current, const std::shared_ptr& transitions, @@ -1384,16 +1380,6 @@ t_ctx2::compute_expressions( regex_mapping ); - // delta: for each numerical column, the numerical delta between the - // previous value and the current value in the row. - expr->compute( - delta, - pkey_map, - m_expression_tables->m_delta, - expression_vocab, - regex_mapping - ); - // prev: the values of the updated rows before this update was applied expr->compute( prev, @@ -1423,7 +1409,6 @@ t_ctx2::compute_expressions( m_expression_tables->m_flattened, m_expression_tables->m_prev, m_expression_tables->m_current, - m_expression_tables->m_delta, flattened, existed ); diff --git a/rust/perspective-server/cpp/perspective/src/cpp/context_unit.cpp b/rust/perspective-server/cpp/perspective/src/cpp/context_unit.cpp index 7bf263552a..b7d4598b39 100644 --- a/rust/perspective-server/cpp/perspective/src/cpp/context_unit.cpp +++ b/rust/perspective-server/cpp/perspective/src/cpp/context_unit.cpp @@ -62,7 +62,6 @@ t_ctxunit::step_end() {} void t_ctxunit::notify( const t_data_table& flattened, - const t_data_table& delta, const t_data_table& prev, const t_data_table& curr, const t_data_table& transitions, @@ -335,14 +334,10 @@ t_ctxunit::get_pkeys(const std::vector>& cells * @return t_tscalar */ t_tscalar -t_ctxunit::get_column_name(t_index idx) { - std::string empty; - - if (idx >= get_column_count()) { - return m_symtable.get_interned_tscalar(empty.c_str()); - } - - return m_symtable.get_interned_tscalar(m_config.col_at(idx).c_str()); +t_ctxunit::get_column_name(t_index idx) const { + t_tscalar rval; + rval.set(idx < get_column_count() ? m_config.col_at(idx).c_str() : ""); + return rval; } /** @@ -382,6 +377,7 @@ t_ctxunit::get_column_names() const { void t_ctxunit::reset() { m_has_delta = false; + ++m_storage_generation; } bool diff --git a/rust/perspective-server/cpp/perspective/src/cpp/context_zero.cpp b/rust/perspective-server/cpp/perspective/src/cpp/context_zero.cpp index 8e06c2829c..bb02d87466 100644 --- a/rust/perspective-server/cpp/perspective/src/cpp/context_zero.cpp +++ b/rust/perspective-server/cpp/perspective/src/cpp/context_zero.cpp @@ -93,7 +93,6 @@ t_ctx0::step_end() { void t_ctx0::notify( const t_data_table& flattened, - const t_data_table& delta, const t_data_table& prev, const t_data_table& curr, const t_data_table& transitions, @@ -497,14 +496,10 @@ t_ctx0::reset_sortby() { } t_tscalar -t_ctx0::get_column_name(t_index idx) { - std::string empty; - - if (idx >= get_column_count()) { - return m_symtable.get_interned_tscalar(empty.c_str()); - } - - return m_symtable.get_interned_tscalar(m_config.col_at(idx).c_str()); +t_ctx0::get_column_name(t_index idx) const { + t_tscalar rval; + rval.set(idx < get_column_count() ? m_config.col_at(idx).c_str() : ""); + return rval; } std::vector @@ -559,8 +554,19 @@ t_ctx0::get_sort_by() const { return m_traversal->get_sort_by(); } +bool +t_ctx0::has_pkey(t_tscalar pkey) const { + return m_traversal->get_row_idx(pkey) >= 0; +} + +std::vector +t_ctx0::get_member_pkeys() const { + return m_traversal->get_pkeys(); +} + void t_ctx0::reset(bool reset_expressions) { + ++m_storage_generation; m_traversal->reset(); m_deltas = std::make_shared(); m_has_delta = false; @@ -783,7 +789,6 @@ t_ctx0::compute_expressions( const std::shared_ptr& master, const t_gstate::t_mapping& pkey_map, const std::shared_ptr& flattened, - const std::shared_ptr& delta, const std::shared_ptr& prev, const std::shared_ptr& current, const std::shared_ptr& transitions, @@ -824,16 +829,6 @@ t_ctx0::compute_expressions( regex_mapping ); - // delta: for each numerical column, the numerical delta between the - // previous value and the current value in the row. - expr->compute( - delta, - pkey_map, - m_expression_tables->m_delta, - expression_vocab, - regex_mapping - ); - // prev: the values of the updated rows before this update was applied expr->compute( prev, @@ -863,7 +858,6 @@ t_ctx0::compute_expressions( m_expression_tables->m_flattened, m_expression_tables->m_prev, m_expression_tables->m_current, - m_expression_tables->m_delta, flattened, existed ); diff --git a/rust/perspective-server/cpp/perspective/src/cpp/dense_tree_context.cpp b/rust/perspective-server/cpp/perspective/src/cpp/dense_tree_context.cpp index a9247d21d4..bd61b5b920 100644 --- a/rust/perspective-server/cpp/perspective/src/cpp/dense_tree_context.cpp +++ b/rust/perspective-server/cpp/perspective/src/cpp/dense_tree_context.cpp @@ -34,6 +34,15 @@ t_dtree_ctx::t_dtree_ctx( m_aggspecs.emplace_back("psp_strand_count_sum", AGGTYPE_SUM, depvec); + for (const auto& colname : m_strand_deltas->get_schema().m_columns) { + if (colname.rfind("psp_valid_", 0) != 0) { + continue; + } + + std::vector valid_dep = {t_dep(colname, DEPTYPE_COLUMN)}; + m_aggspecs.emplace_back(colname, AGGTYPE_VALID_COUNT, valid_dep); + } + t_uindex aggidx = 0; for (const auto& spec : m_aggspecs) { m_aggspecmap[spec.name()] = aggidx; @@ -86,7 +95,10 @@ t_dtree_ctx::build_aggregates() { std::vector> icolumns; icolumns.reserve(deps.size()); for (const auto& d : deps) { - icolumns.push_back(tbl->get_const_column(d.name())); + icolumns.push_back(tbl->get_const_column( + aggspec.agg() == AGGTYPE_SUM_ABS ? abs_strand_colname(d.name()) + : d.name() + )); } auto output_col = m_aggregates->get_column(aggspec.name()); diff --git a/rust/perspective-server/cpp/perspective/src/cpp/derived_source.cpp b/rust/perspective-server/cpp/perspective/src/cpp/derived_source.cpp new file mode 100644 index 0000000000..d2d85c5de0 --- /dev/null +++ b/rust/perspective-server/cpp/perspective/src/cpp/derived_source.cpp @@ -0,0 +1,1354 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +#include +#include +#include +#include +#include +#include +#include + +namespace perspective { + +namespace { + std::int64_t + wide_key(t_uindex depth, t_uindex idx) { + return (static_cast(depth) << 40) + | static_cast(idx); + } + + void + check_unique(const std::vector& names) { + tsl::hopscotch_set seen; + for (const auto& name : names) { + if (!seen.insert(name).second) { + PSP_COMPLAIN_AND_ABORT( + "Duplicate column '" + name + "' in derived table schema" + ); + } + } + } + + t_dtype + pivot_dtype( + const t_schema& parent_schema, + const t_view_config& config, + const std::string& pivot + ) { + if (parent_schema.has_column(pivot)) { + return parent_schema.get_dtype(pivot); + } + + for (const auto& expr : config.get_expressions()) { + if (expr->get_expression_alias() == pivot) { + return expr->get_dtype(); + } + } + + return DTYPE_STR; + } + + bool + is_pct(const t_aggspec& spec) { + return spec.agg() == AGGTYPE_PCT_SUM_PARENT + || spec.agg() == AGGTYPE_PCT_SUM_GRAND_TOTAL; + } + + /// Whether `extract_aggregate` returns the stored aggregate unchanged, + /// so a child may read the aggtable column in place. + bool + identity_extract(const t_aggspec& spec, t_dtype stored_dtype) { + return !is_pct(spec) && spec.agg() != AGGTYPE_ABS_SUM + && stored_dtype != DTYPE_F64PAIR; + } + + t_dtype + aggregate_dtype(const t_stree& tree, const t_aggspec& spec) { + t_dtype dtype = tree.get_aggtable()->get_schema().get_dtype(spec.name()); + if (dtype == DTYPE_F64PAIR || is_pct(spec)) { + return DTYPE_FLOAT64; + } + + return dtype; + } + + void + put(t_column* column, + t_uindex idx, + t_derived_cell_state state, + const t_tscalar& value) { + if (column == nullptr) { + return; + } + + switch (state) { + case DERIVED_CELL_SET: { + column->set_scalar(idx, value); + } break; + case DERIVED_CELL_CLEAR: { + column->unset(idx); + } break; + case DERIVED_CELL_KEEP: { + } break; + } + } + + void + put_scalar(t_column* column, t_uindex idx, const t_tscalar& value) { + put(column, + idx, + value.is_valid() && !value.is_none() ? DERIVED_CELL_SET + : DERIVED_CELL_CLEAR, + value); + } + + std::vector + node_path(const t_stree& tree, t_uindex idx) { + std::vector path; + tree.get_path(idx, path); + std::reverse(path.begin(), path.end()); + return path; + } + + /// Root-to-node paths of live tree nodes under a caller-chosen key + /// that must identify the tree as well as the node when several trees + /// share an index space; a reference is valid until the next `get`. + class t_path_cache { + public: + const std::vector& + get(const t_stree& tree, std::int64_t key, t_uindex idx) { + auto iter = m_paths.find(key); + if (iter == m_paths.end()) { + iter = m_paths.emplace(key, node_path(tree, idx)).first; + } + + return iter->second; + } + + void + erase(std::int64_t key) { + m_paths.erase(key); + } + + void + clear() { + m_paths.clear(); + } + + private: + tsl::hopscotch_map> m_paths; + }; + + /** + * @brief Add the nodes whose `pct sum` values move with a touched node. + */ + void + widen_pct( + const t_stree& tree, + bool pct_parent, + bool pct_grand, + tsl::hopscotch_set& touched + ) { + if (pct_grand && touched.contains(0)) { + tree.for_each_node([&touched](const t_stnode& node) { + touched.insert(node.m_idx); + }); + return; + } + + if (!pct_parent) { + return; + } + + std::vector parents(touched.begin(), touched.end()); + std::vector children; + for (t_uindex idx : parents) { + if (!tree.node_exists(idx)) { + continue; + } + + children.clear(); + tree.get_child_indices(idx, children); + for (t_index child : children) { + touched.insert(static_cast(child)); + } + } + } + + tsl::hopscotch_set + touched_nodes( + const t_stree& tree, + const t_stree_capture& capture, + bool pct_parent, + bool pct_grand + ) { + tsl::hopscotch_set touched; + for (const auto& kv : capture.m_changed) { + touched.insert(kv.first); + } + + for (t_uindex idx : capture.m_created) { + touched.insert(idx); + } + + widen_pct(tree, pct_parent, pct_grand, touched); + return touched; + } +} // namespace + +std::string +t_derived_source::index() const { + return ""; +} + +std::uint32_t +t_derived_source::limit() const { + return std::numeric_limits::max(); +} + +const std::shared_ptr
& +t_derived_source::child() const { + return m_child; +} + +void +t_derived_source::detach() {} + +std::shared_ptr +t_derived_source::alias_column(t_uindex cidx) const { + return nullptr; +} + +bool +t_derived_source::previous( + const t_derived_member& member, t_uindex cidx, t_tscalar& out +) const { + return false; +} + +t_index +t_derived_source::bound_row(const t_tscalar& pkey) const { + const auto& mapping = m_child->get_gnode()->get_pkey_map(); + auto iter = mapping.find(pkey); + if (iter == mapping.end()) { + return -1; + } + + return static_cast(iter->second); +} + +bool +t_derived_source::take_replaced() { + t_uindex generation = storage_generation(); + bool replaced = generation != m_generation; + m_generation = generation; + return replaced; +} + +void +t_derived_source::init_pct_flags(const t_view_config& config) { + for (const auto& spec : config.get_aggspecs()) { + m_pct_parent = m_pct_parent || spec.agg() == AGGTYPE_PCT_SUM_PARENT; + m_pct_grand = + m_pct_grand || spec.agg() == AGGTYPE_PCT_SUM_GRAND_TOTAL; + } +} + +void +t_derived_source::attach(const std::shared_ptr
& child) { + m_child = child; + t_schema schema = child->get_schema(); + m_columns = schema.columns(); + resolve(schema); + take_replaced(); + m_attached = false; + step(false); +} + +bool +t_derived_source::step(bool notify) { +#ifdef PSP_PARALLEL_FOR + PSP_WRITE_LOCK(*m_child->get_pool()->get_lock()); +#endif + bool full = take_replaced() || !m_attached; + auto gnode = m_child->get_gnode(); + if (full && m_attached) { + gnode->reset(); + } + + m_attached = true; + std::vector members; + collect(full, members); + + std::shared_ptr master = gnode->get_table_sptr(); + const t_schema& schema = master->get_schema(); + t_uindex num_columns = m_columns.size(); + if (full) { + for (t_uindex cidx = 0; cidx < num_columns; ++cidx) { + auto column = alias_column(cidx); + if (column + && column->get_dtype() == schema.get_dtype(m_columns[cidx])) { + gnode->set_derived_alias(m_columns[cidx], std::move(column)); + } + } + } + + std::vector aliased(num_columns); + for (t_uindex cidx = 0; cidx < num_columns; ++cidx) { + aliased[cidx] = gnode->is_derived_alias(m_columns[cidx]); + } + + t_uindex size = members.size(); + auto flattened = std::make_shared(schema); + flattened->init(); + flattened->extend(size); + auto prev_state = std::make_shared(schema); + prev_state->init(); + if (!full) { + prev_state->extend(size); + } + + t_column* pkey_col = flattened->_get_column("psp_pkey"); + t_column* okey_col = flattened->_get_column("psp_okey"); + t_column* op_col = flattened->_get_column("psp_op"); + const t_column* master_okey = master->_get_const_column("psp_okey"); + t_column* prev_okey = prev_state->_get_column("psp_okey"); + std::vector columns(num_columns); + std::vector master_cols(num_columns); + std::vector prev_cols(num_columns); + for (t_uindex cidx = 0; cidx < num_columns; ++cidx) { + const std::string& name = m_columns[cidx]; + columns[cidx] = + full && aliased[cidx] ? nullptr : flattened->_get_column(name); + master_cols[cidx] = master->_get_const_column(name); + prev_cols[cidx] = prev_state->_get_column(name); + } + + std::vector rows(size); + std::vector bound(size); + t_uindex widx = 0; + for (const t_derived_member& member : members) { + t_index bound_idx = bound_row(member.m_pkey); + if (member.m_deleted && bound_idx < 0) { + continue; + } + + pkey_col->set_scalar(widx, member.m_pkey); + okey_col->set_scalar(widx, member.m_pkey); + op_col->set_nth( + widx, member.m_deleted ? OP_DELETE : OP_INSERT + ); + + rows[widx] = + member.m_deleted ? static_cast(bound_idx) : member.m_row; + bound[widx] = bound_idx; + + if (bound_idx >= 0) { + if (master_okey->is_valid(bound_idx)) { + prev_okey->set_scalar(widx, master_okey->get_scalar(bound_idx)); + } + + for (t_uindex cidx = 0; cidx < num_columns; ++cidx) { + t_tscalar value; + if (!aliased[cidx] || !previous(member, cidx, value)) { + if (!master_cols[cidx]->is_valid(bound_idx)) { + continue; + } + + value = master_cols[cidx]->get_scalar(bound_idx); + } + + if (value.is_valid() && !value.is_none()) { + prev_cols[cidx]->set_scalar(widx, value); + } + } + } + + if (!member.m_deleted) { + fill(member, widx, columns); + } + + ++widx; + } + + if (widx != size) { + flattened->set_size(widx); + prev_state->set_size(widx); + rows.resize(widx); + bound.resize(widx); + } + + t_derived_step derived_step; + derived_step.m_flattened = flattened; + derived_step.m_prev_state = prev_state; + derived_step.m_rows = std::move(rows); + derived_step.m_bound = std::move(bound); + derived_step.m_notify = notify; + return gnode->process_derived(derived_step); +} + +/** + * @brief The source for a `group_by` view, one row per tree node. + */ +class t_tree_derived_source : public t_derived_source { +public: + t_tree_derived_source( + std::shared_ptr> view, std::shared_ptr
parent + ) : + m_view(std::move(view)), + m_parent(std::move(parent)) { + auto config = m_view->get_view_config(); + m_num_pivots = config->get_row_pivots().size(); + m_num_visible = config->get_columns().size(); + init_pct_flags(*config); + } + + t_schema + infer_schema() const override { + auto config = m_view->get_view_config(); + t_schema parent_schema = m_parent->get_schema(); + std::vector names; + std::vector types; + const auto pivots = config->get_row_pivots(); + for (t_uindex depth = 0; depth < pivots.size(); ++depth) { + names.push_back(pivots[depth]); + types.push_back(pivot_dtype(parent_schema, *config, pivots[depth])); + } + + const auto aggspecs = config->get_aggspecs(); + for (t_uindex aggnum = 0; aggnum < m_num_visible; ++aggnum) { + names.push_back(config->readable_aggregate_name(aggspecs[aggnum])); + types.push_back(aggregate_dtype(*tree(), aggspecs[aggnum])); + } + + check_unique(names); + return {names, types}; + } + + t_dtype + pkey_dtype() const override { + return DTYPE_INT64; + } + + void + detach() override { + tree()->remove_capture(m_capture); + } + +protected: + void + resolve(const t_schema& schema) override { + auto config = m_view->get_view_config(); + const auto pivots = config->get_row_pivots(); + const auto aggspecs = config->get_aggspecs(); + m_roles.assign(m_columns.size(), {ROLE_NONE, 0}); + m_agg_names.assign(m_columns.size(), ""); + for (t_uindex cidx = 0; cidx < m_columns.size(); ++cidx) { + for (t_uindex aggnum = 0; aggnum < m_num_visible; ++aggnum) { + if (config->readable_aggregate_name(aggspecs[aggnum]) + == m_columns[cidx]) { + m_roles[cidx] = {ROLE_AGGREGATE, aggnum}; + m_agg_names[cidx] = aggspecs[aggnum].name(); + } + } + + for (t_uindex depth = 0; depth < pivots.size(); ++depth) { + if (pivots[depth] == m_columns[cidx]) { + m_roles[cidx] = {ROLE_KEY, depth}; + } + } + } + } + + t_uindex + storage_generation() const override { + return m_view->get_context()->get_storage_generation(); + } + + void + collect(bool full, std::vector& members) override { + t_stree* tr = tree(); + snapshot_aggregates(*tr); + if (full) { + m_paths.clear(); + tr->remove_capture(m_capture); + m_capture = std::make_shared(); + tr->add_capture(m_capture); + tr->for_each_node([&](const t_stnode& node) { + if (selected(node.m_depth)) { + members.push_back(member(*tr, node)); + } + }); + return; + } + + t_stree_capture capture; + std::swap(capture, *m_capture); + for (const auto& dropped : capture.m_dropped) { + m_paths.erase(static_cast(dropped.m_idx)); + if (!selected(dropped.m_depth)) { + continue; + } + + t_derived_member out; + out.m_pkey.set(static_cast(dropped.m_idx)); + out.m_deleted = true; + for (t_uindex aggnum = 0; aggnum < dropped.m_aggregates.size(); + ++aggnum) { + out.m_cells.emplace_back(aggnum, dropped.m_aggregates[aggnum]); + } + + members.push_back(out); + } + + for (t_uindex idx : + touched_nodes(*tr, capture, m_pct_parent, m_pct_grand)) { + const t_stnode* node = tr->find_node(idx); + if (node == nullptr || !selected(node->m_depth)) { + continue; + } + + members.push_back(member(*tr, *node)); + auto changed = capture.m_changed.find(idx); + if (changed != capture.m_changed.end()) { + members.back().m_cells = changed->second; + } + } + } + + std::shared_ptr + alias_column(t_uindex cidx) const override { + const auto& role = m_roles[cidx]; + if (role.first != ROLE_AGGREGATE) { + return nullptr; + } + + const auto aggspecs = m_view->get_view_config()->get_aggspecs(); + auto column = tree()->get_aggtable()->get_column(m_agg_names[cidx]); + if (!identity_extract(aggspecs[role.second], column->get_dtype())) { + return nullptr; + } + + return column; + } + + bool + previous( + const t_derived_member& member, t_uindex cidx, t_tscalar& out + ) const override { + const auto& role = m_roles[cidx]; + for (const auto& cell : member.m_cells) { + if (cell.first == role.second) { + out = cell.second; + return true; + } + } + + return false; + } + + void + fill( + const t_derived_member& member, + t_uindex idx, + const std::vector& columns + ) const override { + const t_stree* tr = tree(); + for (t_uindex cidx = 0; cidx < columns.size(); ++cidx) { + if (columns[cidx] == nullptr) { + continue; + } + + const auto& role = m_roles[cidx]; + switch (role.first) { + case ROLE_KEY: { + if (role.second < member.m_path.size()) { + put_scalar(columns[cidx], idx, member.m_path[role.second]); + } else { + put(columns[cidx], idx, DERIVED_CELL_CLEAR, mknone()); + } + } break; + case ROLE_AGGREGATE: { + const t_column* direct = m_direct[cidx]; + put_scalar( + columns[cidx], + idx, + direct != nullptr + ? direct->get_scalar(member.m_row) + : tr->get_aggregate(member.m_handle, role.second) + ); + } break; + case ROLE_NONE: { + put(columns[cidx], idx, DERIVED_CELL_CLEAR, mknone()); + } break; + } + } + } + +private: + enum t_role : std::uint8_t { ROLE_NONE, ROLE_KEY, ROLE_AGGREGATE }; + + t_stree* + tree() const { + return m_view->get_context()->get_trees()[0]; + } + + bool + selected(t_uindex depth) const { + auto config = m_view->get_view_config(); + if (config->is_total_only()) { + return depth == 0; + } + + if (config->is_leaves_only()) { + return depth == m_num_pivots; + } + + return true; + } + + void + snapshot_aggregates(const t_stree& tr) { + const auto aggspecs = m_view->get_view_config()->get_aggspecs(); + auto aggtable = tr.get_aggtable(); + m_direct.assign(m_columns.size(), nullptr); + for (t_uindex cidx = 0; cidx < m_columns.size(); ++cidx) { + const auto& role = m_roles[cidx]; + if (role.first != ROLE_AGGREGATE) { + continue; + } + + const t_column* column = + aggtable->_get_const_column(m_agg_names[cidx]); + if (identity_extract(aggspecs[role.second], column->get_dtype())) { + m_direct[cidx] = column; + } + } + } + + t_derived_member + member(const t_stree& tr, const t_stnode& node) { + t_derived_member out; + out.m_pkey.set(static_cast(node.m_idx)); + out.m_row = node.m_aggidx; + out.m_handle = node.m_idx; + if (node.m_depth > 0) { + out.m_path = m_paths.get( + tr, static_cast(node.m_idx), node.m_idx + ); + } + + return out; + } + + std::shared_ptr> m_view; + std::shared_ptr
m_parent; + t_uindex m_num_pivots = 0; + t_uindex m_num_visible = 0; + std::vector> m_roles; + std::vector m_agg_names; + std::vector m_direct; + t_path_cache m_paths; + std::shared_ptr m_capture; +}; + +/** + * @brief The source for a `split_by` view, one row per row node and one + * column per column path and aggregate. + */ +class t_wide_derived_source : public t_derived_source { +public: + t_wide_derived_source( + std::shared_ptr> view, std::shared_ptr
parent + ) : + m_view(std::move(view)), + m_parent(std::move(parent)) { + auto config = m_view->get_view_config(); + m_column_only = config->is_column_only(); + m_num_row_pivots = config->get_row_pivots().size(); + m_num_column_pivots = config->get_column_pivots().size(); + m_num_visible = config->get_columns().size(); + init_pct_flags(*config); + } + + t_schema + infer_schema() const override { + auto config = m_view->get_view_config(); + t_schema parent_schema = m_parent->get_schema(); + std::vector names; + std::vector types; + const auto pivots = key_pivots(); + for (t_uindex depth = 0; depth < pivots.size(); ++depth) { + names.push_back(pivots[depth]); + types.push_back(pivot_dtype(parent_schema, *config, pivots[depth])); + } + + const auto aggspecs = config->get_aggspecs(); + const t_stree* tr = trees().back(); + const tsl::hopscotch_set keys(names.begin(), names.end()); + tsl::hopscotch_set seen; + for (const auto& path : m_view->column_names()) { + if (path.empty() || !column_selected(path.size() - 1)) { + continue; + } + + std::string aggregate = path.back().to_string(); + for (t_uindex aggnum = 0; aggnum < m_num_visible; ++aggnum) { + if (aggspecs[aggnum].name() != aggregate) { + continue; + } + + std::string name; + for (t_uindex pidx = 0; pidx + 1 < path.size(); ++pidx) { + name += path[pidx].to_string() + "|"; + } + + name += config->readable_aggregate_name(aggspecs[aggnum]); + if (keys.count(name) > 0) { + PSP_COMPLAIN_AND_ABORT( + "Duplicate column '" + name + + "' in derived table schema" + ); + } + + if (seen.insert(name).second) { + names.push_back(name); + types.push_back(aggregate_dtype(*tr, aggspecs[aggnum])); + } + } + } + + return {names, types}; + } + + t_dtype + pkey_dtype() const override { + return DTYPE_INT64; + } + + void + detach() override { + auto all = trees(); + for (t_uindex depth = 0; depth < m_captures.size(); ++depth) { + all[depth]->remove_capture(m_captures[depth]); + } + } + +protected: + void + resolve(const t_schema& schema) override { + const auto config = m_view->get_view_config(); + const auto aggspecs = config->get_aggspecs(); + const auto pivots = key_pivots(); + m_key_columns.assign(pivots.size(), -1); + m_cell_columns.clear(); + m_cell_names.clear(); + for (t_uindex aggnum = 0; aggnum < m_num_visible; ++aggnum) { + m_cell_names.push_back( + config->readable_aggregate_name(aggspecs[aggnum]) + ); + } + + for (t_uindex cidx = 0; cidx < m_columns.size(); ++cidx) { + bool is_key = false; + for (t_uindex depth = 0; depth < pivots.size(); ++depth) { + if (pivots[depth] == m_columns[cidx]) { + m_key_columns[depth] = static_cast(cidx); + is_key = true; + } + } + + if (!is_key) { + m_cell_columns[m_columns[cidx]] = cidx; + } + } + } + + t_uindex + storage_generation() const override { + return m_view->get_context()->get_storage_generation(); + } + + void + collect(bool full, std::vector& members) override { + if (full) { + m_free_rows.clear(); + m_freed_rows.clear(); + m_next_row = 0; + m_paths.clear(); + m_cell_index.clear(); + } else { + m_free_rows.insert( + m_free_rows.end(), m_freed_rows.begin(), m_freed_rows.end() + ); + m_freed_rows.clear(); + } + + tsl::hopscotch_map index; + auto all = trees(); + m_captures.resize(all.size()); + for (t_uindex depth = 0; depth < all.size(); ++depth) { + if (!row_selected(depth)) { + continue; + } + + t_stree* tr = all[depth]; + if (full) { + tr->remove_capture(m_captures[depth]); + m_captures[depth] = std::make_shared(); + tr->add_capture(m_captures[depth]); + tr->for_each_node([&](const t_stnode& node) { + add_node(*tr, depth, node.m_idx, members, index); + }); + continue; + } + + t_stree_capture capture; + std::swap(capture, *m_captures[depth]); + for (const auto& dropped : capture.m_dropped) { + forget_node(all.size(), dropped.m_idx); + add_dropped(*tr, depth, dropped, members, index); + } + + for (t_uindex idx : + touched_nodes(*tr, capture, m_pct_parent, m_pct_grand)) { + if (tr->node_exists(idx)) { + add_node(*tr, depth, idx, members, index); + } + } + } + } + + void + fill( + const t_derived_member& member, + t_uindex idx, + const std::vector& columns + ) const override { + for (t_uindex depth = 0; depth < m_key_columns.size(); ++depth) { + if (m_key_columns[depth] < 0) { + continue; + } + + t_column* column = columns[m_key_columns[depth]]; + if (depth < member.m_path.size()) { + put_scalar(column, idx, member.m_path[depth]); + } else { + put(column, idx, DERIVED_CELL_CLEAR, mknone()); + } + } + + for (const auto& cell : member.m_cells) { + put_scalar(columns[cell.first], idx, cell.second); + } + } + +private: + using t_cell_index = std::vector>; + + std::vector + trees() const { + return m_view->get_context()->get_trees(); + } + + std::vector + key_pivots() const { + if (m_column_only) { + return {}; + } + + return m_view->get_view_config()->get_row_pivots(); + } + + bool + row_selected(t_uindex depth) const { + auto config = m_view->get_view_config(); + if (config->is_total_only() || m_num_row_pivots == 0) { + return depth == 0; + } + + if (m_column_only || config->is_leaves_only()) { + return depth == m_num_row_pivots; + } + + return true; + } + + bool + column_selected(t_uindex depth) const { + if (m_view->get_view_config()->is_split_rollup()) { + return true; + } + + return depth == m_num_column_pivots; + } + + t_uindex + allocate_row() { + if (!m_free_rows.empty()) { + t_uindex row = m_free_rows.back(); + m_free_rows.pop_back(); + return row; + } + + return m_next_row++; + } + + void + forget_node(t_uindex num_depths, t_uindex idx) { + for (t_uindex depth = 0; depth < num_depths; ++depth) { + m_paths.erase(wide_key(depth, idx)); + m_cell_index.erase(wide_key(depth, idx)); + } + } + + t_derived_member& + ensure_row( + const t_stree& tr, + t_uindex depth, + t_uindex ridx, + std::vector& members, + tsl::hopscotch_map& index + ) { + std::int64_t key = wide_key(depth, ridx); + auto iter = index.find(key); + if (iter != index.end()) { + return members[iter->second]; + } + + t_derived_member out; + out.m_pkey.set(key); + out.m_handle = ridx; + out.m_depth = depth; + out.m_path = m_paths.get(tr, key, ridx); + t_index bound = bound_row(out.m_pkey); + out.m_row = bound >= 0 ? static_cast(bound) : allocate_row(); + index[key] = members.size(); + members.push_back(std::move(out)); + return members.back(); + } + + t_cell_index + resolve_cells(const std::vector& path, t_uindex depth) const { + std::string prefix; + for (t_uindex pidx = depth; pidx < path.size(); ++pidx) { + prefix += path[pidx].to_string() + "|"; + } + + t_cell_index cells; + for (t_uindex aggnum = 0; aggnum < m_num_visible; ++aggnum) { + auto iter = m_cell_columns.find(prefix + m_cell_names[aggnum]); + if (iter != m_cell_columns.end()) { + cells.emplace_back(aggnum, iter->second); + } + } + + return cells; + } + + const t_cell_index& + cells_for(const t_stree& tr, t_uindex depth, t_uindex idx) { + std::int64_t key = wide_key(depth, idx); + auto iter = m_cell_index.find(key); + if (iter == m_cell_index.end()) { + iter = m_cell_index + .emplace( + key, resolve_cells(m_paths.get(tr, key, idx), depth) + ) + .first; + } + + return iter->second; + } + + void + add_node( + const t_stree& tr, + t_uindex depth, + t_uindex idx, + std::vector& members, + tsl::hopscotch_map& index + ) { + t_uindex node_depth = tr.get_depth(idx); + if (node_depth < depth) { + return; + } + + t_uindex ridx = idx; + for (t_uindex step = node_depth; step > depth; --step) { + ridx = tr.get_parent_idx(ridx); + } + + t_derived_member& row = ensure_row(tr, depth, ridx, members, index); + if (!column_selected(node_depth - depth)) { + return; + } + + for (const auto& [aggnum, cidx] : cells_for(tr, depth, idx)) { + row.m_cells.emplace_back(cidx, tr.get_aggregate(idx, aggnum)); + } + } + + void + add_dropped( + const t_stree& tr, + t_uindex depth, + const t_stree_dropped& dropped, + std::vector& members, + tsl::hopscotch_map& index + ) { + if (dropped.m_depth < depth) { + return; + } + + if (dropped.m_depth == depth) { + t_derived_member out; + out.m_pkey.set(wide_key(depth, dropped.m_idx)); + out.m_deleted = true; + t_index bound = bound_row(out.m_pkey); + if (bound >= 0) { + m_freed_rows.push_back(static_cast(bound)); + members.push_back(std::move(out)); + } + + return; + } + + if (!column_selected(dropped.m_depth - depth)) { + return; + } + + std::vector row_path( + dropped.m_path.begin(), dropped.m_path.begin() + depth + ); + std::reverse(row_path.begin(), row_path.end()); + t_index ridx = depth == 0 ? 0 : tr.resolve_path(0, row_path); + if (ridx == INVALID_INDEX || !tr.node_exists(ridx)) { + return; + } + + t_derived_member& row = ensure_row( + tr, depth, static_cast(ridx), members, index + ); + for (const auto& [aggnum, cidx] : resolve_cells(dropped.m_path, depth)) { + row.m_cells.emplace_back(cidx, mknone()); + } + } + + std::shared_ptr> m_view; + std::shared_ptr
m_parent; + bool m_column_only = false; + t_uindex m_num_row_pivots = 0; + t_uindex m_num_column_pivots = 0; + t_uindex m_num_visible = 0; + std::vector m_key_columns; + std::vector m_cell_names; + tsl::hopscotch_map m_cell_columns; + tsl::hopscotch_map m_cell_index; + t_path_cache m_paths; + std::vector m_free_rows; + std::vector m_freed_rows; + t_uindex m_next_row = 0; + std::vector> m_captures; +}; + +namespace { + bool + flat_has_member( + const t_ctx0& ctx, const t_gstate::t_mapping&, const t_tscalar& pkey + ) { + return ctx.has_pkey(pkey); + } + + bool + flat_has_member( + const t_ctxunit&, + const t_gstate::t_mapping& mapping, + const t_tscalar& pkey + ) { + return mapping.find(pkey) != mapping.end(); + } + + std::shared_ptr + flat_expression_master(const t_ctx0& ctx) { + return ctx.get_expression_tables()->m_master; + } + + std::shared_ptr + flat_expression_master(const t_ctxunit&) { + return nullptr; + } + + std::shared_ptr + flat_expression_prev(const t_ctx0& ctx) { + return ctx.get_expression_tables()->m_prev; + } + + std::shared_ptr + flat_expression_prev(const t_ctxunit&) { + return nullptr; + } + + std::vector + flat_members(const t_ctx0& ctx, const t_gstate::t_mapping&) { + return ctx.get_member_pkeys(); + } + + std::vector + flat_members(const t_ctxunit&, const t_gstate::t_mapping& mapping) { + std::vector pkeys; + pkeys.reserve(mapping.size()); + for (const auto& kv : mapping) { + pkeys.push_back(kv.first); + } + + return pkeys; + } +} // namespace + +/** + * @brief The source for an unpivoted view, one row per row passing its filter. + */ +template +class t_flat_derived_source : public t_derived_source { +public: + t_flat_derived_source( + std::shared_ptr> view, std::shared_ptr
parent + ) : + m_view(std::move(view)), + m_parent(std::move(parent)) {} + + t_schema + infer_schema() const override { + t_schema parent_schema = m_parent->get_schema(); + auto expressions = flat_expression_master(*(m_view->get_context())); + std::vector names; + std::vector types; + for (const auto& name : m_view->get_view_config()->get_columns()) { + if (parent_schema.has_column(name)) { + names.push_back(name); + types.push_back(parent_schema.get_dtype(name)); + } else if (expressions && expressions->get_schema().has_column(name)) { + names.push_back(name); + types.push_back(expressions->get_schema().get_dtype(name)); + } + } + + return {names, types}; + } + + t_dtype + pkey_dtype() const override { + return m_parent->get_gnode()->get_table_sptr()->get_schema().get_dtype( + "psp_pkey" + ); + } + + std::string + index() const override { + const std::string& index = m_parent->get_index(); + const auto columns = m_view->get_view_config()->get_columns(); + if (std::find(columns.begin(), columns.end(), index) == columns.end()) { + return ""; + } + + return index; + } + + std::uint32_t + limit() const override { + return m_parent->get_limit(); + } + +protected: + void + resolve(const t_schema& schema) override {} + + t_uindex + storage_generation() const override { + return m_view->get_context()->get_storage_generation(); + } + + void + collect(bool full, std::vector& members) override { + auto gnode = m_parent->get_gnode(); + std::shared_ptr master = gnode->get_table_sptr(); + auto expressions = flat_expression_master(*(m_view->get_context())); + m_sources.assign(m_columns.size(), nullptr); + for (t_uindex cidx = 0; cidx < m_columns.size(); ++cidx) { + const std::string& name = m_columns[cidx]; + if (master->get_schema().has_column(name)) { + m_sources[cidx] = master->_get_const_column(name); + } else if (expressions && expressions->get_schema().has_column(name)) { + m_sources[cidx] = expressions->_get_const_column(name); + } + } + + const auto& mapping = gnode->get_pkey_map(); + const CTX_T& ctx = *(m_view->get_context()); + if (full) { + m_previous.clear(); + for (const t_tscalar& pkey : flat_members(ctx, mapping)) { + add_member(mapping, pkey, false, members); + } + + return; + } + + const t_data_table* flattened = gnode->_get_otable(PSP_PORT_FLATTENED); + const t_data_table* prev = gnode->_get_otable(PSP_PORT_PREV); + auto expression_prev = flat_expression_prev(ctx); + m_previous.assign(m_columns.size(), nullptr); + for (t_uindex cidx = 0; cidx < m_columns.size(); ++cidx) { + const std::string& name = m_columns[cidx]; + if (prev->get_schema().has_column(name)) { + m_previous[cidx] = prev->_get_const_column(name); + } else if (expression_prev + && expression_prev->get_schema().has_column(name)) { + m_previous[cidx] = expression_prev->_get_const_column(name); + } + } + + const t_column* pkey_col = flattened->_get_const_column("psp_pkey"); + const t_column* op_col = flattened->_get_const_column("psp_op"); + for (t_uindex idx = 0; idx < flattened->size(); ++idx) { + t_tscalar pkey = pkey_col->get_scalar(idx); + bool removed = + *(op_col->get_nth(idx)) == OP_DELETE + || !flat_has_member(ctx, mapping, pkey); + add_member(mapping, pkey, removed, members); + members.back().m_handle = idx; + } + } + + std::shared_ptr + alias_column(t_uindex cidx) const override { + if (m_parent->get_backing_store() != BACKING_STORE_MEMORY) { + return nullptr; + } + + const std::string& name = m_columns[cidx]; + auto master = m_parent->get_gnode()->get_table_sptr(); + if (master->get_schema().has_column(name)) { + return master->get_column(name); + } + + auto expressions = flat_expression_master(*(m_view->get_context())); + if (expressions && expressions->get_schema().has_column(name)) { + return expressions->get_column(name); + } + + return nullptr; + } + + bool + previous( + const t_derived_member& member, t_uindex cidx, t_tscalar& out + ) const override { + if (cidx >= m_previous.size()) { + return false; + } + + const t_column* source = m_previous[cidx]; + if (source == nullptr || member.m_handle >= source->size()) { + return false; + } + + out = source->is_valid(member.m_handle) + ? source->get_scalar(member.m_handle) + : mknone(); + return true; + } + + void + fill( + const t_derived_member& member, + t_uindex idx, + const std::vector& columns + ) const override { + for (t_uindex cidx = 0; cidx < columns.size(); ++cidx) { + if (columns[cidx] == nullptr) { + continue; + } + + const t_column* source = m_sources[cidx]; + if (source != nullptr && source->is_valid(member.m_row)) { + put(columns[cidx], + idx, + DERIVED_CELL_SET, + source->get_scalar(member.m_row)); + } else { + put(columns[cidx], idx, DERIVED_CELL_CLEAR, mknone()); + } + } + } + +private: + void + add_member( + const t_gstate::t_mapping& mapping, + const t_tscalar& pkey, + bool removed, + std::vector& members + ) const { + t_derived_member out; + out.m_pkey = pkey; + auto iter = mapping.find(pkey); + if (removed || iter == mapping.end()) { + out.m_deleted = true; + } else { + out.m_row = iter->second; + } + + members.push_back(std::move(out)); + } + + std::shared_ptr> m_view; + std::shared_ptr
m_parent; + std::vector m_sources; + std::vector m_previous; +}; + +std::shared_ptr +make_derived_source( + const std::shared_ptr>& view, + const std::shared_ptr
& parent +) { + return std::make_shared>(view, parent); +} + +std::shared_ptr +make_derived_source( + const std::shared_ptr>& view, + const std::shared_ptr
& parent +) { + return std::make_shared>(view, parent); +} + +std::shared_ptr +make_derived_source( + const std::shared_ptr>& view, + const std::shared_ptr
& parent +) { + return std::make_shared(view, parent); +} + +std::shared_ptr +make_derived_source( + const std::shared_ptr>& view, + const std::shared_ptr
& parent +) { + return std::make_shared(view, parent); +} + +} // namespace perspective diff --git a/rust/perspective-server/cpp/perspective/src/cpp/expression_tables.cpp b/rust/perspective-server/cpp/perspective/src/cpp/expression_tables.cpp index 5343d4dbf9..441f4db788 100644 --- a/rust/perspective-server/cpp/perspective/src/cpp/expression_tables.cpp +++ b/rust/perspective-server/cpp/perspective/src/cpp/expression_tables.cpp @@ -58,9 +58,6 @@ t_expression_tables::t_expression_tables( m_current = std::make_shared( "", "", schema, DEFAULT_EMPTY_CAPACITY, BACKING_STORE_MEMORY ); - m_delta = std::make_shared( - "", "", schema, DEFAULT_EMPTY_CAPACITY, BACKING_STORE_MEMORY - ); m_transitions = std::make_shared( "", "", transitions_schema, DEFAULT_EMPTY_CAPACITY, BACKING_STORE_MEMORY ); @@ -69,7 +66,6 @@ t_expression_tables::t_expression_tables( m_flattened->init(); m_prev->init(); m_current->init(); - m_delta->init(); m_transitions->init(); } @@ -128,8 +124,8 @@ t_expression_tables::calculate_transitions( t_tscalar prev_value = prev_column.get_scalar(ridx); t_tscalar curr_value = current_column.get_scalar(ridx); - bool prev_valid = prev_column.is_valid(ridx); - bool curr_valid = current_column.is_valid(ridx); + bool prev_valid = prev_value.is_valid(); + bool curr_valid = curr_value.is_valid(); bool prev_curr_eq = prev_valid && curr_valid && (prev_value == curr_value); @@ -144,15 +140,15 @@ t_expression_tables::calculate_transitions( // Row existed before, and the current value is // the same as the previous value. transition = VALUE_TRANSITION_EQ_TT; + } else if (!prev_valid && curr_valid) { + // Previous value was a null, new value is valid. + transition = VALUE_TRANSITION_NEQ_FT; + } else if (prev_valid && !curr_valid) { + transition = VALUE_TRANSITION_NEQ_TF; } else { - if (!prev_valid && curr_valid) { - // Previous value was a null, new value is valid. - transition = VALUE_TRANSITION_NEQ_FT; - } else { - // Previous value was not null, new value is - // not null, and previous value != new value - transition = VALUE_TRANSITION_NEQ_TT; - } + // Previous value was not null, new value is + // not null, and previous value != new value + transition = VALUE_TRANSITION_NEQ_TT; } } else { // Row did not exist before and was added @@ -170,7 +166,6 @@ t_expression_tables::reserve_transitional_table_size(t_uindex size) const { m_flattened->reserve(size); m_prev->reserve(size); m_current->reserve(size); - m_delta->reserve(size); m_transitions->reserve(size); } @@ -179,7 +174,6 @@ t_expression_tables::set_transitional_table_size(t_uindex size) const { m_flattened->set_size(size); m_prev->set_size(size); m_current->set_size(size); - m_delta->set_size(size); m_transitions->set_size(size); } @@ -188,7 +182,6 @@ t_expression_tables::clear_transitional_tables() const { m_flattened->clear(); m_prev->clear(); m_current->clear(); - m_delta->clear(); m_transitions->clear(); } @@ -198,7 +191,6 @@ t_expression_tables::reset() const { m_flattened->reset(); m_prev->reset(); m_current->reset(); - m_delta->reset(); m_transitions->reset(); } diff --git a/rust/perspective-server/cpp/perspective/src/cpp/extract_aggregate.cpp b/rust/perspective-server/cpp/perspective/src/cpp/extract_aggregate.cpp index ecc607e9c3..cb9e4d75cf 100644 --- a/rust/perspective-server/cpp/perspective/src/cpp/extract_aggregate.cpp +++ b/rust/perspective-server/cpp/perspective/src/cpp/extract_aggregate.cpp @@ -27,6 +27,12 @@ extract_aggregate( case AGGTYPE_PCT_SUM_PARENT: { t_tscalar cv = aggcol->get_scalar(ridx); + if (!cv.is_valid()) { + t_tscalar ret; + ret.set(t_none()); + return ret; + } + if (pridx == INVALID_INDEX) { return mktscalar(100.0); } @@ -42,6 +48,13 @@ extract_aggregate( } break; case AGGTYPE_PCT_SUM_GRAND_TOTAL: { t_tscalar cv = aggcol->get_scalar(ridx); + + if (!cv.is_valid()) { + t_tscalar ret; + ret.set(t_none()); + return ret; + } + double pv = aggcol->get_scalar(ROOT_AGGIDX).to_double(); t_tscalar ret; if (pv == 0) { @@ -51,11 +64,14 @@ extract_aggregate( } return ret; } break; + case AGGTYPE_ABS_SUM: { + return aggcol->get_scalar(ridx).abs(); + } break; case AGGTYPE_SUM: case AGGTYPE_SUM_ABS: - case AGGTYPE_ABS_SUM: case AGGTYPE_GMV: - case AGGTYPE_SUM_NOT_NULL: + case AGGTYPE_SUM_OR_ZERO: + case AGGTYPE_VALID_COUNT: case AGGTYPE_MUL: case AGGTYPE_COUNT: case AGGTYPE_ANY: diff --git a/rust/perspective-server/cpp/perspective/src/cpp/gnode.cpp b/rust/perspective-server/cpp/perspective/src/cpp/gnode.cpp index 2e828d1178..3c41f942e1 100644 --- a/rust/perspective-server/cpp/perspective/src/cpp/gnode.cpp +++ b/rust/perspective-server/cpp/perspective/src/cpp/gnode.cpp @@ -93,7 +93,6 @@ t_gnode::t_gnode( m_input_schema, m_output_schema, m_output_schema, - m_output_schema, trans_schema, existed_schema }; @@ -336,120 +335,60 @@ t_gnode::_process_mask_existed_rows(t_process_state& process_state) { return mask; } -t_process_table_result -t_gnode::_process_table(t_uindex port_id) { - m_was_updated = false; - m_removed_pkeys = nullptr; - - t_process_table_result result; - result.m_flattened_data_table = nullptr; - result.m_should_notify_userspace = false; - - std::shared_ptr flattened = nullptr; - - if (m_input_ports.count(port_id) == 0) { - std::cerr << "Cannot process table on port `" << port_id - << "` as it does not exist." << '\n'; - return result; +void +t_gnode::_take_reset_removes( + const t_column* pkey_col, t_uindex flattened_num_rows +) { + if (!m_reset_pending) { + return; } - std::shared_ptr& input_port = m_input_ports[port_id]; - - // A `reset` with no rows queued behind it (a bare `clear`) still - // produces one step, so listeners see the table empty and its keys gone. - if (input_port->get_table()->size() == 0) { - if (!m_reset_pending) { - return result; - } - - m_reset_pending = false; - m_removed_pkeys = std::move(m_reset_pkeys); - m_reset_pkeys = nullptr; - m_was_updated = true; - result.m_should_notify_userspace = true; - return result; + m_reset_pending = false; + if (!m_reset_pkeys) { + return; } - m_was_updated = true; - flattened = input_port->get_table()->flatten(); - - PSP_GNODE_VERIFY_TABLE(flattened); - PSP_GNODE_VERIFY_TABLE(get_table()); - - t_uindex flattened_num_rows = flattened->num_rows(); - - std::vector row_lookup(flattened_num_rows); - t_column* pkey_col = flattened->_get_column("psp_pkey"); - + tsl::hopscotch_set incoming; + incoming.reserve(flattened_num_rows); for (t_uindex idx = 0; idx < flattened_num_rows; ++idx) { - // See if each primary key in flattened already exist in the dataset - t_tscalar pkey = pkey_col->get_scalar(idx); - row_lookup[idx] = m_gstate->lookup(pkey); + incoming.insert(pkey_col->get_scalar(idx)); } - if (m_reset_pending) { - m_reset_pending = false; - if (m_reset_pkeys) { - tsl::hopscotch_set incoming; - incoming.reserve(flattened_num_rows); - for (t_uindex idx = 0; idx < flattened_num_rows; ++idx) { - incoming.insert(pkey_col->get_scalar(idx)); - } - - const t_column* stash_col = - m_reset_pkeys->_get_column("psp_pkey"); - const t_uindex stash_size = m_reset_pkeys->size(); - t_column* removed_col = nullptr; - for (t_uindex idx = 0; idx < stash_size; ++idx) { - t_tscalar pkey = stash_col->get_scalar(idx); - if (!incoming.contains(pkey)) { - removed_col = append_removed_pkey( - m_removed_pkeys, - removed_col, - stash_col->get_dtype(), - stash_size, - pkey - ); - } - } - - m_reset_pkeys = nullptr; + const t_column* stash_col = m_reset_pkeys->_get_column("psp_pkey"); + const t_uindex stash_size = m_reset_pkeys->size(); + t_column* removed_col = nullptr; + for (t_uindex idx = 0; idx < stash_size; ++idx) { + t_tscalar pkey = stash_col->get_scalar(idx); + if (!incoming.contains(pkey)) { + removed_col = append_removed_pkey( + m_removed_pkeys, + removed_col, + stash_col->get_dtype(), + stash_size, + pkey + ); } } - // first update - master table is empty - if (m_gstate->mapping_size() == 0) { - m_gstate->update_master_table(flattened); - m_oports[PSP_PORT_FLATTENED]->set_table(flattened); - - _compute_expressions(flattened); - - // Update all contexts registered with the gnode with data. - _update_contexts_from_state(m_gstate->get_pkeyed_table()); - - input_port->release(); - release_outputs(); - -#ifdef PSP_GNODE_VERIFY - auto state_table = get_table(); - PSP_GNODE_VERIFY_TABLE(state_table); -#endif - // Make sure user is notified after first update. - result.m_should_notify_userspace = true; - return result; - } + m_reset_pkeys = nullptr; +} - input_port->release_or_clear(); +t_mask +t_gnode::_compute_transitions( + const std::shared_ptr& flattened, + const std::shared_ptr& state_table, + const std::vector& lookup +) { + t_uindex flattened_num_rows = flattened->num_rows(); // Use `t_process_state` to manage intermediate structures t_process_state _process_state; - _process_state.m_state_data_table = get_table_sptr(); + _process_state.m_state_data_table = state_table; _process_state.m_flattened_data_table = flattened; - _process_state.m_lookup = row_lookup; + _process_state.m_lookup = lookup; // Get data tables for process state - _process_state.m_delta_data_table = m_oports[PSP_PORT_DELTA]->get_table(); _process_state.m_prev_data_table = m_oports[PSP_PORT_PREV]->get_table(); _process_state.m_current_data_table = m_oports[PSP_PORT_CURRENT]->get_table(); @@ -458,7 +397,6 @@ t_gnode::_process_table(t_uindex port_id) { _process_state.m_existed_data_table = m_oports[PSP_PORT_EXISTED]->get_table(); - // Clear delta, prev, current, transitions, existed on EACH call. _process_state.clear_transitional_data_tables(); // And re-reserved for the amount of data in `flattened` @@ -483,8 +421,6 @@ t_gnode::_process_table(t_uindex port_id) { _process_state.m_flattened_data_table->_get_column(cname); auto* scolumn = _process_state.m_state_data_table->_get_column(cname); - auto* dcolumn = - _process_state.m_delta_data_table->_get_column(cname); auto* pcolumn = _process_state.m_prev_data_table->_get_column(cname); auto* ccolumn = @@ -499,7 +435,6 @@ t_gnode::_process_table(t_uindex port_id) { _process_column( fcolumn, scolumn, - dcolumn, pcolumn, ccolumn, tcolumn, @@ -510,7 +445,6 @@ t_gnode::_process_table(t_uindex port_id) { _process_column( fcolumn, scolumn, - dcolumn, pcolumn, ccolumn, tcolumn, @@ -521,7 +455,6 @@ t_gnode::_process_table(t_uindex port_id) { _process_column( fcolumn, scolumn, - dcolumn, pcolumn, ccolumn, tcolumn, @@ -532,7 +465,6 @@ t_gnode::_process_table(t_uindex port_id) { _process_column( fcolumn, scolumn, - dcolumn, pcolumn, ccolumn, tcolumn, @@ -543,7 +475,6 @@ t_gnode::_process_table(t_uindex port_id) { _process_column( fcolumn, scolumn, - dcolumn, pcolumn, ccolumn, tcolumn, @@ -554,7 +485,6 @@ t_gnode::_process_table(t_uindex port_id) { _process_column( fcolumn, scolumn, - dcolumn, pcolumn, ccolumn, tcolumn, @@ -565,7 +495,6 @@ t_gnode::_process_table(t_uindex port_id) { _process_column( fcolumn, scolumn, - dcolumn, pcolumn, ccolumn, tcolumn, @@ -576,7 +505,6 @@ t_gnode::_process_table(t_uindex port_id) { _process_column( fcolumn, scolumn, - dcolumn, pcolumn, ccolumn, tcolumn, @@ -587,7 +515,6 @@ t_gnode::_process_table(t_uindex port_id) { _process_column( fcolumn, scolumn, - dcolumn, pcolumn, ccolumn, tcolumn, @@ -598,7 +525,6 @@ t_gnode::_process_table(t_uindex port_id) { _process_column( fcolumn, scolumn, - dcolumn, pcolumn, ccolumn, tcolumn, @@ -609,7 +535,6 @@ t_gnode::_process_table(t_uindex port_id) { _process_column( fcolumn, scolumn, - dcolumn, pcolumn, ccolumn, tcolumn, @@ -620,7 +545,6 @@ t_gnode::_process_table(t_uindex port_id) { _process_column( fcolumn, scolumn, - dcolumn, pcolumn, ccolumn, tcolumn, @@ -631,7 +555,6 @@ t_gnode::_process_table(t_uindex port_id) { _process_column( fcolumn, scolumn, - dcolumn, pcolumn, ccolumn, tcolumn, @@ -642,7 +565,6 @@ t_gnode::_process_table(t_uindex port_id) { _process_column( fcolumn, scolumn, - dcolumn, pcolumn, ccolumn, tcolumn, @@ -657,35 +579,129 @@ t_gnode::_process_table(t_uindex port_id) { } ); + return existed_mask; +} + +t_process_table_result +t_gnode::_process_table(t_uindex port_id) { + m_was_updated = false; + m_removed_pkeys = nullptr; + + t_process_table_result result; + result.m_flattened_data_table = nullptr; + result.m_should_notify_userspace = false; + + std::shared_ptr flattened = nullptr; + + if (m_input_ports.count(port_id) == 0) { + std::cerr << "Cannot process table on port `" << port_id + << "` as it does not exist." << '\n'; + return result; + } + + std::shared_ptr& input_port = m_input_ports[port_id]; + + // A `reset` with no rows queued behind it (a bare `clear`) still + // produces one step, so listeners see the table empty and its keys gone. + if (input_port->get_table()->size() == 0) { + if (!m_reset_pending) { + return result; + } + + m_reset_pending = false; + m_removed_pkeys = std::move(m_reset_pkeys); + m_reset_pkeys = nullptr; + m_was_updated = true; + result.m_should_notify_userspace = true; + return result; + } + + m_was_updated = true; + flattened = input_port->get_table()->flatten(); + + PSP_GNODE_VERIFY_TABLE(flattened); + PSP_GNODE_VERIFY_TABLE(get_table()); + + t_uindex flattened_num_rows = flattened->num_rows(); + + std::vector row_lookup(flattened_num_rows); + t_column* pkey_col = flattened->_get_column("psp_pkey"); + + for (t_uindex idx = 0; idx < flattened_num_rows; ++idx) { + // See if each primary key in flattened already exist in the dataset + t_tscalar pkey = pkey_col->get_scalar(idx); + row_lookup[idx] = m_gstate->lookup(pkey); + } + + _take_reset_removes(pkey_col, flattened_num_rows); + + // first update - master table is empty + if (m_gstate->mapping_size() == 0) { + m_gstate->update_master_table(flattened); + m_oports[PSP_PORT_FLATTENED]->set_table(flattened); + + _compute_expressions(flattened); + + // Update all contexts registered with the gnode with data. + _update_contexts_from_state(m_gstate->get_pkeyed_table()); + + input_port->release(); + release_outputs(); + +#ifdef PSP_GNODE_VERIFY + auto state_table = get_table(); + PSP_GNODE_VERIFY_TABLE(state_table); +#endif + // Make sure user is notified after first update. + result.m_should_notify_userspace = true; + return result; + } + + input_port->release_or_clear(); + + t_mask existed_mask = + _compute_transitions(flattened, get_table_sptr(), row_lookup); + + result.m_flattened_data_table = _finish_step( + flattened, + existed_mask, + row_lookup, + [this](const std::shared_ptr& masked) { + m_gstate->update_master_table(masked); + } + ); + result.m_should_notify_userspace = true; + return result; +} + +std::shared_ptr +t_gnode::_finish_step( + const std::shared_ptr& flattened, + const t_mask& existed_mask, + const std::vector& row_lookup, + const std::function&)>& commit +) { /** - * After all columns have been processed (transitional tables written into), - * `_process_state.m_flattened_data_table` contains the accumulated state - * of the dataset that updates the master table on `m_gstate`, including - * added rows, updated in-place rows, and rows to be removed. + * `flattened` contains the accumulated state of the dataset that updates + * the master table on `m_gstate`, including added rows, updated in-place + * rows, and rows to be removed. * * `existed_mask` is a bitset marked true for `OP_INSERT`, and false for * `OP_DELETE`. If there are any `OP_DELETE`s, the next step returns a * new `t_data_table` with the deleted rows masked out. */ std::shared_ptr flattened_masked; - - if (existed_mask.count() == _process_state.m_flattened_data_table->size()) { - flattened_masked = _process_state.m_flattened_data_table; + if (existed_mask.count() == flattened->size()) { + flattened_masked = flattened; } else { - flattened_masked = - _process_state.m_flattened_data_table->clone(existed_mask); + flattened_masked = flattened->clone(existed_mask); } PSP_GNODE_VERIFY_TABLE(flattened_masked); -#ifdef PSP_GNODE_VERIFY - { - auto updated_table = get_table(); - PSP_GNODE_VERIFY_TABLE(updated_table); + if (commit) { + commit(flattened_masked); } -#endif - - m_gstate->update_master_table(flattened_masked); #ifdef PSP_GNODE_VERIFY { @@ -694,27 +710,23 @@ t_gnode::_process_table(t_uindex port_id) { } #endif - if (flattened_masked.get() == _process_state.m_flattened_data_table.get()) { + if (flattened_masked.get() == flattened.get()) { _process_windows(flattened_masked, row_lookup); } else { std::vector masked_lookup; masked_lookup.reserve(flattened_masked->size()); - for (t_uindex idx = 0; idx < flattened_num_rows; ++idx) { + for (t_uindex idx = 0, end = flattened->num_rows(); idx < end; ++idx) { if (existed_mask.get(idx)) { masked_lookup.push_back(row_lookup[idx]); } } + _process_windows(flattened_masked, masked_lookup); } m_oports[PSP_PORT_FLATTENED]->set_table(flattened_masked); - _compute_expressions(get_table_sptr(), flattened_masked); - - result.m_flattened_data_table = flattened_masked; - result.m_should_notify_userspace = true; - - return result; + return flattened_masked; } template <> @@ -722,7 +734,6 @@ void t_gnode::_process_column( const t_column* fcolumn, const t_column* scolumn, - t_column* dcolumn, t_column* pcolumn, t_column* ccolumn, t_column* tcolumn, @@ -749,7 +760,9 @@ t_gnode::_process_column( const auto* cur_value = fcolumn->get_nth(idx); std::string curs(cur_value); - bool cur_valid = fcolumn->is_valid(idx); + t_status cur_status = *(fcolumn->get_nth_status(idx)); + bool cur_valid = cur_status == STATUS_VALID; + bool cur_cleared = cur_status == STATUS_CLEAR; if (row_pre_existed) { prev_value = scolumn->get_nth(rlookup.m_idx); @@ -783,16 +796,14 @@ t_gnode::_process_column( if (cur_valid) { ccolumn->set_nth(added_count, cur_value); - } - - if (!cur_valid && prev_valid) { + ccolumn->set_valid(added_count, true); + } else if (prev_valid && !cur_cleared) { ccolumn->set_nth(added_count, prev_value); + ccolumn->set_valid(added_count, true); + } else { + ccolumn->clear(added_count); } - ccolumn->set_valid( - added_count, cur_valid ? cur_valid : prev_valid - ); - tcolumn->set_nth(idx, trans); } break; case OP_DELETE: { @@ -867,6 +878,80 @@ t_gnode::init_bulk(const std::shared_ptr& data_table) { _update_contexts_from_state(m_gstate->get_pkeyed_table()); } +void +t_gnode::set_derived_alias( + const std::string& name, std::shared_ptr column +) { + m_gstate->set_alias(name, std::move(column)); +} + +bool +t_gnode::is_derived_alias(const std::string& name) const { + return m_gstate->is_aliased(name); +} + +bool +t_gnode::process_derived(const t_derived_step& step) { + PSP_TRACE_SENTINEL(); + PSP_VERBOSE_ASSERT(m_init, "Cannot `process_derived` on an uninited gnode."); + m_was_updated = false; + m_removed_pkeys = nullptr; + + const std::shared_ptr& flattened = step.m_flattened; + t_uindex flattened_num_rows = flattened->num_rows(); + if (flattened_num_rows == 0) { + if (!m_reset_pending) { + return false; + } + + m_reset_pending = false; + m_removed_pkeys = std::move(m_reset_pkeys); + m_reset_pkeys = nullptr; + m_was_updated = true; + return true; + } + + m_was_updated = true; + t_column* pkey_col = flattened->_get_column("psp_pkey"); + std::vector row_lookup(flattened_num_rows); + std::vector identity_lookup(flattened_num_rows); + for (t_uindex idx = 0; idx < flattened_num_rows; ++idx) { + bool exists = step.m_bound[idx] >= 0; + row_lookup[idx] = t_rlookup( + exists ? static_cast(step.m_bound[idx]) : 0, exists + ); + identity_lookup[idx] = t_rlookup(idx, exists); + } + + _take_reset_removes(pkey_col, flattened_num_rows); + + if (m_gstate->mapping_size() == 0) { + m_gstate->commit_derived(flattened, step.m_rows); + m_oports[PSP_PORT_FLATTENED]->set_table(flattened); + if (step.m_notify && !m_contexts.empty()) { + _compute_expressions(flattened); + _update_contexts_from_state(m_gstate->get_pkeyed_table()); + } + + release_outputs(); + return true; + } + + if (!step.m_notify) { + m_gstate->commit_derived(flattened, step.m_rows); + release_outputs(); + return true; + } + + t_mask existed_mask = + _compute_transitions(flattened, step.m_prev_state, identity_lookup); + m_gstate->commit_derived(flattened, step.m_rows); + auto flattened_masked = + _finish_step(flattened, existed_mask, row_lookup, nullptr); + notify_contexts(flattened_masked); + return true; +} + bool t_gnode::process(t_uindex port_id) { PSP_TRACE_SENTINEL(); @@ -1091,7 +1176,6 @@ t_gnode::notify_context( ) { auto* ctx = ctxh.get(); - std::shared_ptr delta = m_oports[PSP_PORT_DELTA]->get_table(); std::shared_ptr prev = m_oports[PSP_PORT_PREV]->get_table(); std::shared_ptr current = m_oports[PSP_PORT_CURRENT]->get_table(); @@ -1104,9 +1188,7 @@ t_gnode::notify_context( // pass the tables as const references - the destructors for all of the // joined tables will be called after this function finishes executing, // as the contexts do not retain a reference to these tables. - ctx->notify( - *(flattened), *(delta), *(prev), *(current), *(transitions), existed - ); + ctx->notify(*(flattened), *(prev), *(current), *(transitions), existed); ctx->step_end(); } @@ -1406,7 +1488,6 @@ t_gnode::_process_windows( t_uindex n_old = flattened->size(); t_uindex n_new = n_old + extra.size(); - std::shared_ptr delta = m_oports[PSP_PORT_DELTA]->get_table(); std::shared_ptr prev = m_oports[PSP_PORT_PREV]->get_table(); std::shared_ptr current = m_oports[PSP_PORT_CURRENT]->get_table(); @@ -1416,7 +1497,6 @@ t_gnode::_process_windows( m_oports[PSP_PORT_EXISTED]->get_table(); flattened->extend(n_new); - delta->extend(n_new); prev->extend(n_new); current->extend(n_new); transitions->extend(n_new); @@ -1461,7 +1541,6 @@ t_gnode::_process_windows( copy_plans.push_back(plan_table(flattened)); copy_plans.push_back(plan_table(prev)); copy_plans.push_back(plan_table(current)); - std::vector delta_plan = plan_table(delta); std::vector transitions_plan = plan_table(transitions); t_column* existed_col = existed->get_column("psp_existed").get(); t_column* widened_col = existed->get_column("psp_widened").get(); @@ -1500,20 +1579,6 @@ t_gnode::_process_windows( } } - for (const auto& wcol : delta_plan) { - switch (wcol.m_kind) { - case t_wcol_kind::PKEY: - wcol.m_col->set_scalar(row, pkey); - break; - case t_wcol_kind::OP: - wcol.m_col->set_nth(row, OP_INSERT); - break; - default: - wcol.m_col->clear(row); - break; - } - } - for (const auto& wcol : transitions_plan) { std::uint8_t code = VALUE_TRANSITION_EQ_FF; if (wcol.m_kind == t_wcol_kind::MASTER @@ -1675,7 +1740,6 @@ t_gnode::_compute_expressions( const std::shared_ptr& master, const std::shared_ptr& flattened ) { - std::shared_ptr delta = m_oports[PSP_PORT_DELTA]->get_table(); std::shared_ptr prev = m_oports[PSP_PORT_PREV]->get_table(); std::shared_ptr current = m_oports[PSP_PORT_CURRENT]->get_table(); @@ -1697,7 +1761,6 @@ t_gnode::_compute_expressions( master, m_gstate->get_pkey_map(), flattened, - delta, prev, current, transitions, @@ -1712,7 +1775,6 @@ t_gnode::_compute_expressions( master, m_gstate->get_pkey_map(), flattened, - delta, prev, current, transitions, @@ -1727,7 +1789,6 @@ t_gnode::_compute_expressions( master, m_gstate->get_pkey_map(), flattened, - delta, prev, current, transitions, @@ -1742,7 +1803,6 @@ t_gnode::_compute_expressions( master, m_gstate->get_pkey_map(), flattened, - delta, prev, current, transitions, diff --git a/rust/perspective-server/cpp/perspective/src/cpp/gnode_state.cpp b/rust/perspective-server/cpp/perspective/src/cpp/gnode_state.cpp index e2e8ad0022..4a766bd642 100644 --- a/rust/perspective-server/cpp/perspective/src/cpp/gnode_state.cpp +++ b/rust/perspective-server/cpp/perspective/src/cpp/gnode_state.cpp @@ -86,12 +86,12 @@ t_gstate::erase(const t_tscalar& pkey) { return; } - auto columns = m_table->get_columns(); - t_uindex idx = iter->second; - - for (auto* c : columns) { - c->clear(idx); + auto columns = m_table->get_columns(); + for (t_uindex cidx = 0; cidx < columns.size(); ++cidx) { + if (m_alias_mask.empty() || !m_alias_mask[cidx]) { + columns[cidx]->clear(idx); + } } m_mapping.erase(iter); @@ -259,6 +259,140 @@ t_gstate::init_from_table(const std::shared_ptr& source) { #endif } +void +t_gstate::commit_derived( + const std::shared_ptr& flattened, + const std::vector& rows +) { + const t_column* flattened_pkey_col = + flattened->_get_const_column("psp_pkey"); + const t_column* flattened_op_col = + flattened->_get_const_column("psp_op"); + const std::uint8_t* flattened_op_base = + flattened_op_col->get_nth_base(); + t_uindex num_rows = flattened->num_rows(); + t_uindex extent = m_table->size(); + m_pkcol = m_table->get_column("psp_pkey"); + m_opcol = m_table->get_column("psp_op"); + for (t_uindex idx = 0; idx < num_rows; ++idx) { + if (static_cast(flattened_op_base[idx]) == OP_DELETE) { + erase(flattened_pkey_col->get_scalar(idx)); + } else { + extent = std::max(extent, rows[idx] + 1); + } + } + + resize_owned(extent); + + for (t_uindex idx = 0; idx < num_rows; ++idx) { + if (static_cast(flattened_op_base[idx]) == OP_DELETE) { + continue; + } + + t_tscalar pkey = flattened_pkey_col->get_scalar(idx); + m_mapping[m_symtable.get_interned_tscalar(pkey)] = rows[idx]; + m_free.erase(rows[idx]); + m_opcol->set_nth(rows[idx], OP_INSERT); + m_pkcol->set_scalar(rows[idx], pkey); + } + + const t_schema& master_schema = m_table->get_schema(); + t_data_table* master_table = m_table.get(); + parallel_for( + int(master_table->num_columns()), + [flattened, flattened_op_col, &master_schema, master_table, &rows, this]( + int idx + ) { + const std::string& column_name = master_schema.m_columns[idx]; + if (!m_alias_mask.empty() && m_alias_mask[idx]) { + return; + } + + const t_column* flattened_column = + flattened->_get_const_column_safe(column_name); + if (!flattened_column) { + return; + } + + update_master_column( + master_table->_get_column(column_name), + flattened_column, + flattened_op_col, + rows, + flattened->num_rows() + ); + } + ); +} + +void +t_gstate::set_alias(const std::string& name, std::shared_ptr column) { + m_table->set_column(name, std::move(column)); + m_aliased.insert(name); + _refresh_alias_mask(); +} + +void +t_gstate::_refresh_alias_mask() { + if (m_aliased.empty()) { + m_alias_mask.clear(); + return; + } + + const auto& columns = m_table->get_schema().m_columns; + m_alias_mask.assign(columns.size(), false); + for (t_uindex cidx = 0; cidx < columns.size(); ++cidx) { + m_alias_mask[cidx] = is_aliased(columns[cidx]); + } +} + +bool +t_gstate::is_aliased(const std::string& name) const { + return m_aliased.find(name) != m_aliased.end(); +} + +void +t_gstate::drop_aliases() { + const t_schema& schema = m_table->get_schema(); + for (const auto& name : m_aliased) { + t_uindex idx = schema.get_colidx(name); + auto column = m_table->make_column( + name, schema.m_types[idx], schema.m_status_enabled[idx] + ); + + column->init(); + m_table->set_column(name, std::move(column)); + } + + m_aliased.clear(); + m_alias_mask.clear(); +} + +void +t_gstate::resize_owned(t_uindex extent) { + if (extent <= m_table->size()) { + return; + } + + t_uindex capacity = m_table->get_capacity(); + if (extent > capacity) { + capacity = std::max( + extent, static_cast(capacity * PSP_TABLE_GROW_RATIO) + ); + } + + auto columns = m_table->get_columns(); + for (t_uindex cidx = 0; cidx < columns.size(); ++cidx) { + if (m_alias_mask.empty() || !m_alias_mask[cidx]) { + columns[cidx]->reserve(capacity); + columns[cidx]->set_size(extent); + } + } + + m_table->set_capacity(capacity); + m_table->set_table_size(extent); +} + void t_gstate::update_master_table( const std::shared_ptr& flattened @@ -858,6 +992,7 @@ t_gstate::mapping_size() const { void t_gstate::reset() { + drop_aliases(); m_table->reset(); m_mapping.clear(); m_free.clear(); diff --git a/rust/perspective-server/cpp/perspective/src/cpp/pool.cpp b/rust/perspective-server/cpp/perspective/src/cpp/pool.cpp index 131c6f23c6..69446dcdb9 100644 --- a/rust/perspective-server/cpp/perspective/src/cpp/pool.cpp +++ b/rust/perspective-server/cpp/perspective/src/cpp/pool.cpp @@ -43,7 +43,8 @@ t_pool::t_pool() : m_sleep(0) { m_run.clear(); } t_pool::t_pool() : // : m_update_delegate(empty_callback()) #ifdef PSP_PARALLEL_FOR - m_lock(new std::shared_mutex()), + m_lock_owner(std::make_shared()), + m_lock(m_lock_owner.get()), #endif m_sleep(0) { m_run.clear(); @@ -55,11 +56,7 @@ t_pool::t_pool() : m_sleep(0) { m_run.clear(); } #endif -t_pool::~t_pool() { -#ifdef PSP_PARALLEL_FOR - delete m_lock; -#endif -} +t_pool::~t_pool() {} void t_pool::init() { @@ -149,6 +146,22 @@ std::shared_mutex* t_pool::get_lock() const { return m_lock; } + +std::mutex& +t_pool::get_process_lock() { + return m_process_lock; +} + +void +t_pool::adopt_lock(const t_pool& other) { + m_lock_owner = other.m_lock_owner; + m_lock = m_lock_owner.get(); + for (auto* node : m_gnodes) { + if (node != nullptr) { + node->set_lock(m_lock); + } + } +} #endif void diff --git a/rust/perspective-server/cpp/perspective/src/cpp/process_state.cpp b/rust/perspective-server/cpp/perspective/src/cpp/process_state.cpp index 577426939c..3a3d743a5e 100644 --- a/rust/perspective-server/cpp/perspective/src/cpp/process_state.cpp +++ b/rust/perspective-server/cpp/perspective/src/cpp/process_state.cpp @@ -18,7 +18,6 @@ t_process_state::t_process_state() = default; void t_process_state::clear_transitional_data_tables() const { - m_delta_data_table->clear(); m_prev_data_table->clear(); m_current_data_table->clear(); m_transitions_data_table->clear(); @@ -27,7 +26,6 @@ t_process_state::clear_transitional_data_tables() const { void t_process_state::reserve_transitional_data_tables(t_uindex size) const { - m_delta_data_table->reserve(size); m_prev_data_table->reserve(size); m_current_data_table->reserve(size); m_transitions_data_table->reserve(size); @@ -36,7 +34,6 @@ t_process_state::reserve_transitional_data_tables(t_uindex size) const { void t_process_state::set_size_transitional_data_tables(t_uindex size) const { - m_delta_data_table->set_size(size); m_prev_data_table->set_size(size); m_current_data_table->set_size(size); m_transitions_data_table->set_size(size); diff --git a/rust/perspective-server/cpp/perspective/src/cpp/scalar.cpp b/rust/perspective-server/cpp/perspective/src/cpp/scalar.cpp index 1f8f0301e0..4bdbf990be 100644 --- a/rust/perspective-server/cpp/perspective/src/cpp/scalar.cpp +++ b/rust/perspective-server/cpp/perspective/src/cpp/scalar.cpp @@ -806,16 +806,16 @@ t_tscalar::negate() const { } break; case DTYPE_UINT64: { - rval.set(~m_data.m_uint64); + rval.set(-static_cast(m_data.m_uint64)); } break; case DTYPE_UINT32: { - rval.set(~m_data.m_uint32); + rval.set(-static_cast(m_data.m_uint32)); } break; case DTYPE_UINT16: { - rval.set(~m_data.m_uint16); + rval.set(-static_cast(m_data.m_uint16)); } break; case DTYPE_UINT8: { - rval.set(~m_data.m_uint8); + rval.set(-static_cast(m_data.m_uint8)); } break; case DTYPE_FLOAT64: { @@ -940,16 +940,28 @@ t_tscalar::difference(const t_tscalar& other) const { } break; case DTYPE_UINT64: { - rval.set(m_data.m_uint64 - other.m_data.m_uint64); + rval.set( + static_cast(m_data.m_uint64) + - static_cast(other.m_data.m_uint64) + ); } break; case DTYPE_UINT32: { - rval.set(m_data.m_uint32 - other.m_data.m_uint32); + rval.set( + static_cast(m_data.m_uint32) + - static_cast(other.m_data.m_uint32) + ); } break; case DTYPE_UINT16: { - rval.set(m_data.m_uint16 - other.m_data.m_uint16); + rval.set( + static_cast(m_data.m_uint16) + - static_cast(other.m_data.m_uint16) + ); } break; case DTYPE_UINT8: { - rval.set(m_data.m_uint8 - other.m_data.m_uint8); + rval.set( + static_cast(m_data.m_uint8) + - static_cast(other.m_data.m_uint8) + ); } break; case DTYPE_FLOAT64: { diff --git a/rust/perspective-server/cpp/perspective/src/cpp/server.cpp b/rust/perspective-server/cpp/perspective/src/cpp/server.cpp index bd3d61c62a..90d2038c56 100644 --- a/rust/perspective-server/cpp/perspective/src/cpp/server.cpp +++ b/rust/perspective-server/cpp/perspective/src/cpp/server.cpp @@ -68,6 +68,43 @@ tscalar_to_proto(const t_tscalar& scalar, proto::Scalar* out) { } } +/// Encode a column path scalar as `View::write_scalar` does unformatted, so a +/// split value reaches a client in the same encoding as a cell of its type. +static void +column_path_scalar_to_proto(const t_tscalar& scalar, proto::Scalar* out) { + if (scalar.is_none() || !scalar.is_valid()) { + out->set_null(::google::protobuf::NullValue::NULL_VALUE); + return; + } + + switch (scalar.get_dtype()) { + case DTYPE_BOOL: + out->set_bool_(scalar.get()); + break; + case DTYPE_STR: + out->set_string(scalar.get()); + break; + case DTYPE_TIME: + out->set_float_((double)scalar.get()); + break; + case DTYPE_DATE: + out->set_float_((double)scalar.get().as_epoch_ms()); + break; + case DTYPE_FLOAT32: + case DTYPE_FLOAT64: + if (scalar.is_nan()) { + out->set_null(::google::protobuf::NullValue::NULL_VALUE); + } else { + out->set_float_(scalar.to_double()); + } + + break; + default: + out->set_float_(scalar.to_double()); + break; + } +} + std::uint32_t server::ProtoServer::m_client_id = 1; template <> @@ -661,6 +698,67 @@ ServerResources::delete_table(const t_id& id) { } } +void +DerivedTableEngine::register_table( + const t_id& view_id, + const t_id& table_id, + std::shared_ptr source +) { + PSP_WRITE_LOCK(m_lock); + m_view_to_tables.emplace(view_id, table_id); + m_tables[table_id] = t_entry{view_id, std::move(source)}; +} + +void +DerivedTableEngine::unregister_table(const t_id& table_id) { + PSP_WRITE_LOCK(m_lock); + auto iter = m_tables.find(table_id); + if (iter == m_tables.end()) { + return; + } + + auto range = m_view_to_tables.equal_range(iter->second.m_view_id); + for (auto it = range.first; it != range.second; ++it) { + if (it->second == table_id) { + m_view_to_tables.erase(it); + break; + } + } + + iter->second.m_source->detach(); + m_tables.erase(iter); +} + +bool +DerivedTableEngine::has_dependents(const t_id& view_id) const { + PSP_READ_LOCK(m_lock); + return m_view_to_tables.find(view_id) != m_view_to_tables.end(); +} + +std::optional +DerivedTableEngine::parent_view_of(const t_id& table_id) const { + PSP_READ_LOCK(m_lock); + auto iter = m_tables.find(table_id); + if (iter == m_tables.end()) { + return std::nullopt; + } + + return iter->second.m_view_id; +} + +std::vector< + std::pair>> +DerivedTableEngine::get_dependents(const t_id& view_id) const { + PSP_READ_LOCK(m_lock); + std::vector>> rval; + auto range = m_view_to_tables.equal_range(view_id); + for (auto it = range.first; it != range.second; ++it) { + rval.emplace_back(it->second, m_tables.at(it->second).m_source); + } + + return rval; +} + void ServerResources::mark_table_dirty(const t_id& id) { PSP_WRITE_LOCK(m_write_lock); @@ -1357,13 +1455,16 @@ ProtoServer::handle_process_table( ) { if (!m_realtime_mode && needs_poll(req.client_req_case())) { if (entity_type_is_table(req.client_req_case())) { - if (m_resources.is_table_dirty(req.entity_id())) { - auto table = m_resources.get_table(req.entity_id()); - const auto& table_id = req.entity_id(); + const auto table_id = _root_table_id(req.entity_id()); + if (m_resources.is_table_dirty(table_id)) { + auto table = m_resources.get_table(table_id); _process_table(table, table_id, proto_resp); } } else { - auto table_id = m_resources.get_table_id_for_view(req.entity_id()); + auto table_id = _root_table_id( + m_resources.get_table_id_for_view(req.entity_id()) + ); + if (m_resources.is_table_dirty(table_id)) { auto table = m_resources.get_table(table_id); _process_table(table, table_id, proto_resp); @@ -2084,7 +2185,8 @@ ProtoServer::build_view_config( sides = 0; } - bool is_unit_context = table->get_index().empty() && sides == 0 + bool is_unit_context = !table->is_derived() + && table->get_index().empty() && sides == 0 && row_pivots.empty() && column_pivots.empty() && aggregates.empty() && columns.empty() && sort_str.empty() && cfg.expressions().empty() && cfg.windows().empty(); @@ -2122,6 +2224,7 @@ ProtoServer::_handle_request(std::uint32_t client_id, Request&& req) { features->set_sort(true); features->set_on_update(true); features->set_expressions(true); + features->set_view_derivations(true); const auto window_agg = [](const char* name, std::initializer_list frames, @@ -2319,7 +2422,7 @@ ProtoServer::_handle_request(std::uint32_t client_id, Request&& req) { number_opts.add_aggregates()->set_name("pct sum total"); number_opts.add_aggregates()->set_name("stddev"); number_opts.add_aggregates()->set_name("sum abs"); - number_opts.add_aggregates()->set_name("sum not null"); + number_opts.add_aggregates()->set_name("sum or zero"); number_opts.add_aggregates()->set_name("unique"); number_opts.add_aggregates()->set_name("var"); auto args3 = number_opts.add_aggregates(); @@ -2417,10 +2520,13 @@ ProtoServer::_handle_request(std::uint32_t client_id, Request&& req) { std::string index; std::uint32_t limit = std::numeric_limits::max(); + bool has_limit = false; + std::string make_error; std::shared_ptr
table; switch (r.options().make_table_type_case()) { case proto::MakeTableReq_MakeTableOptions::kMakeLimitTable: { limit = r.options().make_limit_table(); + has_limit = true; break; } case proto::MakeTableReq_MakeTableOptions::kMakeIndexTable: { @@ -2453,31 +2559,19 @@ ProtoServer::_handle_request(std::uint32_t client_id, Request&& req) { break; } + if (r.options().has_view_schema() + && r.data().data_case() != proto::MakeTableData::kFromView) { + proto::Response resp; + *resp.mutable_server_error()->mutable_message() = + "`schema` is only valid when making a Table from a View"; + push_resp(std::move(resp)); + break; + } + switch (r.data().data_case()) { case proto::MakeTableData::kFromView: { - auto view = m_resources.get_view(r.data().from_view()); - proto::ViewPort viewport; - auto dims = parse_format_options( - viewport, - view->num_columns(), - view->num_rows(), - view->sides(), - view->get_view_config()->is_column_only(), - 0 - ); - auto arrow = view->to_arrow( - dims.start_row, - dims.end_row, - dims.start_col, - dims.end_col - ); - - table = Table::from_arrow( - index, - std::move(*arrow), - limit, - backing_store, - list_flatten + make_error = make_derived_table( + r, entity_id, !index.empty() || has_limit, proto_resp, table ); break; } @@ -2567,6 +2661,13 @@ ProtoServer::_handle_request(std::uint32_t client_id, Request&& req) { } } + if (!make_error.empty()) { + proto::Response resp; + *resp.mutable_server_error()->mutable_message() = make_error; + push_resp(std::move(resp)); + break; + } + m_resources.host_table(entity_id, table); proto::Response resp; resp.mutable_make_table_resp(); @@ -2779,6 +2880,14 @@ ProtoServer::_handle_request(std::uint32_t client_id, Request&& req) { push_resp(std::move(resp)); break; } + + if (m_resources.get_table(req.entity_id())->is_derived()) { + proto::Response resp; + *resp.mutable_server_error()->mutable_message() = + "Cannot update a read-only derived table"; + push_resp(std::move(resp)); + break; + } auto table = m_resources.get_table(req.entity_id()); table->clear(); const auto& r = req.table_replace_req(); @@ -2820,6 +2929,14 @@ ProtoServer::_handle_request(std::uint32_t client_id, Request&& req) { push_resp(std::move(resp)); break; } + + if (m_resources.get_table(req.entity_id())->is_derived()) { + proto::Response resp; + *resp.mutable_server_error()->mutable_message() = + "Cannot update a read-only derived table"; + push_resp(std::move(resp)); + break; + } const auto& r = req.table_remove_req(); auto table = m_resources.get_table(req.entity_id()); switch (r.data().data_case()) { @@ -2859,6 +2976,14 @@ ProtoServer::_handle_request(std::uint32_t client_id, Request&& req) { push_resp(std::move(resp)); break; } + + if (m_resources.get_table(req.entity_id())->is_derived()) { + proto::Response resp; + *resp.mutable_server_error()->mutable_message() = + "Cannot update a read-only derived table"; + push_resp(std::move(resp)); + break; + } const auto& r = req.table_update_req(); auto table = m_resources.get_table(req.entity_id()); switch (r.data().data_case()) { @@ -3245,43 +3370,19 @@ ProtoServer::_handle_request(std::uint32_t client_id, Request&& req) { auto view = m_resources.get_view(req.entity_id()); const auto& r = req.view_column_paths_req(); + const auto start_col = r.has_start_col() ? r.start_col() : 0; + const auto end_col = + r.has_end_col() ? r.end_col() : view->num_columns(); proto::Response resp; - auto* view_col_paths = - resp.mutable_view_column_paths_resp()->mutable_paths(); - - // // TODO this is a better representation of column_paths but - // // it is not legacy compat. - // for (const auto& col_paths : view->column_paths()) { - // auto* col_path = view_col_paths->Add(); - // auto* paths = col_path->mutable_path(); - // for (const auto& path : col_paths) { - // *paths->Add() = path; - // } - // } - - std::string col; - const auto column_paths = r.has_start_col() ? r.has_end_col() - ? view->column_paths_range(r.start_col(), r.end_col()) - : view->column_paths_range( - r.start_col(), view->num_columns() - ) - : r.has_end_col() ? view->column_paths_range(0, r.end_col()) - : view->column_paths(); - - for (auto& col_paths : column_paths) { - col = ""; - if (!col_paths.empty()) { - auto it = col_paths.begin(); - col += *it; - - for (++it; it != col_paths.end(); ++it) { - col += "|"; - col += *it; - } - } + auto* area = resp.mutable_view_column_paths_resp()->mutable_area(); - *view_col_paths->Add() = col; + for (const auto& level : view->column_paths(start_col, end_col)) { + auto* values = area->Add()->mutable_values(); + values->Reserve(level.size()); + for (const auto& scalar : level) { + column_path_scalar_to_proto(scalar, values->Add()); + } } push_resp(std::move(resp)); @@ -3422,6 +3523,8 @@ ProtoServer::_handle_request(std::uint32_t client_id, Request&& req) { m_join_engine.unregister_join(req.entity_id()); } + m_derived_engine.unregister_table(req.entity_id()); + const auto is_immediate = req.table_delete_req().is_immediate(); if (is_immediate || m_resources.get_table_view_count(req.entity_id()) == 0) { @@ -3478,6 +3581,14 @@ ProtoServer::_handle_request(std::uint32_t client_id, Request&& req) { break; } case proto::Request::kViewDeleteReq: { + if (m_derived_engine.has_dependents(req.entity_id())) { + proto::Response resp; + *resp.mutable_server_error()->mutable_message() = + "Cannot delete view: it is the source for a derived table"; + push_resp(std::move(resp)); + break; + } + for (const auto& sub : m_resources.get_view_on_delete_sub(req.entity_id())) { proto::Response resp; @@ -3579,9 +3690,6 @@ ProtoServer::_handle_request(std::uint32_t client_id, Request&& req) { proto::Response resp; auto* arrow = resp.mutable_view_to_arrow_resp()->mutable_arrow(); - bool legacy_names = r.viewport().has_emit_legacy_row_path_names() - ? r.viewport().emit_legacy_row_path_names() - : true; *arrow = *view->to_arrow( dims.start_row, dims.end_row, @@ -3589,7 +3697,7 @@ ProtoServer::_handle_request(std::uint32_t client_id, Request&& req) { dims.end_col, true, parse_arrow_compression(r.compression()), - legacy_names + r.viewport().machine_column_names() ); push_resp(std::move(resp)); @@ -3752,13 +3860,13 @@ ProtoServer::_poll() { _process_table_unchecked(table, table_id, resp_envs); } + // Derived children stepped above are dirty too by now; take the full + // set before clearing so the join engine sees every changed source. + auto changed = m_resources.get_dirty_tables(); m_resources.mark_all_tables_clean(); - // Build the set of dirty source table IDs so we can tell the join - // engine which side(s) changed, allowing it to skip rebuilding the - // right-side index when only the left table was updated. tsl::hopscotch_set dirty_ids; - for (auto& [_, table_id] : tables) { + for (auto& [_, table_id] : changed) { dirty_ids.insert(table_id); } @@ -3766,7 +3874,7 @@ ProtoServer::_poll() { // to handle chained joins (join of join) in dependency order. tsl::hopscotch_set processed_joins; std::vector worklist; - for (auto& [_, table_id] : tables) { + for (const auto& table_id : dirty_ids) { auto dependents = m_join_engine.get_dependent_join_tables(table_id); for (auto& join_id : dependents) { if (processed_joins.find(join_id) == processed_joins.end()) { @@ -3816,66 +3924,169 @@ ProtoServer::_poll() { } void -ProtoServer::_process_table_unchecked( +ProtoServer::_notify_table( std::shared_ptr
& table, const ServerResources::t_id& table_id, + t_uindex port_id, std::vector>& outs ) { - table->get_pool()->_process([this, table, table_id, &outs](auto port_id) { - const auto removed = table->get_gnode()->get_removed_pkeys(); - const bool has_removes = - !table->get_index().empty() && removed && removed->size() > 0; - std::shared_ptr removed_indices; - - // record changes per port. - auto view_ids = m_resources.get_view_ids(table_id); - for (const auto& view_id : view_ids) { - if (!m_resources.has_view(view_id)) { - continue; - } - - auto view = m_resources.get_view(view_id); - if (has_removes) { - auto remove_subs = m_resources.get_view_on_remove_sub(view_id); - if (!remove_subs.empty() && !removed_indices) { - removed_indices = apachearrow::column_to_arrow_ipc( - *removed->get_const_column("psp_pkey"), - table->get_index(), - removed->size() - ); - } + const auto removed = table->get_gnode()->get_removed_pkeys(); + const bool has_removes = + !table->get_index().empty() && removed && removed->size() > 0; + std::shared_ptr removed_indices; + + // record changes per port. + auto view_ids = m_resources.get_view_ids(table_id); + for (const auto& view_id : view_ids) { + if (!m_resources.has_view(view_id)) { + continue; + } - for (auto& subscription : remove_subs) { - Response out; - out.set_msg_id(subscription.id); - out.set_entity_id(view_id); - auto* r = out.mutable_view_on_remove_resp(); - r->set_port_id(port_id); - *r->mutable_indices() = *removed_indices; - ProtoServerResp resp2; - resp2.data = std::move(out); - resp2.client_id = subscription.client_id; - outs.emplace_back(std::move(resp2)); - } + auto view = m_resources.get_view(view_id); + if (has_removes) { + auto remove_subs = m_resources.get_view_on_remove_sub(view_id); + if (!remove_subs.empty() && !removed_indices) { + removed_indices = apachearrow::column_to_arrow_ipc( + *removed->get_const_column("psp_pkey"), + table->get_index(), + removed->size() + ); } - auto subscriptions = m_resources.get_view_on_update_sub(view_id); - for (auto& subscription : subscriptions) { + for (auto& subscription : remove_subs) { Response out; out.set_msg_id(subscription.id); out.set_entity_id(view_id); - auto* r = out.mutable_view_on_update_resp(); + auto* r = out.mutable_view_on_remove_resp(); r->set_port_id(port_id); - if (view->get_deltas_enabled()) { - *r->mutable_delta() = *view->get_row_delta_as_arrow(); - } - + *r->mutable_indices() = *removed_indices; ProtoServerResp resp2; resp2.data = std::move(out); resp2.client_id = subscription.client_id; outs.emplace_back(std::move(resp2)); } } + + auto subscriptions = m_resources.get_view_on_update_sub(view_id); + for (auto& subscription : subscriptions) { + Response out; + out.set_msg_id(subscription.id); + out.set_entity_id(view_id); + auto* r = out.mutable_view_on_update_resp(); + r->set_port_id(port_id); + if (view->get_deltas_enabled()) { + *r->mutable_delta() = *view->get_row_delta_as_arrow(); + } + + ProtoServerResp resp2; + resp2.data = std::move(out); + resp2.client_id = subscription.client_id; + outs.emplace_back(std::move(resp2)); + } + } + + for (const auto& view_id : view_ids) { + for (auto& [child_id, source] : + m_derived_engine.get_dependents(view_id)) { + bool notify = !m_resources.get_view_ids(child_id).empty(); + if (source->step(notify)) { + m_resources.mark_table_dirty(child_id); + auto child = source->child(); + _notify_table(child, child_id, 0, outs); + } + } + } +} + +std::string +ProtoServer::make_derived_table( + const proto::MakeTableReq& req, + const ServerResources::t_id& table_id, + bool has_identity_options, + std::vector>& outs, + std::shared_ptr
& table +) { + const auto& view_id = req.data().from_view(); + if (!m_resources.has_view(view_id)) { + return "View \"" + view_id + "\" does not exist"; + } + + if (has_identity_options) { + return "`index` and `limit` cannot be set on a Table made from a View"; + } + + auto parent_id = m_resources.get_table_id_for_view(view_id); + auto parent = m_resources.get_table(parent_id); + auto root_id = _root_table_id(parent_id); + auto root = m_resources.get_table(root_id); + _process_table(root, root_id, outs); + +#ifdef PSP_PARALLEL_FOR + std::lock_guard process_guard( + root->get_pool()->get_process_lock() + ); +#endif + + auto source = m_resources.get_view(view_id)->make_derived_source(parent); + t_schema schema = source->infer_schema(); + if (req.options().has_view_schema()) { + std::vector names; + std::vector types; + for (const auto& it : req.options().view_schema().schema()) { + t_dtype dtype = column_type_to_dtype(it.type()); + if (schema.has_column(it.name())) { + t_dtype inferred = schema.get_dtype(it.name()); + if (dtype_to_column_type(inferred) != it.type()) { + return "Column \"" + it.name() + + "\" does not have the type of the View's column"; + } + + dtype = inferred; + } + + names.push_back(it.name()); + types.push_back(dtype); + } + + schema = t_schema(names, types); + } + + table = Table::make_derived( + schema, source->pkey_dtype(), source->index(), source->limit() + ); + +#ifdef PSP_PARALLEL_FOR + table->get_pool()->adopt_lock(*parent->get_pool()); +#endif + source->attach(table); + m_derived_engine.register_table(view_id, table_id, source); + return ""; +} + +ServerResources::t_id +ProtoServer::_root_table_id(const ServerResources::t_id& table_id) { + ServerResources::t_id id = table_id; + while (auto view_id = m_derived_engine.parent_view_of(id)) { + id = m_resources.get_table_id_for_view(*view_id); + } + + return id; +} + +void +ProtoServer::_process_table_unchecked( + std::shared_ptr
& table, + const ServerResources::t_id& table_id, + std::vector>& outs +) { + auto table_ = table; +#ifdef PSP_PARALLEL_FOR + std::lock_guard process_guard( + table->get_pool()->get_process_lock() + ); +#endif + table->get_pool()->_process([this, &table_, &table_id, &outs](auto port_id) { + _notify_table(table_, table_id, port_id, outs); }); } diff --git a/rust/perspective-server/cpp/perspective/src/cpp/sparse_tree.cpp b/rust/perspective-server/cpp/perspective/src/cpp/sparse_tree.cpp index 7b3691447d..8b98a29cb0 100644 --- a/rust/perspective-server/cpp/perspective/src/cpp/sparse_tree.cpp +++ b/rust/perspective-server/cpp/perspective/src/cpp/sparse_tree.cpp @@ -91,6 +91,28 @@ t_stree::t_stree( m_num_row_pivots_in_tree(pivots.size()) { const auto& g_agg_str = cfg.get_grand_agg_str(); m_grand_agg_str = g_agg_str.empty() ? "Grand Aggregate" : g_agg_str; + + std::set valid_sources; + for (const auto& spec : aggspecs) { + if (!aggtype_wants_valid_count(spec.agg())) { + continue; + } + + const auto& dependencies = spec.get_dependencies(); + if (dependencies.empty()) { + continue; + } + + const std::string& source = dependencies[0].name(); + if (!valid_sources.insert(source).second) { + continue; + } + + std::vector valid_dep = {t_dep(source, DEPTYPE_COLUMN)}; + m_aggspecs.emplace_back( + valid_strand_colname(source), AGGTYPE_VALID_COUNT, valid_dep + ); + } } void @@ -148,7 +170,9 @@ t_stree::init() { for (const auto& ci : cinfo) { columns.push_back(ci.m_name); - dtypes.push_back(ci.m_type); + dtypes.push_back( + spec.agg() == AGGTYPE_VALID_COUNT ? DTYPE_INT32 : ci.m_type + ); } } @@ -188,19 +212,50 @@ t_stree::get_sortby_value(t_index idx) const { return iter->m_sort_value; } +namespace { + +/// The strand delta dtype for a source column, signed for unsigned sources. +t_dtype +strand_value_dtype(t_dtype dtype) { + switch (dtype) { + case DTYPE_UINT64: + case DTYPE_UINT32: + case DTYPE_UINT16: + case DTYPE_UINT8: { + return DTYPE_INT64; + } + default: { + return dtype; + } + } +} + +/// The per-row value a transformed strand column derives from a row image. +t_tscalar +strand_image(t_strand_col_kind kind, const t_tscalar& value) { + if (kind == STRAND_COL_ABS) { + return value.abs(); + } + + t_tscalar rval; + rval.set(std::int8_t(value.is_valid() ? 1 : 0)); + return rval; +} + +} // namespace + void t_stree::build_strand_table_phase_1( t_tscalar pkey, t_op op, t_uindex idx, t_uindex npivots, - t_uindex strand_count_idx, - t_uindex aggcolsize, + const std::vector& agg_kinds, bool force_current_row, const std::vector& piv_ccols, const std::vector& piv_tcols, const std::vector& agg_ccols, - const std::vector& agg_dcols, + const std::vector& agg_pcols, std::vector& piv_scols, std::vector& agg_acols, t_column* agg_scount, @@ -250,15 +305,50 @@ t_stree::build_strand_table_phase_1( // "strand_count_idx" (the number of strands in the table), if the pivot // has changed OR force_current_row is true, then use the aggregate // from `current`, else use the `delta`. - for (t_uindex aggidx = 0; aggidx < aggcolsize; ++aggidx) { - if (aggidx != strand_count_idx) { - if (pivots_neq || force_current_row) { - agg_acols[aggidx]->push_back(agg_ccols[aggidx]->get_scalar(idx) - ); - } else { - agg_acols[aggidx]->push_back(agg_dcols[aggidx]->get_scalar(idx) + for (t_uindex aggidx = 0, agg_loop_end = agg_kinds.size(); + aggidx < agg_loop_end; + ++aggidx) { + const t_strand_col& strand_col = agg_kinds[aggidx]; + switch (strand_col.m_kind) { + case STRAND_COL_COUNT: + break; + case STRAND_COL_ABS: + case STRAND_COL_VALID: { + t_tscalar prev_image = strand_image( + strand_col.m_kind, agg_pcols[aggidx]->get_scalar(idx) ); - } + + if (op == OP_DELETE) { + agg_acols[aggidx]->push_back(prev_image.negate()); + } else { + t_tscalar cur_image = strand_image( + strand_col.m_kind, agg_ccols[aggidx]->get_scalar(idx) + ); + + agg_acols[aggidx]->push_back( + pivots_neq || force_current_row + ? cur_image + : cur_image.difference(prev_image) + ); + } + } break; + case STRAND_COL_VALUE: { + if (op == OP_DELETE) { + agg_acols[aggidx]->push_back( + agg_pcols[aggidx]->get_scalar(idx).negate() + ); + } else if (pivots_neq || force_current_row) { + agg_acols[aggidx]->push_back( + agg_ccols[aggidx]->get_scalar(idx) + ); + } else { + agg_acols[aggidx]->push_back( + agg_ccols[aggidx]->get_scalar(idx).difference( + agg_pcols[aggidx]->get_scalar(idx) + ) + ); + } + } break; } } @@ -286,8 +376,7 @@ t_stree::build_strand_table_phase_2( t_tscalar pkey, t_uindex idx, t_uindex npivots, - t_uindex strand_count_idx, - t_uindex aggcolsize, + const std::vector& agg_kinds, const std::vector& piv_pcols, const std::vector& agg_pcols, std::vector& piv_scols, @@ -310,12 +399,20 @@ t_stree::build_strand_table_phase_2( piv_scols[pidx]->push_back(piv_pcols[pidx]->get_scalar(idx)); } - for (t_uindex aggidx = 0; aggidx < aggcolsize; ++aggidx) { - if (aggidx != strand_count_idx) { - agg_acols[aggidx]->push_back( - agg_pcols[aggidx]->get_scalar(idx).negate() - ); + for (t_uindex aggidx = 0, agg_loop_end = agg_kinds.size(); + aggidx < agg_loop_end; + ++aggidx) { + const t_strand_col& strand_col = agg_kinds[aggidx]; + if (strand_col.m_kind == STRAND_COL_COUNT) { + continue; } + + t_tscalar prev_value = agg_pcols[aggidx]->get_scalar(idx); + agg_acols[aggidx]->push_back( + strand_col.m_kind == STRAND_COL_VALUE + ? prev_value.negate() + : strand_image(strand_col.m_kind, prev_value).negate() + ); } agg_scount->push_back(std::int8_t(-1)); @@ -382,11 +479,43 @@ t_stree::build_strand_table_metadata( for (const auto& aggcol : aggcolset) { metadata.m_aggschema.add_column( - aggcol, metadata.m_flattened_schema.get_dtype(aggcol) + aggcol, + strand_value_dtype(metadata.m_flattened_schema.get_dtype(aggcol)) ); + metadata.m_agg_cols.push_back({STRAND_COL_VALUE, aggcol}); + } + + std::set abs_sources; + std::set valid_sources; + for (const auto& aggspec : aggspecs) { + const auto& dependencies = aggspec.get_dependencies(); + if (dependencies.empty()) { + continue; + } + + const std::string& source = dependencies[0].name(); + if (aggspec.agg() == AGGTYPE_SUM_ABS + && abs_sources.insert(source).second) { + metadata.m_aggschema.add_column( + abs_strand_colname(source), + strand_value_dtype( + metadata.m_flattened_schema.get_dtype(source) + ) + ); + metadata.m_agg_cols.push_back({STRAND_COL_ABS, source}); + } + + if (aggtype_wants_valid_count(aggspec.agg()) + && valid_sources.insert(source).second) { + metadata.m_aggschema.add_column( + valid_strand_colname(source), DTYPE_INT8 + ); + metadata.m_agg_cols.push_back({STRAND_COL_VALID, source}); + } } metadata.m_aggschema.add_column("psp_strand_count", DTYPE_INT8); + metadata.m_agg_cols.push_back({STRAND_COL_COUNT, ""}); return metadata; } @@ -407,7 +536,6 @@ t_stree::build_strand_table_metadata( std::pair, std::shared_ptr> t_stree::build_strand_table( const t_data_table& flattened, - const t_data_table& delta, const t_data_table& prev, const t_data_table& current, const t_data_table& transitions, @@ -457,22 +585,26 @@ t_stree::build_strand_table( t_uindex aggcolsize = metadata.m_aggschema.m_columns.size(); std::vector agg_ccols(aggcolsize); std::vector agg_pcols(aggcolsize); - std::vector agg_dcols(aggcolsize); std::vector agg_acols(aggcolsize); - t_uindex strand_count_idx = 0; - for (t_uindex aggidx = 0; aggidx < aggcolsize; ++aggidx) { const std::string& aggcol = metadata.m_aggschema.m_columns[aggidx]; - if (aggcol == "psp_strand_count") { - agg_dcols[aggidx] = nullptr; - agg_ccols[aggidx] = nullptr; - agg_pcols[aggidx] = nullptr; - strand_count_idx = aggidx; - } else { - agg_dcols[aggidx] = delta._get_const_column(aggcol); - agg_ccols[aggidx] = current._get_const_column(aggcol); - agg_pcols[aggidx] = prev._get_const_column(aggcol); + const t_strand_col& strand_col = metadata.m_agg_cols[aggidx]; + switch (strand_col.m_kind) { + case STRAND_COL_COUNT: { + agg_ccols[aggidx] = nullptr; + agg_pcols[aggidx] = nullptr; + } break; + case STRAND_COL_ABS: + case STRAND_COL_VALID: { + agg_ccols[aggidx] = + current._get_const_column(strand_col.m_source); + agg_pcols[aggidx] = prev._get_const_column(strand_col.m_source); + } break; + case STRAND_COL_VALUE: { + agg_ccols[aggidx] = current._get_const_column(aggcol); + agg_pcols[aggidx] = prev._get_const_column(aggcol); + } break; } agg_acols[aggidx] = aggs->_get_column(aggcol); @@ -515,13 +647,12 @@ t_stree::build_strand_table( op, idx, metadata.m_pivsize, - strand_count_idx, - aggcolsize, + metadata.m_agg_cols, true, piv_ccols, piv_tcols, agg_ccols, - agg_dcols, + agg_pcols, piv_scols, agg_acols, agg_scount, @@ -536,8 +667,7 @@ t_stree::build_strand_table( pkey, idx, metadata.m_pivsize, - strand_count_idx, - aggcolsize, + metadata.m_agg_cols, piv_pcols, agg_pcols, piv_scols, @@ -554,13 +684,12 @@ t_stree::build_strand_table( op, idx, metadata.m_pivsize, - strand_count_idx, - aggcolsize, + metadata.m_agg_cols, false, piv_ccols, piv_tcols, agg_ccols, - agg_dcols, + agg_pcols, piv_scols, agg_acols, agg_scount, @@ -578,8 +707,7 @@ t_stree::build_strand_table( pkey, idx, metadata.m_pivsize, - strand_count_idx, - aggcolsize, + metadata.m_agg_cols, piv_pcols, agg_pcols, piv_scols, @@ -608,13 +736,12 @@ t_stree::build_strand_table( op, idx, metadata.m_pivsize, - strand_count_idx, - aggcolsize, + metadata.m_agg_cols, false, piv_ccols, piv_tcols, agg_ccols, - agg_dcols, + agg_pcols, piv_scols, agg_acols, agg_scount, @@ -634,8 +761,7 @@ t_stree::build_strand_table( pkey, idx, metadata.m_pivsize, - strand_count_idx, - aggcolsize, + metadata.m_agg_cols, piv_pcols, agg_pcols, piv_scols, @@ -712,11 +838,20 @@ t_stree::build_strand_table( t_uindex strand_count_idx = 0; for (t_uindex aggidx = 0; aggidx < aggcolsize; ++aggidx) { const std::string& aggcol = metadata.m_aggschema.m_columns[aggidx]; - if (aggcol == "psp_strand_count") { - agg_fcols[aggidx] = nullptr; - strand_count_idx = aggidx; - } else { - agg_fcols[aggidx] = flattened._get_const_column(aggcol); + const t_strand_col& strand_col = metadata.m_agg_cols[aggidx]; + switch (strand_col.m_kind) { + case STRAND_COL_COUNT: { + agg_fcols[aggidx] = nullptr; + strand_count_idx = aggidx; + } break; + case STRAND_COL_ABS: + case STRAND_COL_VALID: { + agg_fcols[aggidx] = + flattened._get_const_column(strand_col.m_source); + } break; + case STRAND_COL_VALUE: { + agg_fcols[aggidx] = flattened._get_const_column(aggcol); + } break; } agg_acols[aggidx] = aggs->_get_column(aggcol); @@ -771,8 +906,13 @@ t_stree::build_strand_table( spkey->push_back(pkey); ++insert_count; } else if (aggidx - 1 != strand_count_idx) { + t_tscalar value = agg_fcols[aggidx - 1]->get_scalar(idx); + const t_strand_col& strand_col = + metadata.m_agg_cols[aggidx - 1]; agg_acols[aggidx - 1]->push_back( - agg_fcols[aggidx - 1]->get_scalar(idx) + strand_col.m_kind == STRAND_COL_VALUE + ? value + : strand_image(strand_col.m_kind, value) ); } } @@ -912,6 +1052,9 @@ t_stree::update_shape_from_static(const t_dtree_ctx& ctx) { ); m_newids.insert(sptidx); + for (const auto& capture : m_captures) { + capture->m_created.push_back(sptidx); + } if (ndepth == dtree.last_level()) { m_newleaves.insert(sptidx); @@ -982,34 +1125,70 @@ t_stree::mark_zero_desc() { } void -t_stree::update_aggs_from_static( +t_stree::_build_agg_info( const t_dtree_ctx& ctx, const t_gstate& gstate, const t_data_table& expression_master_table ) { - const t_data_table& src_aggtable = ctx.get_aggtable(); - - t_agg_update_info agg_update_info; - t_schema aggschema = m_aggregates->get_schema(); + t_agg_update_info& info = m_agg_info; + info = t_agg_update_info(); + const t_schema& aggschema = m_aggregates->get_schema(); + const t_schema& expression_schema = expression_master_table.get_schema(); + const t_schema& master_schema = gstate.get_table()->get_schema(); + info.m_src.assign(aggschema.m_columns.size(), nullptr); for (const auto& colname : aggschema.m_columns) { - agg_update_info.m_src.push_back( - src_aggtable._get_const_column(colname) - ); - agg_update_info.m_dst.push_back(m_aggregates->_get_column(colname) - ); - agg_update_info.m_aggspecs.push_back(ctx.get_aggspec(colname)); + info.m_dst.push_back(m_aggregates->_get_column(colname)); + info.m_aggspecs.push_back(ctx.get_aggspec(colname)); + + const auto& dependencies = info.m_aggspecs.back().get_dependencies(); + bool numeric_source = true; + if (!dependencies.empty()) { + const std::string& dep = dependencies[0].name(); + if (expression_schema.has_column(dep)) { + numeric_source = + is_numeric_type(expression_schema.get_dtype(dep)); + } else if (master_schema.has_column(dep)) { + numeric_source = + is_numeric_type(master_schema.get_dtype(dep)); + } + } + + info.m_numeric_source.push_back(numeric_source); + } + + std::map valid_positions; + for (t_uindex idx = 0; idx < info.m_aggspecs.size(); ++idx) { + const t_aggspec& spec = info.m_aggspecs[idx]; + if (spec.agg() == AGGTYPE_VALID_COUNT) { + valid_positions[spec.name()] = idx; + } + } + + for (const auto& spec : info.m_aggspecs) { + t_uindex valid_idx = static_cast(-1); + if (aggtype_wants_valid_count(spec.agg()) + && !spec.get_dependencies().empty()) { + auto iter = valid_positions.find( + valid_strand_colname(spec.get_dependencies()[0].name()) + ); + if (iter != valid_positions.end()) { + valid_idx = iter->second; + } + } + + info.m_valid_idx.push_back(valid_idx); } auto is_col_scaled_aggregate = [&](int col_idx) -> bool { - int agg_type = agg_update_info.m_aggspecs[col_idx].agg(); + int agg_type = info.m_aggspecs[col_idx].agg(); return agg_type == AGGTYPE_SCALED_DIV || agg_type == AGGTYPE_SCALED_ADD || agg_type == AGGTYPE_SCALED_MUL; }; size_t col_cnt = aggschema.m_columns.size(); - auto& cols_topo_sorted = agg_update_info.m_dst_topo_sorted; + auto& cols_topo_sorted = info.m_dst_topo_sorted; cols_topo_sorted.clear(); cols_topo_sorted.reserve(col_cnt); @@ -1019,7 +1198,7 @@ t_stree::update_aggs_from_static( tsl::hopscotch_set dst_visited; auto push_column = [&](size_t idx) { if (enable_fix_double_calculation) { - t_column* dst = agg_update_info.m_dst[idx]; + t_column* dst = info.m_dst[idx]; if (dst_visited.find(dst) != dst_visited.end()) { return; } @@ -1032,6 +1211,11 @@ t_stree::update_aggs_from_static( // Move scaled agg columns to the end // This does not handle case where scaled aggregate depends on other // scaled aggregate ( not sure if that is possible ) + for (size_t i = 0; i < col_cnt; ++i) { + if (info.m_aggspecs[i].agg() == AGGTYPE_VALID_COUNT) { + push_column(i); + } + } for (size_t i = 0; i < col_cnt; ++i) { if (!is_col_scaled_aggregate(i)) { push_column(i); @@ -1048,6 +1232,25 @@ t_stree::update_aggs_from_static( push_column(i); } } +} + +void +t_stree::update_aggs_from_static( + const t_dtree_ctx& ctx, + const t_gstate& gstate, + const t_data_table& expression_master_table +) { + if (m_agg_info_table != m_aggregates.get()) { + _build_agg_info(ctx, gstate, expression_master_table); + m_agg_info_table = m_aggregates.get(); + } + + const t_data_table& src_aggtable = ctx.get_aggtable(); + const t_schema& aggschema = m_aggregates->get_schema(); + for (t_uindex idx = 0; idx < aggschema.m_columns.size(); ++idx) { + m_agg_info.m_src[idx] = + src_aggtable._get_const_column(aggschema.m_columns[idx]); + } for (const auto& r : m_tree_unification_records) { if (!node_exists(r.m_sptidx)) { @@ -1056,7 +1259,7 @@ t_stree::update_aggs_from_static( update_agg_table( r.m_sptidx, - agg_update_info, + m_agg_info, r.m_daggidx, r.m_saggidx, r.m_nstrands, @@ -1137,7 +1340,7 @@ t_stree::update_agg_table( const t_gstate& gstate, const t_data_table& expression_master_table ) { - const t_schema& expression_schema = expression_master_table.get_schema(); + bool is_new = m_newids.find(nidx) != m_newids.end(); for (t_uindex idx : info.m_dst_topo_sorted) { const t_column* src = info.m_src[idx]; @@ -1145,19 +1348,16 @@ t_stree::update_agg_table( const t_aggspec& spec = info.m_aggspecs[idx]; t_tscalar new_value = mknone(); t_tscalar old_value = mknone(); - auto is_expr = - expression_schema.has_column(spec.get_dependencies()[0].name()); switch (spec.agg()) { - case AGGTYPE_PCT_SUM_PARENT: - case AGGTYPE_PCT_SUM_GRAND_TOTAL: - case AGGTYPE_SUM: { + case AGGTYPE_SUM_OR_ZERO: { t_tscalar src_scalar = src->get_scalar(src_ridx); t_tscalar dst_scalar = dst->get_scalar(dst_ridx); old_value.set(dst_scalar); // is_nan returns false for non-float types - if (is_expr || old_value.is_nan()) { + if (t_env::force_reaggregate() || !info.m_numeric_source[idx] + || old_value.is_nan()) { // if we previously had a NaN, add can't make it finite // again; recalculate entire sum in case it is now finite @@ -1195,6 +1395,16 @@ t_stree::update_agg_table( new_value.set(dst_scalar.add(src_scalar)); } + t_uindex valid_idx = info.m_valid_idx[idx]; + if (valid_idx != static_cast(-1)) { + t_tscalar count = + info.m_dst[valid_idx]->get_scalar(dst_ridx); + if (!count.is_valid() || count.to_double() == 0) { + new_value.set(std::uint64_t(0)); + new_value.m_type = dst->get_dtype(); + } + } + dst->set_scalar(dst_ridx, new_value); } break; case AGGTYPE_COUNT: { @@ -1763,94 +1973,84 @@ t_stree::update_agg_table( case AGGTYPE_UDF_REDUCER: { // these will be filled in later } break; - case AGGTYPE_SUM_NOT_NULL: { - old_value.set(dst->get_scalar(dst_ridx)); - auto pkeys = get_pkeys(nidx); - - new_value.set( - reduce_from_gstate< - std::function&)>>( - gstate, - expression_master_table, - spec.get_dependencies()[0].name(), - pkeys, - [](std::vector& values) { - if (values.empty()) { - return mknone(); - } - - t_tscalar rval; - rval.set(std::uint64_t(0)); - rval.m_type = values[0].m_type; + case AGGTYPE_VALID_COUNT: { + t_tscalar src_scalar = src->get_scalar(src_ridx); + std::int64_t count = 0; + if (!is_new) { + t_tscalar dst_scalar = dst->get_scalar(dst_ridx); + if (dst_scalar.is_valid()) { + count = + static_cast(dst_scalar.to_double()); + } + } - for (const auto& v : values) { - if (v.is_nan()) { - continue; - } - rval = rval.add(v); - } + if (src_scalar.is_valid()) { + count += static_cast(src_scalar.to_double()); + } - return rval; - } - ) - ); + new_value.set(static_cast(count)); dst->set_scalar(dst_ridx, new_value); } break; - case AGGTYPE_SUM_ABS: { - old_value.set(dst->get_scalar(dst_ridx)); - auto pkeys = get_pkeys(nidx); + case AGGTYPE_PCT_SUM_PARENT: + case AGGTYPE_PCT_SUM_GRAND_TOTAL: + case AGGTYPE_ABS_SUM: + case AGGTYPE_SUM_ABS: + case AGGTYPE_SUM: { + t_tscalar src_scalar = src->get_scalar(src_ridx); + t_tscalar dst_scalar = dst->get_scalar(dst_ridx); + old_value.set(dst_scalar); - new_value.set( - reduce_from_gstate< - std::function&)>>( - gstate, - expression_master_table, - spec.get_dependencies()[0].name(), - pkeys, - [](std::vector& values) { - if (values.empty()) { - return mknone(); - } + bool abs_values = spec.agg() == AGGTYPE_SUM_ABS; - t_tscalar rval; - rval.set(std::uint64_t(0)); - rval.m_type = values[0].m_type; - for (const auto& v : values) { - rval = rval.add(v.abs()); - } - return rval; - } - ) - ); + // is_nan returns false for non-float types + if (t_env::force_reaggregate() || !info.m_numeric_source[idx] + || old_value.is_nan()) { + auto pkeys = get_pkeys(nidx); + new_value.set( + reduce_from_gstate< + std::function&)>>( + gstate, + expression_master_table, + spec.get_dependencies()[0].name(), + pkeys, + [abs_values](std::vector& values) { + t_tscalar rval; + rval.clear(); + for (const auto& v : values) { + if (v.is_nan()) { + continue; + } - dst->set_scalar(dst_ridx, new_value); - } break; - case AGGTYPE_ABS_SUM: { - old_value.set(dst->get_scalar(dst_ridx)); - auto pkeys = get_pkeys(nidx); - new_value.set( - reduce_from_gstate< - std::function&)>>( - gstate, - expression_master_table, - spec.get_dependencies()[0].name(), - pkeys, - [](std::vector& values) { - if (values.empty()) { - return mknone(); - } - t_tscalar rval; - rval.set(std::uint64_t(0)); - rval.m_type = values[0].m_type; - for (const auto& v : values) { - rval = rval.add(v); + rval = rval.add(abs_values ? v.abs() : v); + } + + return rval; } - return rval.abs(); + ) + ); + } else { + new_value.set(dst_scalar.add(src_scalar)); + } + + t_uindex valid_idx = info.m_valid_idx[idx]; + if (valid_idx != static_cast(-1)) { + t_tscalar count = + info.m_dst[valid_idx]->get_scalar(dst_ridx); + if (!count.is_valid() || count.to_double() == 0) { + if (t_env::sum_empty_zero()) { + new_value.set(std::uint64_t(0)); + new_value.m_type = dst->get_dtype(); + } else { + new_value.clear(); + new_value.m_type = dst->get_dtype(); } - ) - ); + } + } + dst->set_scalar(dst_ridx, new_value); } break; + + case AGGTYPE_GMV: { // The user-facing rule is "leaf = sum, parent = sum over // immediate row children of |raw sum of child subtree|". @@ -2109,12 +2309,23 @@ t_stree::update_agg_table( } } // end switch + if (spec.agg() == AGGTYPE_VALID_COUNT) { + continue; + } + bool val_neq = old_value != new_value; - m_has_delta = m_has_delta || val_neq; + m_has_delta = m_has_delta || val_neq || is_new; bool deltas_enabled = m_features.at(CTX_FEAT_DELTA); - if (deltas_enabled && val_neq) { - m_deltas->insert(t_tcdelta(nidx, idx, old_value, new_value)); + if (deltas_enabled && (val_neq || is_new)) { + m_deltas->insert(t_tcdelta( + nidx, idx, is_new ? mknone() : old_value, new_value + )); + } + + if (!m_captures.empty() + && (val_neq || old_value.is_valid() != dst->is_valid(dst_ridx))) { + capture_change(nidx, idx, old_value); } } // end for @@ -2280,6 +2491,9 @@ t_stree::drop_zero_strands() { leaves.push_back(iter->m_idx); } node_ids.push_back(iter->m_aggidx); + if (!m_captures.empty()) { + capture_dropped(*iter); + } } clear_aggregates(node_ids); @@ -2657,6 +2871,73 @@ t_stree::set_deltas_enabled(bool enabled_state) { m_features[CTX_FEAT_DELTA] = enabled_state; } +void +t_stree::add_capture(const std::shared_ptr& capture) { + m_captures.push_back(capture); +} + +void +t_stree::remove_capture(const std::shared_ptr& capture) { + m_captures.erase( + std::remove(m_captures.begin(), m_captures.end(), capture), + m_captures.end() + ); +} + +bool +t_stree::node_exists(t_uindex idx) const { + return m_nodes->get().find(idx) != m_nodes->get().end(); +} + +const t_stnode* +t_stree::find_node(t_uindex idx) const { + const auto& nodes = m_nodes->get(); + auto iter = nodes.find(idx); + return iter == nodes.end() ? nullptr : &*iter; +} + +void +t_stree::for_each_node(const std::function& fn) const { + for (const auto& node : m_nodes->get()) { + fn(node); + } +} + +void +t_stree::capture_change( + t_uindex nidx, t_uindex aggnum, const t_tscalar& old_value +) { + for (const auto& capture : m_captures) { + auto& cells = capture->m_changed[nidx]; + bool seen = false; + for (const auto& cell : cells) { + seen = seen || cell.first == aggnum; + } + + if (!seen) { + cells.emplace_back(aggnum, old_value); + } + } +} + +void +t_stree::capture_dropped(const t_stnode& node) { + t_stree_dropped dropped; + dropped.m_idx = node.m_idx; + dropped.m_aggidx = node.m_aggidx; + dropped.m_depth = node.m_depth; + get_path(node.m_idx, dropped.m_path); + std::reverse(dropped.m_path.begin(), dropped.m_path.end()); + dropped.m_aggregates.reserve(m_aggcols.size()); + for (t_uindex aggnum = 0; aggnum < m_aggcols.size(); ++aggnum) { + dropped.m_aggregates.push_back(get_aggregate(node.m_idx, aggnum)); + } + + for (const auto& capture : m_captures) { + capture->m_dropped.push_back(dropped); + } +} + void t_stree::set_feature_state(t_ctx_feature feature, bool state) { m_features[feature] = state; diff --git a/rust/perspective-server/cpp/perspective/src/cpp/table.cpp b/rust/perspective-server/cpp/perspective/src/cpp/table.cpp index f30f121ba3..67a7b1401f 100644 --- a/rust/perspective-server/cpp/perspective/src/cpp/table.cpp +++ b/rust/perspective-server/cpp/perspective/src/cpp/table.cpp @@ -818,6 +818,39 @@ Table::from_schema( return tbl; } +std::shared_ptr
+Table::make_derived( + const t_schema& schema, + t_dtype pkey_dtype, + const std::string& index, + std::uint32_t limit +) { + auto pool = std::make_shared(); + pool->init(); + + t_data_table data_table(schema); + data_table.init(); + data_table.add_column("psp_pkey", pkey_dtype, true); + data_table.add_column("psp_okey", pkey_dtype, true); + data_table.add_column("psp_op", DTYPE_UINT8, false); + + auto tbl = std::make_shared
( + pool, schema.columns(), schema.types(), limit, index + ); + + tbl->m_derived = true; + tbl->set_gnode(tbl->make_gnode(data_table.get_schema())); + pool->register_gnode(tbl->m_gnode.get()); + tbl->m_gnode->remove_input_port(0); + tbl->m_init = true; + return tbl; +} + +bool +Table::is_derived() const { + return m_derived; +} + void Table::update_arrow(const std::string_view& data, std::uint32_t port_id) { apachearrow::ArrowLoader arrow_loader; diff --git a/rust/perspective-server/cpp/perspective/src/cpp/tree_context_common.cpp b/rust/perspective-server/cpp/perspective/src/cpp/tree_context_common.cpp index 488f92ab21..91f7a19744 100644 --- a/rust/perspective-server/cpp/perspective/src/cpp/tree_context_common.cpp +++ b/rust/perspective-server/cpp/perspective/src/cpp/tree_context_common.cpp @@ -171,7 +171,6 @@ notify_sparse_tree( const std::vector>& tree_sortby, const std::vector& ctx_sortby, const t_data_table& flattened, - const t_data_table& delta, const t_data_table& prev, const t_data_table& current, const t_data_table& transitions, @@ -182,7 +181,7 @@ notify_sparse_tree( ) { auto strand_values = tree->build_strand_table( - flattened, delta, prev, current, transitions, aggregates, config + flattened, prev, current, transitions, aggregates, config ); auto strands = strand_values.first; diff --git a/rust/perspective-server/cpp/perspective/src/cpp/update_task.cpp b/rust/perspective-server/cpp/perspective/src/cpp/update_task.cpp index 35b49648e7..16678360cc 100644 --- a/rust/perspective-server/cpp/perspective/src/cpp/update_task.cpp +++ b/rust/perspective-server/cpp/perspective/src/cpp/update_task.cpp @@ -19,8 +19,7 @@ t_update_task::t_update_task(t_pool& pool) : m_pool(pool) {} void t_update_task::run(std::optional> callback) { - auto work_to_do = m_pool.m_data_remaining.load(); - m_pool.m_data_remaining.store(false); + auto work_to_do = m_pool.m_data_remaining.exchange(false); if (work_to_do) { for (auto* g : m_pool.m_gnodes) { diff --git a/rust/perspective-server/cpp/perspective/src/cpp/view.cpp b/rust/perspective-server/cpp/perspective/src/cpp/view.cpp index 24c20f4b94..1c5eefc8f5 100644 --- a/rust/perspective-server/cpp/perspective/src/cpp/view.cpp +++ b/rust/perspective-server/cpp/perspective/src/cpp/view.cpp @@ -373,31 +373,7 @@ View::column_names_range( template std::vector> -View::column_paths() const { - auto num_column_pivots = m_column_pivots.size(); - auto names = column_names(true, num_column_pivots); - if (!m_hidden_sort.empty()) { - // make a new vector so we don't have to erase while iterating - std::vector> visible_column_paths; - - for (const auto& column : names) { - // Remove undisplayed column names used to sort - std::string name = column.back().to_string(); - if (std::find(m_hidden_sort.begin(), m_hidden_sort.end(), name) - == m_hidden_sort.end()) { - visible_column_paths.push_back(column); - } - } - - return visible_column_paths; - } - - return names; -} - -template -std::vector> -View::column_paths_range(t_uindex start_col, t_uindex end_col) const { +View::column_paths(t_uindex start_col, t_uindex end_col) const { auto num_column_pivots = m_column_pivots.size(); auto names = column_names_range(true, num_column_pivots, start_col, end_col); @@ -414,29 +390,28 @@ View::column_paths_range(t_uindex start_col, t_uindex end_col) const { } } - return visible_column_paths; + names = std::move(visible_column_paths); } - return names; -} + const auto depth = num_column_pivots + 1; + std::vector> area( + depth, std::vector(names.size(), mknone()) + ); -template -std::vector> -View::column_paths_string() const { - auto paths = column_paths(); - std::vector> out; - out.reserve(paths.size()); - for (const auto& path : paths) { - std::vector row; - row.reserve(path.size()); - for (const auto& c : path) { - row.push_back(c.to_string()); + for (t_uindex col = 0; col < names.size(); ++col) { + const auto& path = names[col]; + if (path.empty()) { + continue; + } + + for (t_uindex level = 0; level + 1 < path.size(); ++level) { + area[level][col] = path[level]; } - out.push_back(row); + area[depth - 1][col] = path.back(); } - return out; + return area; } static std::string @@ -798,7 +773,7 @@ View::to_arrow( std::int32_t end_col, bool emit_group_by, t_arrow_compression compression, - bool emit_legacy_row_path_names + bool machine_column_names ) const { PSP_GIL_UNLOCK(); PSP_READ_LOCK(*get_lock()); @@ -806,7 +781,7 @@ View::to_arrow( std::shared_ptr> data_slice = get_data(start_row, end_row, start_col, end_col); return data_slice_to_arrow( - data_slice, emit_group_by, compression, emit_legacy_row_path_names + data_slice, emit_group_by, compression, machine_column_names ); }; @@ -872,7 +847,7 @@ template std::pair, std::shared_ptr> View::data_slice_to_batches( bool emit_group_by, std::shared_ptr> data_slice, - bool emit_legacy_row_path_names + bool machine_column_names ) const { // From the data slice, get all the metadata we need t_get_data_extents extents = data_slice->get_data_extents(); @@ -902,13 +877,8 @@ View::data_slice_to_batches( auto schema = m_table->get_schema(); for (auto rpidx = 0; rpidx < num_row_paths; ++rpidx) { std::string column_name = row_pivots.at(rpidx); - std::string row_path_name; - if (emit_legacy_row_path_names) { - row_path_name = column_name; - row_path_name += " (Group by "; - row_path_name += std::to_string(rpidx + 1); - row_path_name += ")"; - } else { + std::string row_path_name = column_name; + if (machine_column_names) { row_path_name = "__ROW_PATH_"; row_path_name += std::to_string(rpidx); row_path_name += "__"; @@ -1207,6 +1177,22 @@ View::data_slice_to_batches( const std::vector& slice_col_indices = data_slice->get_column_indices(); + const auto aggspecs = m_view_config->get_aggspecs(); + auto leaf_name = [&](const t_tscalar& leaf) -> std::string { + std::string name = leaf.to_string(); + if (machine_column_names || !emit_group_by) { + return name; + } + + for (const auto& spec : aggspecs) { + if (spec.name() == name) { + return m_view_config->readable_aggregate_name(spec); + } + } + + return name; + }; + parallel_for(int(indices.size()), [&](auto iidx) { // for (auto iidx = 0; iidx < indices.size(); iidx++) { auto ccidx = iidx + num_output_row_paths; @@ -1229,10 +1215,12 @@ View::data_slice_to_batches( std::string name; - if (num_sides > 1) { - name = join_column_names(col_path, m_separator); + if (num_sides > 1 && col_path.size() > 1) { + std::vector prefix(col_path.begin(), col_path.end() - 1); + name = join_column_names(prefix, m_separator) + m_separator + + leaf_name(col_path.back()); } else { - name = col_path.at(col_path.size() - 1).to_string(); + name = leaf_name(col_path.back()); } std::shared_ptr arr; @@ -1420,12 +1408,12 @@ View::data_slice_to_arrow( std::shared_ptr> data_slice, bool emit_group_by, t_arrow_compression compression, - bool emit_legacy_row_path_names + bool machine_column_names ) const { std::pair< std::shared_ptr, std::shared_ptr> - pairs = data_slice_to_batches(emit_group_by, data_slice, emit_legacy_row_path_names); + pairs = data_slice_to_batches(emit_group_by, data_slice, machine_column_names); std::shared_ptr batches = pairs.second; std::shared_ptr arrow_schema = pairs.first; arrow::Result> allocated = diff --git a/rust/perspective-server/cpp/perspective/src/cpp/view_config.cpp b/rust/perspective-server/cpp/perspective/src/cpp/view_config.cpp index a5508ade70..70495e09e3 100644 --- a/rust/perspective-server/cpp/perspective/src/cpp/view_config.cpp +++ b/rust/perspective-server/cpp/perspective/src/cpp/view_config.cpp @@ -136,6 +136,7 @@ t_view_config::validate(const std::shared_ptr& schema) { } } + std::unordered_set row_pivots; for (const std::string& col : m_row_pivots) { if (!schema->has_column(col) && expression_aliases.count(col) == 0) { std::stringstream ss; @@ -143,6 +144,13 @@ t_view_config::validate(const std::shared_ptr& schema) { << '\n'; PSP_COMPLAIN_AND_ABORT(ss.str()); } + + if (!row_pivots.insert(col).second) { + std::stringstream ss; + ss << "Duplicate column '" << col << "' found in View group_by." + << '\n'; + PSP_COMPLAIN_AND_ABORT(ss.str()); + } } for (const std::string& col : m_column_pivots) { @@ -277,6 +285,34 @@ t_view_config::get_column_pivots() const { return m_column_pivots; } +std::string +t_view_config::readable_aggregate_name(const t_aggspec& aggspec) const { + const std::string& name = aggspec.name(); + bool is_key = !m_column_only + && std::find(m_row_pivots.begin(), m_row_pivots.end(), name) + != m_row_pivots.end(); + if (!is_key) { + return name; + } + + const auto& spelled = m_aggregates.find(name); + if (spelled != m_aggregates.end() && !spelled->second.empty()) { + std::string qualified = name + " (" + spelled->second.at(0); + if (spelled->second.size() > 1) { + qualified += " by " + spelled->second.at(1); + } + + return qualified + ")"; + } + + std::string qualified = name + " (" + aggspec.agg_str(); + if (aggtype_takes_argument(aggspec.agg())) { + qualified += " by " + aggspec.get_dependencies().at(1).name(); + } + + return qualified + ")"; +} + std::vector t_view_config::get_aggspecs() const { PSP_VERBOSE_ASSERT(m_init, "touching uninited object"); diff --git a/rust/perspective-server/cpp/perspective/src/cpp/window.cpp b/rust/perspective-server/cpp/perspective/src/cpp/window.cpp index ef6b11110b..80ff5ccdb3 100644 --- a/rust/perspective-server/cpp/perspective/src/cpp/window.cpp +++ b/rust/perspective-server/cpp/perspective/src/cpp/window.cpp @@ -1430,7 +1430,6 @@ t_window_engine::compute_update( const std::shared_ptr& expr_flattened, const std::shared_ptr& expr_prev, const std::shared_ptr& expr_current, - const std::shared_ptr& expr_delta, const std::shared_ptr& flattened, const std::shared_ptr& existed ) { @@ -1529,13 +1528,7 @@ t_window_engine::compute_update( } fill_transitional( - pkey_map, - expr_master, - expr_flattened, - expr_prev, - expr_current, - expr_delta, - flattened + pkey_map, expr_master, expr_flattened, expr_current, flattened ); } @@ -1544,9 +1537,7 @@ t_window_engine::fill_transitional( const t_gstate::t_mapping& pkey_map, const std::shared_ptr& expr_master, const std::shared_ptr& expr_flattened, - const std::shared_ptr& expr_prev, const std::shared_ptr& expr_current, - const std::shared_ptr& expr_delta, const std::shared_ptr& flattened ) const { t_uindex num_rows = flattened->size(); @@ -1555,35 +1546,23 @@ t_window_engine::fill_transitional( // Rows-outer so the pkey hash lookup happens ONCE per row, shared by // every spec. struct t_fill_cols { - t_dtype m_dtype; const t_column* m_new; - const t_column* m_prev; t_column* m_flattened; t_column* m_current; - t_column* m_delta; }; std::vector fill_cols; fill_cols.reserve(m_states.size()); for (const auto& state : m_states) { const auto& spec = state.m_spec; auto new_col = expr_master->get_column(spec.m_name); - auto prev_col = expr_prev->get_column(spec.m_name); auto flattened_col = expr_flattened->add_column_sptr(spec.m_name, spec.m_dtype, true); auto current_col = expr_current->add_column_sptr(spec.m_name, spec.m_dtype, true); - auto delta_col = - expr_delta->add_column_sptr(spec.m_name, spec.m_dtype, true); flattened_col->reserve(num_rows); current_col->reserve(num_rows); - delta_col->reserve(num_rows); fill_cols.push_back( - {spec.m_dtype, - new_col.get(), - prev_col.get(), - flattened_col.get(), - current_col.get(), - delta_col.get()} + {new_col.get(), flattened_col.get(), current_col.get()} ); } @@ -1595,32 +1574,12 @@ t_window_engine::fill_transitional( if (!found || !cols.m_new->is_valid(it->second)) { cols.m_flattened->clear(ridx); cols.m_current->clear(ridx); - cols.m_delta->clear(ridx); continue; } t_tscalar value = cols.m_new->get_scalar(it->second); cols.m_flattened->set_scalar(ridx, value); cols.m_current->set_scalar(ridx, value); - - bool prev_valid = cols.m_prev->is_valid(ridx); - if (cols.m_dtype == DTYPE_FLOAT64 && prev_valid) { - write_float( - *cols.m_delta, - ridx, - value.to_double() - - cols.m_prev->get_scalar(ridx).to_double() - ); - } else if (cols.m_dtype == DTYPE_INT64 && prev_valid) { - write_int( - *cols.m_delta, - ridx, - value.get() - - cols.m_prev->get_scalar(ridx).get() - ); - } else { - cols.m_delta->set_scalar(ridx, value); - } } } } diff --git a/rust/perspective-server/cpp/perspective/src/include/perspective/base.h b/rust/perspective-server/cpp/perspective/src/include/perspective/base.h index 9b0d15f586..4c47fcb60f 100644 --- a/rust/perspective-server/cpp/perspective/src/include/perspective/base.h +++ b/rust/perspective-server/cpp/perspective/src/include/perspective/base.h @@ -291,7 +291,8 @@ enum t_aggtype { AGGTYPE_UDF_REDUCER, AGGTYPE_SUM_ABS, AGGTYPE_ABS_SUM, - AGGTYPE_SUM_NOT_NULL, + AGGTYPE_SUM_OR_ZERO, + AGGTYPE_VALID_COUNT, AGGTYPE_MEAN_BY_COUNT, AGGTYPE_IDENTITY, AGGTYPE_DISTINCT_COUNT, @@ -307,6 +308,39 @@ PERSPECTIVE_EXPORT std::optional maybe_str_to_aggtype(const std::string& str); PERSPECTIVE_EXPORT t_aggtype str_to_aggtype(const std::string& str); + +/// The reserved strand-table column carrying `|current| - |prev|` deltas +/// for a `sum abs` over `col`. +inline std::string +abs_strand_colname(const std::string& col) { + return "psp_abs_" + col; +} + +/// The reserved strand-table column carrying validity deltas in +/// {-1, 0, +1} for `col`. +inline std::string +valid_strand_colname(const std::string& col) { + return "psp_valid_" + col; +} + +/// Whether an aggregate's empty-group classification is driven by its +/// source's valid-value count. +inline bool +aggtype_wants_valid_count(t_aggtype agg) { + switch (agg) { + case AGGTYPE_SUM: + case AGGTYPE_SUM_OR_ZERO: + case AGGTYPE_SUM_ABS: + case AGGTYPE_ABS_SUM: + case AGGTYPE_PCT_SUM_PARENT: + case AGGTYPE_PCT_SUM_GRAND_TOTAL: { + return true; + } + default: { + return false; + } + } +} PERSPECTIVE_EXPORT bool is_implemented_aggtype(t_aggtype agg); PERSPECTIVE_EXPORT bool aggtype_takes_argument(t_aggtype agg); @@ -355,7 +389,6 @@ enum t_gnode_type { enum t_gnode_port { PSP_PORT_FLATTENED, // same schema as iport (pkey,op) - PSP_PORT_DELTA, // same schema as state PSP_PORT_PREV, // same schema as state PSP_PORT_CURRENT, // same schema as state PSP_PORT_TRANSITIONS, // same schema as state diff --git a/rust/perspective-server/cpp/perspective/src/include/perspective/config.h b/rust/perspective-server/cpp/perspective/src/include/perspective/config.h index 4ede3e2309..5e0d043125 100644 --- a/rust/perspective-server/cpp/perspective/src/include/perspective/config.h +++ b/rust/perspective-server/cpp/perspective/src/include/perspective/config.h @@ -160,7 +160,7 @@ class PERSPECTIVE_EXPORT t_config { t_uindex get_num_columns() const; - std::string col_at(t_uindex idx) const; + const std::string& col_at(t_uindex idx) const; bool has_pkey_agg() const; diff --git a/rust/perspective-server/cpp/perspective/src/include/perspective/context_base.h b/rust/perspective-server/cpp/perspective/src/include/perspective/context_base.h index 22cd218930..cbd99e4eb9 100644 --- a/rust/perspective-server/cpp/perspective/src/include/perspective/context_base.h +++ b/rust/perspective-server/cpp/perspective/src/include/perspective/context_base.h @@ -68,6 +68,12 @@ class t_ctxbase { bool failed() const; + /** + * @brief A counter that advances each time `reset` replaces this + * context's storage. + */ + t_uindex get_storage_generation() const; + t_ctx_common common() { return t_ctx_common(this); @@ -94,6 +100,7 @@ class t_ctxbase { std::shared_ptr m_gstate; bool m_init; std::vector m_features; + t_uindex m_storage_generation = 0; }; template @@ -192,6 +199,12 @@ t_ctxbase::failed() const { return false; } +template +t_uindex +t_ctxbase::get_storage_generation() const { + return m_storage_generation; +} + template bool t_ctxbase::get_feature_state(t_ctx_feature feature) const { diff --git a/rust/perspective-server/cpp/perspective/src/include/perspective/context_common_decls.h b/rust/perspective-server/cpp/perspective/src/include/perspective/context_common_decls.h index 8cbb63667d..df406288b6 100644 --- a/rust/perspective-server/cpp/perspective/src/include/perspective/context_common_decls.h +++ b/rust/perspective-server/cpp/perspective/src/include/perspective/context_common_decls.h @@ -32,7 +32,6 @@ void notify(const t_data_table& flattened, bool is_registration); void notify( const t_data_table& flattened, - const t_data_table& delta, const t_data_table& prev, const t_data_table& current, const t_data_table& transitions, @@ -125,7 +124,6 @@ void compute_expressions( const std::shared_ptr& master, const t_gstate::t_mapping& pkey_map, const std::shared_ptr& flattened, - const std::shared_ptr& delta, const std::shared_ptr& prev, const std::shared_ptr& current, const std::shared_ptr& transitions, diff --git a/rust/perspective-server/cpp/perspective/src/include/perspective/context_unit.h b/rust/perspective-server/cpp/perspective/src/include/perspective/context_unit.h index 74c8bb1b55..408f7a1dfa 100644 --- a/rust/perspective-server/cpp/perspective/src/include/perspective/context_unit.h +++ b/rust/perspective-server/cpp/perspective/src/include/perspective/context_unit.h @@ -76,7 +76,6 @@ class PERSPECTIVE_EXPORT t_ctxunit : public t_ctxbase { void notify( const t_data_table& flattened, - const t_data_table& delta, const t_data_table& prev, const t_data_table& current, const t_data_table& transitions, @@ -123,7 +122,7 @@ class PERSPECTIVE_EXPORT t_ctxunit : public t_ctxbase { std::shared_ptr get_table() const; - t_tscalar get_column_name(t_index idx); + t_tscalar get_column_name(t_index idx) const; std::vector get_column_names() const; diff --git a/rust/perspective-server/cpp/perspective/src/include/perspective/context_zero.h b/rust/perspective-server/cpp/perspective/src/include/perspective/context_zero.h index 0fd65d992f..34000c6b3a 100644 --- a/rust/perspective-server/cpp/perspective/src/include/perspective/context_zero.h +++ b/rust/perspective-server/cpp/perspective/src/include/perspective/context_zero.h @@ -38,12 +38,23 @@ class PERSPECTIVE_EXPORT t_ctx0 : public t_ctxbase { ~t_ctx0(); #include - t_tscalar get_column_name(t_index idx); + t_tscalar get_column_name(t_index idx) const; std::vector get_column_names() const; const tsl::hopscotch_set& get_delta_pkeys() const; + /** + * @brief Whether the row keyed `pkey` currently passes this context's + * filter. + */ + bool has_pkey(t_tscalar pkey) const; + + /** + * @brief The keys of every row currently passing this context's filter. + */ + std::vector get_member_pkeys() const; + void sort_by(); std::vector get_sort_by() const; diff --git a/rust/perspective-server/cpp/perspective/src/include/perspective/derived_source.h b/rust/perspective-server/cpp/perspective/src/include/perspective/derived_source.h new file mode 100644 index 0000000000..1e94407f8c --- /dev/null +++ b/rust/perspective-server/cpp/perspective/src/include/perspective/derived_source.h @@ -0,0 +1,161 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace perspective { + +/** + * @brief How one cell of a derived table's step is written. + */ +enum t_derived_cell_state : std::uint8_t { + DERIVED_CELL_KEEP, + DERIVED_CELL_CLEAR, + DERIVED_CELL_SET +}; + +/** + * @brief One row of a derived table touched by a step of its source. + */ +struct PERSPECTIVE_EXPORT t_derived_member { + t_tscalar m_pkey; + t_uindex m_row = 0; + bool m_deleted = false; + t_uindex m_handle = 0; + t_uindex m_depth = 0; + std::vector m_path; + std::vector> m_cells; +}; + +/** + * @brief Feeds a read-only derived `Table` from the state of a `View`. + */ +class PERSPECTIVE_EXPORT t_derived_source { +public: + virtual ~t_derived_source() = default; + + /** + * @brief The schema a derived table has when none is supplied. + */ + virtual t_schema infer_schema() const = 0; + + virtual t_dtype pkey_dtype() const = 0; + + virtual std::string index() const; + + virtual std::uint32_t limit() const; + + /** + * @brief Bind `child` to this source and load every current member. + */ + void attach(const std::shared_ptr
& child); + + /** + * @brief Propagate the parent's last step, returning whether the child + * changed, notifying its contexts only when `notify` is set. + */ + bool step(bool notify); + + const std::shared_ptr
& child() const; + + /** + * @brief Stop recording changes on the parent for this source. + */ + virtual void detach(); + +protected: + virtual void resolve(const t_schema& schema) = 0; + + /** + * @brief The parent context's storage generation, which advances when + * its storage is replaced. + */ + virtual t_uindex storage_generation() const = 0; + + bool take_replaced(); + + void init_pct_flags(const t_view_config& config); + + virtual void + collect(bool full, std::vector& members) = 0; + + virtual void fill( + const t_derived_member& member, + t_uindex idx, + const std::vector& columns + ) const = 0; + + /** + * @brief The parent column that child column `cidx` may read in place + * of owning a copy, if any. + */ + virtual std::shared_ptr alias_column(t_uindex cidx) const; + + /** + * @brief The value an aliased column held before this step, when the + * source knows it differs from the current one. + */ + virtual bool previous( + const t_derived_member& member, t_uindex cidx, t_tscalar& out + ) const; + + t_index bound_row(const t_tscalar& pkey) const; + + std::shared_ptr
m_child; + std::vector m_columns; + bool m_attached = false; + t_uindex m_generation = 0; + bool m_pct_parent = false; + bool m_pct_grand = false; +}; + +/** + * @brief Build the source for a view of the given context type. + */ +PERSPECTIVE_EXPORT std::shared_ptr make_derived_source( + const std::shared_ptr>& view, + const std::shared_ptr
& parent +); + +PERSPECTIVE_EXPORT std::shared_ptr make_derived_source( + const std::shared_ptr>& view, + const std::shared_ptr
& parent +); + +PERSPECTIVE_EXPORT std::shared_ptr make_derived_source( + const std::shared_ptr>& view, + const std::shared_ptr
& parent +); + +PERSPECTIVE_EXPORT std::shared_ptr make_derived_source( + const std::shared_ptr>& view, + const std::shared_ptr
& parent +); + +} // namespace perspective diff --git a/rust/perspective-server/cpp/perspective/src/include/perspective/env_vars.h b/rust/perspective-server/cpp/perspective/src/include/perspective/env_vars.h index b4e90318fc..4637c45725 100644 --- a/rust/perspective-server/cpp/perspective/src/include/perspective/env_vars.h +++ b/rust/perspective-server/cpp/perspective/src/include/perspective/env_vars.h @@ -92,6 +92,22 @@ struct PERSPECTIVE_EXPORT t_env { return rv; } + /// Read-once developer toggle: the sum family reports an exact zero + /// instead of null for a group with no valid values. + static inline bool + sum_empty_zero() { + static const bool rv = std::getenv("PSP_SUM_EMPTY_ZERO") != 0; + return rv; + } + + /// Read-once developer toggle: force every sum-family aggregate onto + /// the full re-reduce path. + static inline bool + force_reaggregate() { + static const bool rv = std::getenv("PSP_FORCE_REAGGREGATE") != 0; + return rv; + } + static inline bool log_data_nsparse_strands() { static const bool rv = std::getenv("PSP_LOG_DATA_NSPARSE_STRANDS") != 0; diff --git a/rust/perspective-server/cpp/perspective/src/include/perspective/expression_tables.h b/rust/perspective-server/cpp/perspective/src/include/perspective/expression_tables.h index 532ea88349..98c3a6a1f1 100644 --- a/rust/perspective-server/cpp/perspective/src/include/perspective/expression_tables.h +++ b/rust/perspective-server/cpp/perspective/src/include/perspective/expression_tables.h @@ -68,7 +68,6 @@ struct t_expression_tables { std::shared_ptr m_flattened; std::shared_ptr m_prev; std::shared_ptr m_current; - std::shared_ptr m_delta; std::shared_ptr m_transitions; }; diff --git a/rust/perspective-server/cpp/perspective/src/include/perspective/gnode.h b/rust/perspective-server/cpp/perspective/src/include/perspective/gnode.h index 61c05c8fd2..bbeb1450ae 100644 --- a/rust/perspective-server/cpp/perspective/src/include/perspective/gnode.h +++ b/rust/perspective-server/cpp/perspective/src/include/perspective/gnode.h @@ -30,6 +30,7 @@ #include #include #include +#include #ifdef PSP_PARALLEL_FOR #include @@ -58,6 +59,17 @@ class t_ctx_grouped_pkey; #define PSP_GNODE_VERIFY_TABLE(X) #endif +/** + * @brief The rows a derived table's source touched in one step. + */ +struct PERSPECTIVE_EXPORT t_derived_step { + std::shared_ptr m_flattened; + std::shared_ptr m_prev_state; + std::vector m_rows; + std::vector m_bound; + bool m_notify = true; +}; + /** * @brief The struct returned from `_process_table`, which contains a * pointer to the flattened and processed `t_data_table`, and a boolean showing @@ -112,6 +124,21 @@ class PERSPECTIVE_EXPORT t_gnode { */ void send(t_uindex port_id, const t_data_table& fragments); + /** + * @brief Apply one step of a derived table dictated by its source, with + * the caller holding the pool's write lock. + */ + bool process_derived(const t_derived_step& step); + + /** + * @brief Make the derived master column `name` read `column`, which + * another table owns. + */ + void + set_derived_alias(const std::string& name, std::shared_ptr column); + + bool is_derived_alias(const std::string& name) const; + /** * @brief Bulk-initialize an empty gnode directly from `data_table`, * bypassing the input port, `flatten()`, and `update_master_table()` @@ -319,6 +346,36 @@ class PERSPECTIVE_EXPORT t_gnode { */ t_mask _process_mask_existed_rows(t_process_state& process_state); + /** + * @brief Fill the transitional output tables by diffing `flattened` + * against `state_table` at the rows given by `lookup`. + */ + /** + * @brief Consume a pending reset, reporting stashed keys absent from + * `pkey_col` as removed. + */ + void _take_reset_removes( + const t_column* pkey_col, t_uindex flattened_num_rows + ); + + t_mask _compute_transitions( + const std::shared_ptr& flattened, + const std::shared_ptr& state_table, + const std::vector& lookup + ); + + /** + * @brief Mask out deleted rows, commit through `commit` if given, then + * run windows and expressions, returning the table to notify with. + */ + std::shared_ptr _finish_step( + const std::shared_ptr& flattened, + const t_mask& existed_mask, + const std::vector& row_lookup, + const std::function&)>& + commit + ); + /** * @brief Given a flattened column, the master column from `m_gstate`, and * all transitional columns containing metadata, process and calculate @@ -327,7 +384,6 @@ class PERSPECTIVE_EXPORT t_gnode { * @tparam DATA_T * @param fcolumn * @param scolumn - * @param dcolumn * @param pcolumn * @param ccolumn * @param tcolumn @@ -337,7 +393,6 @@ class PERSPECTIVE_EXPORT t_gnode { void _process_column( const t_column* fcolumn, const t_column* scolumn, - t_column* dcolumn, t_column* pcolumn, t_column* ccolumn, t_column* tcolumn, @@ -457,7 +512,6 @@ t_gnode::notify_context( CTX_T* ctx = ctxh.get(); // Tables from the gnode which do not have the expressions applied yet - std::shared_ptr delta = m_oports[PSP_PORT_DELTA]->get_table(); std::shared_ptr prev = m_oports[PSP_PORT_PREV]->get_table(); std::shared_ptr current = m_oports[PSP_PORT_CURRENT]->get_table(); @@ -479,7 +533,6 @@ t_gnode::notify_context( auto joined_flattened = flattened->join(ctx_expression_tables->m_flattened); - auto joined_delta = delta->join(ctx_expression_tables->m_delta); auto joined_prev = prev->join(ctx_expression_tables->m_prev); auto joined_current = current->join(ctx_expression_tables->m_current); auto joined_transitions = @@ -490,14 +543,13 @@ t_gnode::notify_context( // as the contexts do not retain a reference to these tables. ctx->notify( *joined_flattened, - *joined_delta, *joined_prev, *joined_current, *joined_transitions, existed ); } else { - ctx->notify(*flattened, *delta, *prev, *current, *transitions, existed); + ctx->notify(*flattened, *prev, *current, *transitions, existed); } ctx->step_end(); @@ -566,7 +618,6 @@ template <> void t_gnode::_process_column( const t_column* fcolumn, const t_column* scolumn, - t_column* dcolumn, t_column* pcolumn, t_column* ccolumn, t_column* tcolumn, @@ -578,7 +629,6 @@ void t_gnode::_process_column( const t_column* fcolumn, const t_column* scolumn, - t_column* dcolumn, t_column* pcolumn, t_column* ccolumn, t_column* tcolumn, @@ -602,7 +652,10 @@ t_gnode::_process_column( bool prev_valid = false; DATA_T cur_value = *(fcolumn->get_nth(idx)); - bool cur_valid = fcolumn->is_valid(idx); + + t_status cur_status = *(fcolumn->get_nth_status(idx)); + bool cur_valid = cur_status == STATUS_VALID; + bool cur_cleared = cur_status == STATUS_CLEAR; if (row_pre_existed) { prev_value = *(scolumn->get_nth(rlookup.m_idx)); @@ -623,36 +676,18 @@ t_gnode::_process_column( prev_pkey_eq ); - // Mirrors `t_gstate::update_master_column`: an invalid cell - // is an explicit null if CLEAR (removes this row's - // contribution from additive aggregates), and a no-op if - // INVALID (column omitted from a partial update). A slot's - // raw bits are unspecified when its validity flag is false, - // so only the valid side of a transition may be read (#1256). - DATA_T delta_value; - if (cur_valid) { - delta_value = - cur_value - (prev_valid ? prev_value : DATA_T(0)); - } else if (fcolumn->is_cleared(idx) && prev_valid) { - SUPPRESS_WARNINGS_VC(4146) - delta_value = -prev_value; - RESTORE_WARNINGS_VC() - } else { - delta_value = DATA_T(0); - } - - dcolumn->set_nth(added_count, delta_value); - dcolumn->set_valid(added_count, true); - pcolumn->set_nth(added_count, prev_value); pcolumn->set_valid(added_count, prev_valid); - ccolumn->set_nth( - added_count, cur_valid ? cur_value : prev_value - ); - ccolumn->set_valid( - added_count, cur_valid ? cur_valid : prev_valid - ); + if (cur_valid) { + ccolumn->set_nth(added_count, cur_value); + ccolumn->set_valid(added_count, true); + } else if (prev_valid && !cur_cleared) { + ccolumn->set_nth(added_count, prev_value); + ccolumn->set_valid(added_count, true); + } else { + ccolumn->clear(added_count); + } tcolumn->set_nth(idx, trans); } break; @@ -668,13 +703,6 @@ t_gnode::_process_column( ccolumn->set_nth(added_count, prev_value); ccolumn->set_valid(added_count, prev_valid); - SUPPRESS_WARNINGS_VC(4146) - dcolumn->set_nth( - added_count, prev_valid ? -prev_value : DATA_T(0) - ); - RESTORE_WARNINGS_VC() - dcolumn->set_valid(added_count, true); - tcolumn->set_nth( added_count, VALUE_TRANSITION_NEQ_TDF ); diff --git a/rust/perspective-server/cpp/perspective/src/include/perspective/gnode_state.h b/rust/perspective-server/cpp/perspective/src/include/perspective/gnode_state.h index 148ee9be05..b45e5430b2 100644 --- a/rust/perspective-server/cpp/perspective/src/include/perspective/gnode_state.h +++ b/rust/perspective-server/cpp/perspective/src/include/perspective/gnode_state.h @@ -100,6 +100,23 @@ class PERSPECTIVE_EXPORT t_gstate { */ void init_from_table(const std::shared_ptr& source); + /** + * @brief Apply `flattened` to the master table at the row indices its + * source dictates. + */ + void commit_derived( + const std::shared_ptr& flattened, + const std::vector& rows + ); + + /** + * @brief Replace the master column `name` with a column owned by another + * table, which this state then only reads. + */ + void set_alias(const std::string& name, std::shared_ptr column); + + bool is_aliased(const std::string& name) const; + /** * @brief Given a column in the master data table and the corresponding * column in the `flattened` data table, fill the master column with data @@ -300,6 +317,12 @@ class PERSPECTIVE_EXPORT t_gstate { t_mask get_cpp_mask() const; void _mark_deleted(t_uindex idx); + + void drop_aliases(); + + void resize_owned(t_uindex extent); + + void _refresh_alias_mask(); bool has_pkey(t_tscalar pkey) const; t_dtype get_pkey_dtype() const; @@ -318,6 +341,8 @@ class PERSPECTIVE_EXPORT t_gstate { std::shared_ptr m_table; t_mapping m_mapping; t_free_items m_free; + tsl::hopscotch_set m_aliased; + std::vector m_alias_mask; t_symtable m_symtable; std::shared_ptr m_pkcol; std::shared_ptr m_opcol; diff --git a/rust/perspective-server/cpp/perspective/src/include/perspective/pool.h b/rust/perspective-server/cpp/perspective/src/include/perspective/pool.h index 62986d39e9..f9a44645be 100644 --- a/rust/perspective-server/cpp/perspective/src/include/perspective/pool.h +++ b/rust/perspective-server/cpp/perspective/src/include/perspective/pool.h @@ -57,6 +57,18 @@ class PERSPECTIVE_EXPORT t_pool { #ifdef PSP_PARALLEL_FOR std::shared_mutex* get_lock() const; + + /** + * @brief Share `other`'s lock, so the two pools' tables are read and + * written under one mutex. + */ + void adopt_lock(const t_pool& other); + + /** + * @brief The mutex serializing this pool's `_process` together with the + * notification that follows it. + */ + std::mutex& get_process_lock(); #endif /** @@ -109,7 +121,9 @@ class PERSPECTIVE_EXPORT t_pool { private: #ifdef PSP_PARALLEL_FOR + std::shared_ptr m_lock_owner; std::shared_mutex* m_lock; + std::mutex m_process_lock; #endif std::vector m_gnodes; std::atomic_flag m_run; diff --git a/rust/perspective-server/cpp/perspective/src/include/perspective/process_state.h b/rust/perspective-server/cpp/perspective/src/include/perspective/process_state.h index b36f2ab1d3..dcef96f09a 100644 --- a/rust/perspective-server/cpp/perspective/src/include/perspective/process_state.h +++ b/rust/perspective-server/cpp/perspective/src/include/perspective/process_state.h @@ -48,7 +48,6 @@ struct t_process_state { std::shared_ptr m_state_data_table; std::shared_ptr m_flattened_data_table; - std::shared_ptr m_delta_data_table; std::shared_ptr m_prev_data_table; std::shared_ptr m_current_data_table; std::shared_ptr m_transitions_data_table; diff --git a/rust/perspective-server/cpp/perspective/src/include/perspective/server.h b/rust/perspective-server/cpp/perspective/src/include/perspective/server.h index e714f2b2f5..f11e5c5e79 100644 --- a/rust/perspective-server/cpp/perspective/src/include/perspective/server.h +++ b/rust/perspective-server/cpp/perspective/src/include/perspective/server.h @@ -15,12 +15,15 @@ #include "perspective/base.h" #include "perspective/exports.h" #include "perspective/join_engine.h" +#include "perspective/derived_source.h" #include "perspective/raw_types.h" #include "perspective/schema.h" #include "perspective/view.h" #include "perspective/view_config.h" #include #include +#include +#include #include #include #include @@ -160,7 +163,7 @@ namespace server { t_uindex end_col, bool emit_group_by = true, t_arrow_compression compression = t_arrow_compression::LZ4, - bool emit_legacy_row_path_names = true + bool machine_column_names = false ) const = 0; [[nodiscard]] @@ -226,11 +229,12 @@ namespace server { virtual std::uint32_t sides() const = 0; [[nodiscard]] - virtual std::vector> column_paths() const = 0; + virtual std::shared_ptr + make_derived_source(const std::shared_ptr
& parent) const = 0; [[nodiscard]] - virtual std::vector> - column_paths_range(t_uindex start_col, t_uindex end_col) const = 0; + virtual std::vector> + column_paths(t_uindex start_col, t_uindex end_col) const = 0; [[nodiscard]] virtual std::map @@ -293,7 +297,7 @@ namespace server { t_uindex end_col, bool emit_group_by = true, t_arrow_compression compression = t_arrow_compression::LZ4, - bool emit_legacy_row_path_names = true + bool machine_column_names = false ) const override { return m_view->to_arrow( start_row, @@ -302,7 +306,7 @@ namespace server { end_col, emit_group_by, compression, - emit_legacy_row_path_names + machine_column_names ); } @@ -426,42 +430,16 @@ namespace server { } [[nodiscard]] - std::vector> - column_paths() const override { - std::vector> out; - std::vector> column_paths = - m_view->column_paths(); - - for (const auto& path : column_paths) { - std::vector path_str; - path_str.reserve(path.size()); - for (const auto& scalar : path) { - path_str.push_back(scalar.to_string()); - } - out.push_back(path_str); - } - - return out; + std::shared_ptr + make_derived_source(const std::shared_ptr
& parent + ) const override { + return perspective::make_derived_source(m_view, parent); } [[nodiscard]] - std::vector> - column_paths_range(t_uindex start_col, t_uindex end_col) - const override { - std::vector> out; - std::vector> column_paths = - m_view->column_paths_range(start_col, end_col); - - for (const auto& path : column_paths) { - std::vector path_str; - path_str.reserve(path.size()); - for (const auto& scalar : path) { - path_str.push_back(scalar.to_string()); - } - out.push_back(path_str); - } - - return out; + std::vector> + column_paths(t_uindex start_col, t_uindex end_col) const override { + return m_view->column_paths(start_col, end_col); } [[nodiscard]] @@ -545,6 +523,46 @@ namespace server { * @brief ServerResources is a container for all the resources that the * server requires. */ + /** + * @brief The `Table`s derived from `View`s, and the sources feeding them. + */ + class PERSPECTIVE_EXPORT DerivedTableEngine { + public: + using t_id = std::string; + + void register_table( + const t_id& view_id, + const t_id& table_id, + std::shared_ptr source + ); + + void unregister_table(const t_id& table_id); + + bool has_dependents(const t_id& view_id) const; + + /** + * @brief The id of the `View` that `table_id` is derived from, if + * it is derived at all. + */ + std::optional parent_view_of(const t_id& table_id) const; + + std::vector>> + get_dependents(const t_id& view_id) const; + + private: + struct t_entry { + t_id m_view_id; + std::shared_ptr m_source; + }; + + std::multimap m_view_to_tables; + tsl::hopscotch_map m_tables; + +#ifdef PSP_PARALLEL_FOR + mutable std::shared_mutex m_lock; +#endif + }; + class PERSPECTIVE_EXPORT ServerResources { public: using t_id = std::string; @@ -712,6 +730,36 @@ namespace server { std::vector>& outs ); + /** + * @brief Emit the `on_update` and `on_remove` responses owed by the + * views of a table that just stepped, then step its derived tables. + */ + void _notify_table( + std::shared_ptr
& table, + const ServerResources::t_id& table_id, + t_uindex port_id, + std::vector>& outs + ); + + /** + * @brief The id of the non-derived table at the root of `table_id`'s + * chain of derived tables. + */ + ServerResources::t_id + _root_table_id(const ServerResources::t_id& table_id); + + /** + * @brief Create the read-only `Table` for a `from_view` request, + * returning an error message on failure. + */ + std::string make_derived_table( + const proto::MakeTableReq& req, + const ServerResources::t_id& table_id, + bool has_identity_options, + std::vector>& outs, + std::shared_ptr
& table + ); + void _process_table_unchecked( std::shared_ptr
& table, const ServerResources::t_id& table_id, @@ -725,6 +773,7 @@ namespace server { std::atomic m_cpu_time; ServerResources m_resources; JoinEngine m_join_engine; + DerivedTableEngine m_derived_engine; t_computed_expression_parser m_computed_expression_parser; }; diff --git a/rust/perspective-server/cpp/perspective/src/include/perspective/sparse_tree.h b/rust/perspective-server/cpp/perspective/src/include/perspective/sparse_tree.h index 15a46799bc..c743eab855 100644 --- a/rust/perspective-server/cpp/perspective/src/include/perspective/sparse_tree.h +++ b/rust/perspective-server/cpp/perspective/src/include/perspective/sparse_tree.h @@ -31,6 +31,8 @@ SUPPRESS_WARNINGS_VC(4503) #include #include #include +#include +#include #include #include #include @@ -66,6 +68,19 @@ struct by_idx_lfidx {}; PERSPECTIVE_EXPORT t_tscalar get_dominant(std::vector& values); +/// How one aggschema column is populated during a strand build. +enum t_strand_col_kind : std::uint8_t { + STRAND_COL_VALUE, + STRAND_COL_ABS, + STRAND_COL_VALID, + STRAND_COL_COUNT +}; + +struct t_strand_col { + t_strand_col_kind m_kind; + std::string m_source; +}; + struct t_build_strand_table_metadata { t_schema m_flattened_schema; t_schema m_strand_schema; @@ -73,6 +88,10 @@ struct t_build_strand_table_metadata { t_uindex m_npivotlike; std::vector m_pivot_like_columns; t_uindex m_pivsize; + + /// One entry per `m_aggschema` column, classifying how the strand + /// builders populate it. + std::vector m_agg_cols; }; typedef multi_index_container< @@ -134,11 +153,39 @@ typedef t_idxpkey::index::type::iterator iter_by_idx_pkey; typedef std::pair t_by_idx_pkey_ipair; +/** + * @brief A node removed from a `t_stree`, with its path and last aggregates. + */ +struct PERSPECTIVE_EXPORT t_stree_dropped { + t_uindex m_idx; + t_uindex m_aggidx; + t_depth m_depth; + std::vector m_path; + std::vector m_aggregates; +}; + +/** + * @brief The node changes a `t_stree` recorded for a derived table. + */ +struct PERSPECTIVE_EXPORT t_stree_capture { + tsl::hopscotch_map>> + m_changed; + std::vector m_created; + std::vector m_dropped; +}; + struct PERSPECTIVE_EXPORT t_agg_update_info { std::vector m_src; std::vector m_dst; std::vector m_aggspecs; + /// Whether each aggregate's source column has a numeric dtype. + std::vector m_numeric_source; + + /// Each aggregate's index into its source's `AGGTYPE_VALID_COUNT` + /// column, or `t_uindex(-1)` when it has none. + std::vector m_valid_idx; + std::vector m_dst_topo_sorted; }; @@ -185,13 +232,12 @@ class PERSPECTIVE_EXPORT t_stree { t_op op, t_uindex idx, t_uindex npivots, - t_uindex strand_count_idx, - t_uindex aggcolsize, + const std::vector& agg_kinds, bool force_current_row, const std::vector& piv_ccols, const std::vector& piv_tcols, const std::vector& agg_ccols, - const std::vector& agg_dcols, + const std::vector& agg_pcols, std::vector& piv_scols, std::vector& agg_acols, t_column* agg_scountspar, @@ -205,8 +251,7 @@ class PERSPECTIVE_EXPORT t_stree { t_tscalar pkey, t_uindex idx, t_uindex npivots, - t_uindex strand_count_idx, - t_uindex aggcolsize, + const std::vector& agg_kinds, const std::vector& piv_pcols, const std::vector& agg_pcols, std::vector& piv_scols, @@ -220,7 +265,6 @@ class PERSPECTIVE_EXPORT t_stree { std::pair, std::shared_ptr> build_strand_table( const t_data_table& flattened, - const t_data_table& delta, const t_data_table& prev, const t_data_table& current, const t_data_table& transitions, @@ -327,6 +371,27 @@ class PERSPECTIVE_EXPORT t_stree { const std::shared_ptr& get_deltas() const; + /** + * @brief Record this tree's node changes into `capture` until it is + * removed. + */ + void add_capture(const std::shared_ptr& capture); + + void remove_capture(const std::shared_ptr& capture); + + bool node_exists(t_uindex idx) const; + + /** + * @brief The live node with index `idx`, or null; valid until the tree + * next changes shape. + */ + const t_stnode* find_node(t_uindex idx) const; + + /** + * @brief Call `fn` with every live node in the tree. + */ + void for_each_node(const std::function& fn) const; + void clear(); std::pair first_last_helper( @@ -452,6 +517,21 @@ class PERSPECTIVE_EXPORT t_stree { std::shared_ptr m_nodes; std::shared_ptr m_idxpkey; std::shared_ptr m_idxleaf; + void capture_change( + t_uindex nidx, t_uindex aggnum, const t_tscalar& old_value + ); + + void capture_dropped(const t_stnode& node); + + void _build_agg_info( + const t_dtree_ctx& ctx, + const t_gstate& gstate, + const t_data_table& expression_master_table + ); + + t_agg_update_info m_agg_info; + const t_data_table* m_agg_info_table = nullptr; + t_uindex m_curidx; std::shared_ptr m_aggregates; std::vector m_aggspecs; @@ -467,6 +547,7 @@ class PERSPECTIVE_EXPORT t_stree { std::vector m_features; t_symtable m_symtable; bool m_has_delta; + std::vector> m_captures; std::string m_grand_agg_str; // Used by AGGTYPE_GMV under split_by. For t_ctx1 (group_by only) the diff --git a/rust/perspective-server/cpp/perspective/src/include/perspective/table.h b/rust/perspective-server/cpp/perspective/src/include/perspective/table.h index 93d3a09635..9f6cf1516c 100644 --- a/rust/perspective-server/cpp/perspective/src/include/perspective/table.h +++ b/rust/perspective-server/cpp/perspective/src/include/perspective/table.h @@ -274,6 +274,18 @@ class PERSPECTIVE_EXPORT Table { apachearrow::LIST_FLATTEN_ZIP ); + /** + * @brief Create a read-only `Table` whose rows are dictated by a `View`. + */ + static std::shared_ptr
make_derived( + const t_schema& schema, + t_dtype pkey_dtype, + const std::string& index, + std::uint32_t limit + ); + + bool is_derived() const; + static std::shared_ptr
from_arrow( const std::string& index, std::string&& data, @@ -365,6 +377,7 @@ class PERSPECTIVE_EXPORT Table { */ const std::string m_index; bool m_gnode_set; + bool m_derived = false; const t_backing_store m_backing_store; /** diff --git a/rust/perspective-server/cpp/perspective/src/include/perspective/tree_context_common.h b/rust/perspective-server/cpp/perspective/src/include/perspective/tree_context_common.h index 826c8c6147..68925259af 100644 --- a/rust/perspective-server/cpp/perspective/src/include/perspective/tree_context_common.h +++ b/rust/perspective-server/cpp/perspective/src/include/perspective/tree_context_common.h @@ -40,7 +40,6 @@ PERSPECTIVE_EXPORT void notify_sparse_tree( const std::vector>& tree_sortby, const std::vector& ctx_sortby, const t_data_table& flattened, - const t_data_table& delta, const t_data_table& prev, const t_data_table& current, const t_data_table& transitions, diff --git a/rust/perspective-server/cpp/perspective/src/include/perspective/view.h b/rust/perspective-server/cpp/perspective/src/include/perspective/view.h index 8da6f5f712..6a901252b5 100644 --- a/rust/perspective-server/cpp/perspective/src/include/perspective/view.h +++ b/rust/perspective-server/cpp/perspective/src/include/perspective/view.h @@ -132,21 +132,14 @@ class PERSPECTIVE_EXPORT View { ) const; /** - * @brief The aggregated column names of this View, showing the columns that - * have been composed through the addition of a pivot as they appear in the - * view. - * - * If the view is pivoted, "__ROW_PATH__" will be prepended to the front of - * the vector because it is part of the column path. + * @brief The rectangular column header area for `[start_col, end_col)`, + * transposed as `area[level][col]`, split values left-aligned and the + * column name always on the last of `m_column_pivots.size() + 1` levels. * * @return std::vector>> */ - std::vector> column_paths() const; - std::vector> - column_paths_range(t_uindex start_col, t_uindex end_col) const; - - std::vector> column_paths_string() const; + column_paths(t_uindex start_col, t_uindex end_col) const; /** * @brief @@ -283,7 +276,7 @@ class PERSPECTIVE_EXPORT View { std::int32_t end_col, bool emit_group_by, t_arrow_compression compression, - bool emit_legacy_row_path_names = true + bool machine_column_names = false ) const; /** @@ -319,7 +312,7 @@ class PERSPECTIVE_EXPORT View { std::shared_ptr> data_slice, bool emit_group_b, t_arrow_compression compression, - bool emit_legacy_row_path_names = true + bool machine_column_names = false ) const; /** @@ -435,7 +428,7 @@ class PERSPECTIVE_EXPORT View { std::shared_ptr> data_slice_to_batches( bool emit_group_by, std::shared_ptr> data_slice, - bool emit_legacy_row_path_names = true + bool machine_column_names = false ) const; void _find_hidden_sort(const std::vector& sort); diff --git a/rust/perspective-server/cpp/perspective/src/include/perspective/view_config.h b/rust/perspective-server/cpp/perspective/src/include/perspective/view_config.h index 67e2e06895..010c038937 100644 --- a/rust/perspective-server/cpp/perspective/src/include/perspective/view_config.h +++ b/rust/perspective-server/cpp/perspective/src/include/perspective/view_config.h @@ -110,6 +110,12 @@ class PERSPECTIVE_EXPORT t_view_config { std::vector get_aggspecs() const; + /** + * @brief The name an aggregate takes in human-readable output, qualified + * by its aggregate when its source column is also a `group_by` key. + */ + std::string readable_aggregate_name(const t_aggspec& aggspec) const; + std::vector get_columns() const; std::vector get_fterm() const; diff --git a/rust/perspective-server/cpp/perspective/src/include/perspective/window.h b/rust/perspective-server/cpp/perspective/src/include/perspective/window.h index f4d4c7b8e9..fb26a4f922 100644 --- a/rust/perspective-server/cpp/perspective/src/include/perspective/window.h +++ b/rust/perspective-server/cpp/perspective/src/include/perspective/window.h @@ -184,7 +184,6 @@ class PERSPECTIVE_EXPORT t_window_engine { const std::shared_ptr& expr_flattened, const std::shared_ptr& expr_prev, const std::shared_ptr& expr_current, - const std::shared_ptr& expr_delta, const std::shared_ptr& flattened, const std::shared_ptr& existed ); @@ -286,9 +285,7 @@ class PERSPECTIVE_EXPORT t_window_engine { const t_gstate::t_mapping& pkey_map, const std::shared_ptr& expr_master, const std::shared_ptr& expr_flattened, - const std::shared_ptr& expr_prev, const std::shared_ptr& expr_current, - const std::shared_ptr& expr_delta, const std::shared_ptr& flattened ) const; diff --git a/rust/perspective-viewer/Cargo.toml b/rust/perspective-viewer/Cargo.toml index 74e10e2b0b..cf9a010a81 100644 --- a/rust/perspective-viewer/Cargo.toml +++ b/rust/perspective-viewer/Cargo.toml @@ -46,9 +46,6 @@ lightningcss = "1.0.0-alpha.71" glob = "0.3.0" anyhow = "1.0.66" -[dev-dependencies] -wasm-bindgen-test = "0.3.13" - [dependencies] perspective-client = { version = "5.5.1" } perspective-js = { version = "5.5.1" } diff --git a/rust/perspective-viewer/src/css/column-selector.css b/rust/perspective-viewer/src/css/column-selector.css index 484d37ca6e..4f58113eea 100644 --- a/rust/perspective-viewer/src/css/column-selector.css +++ b/rust/perspective-viewer/src/css/column-selector.css @@ -192,66 +192,66 @@ min-width: 0; } - .column-selector-column { - display: flex; - flex-direction: row-reverse; - align-items: center; - flex: 1 1 auto; - - /* Expression column toolbar buttons */ - span.expression-edit-button, - span.expression-delete-button { - height: 19px; - width: 21px; - font-family: var(--psp-button--font-family, inherit); - cursor: pointer; - padding: 1.5px 2.5px 0; - margin-right: 2px; - margin-left: auto; - border-radius: 3px; - border: 1px solid transparent; - &:hover, - &.is-editing { - background-color: var(--psp--color); - .icon { - background-color: var(--psp--background-color); - } + /* Expression column toolbar buttons */ + span.expression-edit-button, + span.expression-delete-button { + height: 19px; + width: 21px; + font-family: var(--psp-button--font-family, inherit); + cursor: pointer; + padding: 1.5px 2.5px 0; + margin-right: 2px; + margin-left: auto; + border-radius: 3px; + border: 1px solid transparent; + &:hover, + &.is-editing { + background-color: var(--psp--color); + .icon { + background-color: var(--psp--background-color); } } + } - span.expression-edit-button.disabled { - opacity: 0; - pointer-events: none; - } + span.expression-edit-button.disabled { + opacity: 0; + pointer-events: none; + } - span.expression-delete-button { - padding-left: 5px; - margin-right: 8px; - margin-left: auto; + span.expression-delete-button { + padding-left: 5px; + margin-right: 8px; + margin-left: auto; - &:before { - content: var(--column-close--content, "close"); - } + &:before { + content: var(--column-close--content, "close"); } + } - span.expression-delete-button + span.expression-edit-button { - margin-left: 0px; - } + span.expression-delete-button + span.expression-edit-button { + margin-left: 0px; + } - span.expression-edit-button .icon { - background-repeat: no-repeat; - background-color: var(--psp--color); - display: inline-block; - -webkit-mask-size: cover; - mask-size: cover; - -webkit-mask-image: var(--psp-icon--column-settings--mask-image); - mask-image: var(--psp-icon--column-settings--mask-image); + span.expression-edit-button .icon { + background-repeat: no-repeat; + background-color: var(--psp--color); + display: inline-block; + -webkit-mask-size: cover; + mask-size: cover; + -webkit-mask-image: var(--psp-icon--column-settings--mask-image); + mask-image: var(--psp-icon--column-settings--mask-image); - &:before { - content: var(--psp-icon--column-settings--mask-image); - visibility: hidden; - } + &:before { + content: var(--psp-icon--column-settings--mask-image); + visibility: hidden; } + } + + .column-selector-column { + display: flex; + flex-direction: row-reverse; + align-items: center; + flex: 1 1 auto; .column-selector-draggable { cursor: move; diff --git a/rust/perspective-viewer/src/css/config-selector.css b/rust/perspective-viewer/src/css/config-selector.css index bc8aad659f..473ce87a43 100644 --- a/rust/perspective-viewer/src/css/config-selector.css +++ b/rust/perspective-viewer/src/css/config-selector.css @@ -350,6 +350,61 @@ display: inline-block; } + #derived_stack { + display: flex; + flex-direction: column; + margin-bottom: 12px; + } + + .derived_level { + display: flex; + align-items: center; + gap: 4px; + height: 24px; + padding: 0px 6px; + border-left: 2px solid var(--psp-inactive--color, #ababab); + color: var(--psp-inactive--color, #ababab); + font-size: var(--label--font-size, 0.75em); + } + + .derived_label { + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + } + + .derived_snapshot::before { + content: "snapshot"; + opacity: 0.6; + } + + .derived_pop { + margin-left: auto; + cursor: pointer; + -webkit-mask-image: var(--psp-icon--close--mask-image); + mask-image: var(--psp-icon--close--mask-image); + background-color: currentColor; + width: 12px; + height: 12px; + } + + .derived_pop:hover { + color: var(--psp--color, inherit); + } + + .derived_push { + cursor: pointer; + height: 24px; + line-height: 24px; + padding: 0px 6px; + font-size: var(--label--font-size, 0.75em); + color: var(--psp-inactive--color, #ababab); + } + + .derived_push:hover { + color: var(--psp--color, inherit); + } + .pivot_controls { display: flex; justify-content: flex-end; diff --git a/rust/perspective-viewer/src/rust/agent/client/protocol.rs b/rust/perspective-viewer/src/rust/agent/client/protocol.rs index a6b199f2cd..aff8cc8205 100644 --- a/rust/perspective-viewer/src/rust/agent/client/protocol.rs +++ b/rust/perspective-viewer/src/rust/agent/client/protocol.rs @@ -229,92 +229,3 @@ impl ChunkAccumulator { } } } - -#[cfg(test)] -mod tests { - use super::*; - - fn chunk(json: &str) -> ChatChunk { - serde_json::from_str(json).unwrap() - } - - #[test] - fn content_and_both_reasoning_conventions_fold() { - let mut acc = ChunkAccumulator::default(); - acc.push(chunk(r#"{"choices":[{"delta":{"role":"assistant"}}]}"#)); - acc.push(chunk( - r#"{"choices":[{"delta":{"reasoning_content":"th"}}]}"#, - )); - acc.push(chunk(r#"{"choices":[{"delta":{"reasoning":"ink"}}]}"#)); - acc.push(chunk(r#"{"choices":[{"delta":{"content":"Hel"}}]}"#)); - acc.push(chunk( - r#"{"choices":[{"delta":{"content":"lo","reasoning":""}}]}"#, - )); - assert_eq!(acc.progress(), ("Hello", "think")); - let ChatMessage::Assistant { - content, - tool_calls, - reasoning_content, - } = acc.finish() - else { - panic!("not assistant"); - }; - - assert_eq!(content.as_deref(), Some("Hello")); - assert!(tool_calls.is_empty()); - assert_eq!(reasoning_content.as_deref(), Some("think")); - } - - #[test] - fn fragmented_tool_arguments_accumulate_by_index() { - let mut acc = ChunkAccumulator::default(); - acc.push(chunk( - r#"{"choices":[{"delta":{"tool_calls":[ - {"index":0,"id":"c0","function":{"name":"get_schema","arguments":""}}]}}]}"#, - )); - - // Index 1 opens before index 0 finishes; both argument streams - // interleave. - acc.push(chunk( - r#"{"choices":[{"delta":{"tool_calls":[ - {"index":1,"id":"c1","function":{"name":"set_view_config","arguments":"{\"con"}}]}}]}"#, - )); - - acc.push(chunk( - r#"{"choices":[{"delta":{"tool_calls":[ - {"index":1,"function":{"arguments":"fig\":{}}"}}]}}]}"#, - )); - - let ChatMessage::Assistant { tool_calls, .. } = acc.finish() else { - panic!("not assistant"); - }; - - assert_eq!(tool_calls.len(), 2); - assert_eq!(tool_calls[0].id, "c0"); - assert_eq!(tool_calls[0].function.name, "get_schema"); - assert_eq!(tool_calls[1].function.arguments, "{\"config\":{}}"); - assert_eq!(tool_calls[1].kind, "function"); - } - - /// The display-only contract: reasoning deserializes from EITHER wire - /// field but NEVER serializes back into a request. - #[test] - fn assistant_reasoning_reads_both_aliases_and_never_serializes() { - for field in ["reasoning_content", "reasoning"] { - let message: ChatMessage = serde_json::from_str(&format!( - r#"{{"role":"assistant","content":"hi","{field}":"secret"}}"# - )) - .unwrap(); - - let ChatMessage::Assistant { - reasoning_content, .. - } = &message - else { - panic!("not assistant"); - }; - - assert_eq!(reasoning_content.as_deref(), Some("secret")); - assert!(!serde_json::to_string(&message).unwrap().contains("secret")); - } - } -} diff --git a/rust/perspective-viewer/src/rust/agent/markdown/mod.rs b/rust/perspective-viewer/src/rust/agent/markdown/mod.rs index df2ee6dedc..6787615755 100644 --- a/rust/perspective-viewer/src/rust/agent/markdown/mod.rs +++ b/rust/perspective-viewer/src/rust/agent/markdown/mod.rs @@ -226,241 +226,3 @@ impl Writer { } } } - -#[cfg(test)] -mod tests { - use super::events::{Align, Event, Tag}; - - fn parse(text: &str) -> Vec { - super::block::parse(text) - } - - fn start(tag: Tag) -> Event { - Event::Start(tag) - } - - fn text(x: &str) -> Event { - Event::Text(x.to_owned()) - } - - #[test] - fn paragraph_and_emphasis() { - assert_eq!(parse("**Done!** Key *stats* here."), vec![ - start(Tag::Paragraph), - start(Tag::Strong), - text("Done!"), - Event::End, - text(" Key "), - start(Tag::Emphasis), - text("stats"), - Event::End, - text(" here."), - Event::End, - ]); - } - - #[test] - fn snake_case_is_not_emphasis() { - assert_eq!(parse("call get_schema then set_view_config"), vec![ - start(Tag::Paragraph), - text("call get_schema then set_view_config"), - Event::End, - ]); - } - - #[test] - fn nested_strong_emphasis() { - assert_eq!(parse("**bold with *ital* inside**"), vec![ - start(Tag::Paragraph), - start(Tag::Strong), - text("bold with "), - start(Tag::Emphasis), - text("ital"), - Event::End, - text(" inside"), - Event::End, - Event::End, - ]); - } - - #[test] - fn unpaired_delimiters_stay_literal() { - // Space on both sides ⇒ the runs can neither open nor close, so - // they stay in the running text buffer. - assert_eq!(parse("2 * 3 * 4 and a ~ tilde"), vec![ - start(Tag::Paragraph), - text("2 * 3 * 4 and a ~ tilde"), - Event::End, - ]); - } - - #[test] - fn code_spans_and_strikethrough() { - assert_eq!(parse("run `pnpm test` and ~~skip~~ this"), vec![ - start(Tag::Paragraph), - text("run "), - Event::Code("pnpm test".to_owned()), - text(" and "), - start(Tag::Strikethrough), - text("skip"), - Event::End, - text(" this"), - Event::End, - ]); - } - - #[test] - fn heading_rule_and_fence() { - assert_eq!(parse("## Stats\n\n---\n\n```sql\nSELECT 1;\n```"), vec![ - start(Tag::Heading(2)), - text("Stats"), - Event::End, - Event::Rule, - start(Tag::CodeBlock(Some("sql".to_owned()))), - text("SELECT 1;\n"), - Event::End, - ]); - } - - #[test] - fn nested_list_with_code_block_in_item() { - assert_eq!( - parse("- first\n- second:\n ```\n x = 1\n ```\n - inner"), - vec![ - start(Tag::List(None)), - start(Tag::Item), - text("first"), - Event::End, - start(Tag::Item), - text("second:"), - start(Tag::CodeBlock(None)), - text("x = 1\n"), - Event::End, - start(Tag::List(None)), - start(Tag::Item), - text("inner"), - Event::End, - Event::End, - Event::End, - Event::End, - ] - ); - } - - #[test] - fn ordered_list_start() { - assert_eq!(parse("3. three\n4. four"), vec![ - start(Tag::List(Some(3))), - start(Tag::Item), - text("three"), - Event::End, - start(Tag::Item), - text("four"), - Event::End, - Event::End, - ]); - } - - #[test] - fn table_with_alignment_and_pipe_in_code() { - assert_eq!( - parse("| Region | Sales |\n| :--- | ---: |\n| West | `a\\|b` |"), - vec![ - start(Tag::Table(vec![Align::Left, Align::Right])), - start(Tag::TableHead), - start(Tag::TableCell), - text("Region"), - Event::End, - start(Tag::TableCell), - text("Sales"), - Event::End, - Event::End, - start(Tag::TableRow), - start(Tag::TableCell), - text("West"), - Event::End, - start(Tag::TableCell), - Event::Code("a|b".to_owned()), - Event::End, - Event::End, - Event::End, - ] - ); - } - - #[test] - fn dashes_after_paragraph_are_a_rule_not_a_table() { - assert_eq!(parse("Title\n---"), vec![ - start(Tag::Paragraph), - text("Title"), - Event::End, - Event::Rule, - ]); - } - - #[test] - fn links_images_and_autolinks() { - assert_eq!( - parse("[safe](https://example.com \"hi\") ![leak](https://example.com/x.png) "), - vec![ - start(Tag::Paragraph), - start(Tag::Link { - dest_url: "https://example.com".to_owned(), - title: "hi".to_owned(), - }), - text("safe"), - Event::End, - text(" "), - start(Tag::Image { - dest_url: "https://example.com/x.png".to_owned(), - }), - text("leak"), - Event::End, - text(" "), - start(Tag::Link { - dest_url: "https://a.co".to_owned(), - title: String::new(), - }), - text("https://a.co"), - Event::End, - Event::End, - ] - ); - } - - #[test] - fn raw_html_is_literal_text() { - assert_eq!(parse(""), vec![ - start(Tag::Paragraph), - text(""), - Event::End, - ]); - } - - #[test] - fn blockquote_and_breaks() { - assert_eq!(parse("> quoted\n> more\n\nafter \nbreak"), vec![ - start(Tag::BlockQuote), - start(Tag::Paragraph), - text("quoted"), - Event::SoftBreak, - text("more"), - Event::End, - Event::End, - start(Tag::Paragraph), - text("after"), - Event::HardBreak, - text("break"), - Event::End, - ]); - } - - #[test] - fn entities_decode_or_stay_literal() { - assert_eq!(parse("a & b A &unknown; c"), vec![ - start(Tag::Paragraph), - text("a & b A &unknown; c"), - Event::End, - ]); - } -} diff --git a/rust/perspective-viewer/src/rust/components/column_selector.rs b/rust/perspective-viewer/src/rust/components/column_selector.rs index 3ba047f6d2..54fdffe82c 100644 --- a/rust/perspective-viewer/src/rust/components/column_selector.rs +++ b/rust/perspective-viewer/src/rust/components/column_selector.rs @@ -395,6 +395,8 @@ impl Component for ColumnSelector { bool { let min_cols = self.renderer.metadata().min_config_columns.unwrap_or(0); idx < min_cols + && (self + .session + .get_view_config() + .columns + .iter() + .flatten() + .count() + == 1) } fn get_aggregate(&self, name: &str) -> Option { diff --git a/rust/perspective-viewer/src/rust/components/column_selector/config_selector.rs b/rust/perspective-viewer/src/rust/components/column_selector/config_selector.rs index 878f1d3bef..0bb05a1171 100644 --- a/rust/perspective-viewer/src/rust/components/column_selector/config_selector.rs +++ b/rust/perspective-viewer/src/rust/components/column_selector/config_selector.rs @@ -24,7 +24,8 @@ use crate::components::column_dropdown::{ColumnDropDownElement, ColumnDropDownPo use crate::components::dragdrop_list::*; use crate::components::filter_dropdown::{FilterDropDownElement, FilterDropDownPortal}; use crate::config::PluginStaticConfig; -use crate::presentation::Presentation; +use crate::presentation::{ColumnLocator, ColumnSettingsTarget, Presentation}; +use crate::queries::has_column_config; use crate::renderer::*; use crate::session::drag_drop_update::*; use crate::session::*; @@ -36,6 +37,13 @@ use crate::utils::*; pub struct ConfigSelectorProps { pub onselect: Callback<()>, + /// Opens a pivot column's settings, so its format config is reachable. + pub on_open_expr_panel: Callback, + + /// The column whose settings sidebar is open, threaded as a VALUE prop so + /// closing or retargeting the sidebar clears this pill's open indicator. + pub selected_column: Option, + #[prop_or_default] pub ondragenter: Callback<()>, @@ -43,9 +51,11 @@ pub struct ConfigSelectorProps { /// (group_by, sort, filter, etc.) trigger re-renders via normal prop /// diffing rather than a PubSub `view_created` subscription. pub view_config: PtrEqRc, + /// Column currently being dragged — threaded to show `dragdrop-highlight` /// without subscribing to `dragstart_received`/`dragend_received`. pub drag_column: Option, + /// Session metadata snapshot — threaded from `SessionProps`. pub metadata: SessionMetadataRc, @@ -69,6 +79,7 @@ pub struct ConfigSelectorProps { impl PartialEq for ConfigSelectorProps { fn eq(&self, other: &Self) -> bool { self.view_config == other.view_config + && self.selected_column == other.selected_column && self.drag_column == other.drag_column && self.metadata == other.metadata && self.selected_theme == other.selected_theme @@ -578,6 +589,17 @@ impl Component for ConfigSelector { ) }; + let has_config = |name: &str| { + has_column_config(name, config, &ctx.props().metadata, &ctx.props().renderer) + }; + + let editing_column = match &ctx.props().selected_column { + Some(ColumnLocator::Table(x)) + | Some(ColumnLocator::Expression(x)) + | Some(ColumnLocator::Window(x)) => Some(x.clone()), + _ => None, + }; + let transpose = ctx.link().callback(|_| ConfigSelectorMsg::TransposePivots); let column_dropdown = self.column_dropdown.clone(); let mut class = classes!(); @@ -665,6 +687,9 @@ impl Component for ConfigSelector { action={DragTarget::GroupBy} column={group_by.clone()} metadata={metadata.clone()} + on_open_expr_panel={has_config(group_by) + .then(|| ctx.props().on_open_expr_panel.clone())} + is_editing={editing_column.as_deref() == Some(group_by.as_str())} {presentation} opt_session={session} > @@ -716,6 +741,9 @@ impl Component for ConfigSelector { action={ DragTarget::SplitBy } column={ split_by.clone() } metadata={metadata.clone()} + on_open_expr_panel={has_config(split_by) + .then(|| ctx.props().on_open_expr_panel.clone())} + is_editing={editing_column.as_deref() == Some(split_by.as_str())} {presentation} opt_session={session}> diff --git a/rust/perspective-viewer/src/rust/components/column_selector/pivot_column.rs b/rust/perspective-viewer/src/rust/components/column_selector/pivot_column.rs index 3790839bfd..74b8369372 100644 --- a/rust/perspective-viewer/src/rust/components/column_selector/pivot_column.rs +++ b/rust/perspective-viewer/src/rust/components/column_selector/pivot_column.rs @@ -14,9 +14,10 @@ use perspective_client::config::ColumnType; use web_sys::*; use yew::prelude::*; +use super::expr_edit_button::*; use crate::components::dragdrop_list::*; use crate::components::type_icon::TypeIcon; -use crate::presentation::Presentation; +use crate::presentation::{ColumnSettingsTarget, Presentation}; use crate::session::*; use crate::utils::*; @@ -35,6 +36,14 @@ pub struct PivotColumnProps { #[prop_or_default] pub metadata: Option, + /// Opens this column's settings, when the host offers them. + #[prop_or_default] + pub on_open_expr_panel: Option>, + + /// Is this column's settings panel open? + #[prop_or_default] + pub is_editing: bool, + // State #[prop_or_default] pub opt_session: Option, @@ -46,6 +55,8 @@ impl PartialEq for PivotColumnProps { self.column == other.column && self.action == other.action && self.metadata == other.metadata + && self.is_editing == other.is_editing + && self.on_open_expr_panel.is_some() == other.on_open_expr_panel.is_some() } } @@ -103,6 +114,15 @@ impl Component for PivotColumn { { ctx.props().column.clone() } + if let Some(on_open_expr_panel) = &ctx.props().on_open_expr_panel { + + } } diff --git a/rust/perspective-viewer/src/rust/components/column_settings_sidebar.rs b/rust/perspective-viewer/src/rust/components/column_settings_sidebar.rs index 02dd86b533..ee0038d912 100644 --- a/rust/perspective-viewer/src/rust/components/column_settings_sidebar.rs +++ b/rust/perspective-viewer/src/rust/components/column_settings_sidebar.rs @@ -34,6 +34,7 @@ use crate::components::expression_editor::ExpressionEditorProps; use crate::components::type_icon::TypeIconType; use crate::components::window_editor::WindowEditorProps; use crate::presentation::{ColumnLocator, ColumnSettingsTab, Presentation}; +use crate::queries::{has_column_config, is_pivot_only_column}; use crate::renderer::Renderer; use crate::session::{Session, SessionMetadataRc}; use crate::tasks::{ @@ -551,23 +552,35 @@ impl ColumnSettingsPanel { } fn refresh_derived(&mut self, ctx: &yew::prelude::Context) { - self.maybe_ty = ctx + let view_ty = ctx .props() .metadata .locator_view_type(&ctx.props().selected_column); + self.maybe_ty = match ctx.props().selected_column.name() { + Some(name) if is_pivot_only_column(name, &ctx.props().view_config) => { + ctx.props().metadata.get_column_table_type(name).or(view_ty) + }, + _ => view_ty, + }; + self.tabs = { let mut tabs = vec![]; let is_new_expr = ctx.props().selected_column.is_new_expr(); + let selected_name = ctx + .props() + .selected_column + .name() + .map(|x| x.to_string()) + .unwrap_or_default(); + let show_styles = !is_new_expr - && ctx.props().renderer.can_render_column_styles() - && ctx.props().view_config.columns.contains(&Some( - ctx.props() - .selected_column - .name() - .map(|x| x.to_string()) - .unwrap_or_default(), - )); + && has_column_config( + &selected_name, + &ctx.props().view_config, + &ctx.props().metadata, + &ctx.props().renderer, + ); if !is_new_expr && show_styles { tabs.push(ColumnSettingsTab::Style); diff --git a/rust/perspective-viewer/src/rust/components/debug_panel.rs b/rust/perspective-viewer/src/rust/components/debug_panel.rs index 84b01c1c82..e56591aabc 100644 --- a/rust/perspective-viewer/src/rust/components/debug_panel.rs +++ b/rust/perspective-viewer/src/rust/components/debug_panel.rs @@ -27,7 +27,7 @@ use crate::session::*; use crate::utils::*; use crate::workspace::Workspace; -#[derive(Clone, PartialEq, Properties)] +#[derive(Clone, Properties)] pub struct DebugPanelProps { pub presentation: Presentation, pub renderer: Renderer, @@ -46,9 +46,16 @@ pub struct DebugPanelProps { pub on_auto_width: Callback, } +impl PartialEq for DebugPanelProps { + fn eq(&self, rhs: &Self) -> bool { + self.initial_width == rhs.initial_width + } +} + /// Whether the editor holds text the panel has not successfully applied, read -/// synchronously by the config-change listeners so a failed apply's own -/// `view_config_changed` never discards it. +/// synchronously by the config-change listeners and again when a +/// [`DebugPanelProps::set_text`] read lands, so neither a failed apply's own +/// `view_config_changed` nor an in-flight config read discards it. type Dirty = Rc>; #[function_component(DebugPanel)] @@ -71,10 +78,12 @@ pub fn debug_panel(props: &DebugPanelProps) -> Html { } }); - use_effect_with((expr.setter(), props.clone()), { + use_effect_with((), { clone!(error, modified, dirty); - move |(text, state)| { - state.set_text(text.clone()); + let state = props.clone(); + let text = expr.setter(); + move |()| { + state.set_text(text.clone(), dirty.clone()); error.set(None); let sub1 = state .renderer @@ -153,7 +162,7 @@ pub fn debug_panel(props: &DebugPanelProps) -> Html { clone!(modified, dirty); move |_, (text, error, props)| { dirty.set(false); - props.set_text(text.clone()); + props.set_text(text.clone(), dirty.clone()); error.set(None); modified.set(false); } @@ -213,7 +222,9 @@ pub fn debug_panel(props: &DebugPanelProps) -> Html { } impl DebugPanelProps { - fn set_text(&self, setter: UseStateSetter>) { + /// Asynchronously read this panel's config into the editor, unless the + /// editor has unapplied text by the time the read lands. + fn set_text(&self, setter: UseStateSetter>, dirty: Dirty) { let props = self.clone(); ApiFuture::spawn(async move { let config = crate::queries::get_viewer_config( @@ -226,7 +237,10 @@ impl DebugPanelProps { let js_string = js_sys::JSON::stringify_with_replacer_and_space(&json, &JsValue::NULL, &2.into())?; - setter.set(Rc::new(js_string.as_string().unwrap())); + if !dirty.get() { + setter.set(Rc::new(js_string.as_string().unwrap())); + } + Ok(()) }); } @@ -245,7 +259,7 @@ impl DebugPanelProps { } error.set(None); - props.set_text(text.clone()); + props.set_text(text.clone(), dirty.clone()); modified.set(false); } } @@ -301,7 +315,7 @@ impl DebugPanelProps { dirty.set(false); error.set(None); modified.set(false); - props.set_text(text); + props.set_text(text, dirty.clone()); }, Err(e) => { let message = format!("{e}"); diff --git a/rust/perspective-viewer/src/rust/components/main_panel/render.rs b/rust/perspective-viewer/src/rust/components/main_panel/render.rs index dfca9af7f7..f7e20ec645 100644 --- a/rust/perspective-viewer/src/rust/components/main_panel/render.rs +++ b/rust/perspective-viewer/src/rust/components/main_panel/render.rs @@ -406,6 +406,7 @@ impl MainPanel { on_reset={ctx.props().on_reset.clone()} {on_new_panel} session_props={ctx.props().session_props.clone()} + render_error={ctx.props().renderer_props.render_error.clone()} presentation_props={ctx.props().presentation_props.clone()} is_settings_open={ctx.props().is_settings_open} update_count={ctx.props().update_count} diff --git a/rust/perspective-viewer/src/rust/components/settings_panel.rs b/rust/perspective-viewer/src/rust/components/settings_panel.rs index fcdcd361f2..6e4dfe0755 100644 --- a/rust/perspective-viewer/src/rust/components/settings_panel.rs +++ b/rust/perspective-viewer/src/rust/components/settings_panel.rs @@ -22,7 +22,7 @@ use super::plugin_tab::PluginTab; use crate::components::debug_panel::DebugPanel; use crate::config::{PluginStaticConfig, PluginUpdate}; use crate::presentation::{ColumnLocator, ColumnSettingsTarget, OpenColumnSettings, Presentation}; -use crate::queries::classify_column; +use crate::queries::{classify_column, is_pivot_column}; use crate::renderer::*; use crate::session::column_defaults_update::*; use crate::session::*; @@ -166,8 +166,7 @@ pub fn SettingsPanel(props: &SettingsPanelProps) -> Html { let used = config.columns.iter().any(|maybe_col| { maybe_col.as_ref().map(|col| col == n).unwrap_or_default() - }) || config.group_by.iter().any(|col| col == n) - || config.split_by.iter().any(|col| col == n) + }) || is_pivot_column(n, config) || config.filter.iter().any(|col| col.column() == n) || config.sort.iter().any(|col| &col.0 == n); (used && props.renderer.can_render_column_styles()).then_some(locator) @@ -193,7 +192,7 @@ pub fn SettingsPanel(props: &SettingsPanelProps) -> Html { let session_metadata = props.metadata.clone(); let view_config = props.view_config.clone(); Callback::from(move |plugin_name: String| { - if session.is_errored() { + if session.is_errored() || renderer.failure().is_some() { return; } // Pure resolve — the swap itself is committed inside the locked diff --git a/rust/perspective-viewer/src/rust/components/status_bar.rs b/rust/perspective-viewer/src/rust/components/status_bar.rs index c22529592f..a4e8c7de45 100644 --- a/rust/perspective-viewer/src/rust/components/status_bar.rs +++ b/rust/perspective-viewer/src/rust/components/status_bar.rs @@ -50,6 +50,9 @@ pub struct StatusBarProps { /// `has_table_cells`, `error` from session_props; `selected_theme`, /// `available_themes` from presentation_props. pub session_props: SessionProps, + + /// The failure of the active panel's last draw. + pub render_error: Option, pub presentation_props: PresentationProps, /// Derived from root: `settings_open && has_table_loaded`. Used @@ -207,6 +210,7 @@ impl Component for StatusBar { }, StatusBarMsg::ResetTheme => { update_theme( + &ctx.props().session, &ctx.props().renderer, &ctx.props().presentation, &ctx.props().workspace, @@ -216,6 +220,7 @@ impl Component for StatusBar { }, StatusBarMsg::SetTheme(theme_name) => { update_theme( + &ctx.props().session, &ctx.props().renderer, &ctx.props().presentation, &ctx.props().workspace, @@ -261,7 +266,8 @@ impl Component for StatusBar { } = ctx.props(); let has_table = ctx.props().session_props.has_table.clone(); - let is_errored = ctx.props().session_props.is_errored(); + let is_errored = + ctx.props().session_props.is_errored() || ctx.props().render_error.is_some(); let is_settings_open = ctx.props().is_settings_open; let mut is_updating_class_name = classes!(); @@ -374,6 +380,7 @@ impl Component for StatusBar { {session} update_count={ctx.props().update_count} session_props={ctx.props().session_props.clone()} + render_error={ctx.props().render_error.clone()} /> if is_panel_bar {
diff --git a/rust/perspective-viewer/src/rust/components/status_indicator.rs b/rust/perspective-viewer/src/rust/components/status_indicator.rs index 88debcb5a6..5e48c477ed 100644 --- a/rust/perspective-viewer/src/rust/components/status_indicator.rs +++ b/rust/perspective-viewer/src/rust/components/status_indicator.rs @@ -15,7 +15,7 @@ use perspective_js::utils::ApiError; use web_sys::*; use yew::prelude::*; -use crate::renderer::Renderer; +use crate::renderer::{RenderError, Renderer}; use crate::session::{Session, SessionProps, TableLoadState}; use crate::tasks::apply_and_render; use crate::utils::*; @@ -32,6 +32,7 @@ pub struct StatusIndicatorProps { /// TODO(texodus): remove this pub update_count: u32, pub session_props: SessionProps, + pub render_error: Option, } /// An indicator component which displays the current status of the perspective @@ -47,6 +48,8 @@ pub fn StatusIndicator(props: &StatusIndicatorProps) -> Html { err.kind(), err.is_reconnect(), ) + } else if let Some(RenderError(err)) = &props.render_error { + StatusIconState::Errored(err.message(), err.stacktrace(), err.kind(), true) } else if !has_table_cells && matches!(props.session_props.has_table, Some(TableLoadState::Loading)) { @@ -76,6 +79,10 @@ pub fn StatusIndicator(props: &StatusIndicatorProps) -> Html { async move |_: MouseEvent, (session, renderer, state)| { match &state { StatusIconState::Errored(..) => { + if renderer.failure().is_some() { + session.reset_failed_config().await?; + } + session.reconnect().await?; apply_and_render(session, renderer, ViewConfigUpdate::default())?.await?; }, diff --git a/rust/perspective-viewer/src/rust/components/viewer.rs b/rust/perspective-viewer/src/rust/components/viewer.rs index adc13d5af0..97d0458083 100644 --- a/rust/perspective-viewer/src/rust/components/viewer.rs +++ b/rust/perspective-viewer/src/rust/components/viewer.rs @@ -151,7 +151,7 @@ impl Component for PerspectiveViewer { let elem = ctx.props().elem.clone(); let fonts = FontLoaderProps::new(&elem, ctx.link().callback(|()| PreloadFontsUpdate)); let empty_session = Session::new(); - let empty_renderer = Renderer::new(&elem, empty_session.cell()); + let empty_renderer = Renderer::new(&elem); let active_session = ctx .props() .workspace diff --git a/rust/perspective-viewer/src/rust/components/viewer/snapshots.rs b/rust/perspective-viewer/src/rust/components/viewer/snapshots.rs index ffcde99736..b3a608c66f 100644 --- a/rust/perspective-viewer/src/rust/components/viewer/snapshots.rs +++ b/rust/perspective-viewer/src/rust/components/viewer/snapshots.rs @@ -23,7 +23,7 @@ use super::PerspectiveViewer; use crate::presentation::{DragDropProps, PresentationProps}; use crate::renderer::RendererProps; use crate::session::{SessionProps, TableLoadState}; -use crate::tasks::resize_visible_panels; +use crate::tasks::{resize_visible_panels, submit_theme}; impl PerspectiveViewer { pub(super) fn refresh_session_snapshot(&mut self, ctx: &Context) { @@ -105,7 +105,7 @@ impl PerspectiveViewer { continue; } - panel.renderer.set_theme(new_default.clone()); + submit_theme(&panel.session, &panel.renderer, new_default.clone()); if panel.renderer.needs_restyle() { let renderer = panel.renderer.clone(); crate::utils::spawn_owned("default-theme-restyle", async move { diff --git a/rust/perspective-viewer/src/rust/components/viewer/wiring.rs b/rust/perspective-viewer/src/rust/components/viewer/wiring.rs index c84f4f4923..ef2733b2ee 100644 --- a/rust/perspective-viewer/src/rust/components/viewer/wiring.rs +++ b/rust/perspective-viewer/src/rust/components/viewer/wiring.rs @@ -446,6 +446,15 @@ pub(super) fn inject_active_callbacks( *renderer.on_render_limits_changed.borrow_mut() = Some(cb); } + + { + let r = renderer.clone(); + let cb = ctx + .link() + .callback(move |_: ()| UpdateRenderer(Box::new(r.to_props(None)))); + + *renderer.on_render_failure_changed.borrow_mut() = Some(cb); + } } /// Clear the active panel's session/renderer direct callbacks so a @@ -454,6 +463,7 @@ pub(super) fn clear_active_callbacks(session: &Session, renderer: &Renderer) { *session.on_stats_changed.borrow_mut() = None; *session.on_table_errored.borrow_mut() = None; *renderer.on_render_limits_changed.borrow_mut() = None; + *renderer.on_render_failure_changed.borrow_mut() = None; } /// Inject element-level (shared) presentation callbacks. Created once at diff --git a/rust/perspective-viewer/src/rust/config/column_config_schema.rs b/rust/perspective-viewer/src/rust/config/column_config_schema.rs index 455b91fcb0..e6561c1ae8 100644 --- a/rust/perspective-viewer/src/rust/config/column_config_schema.rs +++ b/rust/perspective-viewer/src/rust/config/column_config_schema.rs @@ -523,380 +523,3 @@ pub fn filter_to_schema( .map(|(k, v)| (k.clone(), v.clone())) .collect() } - -#[cfg(test)] -mod tests { - use serde_json::json; - - use super::*; - - fn color(key: &str, default: &str) -> ControlSpec { - ControlSpec::Color { - key: key.to_owned(), - default: default.to_owned(), - } - } - - fn flag(key: &str) -> ControlSpec { - ControlSpec::Bool { - key: key.to_owned(), - default: false, - } - } - - fn group(key: &str, fields: Vec) -> ControlSpec { - ControlSpec::Group { - key: key.to_owned(), - fields, - } - } - - #[test] - fn group_deserializes_recursively() { - let schema: ColumnConfigSchema = serde_json::from_value(json!({ - "fields": [{ - "kind": "Group", - "key": "legend", - "fields": [ - { "kind": "Bool", "key": "legend_on", "default": false }, - { - "kind": "Group", - "key": "inner", - "fields": [{ "kind": "Color", "key": "color", "default": "#ff0000" }] - } - ] - }] - })) - .unwrap(); - - let keys = schema.active_keys(); - assert_eq!( - keys, - HashSet::from(["legend_on".to_owned(), "color".to_owned()]) - ); - - let leaves = schema.leaf_fields(); - assert_eq!(leaves.len(), 2); - assert!( - leaves - .iter() - .all(|s| !matches!(s, ControlSpec::Group { .. })) - ); - } - - #[test] - fn grouped_schema_is_equivalent_to_flat() { - let flat = ColumnConfigSchema { - fields: vec![flag("stack"), color("color", "#0366d6")], - }; - - let grouped = ColumnConfigSchema { - fields: vec![group("series", vec![ - flag("stack"), - color("color", "#0366d6"), - ])], - }; - - assert_eq!(flat.active_keys(), grouped.active_keys()); - assert_eq!(flat.css_kind_of("color"), grouped.css_kind_of("color")); - assert_eq!(flat.css_kind_of("stack"), grouped.css_kind_of("stack")); - } - - #[test] - fn enum_variants_see_through_groups() { - let mode = ControlSpec::Enum { - key: "fg_mode".to_owned(), - default: "color".to_owned(), - variants: vec![ - EnumVariant { - value: "disabled".to_owned(), - label: None, - }, - EnumVariant { - value: "color".to_owned(), - label: None, - }, - ], - }; - - let grouped = ColumnConfigSchema { - fields: vec![group("color", vec![mode, flag("flag")])], - }; - - let values: Vec<&str> = grouped - .enum_variants_of("fg_mode") - .unwrap() - .iter() - .map(|v| v.value.as_str()) - .collect(); - - assert_eq!(values, vec!["disabled", "color"]); - assert!(grouped.enum_variants_of("flag").is_none()); - assert!(grouped.enum_variants_of("missing").is_none()); - } - - #[test] - fn format_controls_group_and_merge() { - let schema = ColumnConfigSchema { - fields: vec![ - ControlSpec::NumberFormat { default: None }, - flag("flag"), - ControlSpec::DatetimeFormat { default: None }, - ], - } - .group_format_controls(); - - assert_eq!(schema.fields.len(), 2); - let ControlSpec::Group { key, fields } = &schema.fields[0] else { - panic!("expected format group first"); - }; - - assert_eq!(key, "format"); - assert!(matches!(fields[0], ControlSpec::NumberFormat { .. })); - assert!(matches!(fields[1], ControlSpec::DatetimeFormat { .. })); - assert!(matches!(&schema.fields[1], ControlSpec::Bool { .. })); - - assert_eq!( - schema.active_keys(), - HashSet::from([ - "number_format".to_owned(), - "date_format".to_owned(), - "flag".to_owned() - ]) - ); - } - - #[test] - fn font_owns_its_size_and_toggle_keys() { - let spec = ControlSpec::Font { - key: "font_family".to_owned(), - default: "inherit".to_owned(), - size: Some(FontSize { - key: "font_size".to_owned(), - default: 12.0, - min: None, - max: None, - step: None, - }), - bold: Some(FontToggle { - key: "bold".to_owned(), - default: false, - }), - italic: None, - }; - - assert_eq!(spec.serialized_keys(), vec![ - "font_family", - "font_size", - "bold" - ]); - } - - #[test] - fn format_grouping_recurses_but_never_double_wraps() { - let schema = ColumnConfigSchema { - fields: vec![ - group("format", vec![ControlSpec::NumberFormat { default: None }]), - group("styling", vec![flag("x"), ControlSpec::DatetimeFormat { - default: None, - }]), - ], - } - .group_format_controls(); - - let ControlSpec::Group { key, fields } = &schema.fields[0] else { - panic!("expected group"); - }; - - assert_eq!(key, "format"); - assert!(matches!(fields[0], ControlSpec::NumberFormat { .. })); - - let ControlSpec::Group { fields, .. } = &schema.fields[1] else { - panic!("expected group"); - }; - - assert!(matches!( - &fields[1], - ControlSpec::Group { key, fields } - if key == "format" && matches!(fields[0], ControlSpec::DatetimeFormat { .. }) - )); - } - - #[test] - fn format_controls_deserialize_without_default_payload() { - let schema: ColumnConfigSchema = serde_json::from_value(json!({ - "fields": [{ "kind": "NumberFormat" }, { "kind": "DatetimeFormat" }] - })) - .unwrap(); - - assert!(matches!(&schema.fields[0], ControlSpec::NumberFormat { - default: None - })); - assert!(matches!(&schema.fields[1], ControlSpec::DatetimeFormat { - default: None - })); - } - - #[test] - fn number_format_default_payload_deserializes_flattened_families() { - let schema: ColumnConfigSchema = serde_json::from_value(json!({ - "fields": [{ - "kind": "NumberFormat", - "default": { - "notation": "compact", - "compactDisplay": "short", - "minimumFractionDigits": 0, - "maximumFractionDigits": 1 - } - }] - })) - .unwrap(); - - let ControlSpec::NumberFormat { - default: Some(default), - } = &schema.fields[0] - else { - panic!("expected NumberFormat with default"); - }; - - assert_eq!( - default._notation, - Some(crate::config::Notation::Compact( - crate::config::CompactDisplay::Short - )) - ); - assert_eq!(default._style, None); - assert_eq!(default.minimum_fraction_digits, Some(0.)); - assert_eq!(default.maximum_fraction_digits, Some(1.)); - assert_eq!( - schema.active_keys(), - HashSet::from(["number_format".to_owned()]) - ); - } - - #[test] - fn datetime_format_default_payload_deserializes_simple_arm() { - let schema: ColumnConfigSchema = serde_json::from_value(json!({ - "fields": [{ - "kind": "DatetimeFormat", - "default": { "dateStyle": "medium", "timeStyle": "disabled" } - }] - })) - .unwrap(); - - let ControlSpec::DatetimeFormat { - default: Some(DatetimeFormatType::Simple(simple)), - } = &schema.fields[0] - else { - panic!("expected DatetimeFormat with Simple default"); - }; - - assert_eq!( - simple.date_style, - crate::config::SimpleDatetimeFormat::Medium - ); - assert_eq!( - simple.time_style, - crate::config::SimpleDatetimeFormat::Disabled - ); - } - - #[test] - fn alignment_tokens_round_trip() { - for align in Alignment::ALL { - assert_eq!(Alignment::parse(align.as_str()), Some(align)); - assert_eq!(serde_json::to_value(align).unwrap(), json!(align.as_str())); - } - - for bad in ["middle", "left-top", "top-center", "middle-left", ""] { - assert_eq!(Alignment::parse(bad), None, "{bad}"); - } - - assert_eq!(Alignment::ALL.iter().filter(|x| x.is_corner()).count(), 4); - assert_eq!(Alignment::TopLeft.humanized(), "Top Left"); - assert_eq!(Alignment::Center.humanized(), "Center"); - } - - #[test] - fn alignment_deserializes_with_optional_default_and_corners() { - let schema: ColumnConfigSchema = serde_json::from_value(json!({ - "fields": [ - { "kind": "Alignment", "key": "align" }, - { "kind": "Alignment", "key": "legend_anchor", "default": "top-right", "corners": true } - ] - })) - .unwrap(); - - assert!(matches!(&schema.fields[0], ControlSpec::Alignment { - default: None, - corners: false, - .. - })); - assert!(matches!(&schema.fields[1], ControlSpec::Alignment { - default: Some(Alignment::TopRight), - corners: true, - .. - })); - assert_eq!( - schema.active_keys(), - HashSet::from(["align".to_owned(), "legend_anchor".to_owned()]) - ); - assert!( - serde_json::from_value::(json!({ - "fields": [{ "kind": "Alignment", "key": "x", "default": "middle" }] - })) - .is_err() - ); - } - - #[test] - fn canonicalize_defaults_drops_non_corner_default_of_corners_field() { - let schema = ColumnConfigSchema { - fields: vec![ - ControlSpec::Alignment { - key: "bad".to_owned(), - default: Some(Alignment::Top), - corners: true, - }, - ControlSpec::Alignment { - key: "ok".to_owned(), - default: Some(Alignment::Top), - corners: false, - }, - ControlSpec::Alignment { - key: "unset".to_owned(), - default: None, - corners: true, - }, - ], - } - .canonicalize_defaults(); - - assert_eq!( - schema.active_keys(), - HashSet::from(["ok".to_owned(), "unset".to_owned()]) - ); - } - - #[test] - fn canonicalize_defaults_recurses_and_drops_empty_groups() { - let schema = ColumnConfigSchema { - fields: vec![ - group("ok", vec![color("good", "RGB(255,0,0)"), flag("flag")]), - group("doomed", vec![color("bad", "not-a-color")]), - ], - } - .canonicalize_defaults(); - - assert_eq!(schema.fields.len(), 1); - let ControlSpec::Group { key, fields } = &schema.fields[0] else { - panic!("expected group"); - }; - - assert_eq!(key, "ok"); - assert!(matches!( - &fields[0], - ControlSpec::Color { default, .. } if default == "#ff0000" - )); - } -} diff --git a/rust/perspective-viewer/src/rust/config/number_string_format.rs b/rust/perspective-viewer/src/rust/config/number_string_format.rs index 4255dabb23..bd53cf2e64 100644 --- a/rust/perspective-viewer/src/rust/config/number_string_format.rs +++ b/rust/perspective-viewer/src/rust/config/number_string_format.rs @@ -422,97 +422,3 @@ impl CustomNumberFormatConfig { } } } - -#[cfg(test)] -mod tests { - use super::*; - - fn sig(min: f64, max: f64) -> CustomNumberFormatConfig { - CustomNumberFormatConfig { - minimum_significant_digits: Some(min), - maximum_significant_digits: Some(max), - ..Default::default() - } - } - - #[test] - fn resolve_overlays_partial_default_on_builtin() { - let spec = CustomNumberFormatConfig { - _notation: Some(Notation::Compact(CompactDisplay::Short)), - minimum_fraction_digits: Some(0.), - maximum_fraction_digits: Some(1.), - ..Default::default() - }; - - let defaults = NumberFormatDefaults::resolve(Some(&spec), true); - assert_eq!(defaults.notation, Notation::Compact(CompactDisplay::Short)); - assert_eq!(defaults.fraction, (0., 1.)); - assert_eq!(defaults.style, NumberFormatStyle::Decimal); - assert_eq!(defaults.significant, (1., 21.)); - assert_eq!( - NumberFormatDefaults::resolve(None, true), - NumberFormatDefaults::builtin(true) - ); - } - - #[test] - fn filter_preserves_max_significant_digits() { - let filtered = sig(1., 5.).filter_default(&NumberFormatDefaults::builtin(true)); - assert_eq!(filtered.minimum_significant_digits, Some(1.)); - assert_eq!(filtered.maximum_significant_digits, Some(5.)); - - let filtered = sig(3., 21.).filter_default(&NumberFormatDefaults::builtin(true)); - assert_eq!(filtered.minimum_significant_digits, Some(3.)); - assert_eq!(filtered.maximum_significant_digits, Some(21.)); - } - - #[test] - fn filter_elides_values_equal_to_resolved_defaults() { - let defaults = NumberFormatDefaults { - fraction: (0., 1.), - notation: Notation::Compact(CompactDisplay::Short), - ..NumberFormatDefaults::builtin(true) - }; - - let config = CustomNumberFormatConfig { - _notation: Some(Notation::Compact(CompactDisplay::Short)), - minimum_fraction_digits: Some(0.), - maximum_fraction_digits: Some(1.), - ..Default::default() - }; - - assert_eq!( - config.filter_default(&defaults), - CustomNumberFormatConfig::default() - ); - } - - #[test] - fn filter_serializes_builtin_values_under_override() { - let defaults = NumberFormatDefaults { - notation: Notation::Compact(CompactDisplay::Short), - ..NumberFormatDefaults::builtin(true) - }; - - let config = CustomNumberFormatConfig { - _notation: Some(Notation::Standard), - ..Default::default() - }; - - let filtered = config.filter_default(&defaults); - assert_eq!(filtered._notation, Some(Notation::Standard)); - } - - #[test] - fn filter_bakes_fraction_digits_for_percent_style() { - let config = CustomNumberFormatConfig { - _style: Some(NumberFormatStyle::Percent), - ..Default::default() - }; - - let filtered = config.filter_default(&NumberFormatDefaults::builtin(true)); - assert_eq!(filtered._style, Some(NumberFormatStyle::Percent)); - assert_eq!(filtered.minimum_fraction_digits, Some(2.)); - assert_eq!(filtered.maximum_fraction_digits, Some(2.)); - } -} diff --git a/rust/perspective-viewer/src/rust/config/plugin_static_config.rs b/rust/perspective-viewer/src/rust/config/plugin_static_config.rs index 6367cb59c0..06cb8dc33a 100644 --- a/rust/perspective-viewer/src/rust/config/plugin_static_config.rs +++ b/rust/perspective-viewer/src/rust/config/plugin_static_config.rs @@ -215,45 +215,3 @@ impl PluginStaticConfig { .unwrap_or_default() } } - -#[cfg(test)] -mod tests { - use super::*; - - fn plugin(names: &[&str]) -> PluginStaticConfig { - PluginStaticConfig { - config_column_names: names.iter().map(|x| (*x).to_owned()).collect(), - ..Default::default() - } - } - - /// The tail rule has ONE definition: `is_swap` and the role a - /// trailing column repeats must never disagree about where the - /// positional slots end. - #[test] - fn positional_slots_and_tail_agree() { - // `Y Line` — the only named slot IS the tail, so every column - // is another Y series and none of them swap. - let y_line = plugin(&["Y Axis"]); - assert_eq!(y_line.positional_columns(), 0); - assert_eq!(y_line.tail_column_role(), Some("Y Axis")); - assert!(!y_line.is_swap(0)); - - // `X/Y Line` — two pinned axes, then tooltips. - let xy_line = plugin(&["X Axis", "Y Axis", "Tooltip"]); - assert_eq!(xy_line.positional_columns(), 2); - assert_eq!(xy_line.tail_column_role(), Some("Tooltip")); - assert!(xy_line.is_swap(0)); - assert!(xy_line.is_swap(1)); - assert!(!xy_line.is_swap(2)); - - for index in 0..4 { - assert_eq!(xy_line.is_swap(index), index < xy_line.positional_columns()); - } - - let unnamed = plugin(&[]); - assert_eq!(unnamed.positional_columns(), 0); - assert_eq!(unnamed.tail_column_role(), None); - assert!(!unnamed.is_swap(0)); - } -} diff --git a/rust/perspective-viewer/src/rust/config/workspace_config.rs b/rust/perspective-viewer/src/rust/config/workspace_config.rs index 78cd497896..add9b55ffc 100644 --- a/rust/perspective-viewer/src/rust/config/workspace_config.rs +++ b/rust/perspective-viewer/src/rust/config/workspace_config.rs @@ -138,129 +138,3 @@ pub fn validate_palette( }) .collect() } - -#[cfg(test)] -mod tests { - use serde_json::json; - - use super::*; - - #[test] - fn palette_serializes_only_when_present() { - let config = WorkspaceConfig { - version: "x".to_owned(), - active: None, - layout: None, - panels: BTreeMap::new(), - global_filters: vec![], - masters: vec![], - palette: BTreeMap::new(), - }; - - assert_eq!( - serde_json::to_value(&config).unwrap(), - json!({ "version": "x", "layout": null, "panels": {} }) - ); - - let mut palette = BTreeMap::new(); - palette.insert("--psp-user--color-hot".to_owned(), "#ff0000".to_owned()); - - let config = WorkspaceConfig { palette, ..config }; - assert_eq!( - serde_json::to_value(&config).unwrap(), - json!({ - "version": "x", - "layout": null, - "panels": {}, - "palette": { "--psp-user--color-hot": "#ff0000" }, - }) - ); - } - - #[test] - fn update_absent_fields_are_missing_and_null_resets() { - let update: WorkspaceConfigUpdate = serde_json::from_value(json!({})).unwrap(); - assert!(update.panels.is_none()); - assert!(update.layout.is_none()); - assert_eq!(update.active, OptionalUpdate::Missing); - assert_eq!(update.global_filters, OptionalUpdate::Missing); - assert_eq!(update.masters, OptionalUpdate::Missing); - assert_eq!(update.palette, OptionalUpdate::Missing); - - let update: WorkspaceConfigUpdate = serde_json::from_value(json!({ - "panels": null, - "active": null, - "global_filters": null, - "masters": null, - "palette": null, - })) - .unwrap(); - - assert!(update.panels.is_none()); - assert_eq!(update.active, OptionalUpdate::SetDefault); - assert_eq!(update.global_filters, OptionalUpdate::SetDefault); - assert_eq!(update.masters, OptionalUpdate::SetDefault); - assert_eq!(update.palette, OptionalUpdate::SetDefault); - - let update: WorkspaceConfigUpdate = serde_json::from_value(json!({ - "panels": {}, - "global_filters": [["Region", "==", "West"]], - "masters": ["a"], - })) - .unwrap(); - - assert_eq!(update.panels.map(|p| p.len()), Some(0)); - assert_eq!(update.masters, OptionalUpdate::Update(vec!["a".to_owned()])); - assert!(matches!(update.global_filters, OptionalUpdate::Update(ref f) if f.len() == 1)); - } - - #[test] - fn update_palette_defaults_empty_and_validates_by_prefix() { - let update: WorkspaceConfigUpdate = - serde_json::from_value(json!({ "panels": {} })).unwrap(); - assert_eq!(update.palette, OptionalUpdate::Missing); - - let update: WorkspaceConfigUpdate = serde_json::from_value(json!({ - "panels": {}, - "palette": { - "--psp-user--gradient-1": "linear-gradient(#000, #fff)", - "--psp-user--palette-warm": "linear-gradient(90deg, RGB(255,0,0), #ff0)", - "--psp-user--color-hot": "#F00", - }, - })) - .unwrap(); - - let OptionalUpdate::Update(palette) = update.palette else { - panic!("palette missing"); - }; - - let valid = validate_palette(palette).unwrap(); - assert_eq!( - valid.get("--psp-user--gradient-1").unwrap(), - "linear-gradient(to right, #000000 0%, #ffffff 100%)" - ); - assert_eq!( - valid.get("--psp-user--palette-warm").unwrap(), - "linear-gradient(to right, #ff0000, #ffff00)" - ); - assert_eq!(valid.get("--psp-user--color-hot").unwrap(), "#ff0000"); - - let bad = |name: &str, value: &str| { - let mut map = BTreeMap::new(); - map.insert(name.to_owned(), value.to_owned()); - validate_palette(map).unwrap_err() - }; - - assert!(bad("--psp-user--other-1", "#ff0000").contains("--psp-user--other-1")); - assert!(bad("--psp-charts--gradient", "#ff0000").contains("must start with")); - assert!( - bad( - "--psp-user--palette-1", - "linear-gradient(#000 0%, #fff 100%)" - ) - .contains("--psp-user--palette-1") - ); - assert!(bad("--psp-user--gradient-1", "#ff0000").contains("linear-gradient")); - assert!(bad("--psp-user--color-1", "red").contains("red")); - } -} diff --git a/rust/perspective-viewer/src/rust/custom_elements/viewer.rs b/rust/perspective-viewer/src/rust/custom_elements/viewer.rs index ff218840f7..66c44877d8 100644 --- a/rust/perspective-viewer/src/rust/custom_elements/viewer.rs +++ b/rust/perspective-viewer/src/rust/custom_elements/viewer.rs @@ -37,9 +37,7 @@ use crate::js::*; use crate::presentation::*; use crate::queries::*; use crate::root::Root; -use crate::session::{ - BindPlan, Disposal, MissingTable, OpKind, ResetOptions, StepOutcome, TableLoadState, -}; +use crate::session::{BindPlan, Disposal, MissingTable, OpKind, ResetOptions, TableLoadState}; use crate::tasks::*; use crate::utils::*; use crate::workspace::{Panel, PanelId, Workspace}; @@ -697,13 +695,13 @@ impl PerspectiveViewerElement { }, Ok(Some(Discard::Unclaimed(panel))) => { eject_panel(panel, Disposal::Reject).await?; - Ok(StepOutcome::Done) + Ok(None) }, Ok(Some(Discard::Evicted(panel))) => { eject_panel(panel, Disposal::Resolve).await?; Err(ApiError::new(CREATE_REQUIRES_TABLE)) }, - Ok(None) => Ok(StepOutcome::Done), + Ok(None) => Ok(None), } }) } @@ -1025,6 +1023,7 @@ impl PerspectiveViewerElement { && presentation.is_visible() && !panel.renderer.is_plugin_activated()? && panel.session.get_error().is_none() + && panel.renderer.failure().is_none() && matches!(panel.session.has_table(), Some(TableLoadState::Loaded)) { set_panel_paused(&panel.session, &panel.renderer, &presentation, true) @@ -1319,12 +1318,12 @@ impl PerspectiveViewerElement { }; let reset_effect = self.workspace.effects().guard(); - let reset = panel.session.submit(OpKind::Restore { fields: None }, { + let reset = panel.session.submit(OpKind::Restore { update: None }, { let session = panel.session.clone(); move |_ctx| { Box::pin(async move { session.reset(ResetOptions::default()).await?; - Ok(StepOutcome::Done) + Ok(None) }) } }); @@ -1876,7 +1875,7 @@ impl PerspectiveViewerElement { // alone must repaint nothing. let stale = theme.as_ref().is_none_or(|x| !available.contains(x)); if (stale || theme == previous) && theme != active { - panel.renderer.set_theme(active.clone()); + submit_theme(&panel.session, &panel.renderer, active.clone()); if panel.renderer.needs_restyle() { panel.renderer.restyle_all().await?; } diff --git a/rust/perspective-viewer/src/rust/exprtk/tokenize.rs b/rust/perspective-viewer/src/rust/exprtk/tokenize.rs index 495d1e71cc..463b62bcaa 100644 --- a/rust/perspective-viewer/src/rust/exprtk/tokenize.rs +++ b/rust/perspective-viewer/src/rust/exprtk/tokenize.rs @@ -151,57 +151,3 @@ pub fn tokenize(input: &str) -> Vec> { tokens } - -#[cfg(test)] -mod tests { - use wasm_bindgen_test::*; - - use super::*; - - #[wasm_bindgen_test] - fn test_simple() { - let s = "123 abc 'hello' \"Sales\""; - assert_eq!(tokenize(s), vec![ - Literal("123"), - Whitespace(" "), - Symbol("abc"), - Whitespace(" "), - Column("'hello'"), - Whitespace(" "), - Literal("\"Sales\"") - ]); - } - - #[wasm_bindgen_test] - fn test_complex_string() { - let s = "'this is 'a \"test of\" strings"; - assert_eq!(tokenize(s), vec![ - Column("'this is '"), - Symbol("a"), - Whitespace(" "), - Literal("\"test of\""), - Whitespace(" "), - Symbol("strings"), - ]); - } - - #[wasm_bindgen_test] - fn test_comment_newline() { - let s = "// Title\n1 + 2"; - assert_eq!(tokenize(s), vec![ - Comment("// Title"), - Break("\n"), - Literal("1"), - Whitespace(" "), - Operator("+"), - Whitespace(" "), - Literal("2"), - ]); - } - - #[wasm_bindgen_test] - fn test_escape_strings() { - let s = "'test\\/'"; - assert_eq!(tokenize(s), vec![Literal("'test\\/'"),]); - } -} diff --git a/rust/perspective-viewer/src/rust/js/plugin.rs b/rust/perspective-viewer/src/rust/js/plugin.rs index 94d3712114..c1bfabe540 100644 --- a/rust/perspective-viewer/src/rust/js/plugin.rs +++ b/rust/perspective-viewer/src/rust/js/plugin.rs @@ -51,7 +51,7 @@ extern "C" { /// call — the view re-renders and re-queries the schema once the /// async fetch resolves. #[wasm_bindgen(method, catch, js_name = column_config_schema)] - pub fn _column_config_schema(this: &JsPerspectiveViewerPlugin, view_type: &str, group: Option<&str>, column_name: &str, current_value: &JsValue, view_config: &JsValue, plugin_config: &JsValue) -> ApiResult; + pub fn _column_config_schema(this: &JsPerspectiveViewerPlugin, view_type: &str, group: Option<&str>, column_name: &str, current_value: &JsValue, view_config: &JsValue, plugin_config: &JsValue, role: &str) -> ApiResult; /// The plugin-level settings schema as it applies to `current_value`, the /// `plugin_config` state the schema should describe. diff --git a/rust/perspective-viewer/src/rust/queries/column_locator.rs b/rust/perspective-viewer/src/rust/queries/column_locator.rs index 40ec4f3997..041e330d2b 100644 --- a/rust/perspective-viewer/src/rust/queries/column_locator.rs +++ b/rust/perspective-viewer/src/rust/queries/column_locator.rs @@ -10,7 +10,7 @@ // ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ // ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ -use perspective_client::config::ViewConfig; +use perspective_client::config::{ColumnType, ViewConfig}; use crate::presentation::{ColumnLocator, ColumnSettingsTarget, OpenColumnSettings}; use crate::renderer::Renderer; @@ -37,11 +37,62 @@ pub fn classify_column( } } +/// Whether `name` is one of `view_config`'s pivots, whose own raw values a +/// plugin draws as a header label or axis tick and therefore formats with its +/// TABLE type rather than an aggregate's view type. +pub fn is_pivot_column(name: &str, view_config: &ViewConfig) -> bool { + view_config.group_by.iter().any(|col| col == name) + || view_config.split_by.iter().any(|col| col == name) +} + +/// Whether `name` is configured AS a pivot: a pivot that is also in `columns` +/// is configured as that aggregate instead, so its pivot copy is not +/// separately configurable. +pub fn is_pivot_only_column(name: &str, view_config: &ViewConfig) -> bool { + is_pivot_column(name, view_config) + && !view_config + .columns + .iter() + .any(|col| col.as_deref() == Some(name)) +} + +/// Whether the active plugin offers `name` any column config, gating every +/// affordance that opens the Style tab. +// TODO This hardcodes the set of types a plugin offers a pivot a format control +// for, instead of asking it. `get_column_config_schema` answers exactly +// (`!schema.leaf_fields().is_empty()`) but costs a plugin call per pivot pill +// per render; switch to it if pills ever outgrow a handful, or if a plugin's +// pivot controls stop being format-only. +pub fn has_column_config( + name: &str, + view_config: &ViewConfig, + metadata: &SessionMetadata, + renderer: &Renderer, +) -> bool { + if !renderer.can_render_column_styles() { + return false; + } + + if view_config + .columns + .iter() + .any(|col| col.as_deref() == Some(name)) + { + return true; + } + + is_pivot_column(name, view_config) + && matches!( + metadata.get_column_table_type(name), + Some(ColumnType::Integer | ColumnType::Float | ColumnType::Date | ColumnType::Datetime) + ) +} + /// Gets a [`ColumnLocator`] for the current UI's column settings state, /// or [`None`] if it is not currently active. /// -/// Table columns only have a useful sidebar (the Style tab) -/// when they're in `view_config.columns`. +/// Table columns only have a useful sidebar (the Style tab) when they're in +/// `view_config.columns` or are a pivot. pub fn get_current_column_locator( open_column_settings: &OpenColumnSettings, renderer: &Renderer, @@ -54,54 +105,10 @@ pub fn get_current_column_locator( let locator = classify_column(name, view_config, metadata)?; match locator { ColumnLocator::Table(_) => { - let in_columns = view_config.columns.iter().any(|maybe_col| { - maybe_col - .as_ref() - .map(|col| col == name) - .unwrap_or_default() - }); - - (in_columns && renderer.can_render_column_styles()).then_some(locator) + has_column_config(name, view_config, metadata, renderer).then_some(locator) }, locator => Some(locator), } }, } } - -#[cfg(test)] -mod tests { - use std::collections::HashMap; - - use perspective_client::config::Expressions; - - use super::*; - - fn config_with_expression(name: &str, expr: &str) -> ViewConfig { - ViewConfig { - expressions: Expressions(HashMap::from([(name.to_owned(), expr.to_owned())])), - ..ViewConfig::default() - } - } - - #[test] - fn classifies_from_the_config_before_metadata_catches_up() { - let config = config_with_expression("x", "1 + 1"); - let metadata = SessionMetadata::default(); - assert_eq!( - classify_column("x", &config, &metadata), - Some(ColumnLocator::Expression("x".to_owned())) - ); - assert_eq!(classify_column("nope", &config, &metadata), None); - } - - #[test] - fn unnamed_expressions_are_keyed_by_their_text() { - let config = config_with_expression("\"Sales\" + 1", "\"Sales\" + 1"); - let metadata = SessionMetadata::default(); - assert_eq!( - classify_column("\"Sales\" + 1", &config, &metadata), - Some(ColumnLocator::Expression("\"Sales\" + 1".to_owned())) - ); - } -} diff --git a/rust/perspective-viewer/src/rust/queries/plugin_column_styles.rs b/rust/perspective-viewer/src/rust/queries/plugin_column_styles.rs index 697ed0449b..d9955e3831 100644 --- a/rust/perspective-viewer/src/rust/queries/plugin_column_styles.rs +++ b/rust/perspective-viewer/src/rust/queries/plugin_column_styles.rs @@ -14,8 +14,9 @@ use itertools::Itertools; use perspective_client::config::ViewConfig; use perspective_js::utils::{ApiResult, JsValueSerdeExt}; +use super::column_locator::is_pivot_only_column; use crate::config::ColumnConfigSchema; -use crate::renderer::Renderer; +use crate::renderer::{ColumnRole, Renderer}; use crate::session::SessionMetadata; /// Queries the active plugin for its plugin-scoped [`ColumnConfigSchema`] as @@ -58,9 +59,22 @@ pub fn get_column_config_schema( .and_then(|(idx, _)| names.get(idx)) .map(|s| s.as_str()); - let view_type = if let Some(x) = metadata.get_column_view_type(column_name) { - x + let role = if !is_pivot_only_column(column_name, view_config) { + ColumnRole::Column + } else if view_config.split_by.iter().any(|x| x == column_name) { + ColumnRole::SplitBy } else { + ColumnRole::GroupBy + }; + + let view_type = match role { + ColumnRole::Column => metadata.get_column_view_type(column_name), + _ => metadata + .get_column_table_type(column_name) + .or_else(|| metadata.get_column_view_type(column_name)), + }; + + let Some(view_type) = view_type else { return Ok(ColumnConfigSchema { fields: vec![] }); }; @@ -82,6 +96,7 @@ pub fn get_column_config_schema( ¤t_js, &view_config_js, &plugin_config_js, + role.as_str(), )?; serde_wasm_bindgen::from_value::(raw) diff --git a/rust/perspective-viewer/src/rust/renderer.rs b/rust/perspective-viewer/src/rust/renderer.rs index 34a8e5e19e..68e88af5c2 100644 --- a/rust/perspective-viewer/src/rust/renderer.rs +++ b/rust/perspective-viewer/src/rust/renderer.rs @@ -25,6 +25,7 @@ mod plugin_store; mod props; mod registry; mod render_timer; +mod state; use std::cell::{Cell, RefCell}; use std::collections::HashMap; @@ -32,7 +33,7 @@ use std::ops::Deref; use std::rc::Rc; use perspective_client::ViewWindow; -use perspective_js::utils::{ApiFuture, ApiResult, JsValueSerdeExt}; +use perspective_js::utils::{ApiError, ApiFuture, ApiResult, JsValueSerdeExt}; use wasm_bindgen::prelude::*; use web_sys::*; use yew::html::ImplicitClone; @@ -41,16 +42,17 @@ use yew::prelude::*; use self::activate::*; pub use self::limits::RenderLimits; pub use self::plugin_config::{ - ColumnConfigMap, PluginScopedConfig, ValidatedColumnsConfig, ValidatedPluginConfig, - apply_columns_config_to, apply_plugin_config_to, + ColumnConfigMap, ColumnRole, PluginScopedConfig, PreparedRenderer, ValidatedColumnsConfig, + ValidatedPluginConfig, }; use self::plugin_store::*; pub use self::props::RendererProps; pub use self::registry::*; use self::render_timer::*; +pub use self::state::{PluginRef, RenderError, RendererState, Staged, StagedEdit}; +use self::state::{RenderOutcome, StagedEdits, project_bucket}; use crate::config::*; use crate::js::plugin::*; -use crate::session::{PanelCell, PluginRef}; use crate::utils::*; /// Minimum geometry delta (px) considered a real size/position change by the @@ -101,8 +103,13 @@ impl Drop for ContextPin { /// Immutable state pub struct RendererData { - /// This panel's committed [`PanelState`]. - cell: PanelCell, + state: RefCell>, + outcome: RefCell, + + /// Injected callback from the root component, fired when a render failure + /// is raised or cleared. + pub on_render_failure_changed: RefCell>>, + staged: Rc, plugin_data: RefCell, draw_lock: DebounceMutex, pub plugin_changed: PubSub, @@ -242,10 +249,13 @@ impl Deref for RendererData { } impl Renderer { - pub fn new(viewer_elem: &HtmlElement, cell: PanelCell) -> Self { + pub fn new(viewer_elem: &HtmlElement) -> Self { let draw_lock = DebounceMutex::default(); Self(Rc::new(RendererData { - cell, + state: RefCell::default(), + outcome: RefCell::default(), + on_render_failure_changed: RefCell::default(), + staged: Rc::default(), plugin_data: RefCell::new(RendererMutData { viewer_elem: viewer_elem.clone(), plugin_store: PluginStore::default(), @@ -289,16 +299,58 @@ impl Renderer { self.0.slot_name.borrow().clone() } - /// Set this panel's theme name. Callers must pass a CONCRETE name — - /// resolve "the default" through `Presentation` before calling, never by - /// leaving this empty. - pub fn set_theme(&self, name: Option) { - self.cell.submit_theme(name); + /// Write this panel's concrete theme name. + pub fn commit_theme(&self, name: Option) { + self.swap(self.state().with_theme(name)); } - /// Write this panel's theme name NOW. - pub fn commit_theme(&self, name: Option) { - self.cell.swap(self.cell.state().with_theme(name)); + /// The committed state, which is what a running step reads. + pub fn state(&self) -> Rc { + self.0.state.borrow().clone() + } + + pub(crate) fn swap(&self, next: RendererState) { + *self.0.state.borrow_mut() = Rc::new(next); + self.set_outcome(RenderOutcome::Never); + } + + /// The failure of the last draw, which stands until the next commit. + pub fn failure(&self) -> Option { + match &*self.0.outcome.borrow() { + RenderOutcome::Failed(error) => Some(error.clone()), + _ => None, + } + } + + /// Record that the draw of the committed state failed. + pub fn fail(&self, error: ApiError) -> ApiResult<()> { + self.set_outcome(RenderOutcome::Failed(RenderError(error.clone()))); + Err(error) + } + + /// Record that the draw of the committed state landed. + pub fn landed(&self) { + self.set_outcome(RenderOutcome::Ok); + } + + fn set_outcome(&self, next: RenderOutcome) { + let was_failed = self.failure().is_some(); + let is_failed = matches!(next, RenderOutcome::Failed(_)); + *self.0.outcome.borrow_mut() = next; + if was_failed != is_failed + && let Some(cb) = self.0.on_render_failure_changed.borrow().as_ref() + { + cb.emit(()); + } + } + + /// Show `edit` to the UI until the returned guard is dropped. + pub fn stage(&self, edit: StagedEdit) -> Staged { + self.0.staged.stage(edit) + } + + fn projected_bucket(&self, name: &str) -> PluginScopedConfig { + project_bucket(self.state().bucket(name), self.0.staged.pending()) } /// [`Self::commit_theme`] plus a synchronous [`Self::stamp_theme`]. @@ -310,15 +362,18 @@ impl Renderer { /// This panel's theme name, as the UI sees it: the committed theme, or the /// latest pending pick. pub fn theme(&self) -> Option { - self.cell.projected_theme() - } + let pending = self + .0 + .staged + .pending() + .into_iter() + .filter_map(|edit| match edit { + StagedEdit::Theme(theme) => Some(theme), + _ => None, + }) + .next_back(); - /// [`Self::set_theme`] plus a synchronous [`Self::stamp_theme`] — the - /// shared "stamp-with-commit" head of every per-panel theme mutation - /// site. - pub fn set_theme_stamped(&self, theme: Option) { - self.set_theme(theme); - self.stamp_theme(None); + pending.unwrap_or_else(|| self.state().theme.clone()) } /// Whether the active plugin's captured `--psp-*` CSS is STALE — the @@ -388,13 +443,12 @@ impl Renderer { timer: MovingWindowRenderTimer::default(), }; - self.cell.swap(self.cell.state().without_plugins()); + self.swap(self.state().without_plugins()); Ok(()) } pub fn metadata(&self) -> Rc { - self.cell - .state() + self.state() .plugin .as_ref() .map(|plugin| plugin.static_config.clone()) @@ -655,7 +709,7 @@ impl Renderer { .cloned() .ok_or("No Plugin")?; - self.cell.swap(self.cell.state().with_plugin(PluginRef { + self.swap(self.state().with_plugin(PluginRef { idx, static_config: config, })); @@ -694,7 +748,7 @@ impl Renderer { // Push the newly-activated plugin's stored bucket through // `plugin.restore` so the swap immediately reflects any // viewer-owned per-column and plugin-level config. - let bucket = self.cell.state().bucket(&self.metadata().name); + let bucket = self.state().bucket(&self.metadata().name); let token = JsValue::from_serde_ext(&bucket.plugin).unwrap_or(JsValue::NULL); if let Err(e) = plugin.restore(&token, Some(&bucket.columns)) { tracing::warn!("plugin.restore on swap failed: {:?}", e); @@ -711,11 +765,11 @@ impl Renderer { /// The COMMITTED panel theme, pending theme edits excluded. pub fn committed_theme(&self) -> Option { - self.cell.state().chrome.theme.clone() + self.state().theme.clone() } fn selected_idx(&self) -> Option { - self.cell.state().plugin.as_ref().map(|plugin| plugin.idx) + self.state().plugin.as_ref().map(|plugin| plugin.idx) } pub fn render_timer(&self) -> MovingWindowRenderTimer { @@ -772,6 +826,7 @@ impl Renderer { is_chart, plugin_config, columns_config, + render_error: self.failure(), } } else { RendererProps { @@ -782,6 +837,7 @@ impl Renderer { is_chart: false, plugin_config: PtrEqRc::default(), columns_config: PtrEqRc::default(), + render_error: self.failure(), } } } diff --git a/rust/perspective-viewer/src/rust/renderer/plugin_config.rs b/rust/perspective-viewer/src/rust/renderer/plugin_config.rs index 9abbfb5ca0..ad564f676b 100644 --- a/rust/perspective-viewer/src/rust/renderer/plugin_config.rs +++ b/rust/perspective-viewer/src/rust/renderer/plugin_config.rs @@ -25,9 +25,10 @@ use serde_json::Value; use wasm_bindgen::prelude::*; use super::Renderer; +use super::state::PluginRef; use crate::config::*; use crate::js::plugin::JsPerspectiveViewerPlugin; -use crate::queries::resolve_abs_max; +use crate::queries::{is_pivot_only_column, resolve_abs_max}; use crate::session::Session; use crate::utils::{CssKind, CssLiteralUse, parse_var_ref, resolve_css_refs}; @@ -36,6 +37,25 @@ type ConfigMap = serde_json::Map; /// A schema query's answer: `Ok(None)` when the plugin declares no schema. pub type SchemaResult = Result, ValidationError>; +/// Where in the `ViewConfig` a column being configured appears, which is a +/// different axis from its plugin column slot. +#[derive(Clone, Copy, Debug, PartialEq)] +pub enum ColumnRole { + Column, + GroupBy, + SplitBy, +} + +impl ColumnRole { + pub fn as_str(&self) -> &'static str { + match self { + Self::Column => "column", + Self::GroupBy => "group_by", + Self::SplitBy => "split_by", + } + } +} + /// Everything a column's schema is a function of, besides the column itself. struct ColumnSchemaEnv<'a> { view_config: &'a ViewConfig, @@ -69,6 +89,24 @@ pub struct PluginScopedConfig { pub plugin: serde_json::Map, } +/// The renderer's part of a restore that has passed every check. +pub struct PreparedRenderer { + /// The plugin selection to commit, when the restore ends on another plugin. + pub plugin: Option, + + /// The plugin whose bucket the validated updates belong to. + pub target_name: String, + pub plugin_config: ValidatedPluginConfig, + pub columns_config: ValidatedColumnsConfig, + pub theme: Option>, +} + +/// Which of the target plugin's buckets a [`Renderer::commit`] changed. +pub struct RendererChanges { + pub plugin_config: bool, + pub columns_config: bool, +} + /// The plugin a bucket update is validated against and written to — named /// EXPLICITLY, so a validation that precedes a plugin swap can name the swap /// target rather than whichever plugin happens to be active. @@ -303,10 +341,10 @@ impl Renderer { /// Rewrite the named plugin's bucket: `f` edits a COPY, which is then /// swapped in as one new [`crate::session::PanelState`]. fn edit_bucket(&self, name: &str, f: impl FnOnce(&mut PluginScopedConfig) -> R) -> R { - let state = self.cell.state(); + let state = self.state(); let mut bucket = state.bucket(name); let result = f(&mut bucket); - self.cell.swap(state.with_bucket(name, bucket)); + self.swap(state.with_bucket(name, bucket)); result } @@ -367,7 +405,7 @@ impl Renderer { return Ok((self.target_at(idx)?, Some(idx))); } - if let Some(plugin) = self.cell.state().plugin.as_ref() { + if let Some(plugin) = self.state().plugin.as_ref() { return Ok((self.target_at(plugin.idx)?, None)); } @@ -378,6 +416,39 @@ impl Renderer { Ok((self.target_at(idx)?, Some(idx))) } + /// The COMMIT half of a restore, as one swap of this renderer's state. + pub fn commit(&self, prepared: PreparedRenderer) -> RendererChanges { + let PreparedRenderer { + plugin, + target_name, + plugin_config, + columns_config, + theme, + } = prepared; + + let mut next = (*self.state()).clone(); + if let Some(plugin) = plugin { + next = next.with_plugin(plugin); + } + + let mut bucket = next.bucket(&target_name); + let changes = RendererChanges { + plugin_config: apply_plugin_config_to(&mut bucket, plugin_config), + columns_config: apply_columns_config_to(&mut bucket, columns_config), + }; + + if changes.plugin_config || changes.columns_config { + next = next.with_bucket(&target_name, bucket); + } + + if let Some(theme) = theme { + next = next.with_theme(theme); + } + + self.swap(next); + changes + } + /// Validate BOTH bucket updates of a restore against `target` and the view /// the restore would build (`view_config`, whose columns have the types in /// `view_schema`) — before either is written, and without a `View`. @@ -392,7 +463,7 @@ impl Renderer { plugin_config: PluginConfigUpdate, columns_config: ColumnConfigUpdate, ) -> ApiResult<(ValidatedPluginConfig, ValidatedColumnsConfig)> { - let current = self.cell.state().bucket(&target.static_config.name); + let current = self.state().bucket(&target.static_config.name); let plugin = validate_plugin_config(¤t.plugin, plugin_config, &|merged| { if !declares(target, "plugin_config_schema") { return Ok(None); @@ -441,7 +512,7 @@ impl Renderer { /// — pending style edits included. pub fn all_columns_configs(&self) -> ColumnConfigMap { self.active_plugin_name() - .map(|n| self.cell.projected_bucket(&n).columns) + .map(|n| self.projected_bucket(&n).columns) .unwrap_or_default() } @@ -449,7 +520,7 @@ impl Renderer { /// op's step reads, since the edits queued behind it have not happened. pub fn committed_columns_configs(&self) -> ColumnConfigMap { self.active_plugin_name() - .map(|n| self.cell.state().bucket(&n).columns) + .map(|n| self.state().bucket(&n).columns) .unwrap_or_default() } @@ -595,11 +666,7 @@ impl Renderer { column_name: &str, ) -> Option> { let n = self.active_plugin_name()?; - self.cell - .projected_bucket(&n) - .columns - .get(column_name) - .cloned() + self.projected_bucket(&n).columns.get(column_name).cloned() } /// Wholesale update the active plugin's per-column config map: @@ -653,13 +720,7 @@ impl Renderer { return; }; - let current_value = self - .cell - .state() - .bucket(&n) - .columns - .get(&column_name) - .cloned(); + let current_value = self.state().bucket(&n).columns.get(&column_name).cloned(); if let Ok(schema) = self.query_column_config_schema( view_config, @@ -722,7 +783,7 @@ impl Renderer { /// it — pending settings edits included. pub fn get_plugin_config(&self) -> serde_json::Map { self.active_plugin_name() - .map(|n| self.cell.projected_bucket(&n).plugin) + .map(|n| self.projected_bucket(&n).plugin) .unwrap_or_default() } @@ -730,7 +791,7 @@ impl Renderer { /// op's step reads. pub fn committed_plugin_config(&self) -> serde_json::Map { self.active_plugin_name() - .map(|n| self.cell.state().bucket(&n).plugin) + .map(|n| self.state().bucket(&n).plugin) .unwrap_or_default() } @@ -820,6 +881,14 @@ impl Renderer { .and_then(|idx| names.get(idx)) .map(|s| s.as_str()); + let role = if !is_pivot_only_column(column_name, view_config) { + ColumnRole::Column + } else if view_config.split_by.iter().any(|x| x == column_name) { + ColumnRole::SplitBy + } else { + ColumnRole::GroupBy + }; + let view_type = match view_schema { Some(view_schema) => view_schema.get(column_name).copied(), None => { @@ -831,6 +900,14 @@ impl Renderer { }, }; + let view_type = match role { + ColumnRole::Column => view_type, + _ => session + .metadata() + .get_column_table_type(column_name) + .or(view_type), + }; + let Some(view_type) = view_type else { return Ok(ColumnConfigSchema { fields: vec![] }); }; @@ -845,6 +922,7 @@ impl Renderer { ¤t_js, &view_config_js, &plugin_config_js, + role.as_str(), )?; let abs_max = column_stats @@ -1049,415 +1127,3 @@ fn css_matches(kind: CssKind, value: &Value, default: &str) -> bool { .and_then(|src| kind.canonicalize(src).ok()) .is_some_and(|canonical| canonical == default) } - -#[cfg(test)] -mod tests { - use serde_json::json; - - use super::*; - - #[test] - fn palette_default_matches_canonically() { - let spec = ControlSpec::Palette { - key: "palette".to_owned(), - default: "linear-gradient(to right, #0366d6, #ff7f0e)".to_owned(), - max: None, - }; - - assert!(matches_declared_default( - &spec, - "palette", - &json!("linear-gradient(to right, #0366d6, #ff7f0e)") - )); - - assert!(matches_declared_default( - &spec, - "palette", - &json!("linear-gradient(90deg, RGB(3,102,214), #FF7F0E)") - )); - - assert!(!matches_declared_default( - &spec, - "palette", - &json!("linear-gradient(to right, #ff7f0e, #0366d6)") - )); - - assert!(!matches_declared_default( - &spec, - "palette", - &json!("var(--psp-user--palette-1)") - )); - - assert!(!matches_declared_default( - &spec, - "palette", - &json!(["#0366d6"]) - )); - assert!(!matches_declared_default( - &spec, - "other", - &json!("linear-gradient(to right, #0366d6, #ff7f0e)") - )); - } - - #[test] - fn gradient_default_matches_canonical_values() { - let spec = ControlSpec::GradientStops { - key: "gradient".to_owned(), - default: "linear-gradient(to right, #0366d6 0%, #ff7f0e 33.3%)".to_owned(), - discrete: false, - }; - - assert!(matches_declared_default( - &spec, - "gradient", - &json!("linear-gradient(#0366d6, #ff7f0e 33.3%)") - )); - - assert!(!matches_declared_default( - &spec, - "gradient", - &json!("linear-gradient(to right, #0366d6 0%, #ff7f0e 33.4%)") - )); - - let color = ControlSpec::Color { - key: "color".to_owned(), - default: "#ff7f0e".to_owned(), - }; - - assert!(matches_declared_default(&color, "color", &json!("#FF7F0E"))); - assert!(matches_declared_default( - &color, - "color", - &json!("rgb(255, 127, 14)") - )); - assert!(!matches_declared_default( - &color, - "color", - &json!("var(--psp-user--color-1)") - )); - } - - #[test] - fn alignment_strips_only_a_declared_default() { - let schema = ColumnConfigSchema { - fields: vec![ - ControlSpec::Alignment { - key: "align".to_owned(), - default: None, - corners: false, - }, - ControlSpec::Alignment { - key: "legend_anchor".to_owned(), - default: Some(Alignment::TopRight), - corners: true, - }, - ], - }; - - let mut map = json!({ "align": "center", "legend_anchor": "top-right" }) - .as_object() - .unwrap() - .clone(); - - strip_default_values(&schema, &mut map); - assert_eq!( - map, - json!({ "align": "center" }).as_object().unwrap().clone() - ); - } - - #[test] - fn strip_default_values_sees_through_groups() { - let leaves = vec![ - ControlSpec::Bool { - key: "flag".to_owned(), - default: false, - }, - ControlSpec::Number { - key: "size".to_owned(), - default: 3.0, - include: None, - min: None, - max: None, - step: None, - default_stat: None, - }, - ]; - - let flat = ColumnConfigSchema { - fields: leaves.clone(), - }; - - let grouped = ColumnConfigSchema { - fields: vec![ControlSpec::Group { - key: "section".to_owned(), - fields: leaves, - }], - }; - - let src = json!({ "flag": false, "size": 4.0, "foreign": 1 }) - .as_object() - .unwrap() - .clone(); - - let mut a = src.clone(); - let mut b = src; - strip_default_values(&flat, &mut a); - strip_default_values(&grouped, &mut b); - assert_eq!(a, b); - assert_eq!( - a, - json!({ "size": 4.0, "foreign": 1 }) - .as_object() - .unwrap() - .clone() - ); - } - - #[test] - fn normalize_values_rejects_enum_values_outside_the_variants() { - let schema = ColumnConfigSchema { - fields: vec![ControlSpec::Group { - key: "color".to_owned(), - fields: vec![ControlSpec::Enum { - key: "fg_mode".to_owned(), - default: "color".to_owned(), - variants: vec![ - EnumVariant { - value: "disabled".to_owned(), - label: None, - }, - EnumVariant { - value: "color".to_owned(), - label: None, - }, - ], - }], - }], - }; - - let mut map = json!({ "fg_mode": "color", "other": "series" }) - .as_object() - .unwrap() - .clone(); - - assert!(normalize_values(&schema, &mut map).is_empty()); - assert_eq!(map.len(), 2); - - let mut map = json!({ "fg_mode": "series" }).as_object().unwrap().clone(); - let errors = normalize_values(&schema, &mut map); - assert_eq!(errors.len(), 1); - assert_eq!(errors[0].0, "fg_mode"); - assert_eq!(errors[0].1, "\"series\" is not one of `disabled`, `color`"); - assert!(map.is_empty()); - - let mut map = json!({ "fg_mode": 3 }).as_object().unwrap().clone(); - let errors = normalize_values(&schema, &mut map); - assert_eq!(errors.len(), 1); - assert!(map.is_empty()); - } - - fn enum_schema() -> ColumnConfigSchema { - ColumnConfigSchema { - fields: vec![ - ControlSpec::Enum { - key: "fg_mode".to_owned(), - default: "color".to_owned(), - variants: vec![ - EnumVariant { - value: "disabled".to_owned(), - label: None, - }, - EnumVariant { - value: "color".to_owned(), - label: None, - }, - ], - }, - ControlSpec::Bool { - key: "flag".to_owned(), - default: false, - }, - ], - } - } - - fn obj(value: serde_json::Value) -> ConfigMap { - value.as_object().unwrap().clone() - } - - #[test] - fn validate_plugin_config_rejects_enum_values_outside_the_variants() { - let err = validate_plugin_config( - &ConfigMap::new(), - OptionalUpdate::Update(obj(json!({ "fg_mode": "series" }))), - &|_| Ok(Some(enum_schema())), - ) - .err() - .expect("rejected"); - - assert!( - err.0 - .contains("Invalid `plugin_config.fg_mode`: \"series\" is not one of"), - "{}", - err.0 - ); - } - - #[test] - fn validate_plugin_config_splits_defaults_and_prunes_to_active_keys() { - let validated = validate_plugin_config( - &obj(json!({ "fg_mode": "disabled" })), - OptionalUpdate::Update(obj(json!({ "fg_mode": "color", "flag": true, "ghost": 1 }))), - &|merged| { - assert_eq!(merged["fg_mode"], "color"); - assert_eq!(merged["flag"], true); - Ok(Some(enum_schema())) - }, - ) - .unwrap(); - - let ValidatedPluginConfig::Set { - map, - remove, - active, - } = validated - else { - panic!("expected Set"); - }; - - assert_eq!(map, obj(json!({ "flag": true }))); - assert_eq!(remove, vec!["fg_mode".to_owned()]); - assert_eq!( - active.unwrap(), - HashSet::from(["fg_mode".to_owned(), "flag".to_owned()]) - ); - } - - #[test] - fn validate_plugin_config_passes_through_without_a_schema() { - let update = obj(json!({ "fg_mode": "series", "ghost": 1 })); - let validated = validate_plugin_config( - &ConfigMap::new(), - OptionalUpdate::Update(update.clone()), - &|_| Ok(None), - ) - .unwrap(); - - assert_eq!(validated, ValidatedPluginConfig::Set { - map: update, - remove: vec![], - active: None, - }); - assert_eq!( - validate_plugin_config(&ConfigMap::new(), OptionalUpdate::SetDefault, &|_| { - Ok(Some(enum_schema())) - }) - .unwrap(), - ValidatedPluginConfig::Clear - ); - } - - #[test] - fn validate_columns_config_rejects_and_names_the_column() { - let err = validate_columns_config( - OptionalUpdate::Update(HashMap::from([( - "a".to_owned(), - obj(json!({ "fg_mode": "series" })), - )])), - &|_, _| Ok(Some(enum_schema())), - &|_| Some(ColumnType::Integer), - &|_, _| {}, - ) - .err() - .expect("rejected"); - - let msg = err.0; - assert!( - msg.contains("Invalid `columns_config[\"a\"].fg_mode`"), - "{msg}" - ); - assert!(msg.contains("for a integer column"), "{msg}"); - } - - #[test] - fn validate_columns_config_strips_defaults_and_keeps_unschematized_columns() { - let validated = validate_columns_config( - OptionalUpdate::Update(HashMap::from([ - ( - "a".to_owned(), - obj(json!({ "fg_mode": "color", "flag": true, "ghost": 1 })), - ), - ("b".to_owned(), obj(json!({ "anything": "goes" }))), - ])), - &|col, _| Ok((col == "a").then(enum_schema)), - &|_| None, - &|_, _| {}, - ) - .unwrap(); - - let ValidatedColumnsConfig::Set(mut entries) = validated else { - panic!("expected Set"); - }; - - entries.sort_by(|x, y| x.0.cmp(&y.0)); - assert_eq!(entries, vec![ - ("a".to_owned(), obj(json!({ "flag": true }))), - ("b".to_owned(), obj(json!({ "anything": "goes" }))), - ]); - } - - #[test] - fn normalize_values_canonicalizes_and_reports() { - let schema = ColumnConfigSchema { - fields: vec![ - ControlSpec::Color { - key: "color".to_owned(), - default: "#000000".to_owned(), - }, - ControlSpec::Palette { - key: "palette".to_owned(), - default: "linear-gradient(to right, #000000)".to_owned(), - max: None, - }, - ControlSpec::GradientStops { - key: "gradient".to_owned(), - default: "linear-gradient(to right, #000000 0%, #ffffff 100%)".to_owned(), - discrete: false, - }, - ControlSpec::Bool { - key: "flag".to_owned(), - default: false, - }, - ], - }; - - let mut map = json!({ - "color": "RGB(255,0,0)", - "palette": "var(--psp-user--palette-warm)", - "gradient": [{ "color": "#000000", "offset": 0 }], - "flag": true, - "foreign": "linear-gradient(red, blue)", - }) - .as_object() - .unwrap() - .clone(); - - let errors = normalize_values(&schema, &mut map); - assert_eq!(errors.len(), 1); - assert_eq!(errors[0].0, "gradient"); - assert_eq!( - map, - json!({ - "color": "#ff0000", - "palette": "var(--psp-user--palette-warm)", - "flag": true, - "foreign": "linear-gradient(red, blue)", - }) - .as_object() - .unwrap() - .clone() - ); - } -} diff --git a/rust/perspective-viewer/src/rust/renderer/props.rs b/rust/perspective-viewer/src/rust/renderer/props.rs index c27d5491cb..6120fe7a04 100644 --- a/rust/perspective-viewer/src/rust/renderer/props.rs +++ b/rust/perspective-viewer/src/rust/renderer/props.rs @@ -15,6 +15,7 @@ use std::rc::Rc; use crate::config::PluginStaticConfig; use crate::renderer::limits::RenderLimits; use crate::renderer::plugin_config::ColumnConfigMap; +use crate::renderer::state::RenderError; use crate::utils::PtrEqRc; /// Value-semantic snapshot of the renderer state read by components. @@ -55,6 +56,9 @@ pub struct RendererProps { /// Snapshot of the active plugin's `columns_config` bucket, re-read by /// `Renderer::to_props` on `columns_config_changed` and `plugin_changed`. pub columns_config: PtrEqRc, + + /// The failure of the last draw, which stands until the next commit. + pub render_error: Option, } impl RendererProps { diff --git a/rust/perspective-viewer/src/rust/renderer/registry.rs b/rust/perspective-viewer/src/rust/renderer/registry.rs index 2b1803da57..ae2a124acb 100644 --- a/rust/perspective-viewer/src/rust/renderer/registry.rs +++ b/rust/perspective-viewer/src/rust/renderer/registry.rs @@ -147,11 +147,6 @@ pub impl LocalKey>>> { }); }); } - - #[cfg(test)] - fn reset(&'static self) { - self.with(|plugins| plugins.borrow_mut().clear()); - } } fn register_default() { diff --git a/rust/perspective-viewer/src/rust/renderer/state.rs b/rust/perspective-viewer/src/rust/renderer/state.rs new file mode 100644 index 0000000000..209bb31aa4 --- /dev/null +++ b/rust/perspective-viewer/src/rust/renderer/state.rs @@ -0,0 +1,205 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +use std::cell::RefCell; +use std::collections::HashMap; +use std::rc::Rc; + +use perspective_js::utils::ApiError; + +use super::plugin_config::PluginScopedConfig; +use crate::config::{ColumnConfigFieldUpdate, PluginStaticConfig}; + +/// How a panel is drawn, as one immutable value replaced whole. +#[derive(Clone, Default)] +pub struct RendererState { + /// The selected plugin, or `None` until the first run selects one. + pub plugin: Option, + + /// Every plugin's own `plugin_config` and `columns_config`, keyed by + /// plugin name. + pub buckets: Rc>, + + /// This panel's concrete theme name. + pub theme: Option, +} + +/// A plugin selection: its index in the plugin store and its static config. +#[derive(Clone)] +pub struct PluginRef { + pub idx: usize, + pub static_config: Rc, +} + +impl RendererState { + pub fn with_theme(&self, theme: Option) -> Self { + Self { + theme, + ..self.clone() + } + } + + pub fn with_plugin(&self, plugin: PluginRef) -> Self { + Self { + plugin: Some(plugin), + ..self.clone() + } + } + + /// This state with no plugin selected and every bucket forgotten. + pub fn without_plugins(&self) -> Self { + Self { + plugin: None, + buckets: Rc::default(), + ..self.clone() + } + } + + /// The named plugin's bucket, empty when it has never been written. + pub fn bucket(&self, name: &str) -> PluginScopedConfig { + self.buckets.get(name).cloned().unwrap_or_default() + } + + /// This state with the named plugin's bucket replaced. + pub fn with_bucket(&self, name: &str, bucket: PluginScopedConfig) -> Self { + let mut buckets = (*self.buckets).clone(); + buckets.insert(name.to_owned(), bucket); + Self { + buckets: Rc::new(buckets), + ..self.clone() + } + } +} + +/// A failure to draw a valid committed state. +#[derive(Clone)] +pub struct RenderError(pub ApiError); + +impl PartialEq for RenderError { + fn eq(&self, other: &Self) -> bool { + self.0.to_string() == other.0.to_string() + } +} + +impl std::fmt::Debug for RenderError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_tuple("RenderError") + .field(&self.0.to_string()) + .finish() + } +} + +/// How the last draw of the committed state went, which any commit resets. +#[derive(Clone, Default)] +pub enum RenderOutcome { + #[default] + Never, + Ok, + Failed(RenderError), +} + +/// A UI edit of the renderer's state that is queued but not yet committed. +#[derive(Clone)] +pub enum StagedEdit { + /// One plugin-level settings field of the selected plugin. + PluginField(ColumnConfigFieldUpdate), + + /// One style field of one column, for the selected plugin. + ColumnField { + column: String, + update: ColumnConfigFieldUpdate, + }, + + /// Keys merged into the selected plugin's `plugin_config`. + PluginConfig(serde_json::Map), + Theme(Option), +} + +/// The renderer's staged edits, in the order they were made. +#[derive(Default)] +pub(super) struct StagedEdits { + next_id: std::cell::Cell, + edits: RefCell>, +} + +impl StagedEdits { + pub fn stage(self: &Rc, edit: StagedEdit) -> Staged { + let id = self.next_id.get(); + self.next_id.set(id + 1); + self.edits.borrow_mut().push((id, edit)); + Staged { + id, + edits: Rc::downgrade(self), + } + } + + pub fn pending(&self) -> Vec { + self.edits.borrow().iter().map(|x| x.1.clone()).collect() + } +} + +/// Keeps one [`StagedEdit`] visible to the UI until its op leaves the queue. +pub struct Staged { + id: u64, + edits: std::rc::Weak, +} + +impl Drop for Staged { + fn drop(&mut self) { + if let Some(edits) = self.edits.upgrade() { + edits.edits.borrow_mut().retain(|x| x.0 != self.id); + } + } +} + +/// `bucket` with every pending bucket edit applied, in the order made. +pub(super) fn project_bucket( + mut bucket: PluginScopedConfig, + pending: Vec, +) -> PluginScopedConfig { + for edit in pending { + match edit { + StagedEdit::PluginField(update) => { + for key in &update.keys { + match update.value.get(key) { + Some(value) => { + bucket.plugin.insert(key.clone(), value.clone()); + }, + None => { + bucket.plugin.remove(key); + }, + } + } + }, + StagedEdit::ColumnField { column, update } => { + let entry = bucket.columns.entry(column.clone()).or_default(); + for key in &update.keys { + entry.remove(key); + } + + for (key, value) in update.value { + if update.keys.contains(&key) { + entry.insert(key, value); + } + } + + if entry.is_empty() { + bucket.columns.remove(&column); + } + }, + StagedEdit::PluginConfig(map) => bucket.plugin.extend(map), + StagedEdit::Theme(_) => {}, + } + } + + bucket +} diff --git a/rust/perspective-viewer/src/rust/session.rs b/rust/perspective-viewer/src/rust/session.rs index 33fb2d83c5..e5ade9bbfb 100644 --- a/rust/perspective-viewer/src/rust/session.rs +++ b/rust/perspective-viewer/src/rust/session.rs @@ -39,9 +39,9 @@ use yew::prelude::*; use self::metadata::*; pub use self::metadata::{MetadataRef, SessionMetadata, SessionMetadataRc}; use self::op_queue::OpQueue; -pub use self::op_queue::{EditDelta, OpCtx, OpKind, StepFuture, StepOutcome, Ticket, view_fields}; +pub use self::op_queue::{EditDelta, OpCtx, OpKind, StepFuture, Ticket}; use self::panel_state::effective; -pub use self::panel_state::{Binding, OverlayClause, PanelState, PluginRef}; +pub use self::panel_state::{Binding, OverlayClause, PanelState}; pub use self::props::{SessionProps, TableLoadState}; pub use self::view_subscription::ViewStats; use self::view_subscription::*; @@ -204,9 +204,6 @@ pub struct SessionData { view_sub: Option, stats: Option, is_paused: bool, - - /// How the last config-driven run of the committed state went. - rendered: Rendered, } /// A panel's state and whether it can still be written. @@ -264,132 +261,15 @@ impl Deref for ViewConfigRef { } } -/// The [`crate::renderer::Renderer`]'s handle on its panel's committed -/// [`PanelState`] — read the current value, or swap in the next one. -#[derive(Clone)] -pub struct PanelCell(Session); - -impl PanelCell { - /// The COMMITTED state — what a running step reads. - pub fn state(&self) -> Rc { - self.0.borrow().state.clone() - } - - /// The theme as the UI sees it: the committed theme, or the latest pending - /// pick. - pub fn projected_theme(&self) -> Option { - let pending = self - .0 - .0 - .queue - .pending_edits() - .into_iter() - .filter_map(|delta| match delta { - EditDelta::Theme(theme) => Some(theme), - _ => None, - }) - .next_back(); - - pending.unwrap_or_else(|| self.state().chrome.theme.clone()) - } - - /// The SELECTED plugin's bucket as the UI and `save()` see it: the - /// committed bucket with every pending style / settings edit applied, in - /// submit order. - pub fn projected_bucket(&self, name: &str) -> crate::renderer::PluginScopedConfig { - let mut bucket = self.state().bucket(name); - for delta in self.0.0.queue.pending_edits() { - match delta { - EditDelta::PluginField(update) => { - for key in &update.keys { - match update.value.get(key) { - Some(value) => { - bucket.plugin.insert(key.clone(), value.clone()); - }, - None => { - bucket.plugin.remove(key); - }, - } - } - }, - EditDelta::ColumnField { column, update } => { - let entry = bucket.columns.entry(column.clone()).or_default(); - for key in &update.keys { - entry.remove(key); - } - - for (key, value) in update.value { - if update.keys.contains(&key) { - entry.insert(key, value); - } - } - - if entry.is_empty() { - bucket.columns.remove(&column); - } - }, - EditDelta::PluginConfig(map) => bucket.plugin.extend(map), - EditDelta::View(_) | EditDelta::Theme(_) | EditDelta::Title(_) => {}, - } - } - - bucket - } - - /// Submit a theme pick — a UI edit, committed by the drain in its turn. - pub fn submit_theme(&self, theme: Option) { - let cell = self.clone(); - let _ticket = self.0.submit( - OpKind::Edit { - delta: EditDelta::Theme(theme.clone()), - fields: None, - }, - move |_ctx| { - Box::pin(async move { - cell.swap(cell.state().with_theme(theme)); - Ok(StepOutcome::Done) - }) - }, - ); - } - - /// Submit an op on this panel's queue (see [`Session::submit`]). - pub fn submit(&self, kind: OpKind, step: impl FnOnce(OpCtx) -> StepFuture + 'static) -> Ticket { - self.0.submit(kind, step) - } - - pub fn swap(&self, next: PanelState) { - self.0.borrow_mut().swap(next); - } -} - -/// The outcome of the last config-driven run. -#[derive(Clone, Default)] -pub enum Rendered { - #[default] - Never, - Ok, - - /// The run of THIS state failed. - Failed(Rc, TableErrorState), -} - impl SessionData { fn error(&self) -> Option<&TableErrorState> { - self.state.lost().or(match &self.rendered { - Rendered::Failed(_, error) => Some(error), - _ => None, - }) + self.state.lost() } fn clear_errors(&mut self) { if self.state.lost().is_some() { self.swap(self.state.recovered()); } - - if matches!(self.rendered, Rendered::Failed(..)) { - self.rendered = Rendered::Never; - } } fn swap(&mut self, next: PanelState) { @@ -567,12 +447,6 @@ impl Session { MetadataRef(self.borrow().metadata().clone()) } - /// A handle on this panel's committed [`PanelState`] for the panel's - /// [`crate::renderer::Renderer`], which owns none of it. - pub fn cell(&self) -> PanelCell { - PanelCell(self.clone()) - } - pub(crate) fn metadata_mut(&self) -> MetadataMutRef<'_> { std::cell::RefMut::map(self.borrow_mut(), |x| x.metadata_mut()) } @@ -601,12 +475,12 @@ impl Session { let _ticket = self.submit( OpKind::Edit { delta: EditDelta::Title(title.clone()), - fields: None, + swaps_plugin: false, }, move |_ctx| { Box::pin(async move { session.commit_title(title); - Ok(StepOutcome::Done) + Ok(None) }) }, ); @@ -903,7 +777,7 @@ impl Session { Some(perspective_js::Table::from(self.borrow().table().cloned()?).into()) } - /// Whether the binding is lost OR the last run failed. + /// Whether the binding is lost. pub(crate) fn is_errored(&self) -> bool { self.borrow().error().is_some() } @@ -912,37 +786,14 @@ impl Session { self.borrow().error().map(|x| x.0.clone()) } - /// The error a config-driven run must not proceed past: a lost binding, or - /// a failed run of EXACTLY the state now committed (re-running it would - /// only fail again). - pub(crate) fn blocking_error(&self) -> Option { - let data = self.borrow(); - if let Some(error) = data.state.lost() { - return Some(error.0.clone()); - } - - match &data.rendered { - Rendered::Failed(state, error) if Rc::ptr_eq(state, &data.state) => { - Some(error.0.clone()) - }, - _ => None, - } - } - - /// Record that a config-driven run of the committed state is starting (any - /// earlier failure is stale) or has landed. - pub(crate) fn set_rendered(&self, ok: bool) { - self.borrow_mut().rendered = if ok { Rendered::Ok } else { Rendered::Never }; - } - /// Recover from an error state (the overlay's button) — a writer, so an op /// on the queue like any other. pub async fn reconnect(&self) -> ApiResult<()> { let session = self.clone(); - self.submit(OpKind::Restore { fields: None }, move |_ctx| { + self.submit(OpKind::Restore { update: None }, move |_ctx| { Box::pin(async move { session.reconnect_step().await?; - Ok(StepOutcome::Done) + Ok(None) }) }) .settle() @@ -1413,10 +1264,10 @@ impl Session { bound.map(|bound| (bound.table.clone(), bound.metadata.clone())), ) }, - BindPlan::Bind { client, table, .. } => { + BindPlan::Bind { table, client, .. } => { let metadata = Rc::new(SessionMetadata::from_table(&table).await?); let binding = PreparedBinding::Bind { - client, + client: client.clone(), table: table.clone(), metadata: metadata.clone(), }; @@ -1509,13 +1360,12 @@ impl Session { }) } - /// The COMMIT half of [`Self::prepare_view`], as ONE swap: `state` with the - /// prepared binding, config, description and expression metadata, then - /// whatever else of the panel `rest` replaces. + /// The COMMIT half of [`Self::prepare_view`], as one swap of this session's + /// state. pub(crate) fn commit_view( &self, view: PreparedView, - rest: impl FnOnce(PanelState) -> PanelState, + title: Option>, ) -> BindingEffects { let state = self.borrow().state.clone(); let title_before = state.chrome.title.clone(); @@ -1553,14 +1403,16 @@ impl Session { next = next.with_description(Some((view.effective, description))); } - let next = rest(next); + let next = match title { + Some(title) => next.with_title(title), + None => next, + }; + let title_after = next.chrome.title.clone(); let bound = next.bound().is_some(); self.borrow_mut().swap(next); let outgoing = if rebinds { - let mut data = self.borrow_mut(); - data.rendered = Rendered::Never; - data.view_sub.take() + self.borrow_mut().view_sub.take() } else { None }; @@ -1775,41 +1627,24 @@ impl Session { Ok(table.view(Some(view_config.into())).await?) } - /// Record a failed pipeline run: error state plus a reconnect affordance - /// that resets the config (the error screen's reset button). Replaces - /// the old `validate()` error path — the committed config is NOT rolled - /// back (I4: it holds exactly what the caller committed; the failure - /// belongs to the run). - pub async fn set_run_error(&self, err: ApiError) -> ApiResult<()> { + /// Reset the config a failed draw could not render, as one queued op. + pub async fn reset_failed_config(&self) -> ApiResult<()> { let session = self.clone(); - let poll_loop = LocalPollLoop::new(move |()| { - ApiFuture::spawn(session.reset(ResetOptions { - config: true, - expressions: true, - ..ResetOptions::default() - })); - Ok(JsValue::UNDEFINED) - }); - - let error = TableErrorState( - err.clone(), - Some(ReconnectCallback::new(move || { - clone!(poll_loop); - Box::pin(async move { - poll_loop.poll(()).await; - Ok(()) - }) - })), - ); - - let state = self.borrow().state.clone(); - self.borrow_mut().rendered = Rendered::Failed(state, error); - - if let Some(cb) = self.on_table_errored.borrow().as_ref() { - cb.emit(()); - } + self.submit(OpKind::Restore { update: None }, move |_ctx| { + Box::pin(async move { + session + .reset(ResetOptions { + config: true, + expressions: true, + ..ResetOptions::default() + }) + .await?; - Err(err) + Ok(None) + }) + }) + .settle() + .await } fn update_stats(&self, stats: ViewStats) { @@ -1920,9 +1755,6 @@ pub(crate) enum BindPlan { Keep, /// Bind `table`. - /// - /// Boxed: `Table` is a value handle several times the size of every other - /// variant here. Bind { client: Client, table: Box, diff --git a/rust/perspective-viewer/src/rust/session/metadata.rs b/rust/perspective-viewer/src/rust/session/metadata.rs index 80caea3ce7..074cd9b074 100644 --- a/rust/perspective-viewer/src/rust/session/metadata.rs +++ b/rust/perspective-viewer/src/rust/session/metadata.rs @@ -87,15 +87,18 @@ pub struct SessionMetadataState { impl SessionMetadata { /// Creates a new `SessionMetadata` from a `JsPerspectiveTable`. pub(super) async fn from_table(table: &perspective_client::Table) -> ApiResult { - let features = table.get_features().await?.clone(); - let column_names = table.columns().await?; - let table_schema = table.schema().await?; - let edit_port = table.make_port().await? as f64; + let (features, column_names, table_schema, edit_port) = futures::join!( + table.get_features(), + table.columns(), + table.schema(), + table.make_port() + ); + Ok(Self(Some(SessionMetadataState { - features, - column_names, - table_schema, - edit_port, + features: features?, + column_names: column_names?, + table_schema: table_schema?, + edit_port: edit_port? as f64, ..SessionMetadataState::default() }))) } diff --git a/rust/perspective-viewer/src/rust/session/op_queue.rs b/rust/perspective-viewer/src/rust/session/op_queue.rs index 976a0724d0..9bf47d388b 100644 --- a/rust/perspective-viewer/src/rust/session/op_queue.rs +++ b/rust/perspective-viewer/src/rust/session/op_queue.rs @@ -15,61 +15,47 @@ //! time in SUBMIT order. use std::cell::{Cell, RefCell}; -use std::collections::{BTreeSet, VecDeque}; +use std::collections::VecDeque; +use std::rc::Rc; use futures::channel::oneshot; use futures::future::LocalBoxFuture; use perspective_client::config::ViewConfigUpdate; use perspective_js::utils::*; -use crate::config::ColumnConfigFieldUpdate; +use crate::config::{OptionalUpdate, ViewerConfigUpdate}; use crate::utils::InFlightGuard; -/// What an op's step hands back to the drain. -pub enum StepOutcome { - /// The op is finished; its ticket resolves now. - Done, +/// The render an op's ticket waits on after its write is finished. +pub type RenderFuture = LocalBoxFuture<'static, ApiResult<()>>; - /// The op's WRITE is finished, and the drain may run the next op; its - /// ticket resolves when this render lands. - Render(LocalBoxFuture<'static, ApiResult<()>>), -} - -pub type StepFuture = LocalBoxFuture<'static, ApiResult>; +/// An op's write, which yields the render its ticket resolves with, if any. +pub type StepFuture = LocalBoxFuture<'static, ApiResult>>; type Step = Box StepFuture>; -/// The top-level config keys an op OVERWRITES. -pub type Fields = Option>; - /// What a UI edit changes — enough to PROJECT it onto the committed state /// before the drain commits it. #[derive(Clone)] pub enum EditDelta { - /// Boxed: `ViewConfigUpdate` is several times the size of every other - /// variant, and every queued op pays for it. View(Box), - - /// One plugin-level settings field of the selected plugin. - PluginField(ColumnConfigFieldUpdate), - - /// One style field of one column, for the selected plugin. - ColumnField { - column: String, - update: ColumnConfigFieldUpdate, - }, - - /// Keys merged into the selected plugin's `plugin_config`. - PluginConfig(serde_json::Map), - Theme(Option), Title(Option), + + /// An edit of the renderer's own state, which the renderer projects. + Renderer, } pub enum OpKind { - /// A UI edit. - Edit { delta: EditDelta, fields: Fields }, + /// A UI edit, which a later op may supersede unless it swaps the plugin. + Edit { + delta: EditDelta, + swaps_plugin: bool, + }, - /// A restore-family op. - Restore { fields: Fields }, + /// A restore-family op, which a later op may supersede when it carries + /// its `update`. + Restore { + update: Option>, + }, /// The element's global filter, broadcast to this panel. Overlay, @@ -79,10 +65,19 @@ pub enum OpKind { } impl OpKind { - fn fields(&self) -> &Fields { + fn writes(&self) -> Option> { match self { - OpKind::Edit { fields, .. } | OpKind::Restore { fields } => fields, - OpKind::Load { .. } | OpKind::Overlay => &None, + OpKind::Edit { + delta: EditDelta::View(update), + swaps_plugin: false, + } => Some((update, None)), + OpKind::Restore { + update: Some(update), + } => Some((&update.view_config, Some(update))), + OpKind::Edit { .. } + | OpKind::Restore { .. } + | OpKind::Load { .. } + | OpKind::Overlay => None, } } @@ -95,8 +90,8 @@ impl OpKind { (OpKind::Load { .. }, OpKind::Load { .. }) => true, (OpKind::Load { table_known }, _) => *table_known, (_, OpKind::Load { .. }) => false, - (later, earlier) => match (later.fields(), earlier.fields()) { - (Some(later), Some(earlier)) => earlier.is_subset(later), + (later, earlier) => match (later.writes(), earlier.writes()) { + (Some(later), Some(earlier)) => overwrites(later, earlier), _ => false, }, } @@ -155,20 +150,13 @@ struct Entry { impl Entry { fn resolve(self, result: ApiResult<()>) { - resolve(self.replies, self.guards, result) - } -} - -fn resolve( - replies: Vec>>, - guards: Vec, - result: ApiResult<()>, -) { - for reply in replies { - let _ = reply.send(result.clone()); + { + let replies = self.replies; + for reply in replies { + let _ = reply.send(result.clone()); + } + } } - - drop(guards); } /// One unsettled op's accounts, released together when its ticket resolves — on @@ -319,9 +307,23 @@ impl OpQueue { self.load_running.set(false); match outcome { - Ok(StepOutcome::Done) => resolve(replies, guards, Ok(())), - Ok(StepOutcome::Render(render)) => ApiFuture::spawn(async move { - resolve(replies, guards, render.await); + Ok(None) => { + let result = Ok(()); + for reply in replies { + let _ = reply.send(result.clone()); + } + + drop(guards); + }, + Ok(Some(render)) => ApiFuture::spawn(async move { + { + let result = render.await; + for reply in replies { + let _ = reply.send(result.clone()); + } + + drop(guards); + }; Ok(()) }), Err(error) => { @@ -330,7 +332,14 @@ impl OpQueue { on_exit(Exit::Rejected); } - resolve(replies, guards, Err(error)); + { + let result = Err(error); + for reply in replies { + let _ = reply.send(result.clone()); + } + + drop(guards); + }; }, } } @@ -347,8 +356,27 @@ impl OpQueue { } } -/// The top-level view-config keys `update` overwrites. -pub fn view_fields(update: &ViewConfigUpdate) -> BTreeSet<&'static str> { +type Writes<'a> = (&'a ViewConfigUpdate, Option<&'a ViewerConfigUpdate>); + +/// Whether `later` overwrites every top-level key `earlier` writes, which a +/// merging `plugin_config` / `columns_config` update never does. +fn overwrites((later_view, later): Writes, (earlier_view, earlier): Writes) -> bool { + let empty = ViewerConfigUpdate::default(); + let later = later.unwrap_or(&empty); + let ViewerConfigUpdate { + version, + plugin, + plugin_config, + columns_config, + settings, + theme, + title, + table, + view_config, + } = earlier.unwrap_or(&empty); + + let _ = (version, view_config); + let ViewConfigUpdate { group_by, split_by, @@ -362,32 +390,41 @@ pub fn view_fields(update: &ViewConfigUpdate) -> BTreeSet<&'static str> { filter_op, group_rollup_mode, split_rollup_mode, - } = update; + } = earlier_view; - let mut fields = BTreeSet::new(); - let mut set = |name: &'static str, present: bool| { - if present { - fields.insert(name); - } - }; - - set( - "group_by", - group_by.is_some() || group_rollup_mode.is_some(), - ); - set( - "group_rollup_mode", - group_by.is_some() || group_rollup_mode.is_some(), - ); - set("split_by", split_by.is_some()); - set("columns", columns.is_some()); - set("filter", filter.is_some()); - set("sort", sort.is_some()); - set("expressions", expressions.is_some()); - set("windows", windows.is_some()); - set("aggregates", aggregates.is_some()); - set("group_by_depth", group_by_depth.is_some()); - set("filter_op", filter_op.is_some()); - set("split_rollup_mode", split_rollup_mode.is_some()); - fields + fn merges(update: &OptionalUpdate) -> bool { + matches!(update, OptionalUpdate::Update(_)) + } + + fn kept(earlier: &OptionalUpdate, later: &OptionalUpdate) -> bool { + !matches!(earlier, OptionalUpdate::Missing) && matches!(later, OptionalUpdate::Missing) + } + + let merging = merges(plugin_config) + || merges(columns_config) + || merges(&later.plugin_config) + || merges(&later.columns_config); + + let groups = |x: &ViewConfigUpdate| x.group_by.is_some() || x.group_rollup_mode.is_some(); + let kept_view = (group_by.is_some() || group_rollup_mode.is_some()) && !groups(later_view) + || split_by.is_some() && later_view.split_by.is_none() + || columns.is_some() && later_view.columns.is_none() + || filter.is_some() && later_view.filter.is_none() + || sort.is_some() && later_view.sort.is_none() + || expressions.is_some() && later_view.expressions.is_none() + || windows.is_some() && later_view.windows.is_none() + || aggregates.is_some() && later_view.aggregates.is_none() + || group_by_depth.is_some() && later_view.group_by_depth.is_none() + || filter_op.is_some() && later_view.filter_op.is_none() + || split_rollup_mode.is_some() && later_view.split_rollup_mode.is_none(); + + !merging + && !kept_view + && !kept(plugin, &later.plugin) + && !kept(plugin_config, &later.plugin_config) + && !kept(columns_config, &later.columns_config) + && !kept(settings, &later.settings) + && !kept(theme, &later.theme) + && !kept(title, &later.title) + && !kept(table, &later.table) } diff --git a/rust/perspective-viewer/src/rust/session/panel_state.rs b/rust/perspective-viewer/src/rust/session/panel_state.rs index 78fa530344..17e23a216c 100644 --- a/rust/perspective-viewer/src/rust/session/panel_state.rs +++ b/rust/perspective-viewer/src/rust/session/panel_state.rs @@ -10,7 +10,6 @@ // ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ // ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ -use std::collections::HashMap; use std::rc::Rc; use perspective_client::config::{ColumnType, Filter, ViewConfig}; @@ -18,8 +17,6 @@ use perspective_client::{Client, Description, Table}; use super::TableErrorState; use super::metadata::SessionMetadata; -use crate::config::PluginStaticConfig; -use crate::renderer::PluginScopedConfig; /// One panel's committed state, as ONE immutable value: a holder of an /// `Rc` sees a consistent snapshot forever, and a change is a new @@ -27,13 +24,6 @@ use crate::renderer::PluginScopedConfig; #[derive(Clone, Default)] pub struct PanelState { pub chrome: Chrome, - - /// The selected plugin, or `None` until the first run selects one. - pub plugin: Option, - - /// Every plugin's own `plugin_config` / `columns_config`, keyed by plugin - /// name, so a swap away and back finds a plugin's config as it left it. - pub buckets: Rc>, pub binding: Binding, /// The element's global filter as broadcast to this panel — the WHOLE set. @@ -89,18 +79,6 @@ pub fn effective( #[derive(Clone, Default)] pub struct Chrome { pub title: Option, - - /// This panel's theme name — CONCRETE, resolved once at creation from the - /// config's `theme`, else the host's, else the registry default. - pub theme: Option, -} - -/// A plugin selection: its index in the renderer's plugin store and the static -/// config that index resolved to. -#[derive(Clone)] -pub struct PluginRef { - pub idx: usize, - pub static_config: Rc, } /// What this panel is bound to. @@ -342,122 +320,8 @@ impl PanelState { pub fn with_title(&self, title: Option) -> Self { Self { - chrome: Chrome { - title, - ..self.chrome.clone() - }, - ..self.clone() - } - } - - pub fn with_theme(&self, theme: Option) -> Self { - Self { - chrome: Chrome { - theme, - ..self.chrome.clone() - }, - ..self.clone() - } - } - - pub fn with_plugin(&self, plugin: PluginRef) -> Self { - Self { - plugin: Some(plugin), - ..self.clone() - } - } - - /// This state with no plugin selected and every bucket forgotten — a - /// deleted renderer's selection and buckets go with it. - pub fn without_plugins(&self) -> Self { - Self { - plugin: None, - buckets: Rc::default(), - ..self.clone() - } - } - - /// The named plugin's bucket, empty when it has never been written. - pub fn bucket(&self, name: &str) -> PluginScopedConfig { - self.buckets.get(name).cloned().unwrap_or_default() - } - - /// This state with the named plugin's bucket replaced. - pub fn with_bucket(&self, name: &str, bucket: PluginScopedConfig) -> Self { - let mut buckets = (*self.buckets).clone(); - buckets.insert(name.to_owned(), bucket); - Self { - buckets: Rc::new(buckets), + chrome: Chrome { title }, ..self.clone() } } } - -#[cfg(test)] -mod tests { - use super::*; - - fn clause(column: &str, column_type: Option) -> OverlayClause { - OverlayClause { - filter: serde_json::from_value(serde_json::json!([column, "==", "x"])).unwrap(), - column_type, - } - } - - fn listener(name: &str) -> Option { - match name { - "region" => Some(ColumnType::String), - "sales" => Some(ColumnType::Float), - _ => None, - } - } - - #[test] - fn a_clause_applies_when_the_listener_has_its_column_and_type() { - let overlay = [clause("region", Some(ColumnType::String))]; - let (config, skipped) = effective(&ViewConfig::default(), &overlay, &listener); - assert_eq!(config.filter, vec![overlay[0].filter.clone()]); - assert!(skipped.is_empty()); - } - - #[test] - fn a_clause_is_skipped_when_the_listener_types_the_column_differently() { - let overlay = [clause("sales", Some(ColumnType::String))]; - let (config, skipped) = effective(&ViewConfig::default(), &overlay, &listener); - assert!(config.filter.is_empty()); - assert_eq!(skipped, vec![0]); - } - - #[test] - fn a_clause_is_skipped_when_the_listener_lacks_the_column() { - let overlay = [ - clause("nope", Some(ColumnType::String)), - clause("region", Some(ColumnType::String)), - ]; - - let (config, skipped) = effective(&ViewConfig::default(), &overlay, &listener); - assert_eq!(config.filter, vec![overlay[1].filter.clone()]); - assert_eq!(skipped, vec![0]); - } - - #[test] - fn an_untyped_clause_matches_on_name() { - let overlay = [clause("sales", None), clause("nope", None)]; - let (config, skipped) = effective(&ViewConfig::default(), &overlay, &listener); - assert_eq!(config.filter, vec![overlay[0].filter.clone()]); - assert_eq!(skipped, vec![1]); - } - - #[test] - fn the_overlay_follows_the_panel_s_own_filters() { - let own = clause("sales", None).filter; - let config = ViewConfig { - filter: vec![own.clone()], - ..ViewConfig::default() - }; - - let overlay = [clause("region", Some(ColumnType::String))]; - let (config, _) = effective(&config, &overlay, &listener); - assert_eq!(config.filter, vec![own, overlay[0].filter.clone()]); - } -} diff --git a/rust/perspective-viewer/src/rust/tasks/apply_global_filters.rs b/rust/perspective-viewer/src/rust/tasks/apply_global_filters.rs index 5034b5d5a9..6580489ac8 100644 --- a/rust/perspective-viewer/src/rust/tasks/apply_global_filters.rs +++ b/rust/perspective-viewer/src/rust/tasks/apply_global_filters.rs @@ -12,12 +12,13 @@ use std::rc::Rc; +use futures::FutureExt; use perspective_client::clone; use perspective_js::utils::ApiFuture; use super::pipeline::{RunCommit, render_run}; use super::transactional_restore::{commit_edit, prepare_overlay}; -use crate::session::{OpKind, OverlayClause, StepOutcome}; +use crate::session::{OpKind, OverlayClause}; use crate::utils::spawn_owned; use crate::workspace::{Panel, PanelId, Workspace}; @@ -40,20 +41,16 @@ pub fn broadcast_overlay(workspace: &Workspace, panel: &Panel) -> ApiFuture<()> let ticket = panel.session.submit(OpKind::Overlay, move |_ctx| { Box::pin(async move { if *session.committed_overlay() == *overlay { - return Ok(StepOutcome::Done); + return Ok(None); } let bound = session.get_table().is_some(); let prepared = prepare_overlay(&session, &renderer, overlay).await?; let committed = commit_edit(&session, &renderer, prepared); Ok(if bound { - StepOutcome::Render(Box::pin(render_run( - session, - renderer, - RunCommit::Done(committed), - ))) + Some(render_run(session, renderer, RunCommit::Done(committed)).boxed_local()) } else { - StepOutcome::Done + None }) }) }); diff --git a/rust/perspective-viewer/src/rust/tasks/auto_pause.rs b/rust/perspective-viewer/src/rust/tasks/auto_pause.rs index 7c12985e1b..b2a05c4b04 100644 --- a/rust/perspective-viewer/src/rust/tasks/auto_pause.rs +++ b/rust/perspective-viewer/src/rust/tasks/auto_pause.rs @@ -23,7 +23,7 @@ use crate::config::ViewerConfigUpdate; use crate::js::*; use crate::presentation::Presentation; use crate::renderer::*; -use crate::session::{OpKind, Session, StepOutcome}; +use crate::session::{OpKind, Session}; use crate::utils::*; use crate::workspace::Workspace; use crate::*; @@ -149,7 +149,7 @@ pub(crate) async fn set_panel_paused( ) -> ApiResult<()> { if visible { if session.set_pause(false) { - let ticket = session.submit(OpKind::Restore { fields: None }, { + let ticket = session.submit(OpKind::Restore { update: None }, { clone!(session, renderer, presentation); move |_ctx| { Box::pin(async move { @@ -162,7 +162,7 @@ pub(crate) async fn set_panel_paused( ) .await?; - Ok(StepOutcome::Done) + Ok(None) }) } }); @@ -170,8 +170,7 @@ pub(crate) async fn set_panel_paused( let result = ticket.settle().await; if let Err(e) = result.ignore_view_delete() { - session.set_run_error(e.clone()).await?; - return Err(e); + return renderer.fail(e); } } } else { diff --git a/rust/perspective-viewer/src/rust/tasks/create_panel.rs b/rust/perspective-viewer/src/rust/tasks/create_panel.rs index 6e36f03551..35fdc60e10 100644 --- a/rust/perspective-viewer/src/rust/tasks/create_panel.rs +++ b/rust/perspective-viewer/src/rust/tasks/create_panel.rs @@ -189,7 +189,7 @@ pub(crate) fn create_panel_model( placement: Placement, ) -> (PanelId, Session, Renderer, ViewerConfigUpdate) { let session = Session::new(); - let renderer = Renderer::new(elem, session.cell()); + let renderer = Renderer::new(elem); let id = id.unwrap_or_else(|| workspace.generate_id()); renderer.set_slot_name(id.as_str()); let subs = wire_panel_subs(elem, presentation, &session, &renderer); diff --git a/rust/perspective-viewer/src/rust/tasks/pipeline.rs b/rust/perspective-viewer/src/rust/tasks/pipeline.rs index a463882c00..9eb98011d3 100644 --- a/rust/perspective-viewer/src/rust/tasks/pipeline.rs +++ b/rust/perspective-viewer/src/rust/tasks/pipeline.rs @@ -12,6 +12,7 @@ use std::rc::Rc; +use futures::FutureExt; use perspective_client::config::ViewConfigUpdate; use perspective_client::{View, clone}; use perspective_js::utils::*; @@ -21,9 +22,7 @@ use yew::prelude::*; use super::transactional_restore::{commit_edit, prepare_edit}; use crate::presentation::Presentation; use crate::renderer::{RenderContext, Renderer}; -use crate::session::{ - BindDisposition, BindingEffects, Disposal, EditDelta, OpKind, Session, StepOutcome, view_fields, -}; +use crate::session::{BindDisposition, BindingEffects, Disposal, EditDelta, OpKind, Session}; use crate::utils::RenderGuard; /// Snapshot → validate → bind → cache + pin the [`RenderContext`]. The core @@ -133,10 +132,9 @@ pub fn update_plugin_and_render( plugin_idx: Option, ) -> ApiResult> { session.check_edit(&update)?; - let fields = plugin_idx.is_none().then(|| view_fields(&update)); let kind = OpKind::Edit { delta: EditDelta::View(Box::new(update.clone())), - fields, + swaps_plugin: plugin_idx.is_some(), }; let ticket = session.submit(kind, { @@ -145,11 +143,9 @@ pub fn update_plugin_and_render( Box::pin(async move { let prepared = prepare_edit(&session, &renderer, update, plugin_idx).await?; let committed = commit_edit(&session, &renderer, prepared); - Ok(StepOutcome::Render(Box::pin(render_run( - session, - renderer, - RunCommit::Done(committed), - )))) + Ok(Some( + render_run(session, renderer, RunCommit::Done(committed)).boxed_local(), + )) }) } }); @@ -234,7 +230,7 @@ impl Default for RunSpec { /// /// A pre-existing session error skips the run for `Internal` origins and /// fails it for `Public` ones (the caller asked and must hear the failure — -/// I6). Returns the RAW run result; error-reporting policy (`set_run_error` +/// I6). Returns the RAW run result; error-reporting policy (`Renderer::fail` /// vs. propagate) belongs to the caller. pub(crate) async fn locked_run( session: &Session, @@ -301,15 +297,17 @@ pub(crate) async fn run_locked( }; let plugin = renderer.active_plugin().ok_or("No Plugin")?; renderer.stamp_theme(Some(&plugin)); - if let Some(error) = session.blocking_error() { + let blocking = session + .get_error() + .or_else(|| renderer.failure().map(|x| x.0)); + + if let Some(error) = blocking { return match spec.origin { RunOrigin::Public => Err(error), RunOrigin::Internal => Ok(()), }; } - session.set_rendered(false); - let (disposition, _pin) = if session.get_table().is_some() { bind_snapshot(&guard, &session, &renderer).await? } else { @@ -355,7 +353,7 @@ pub(crate) async fn run_locked( dispatch_bound(&guard, &renderer, disposition, changed, spec.origin).await?; } - session.set_rendered(true); + renderer.landed(); Ok(()) } } @@ -379,7 +377,7 @@ pub(super) async fn render_run( .await .ignore_view_delete() { - Err(e) => session.set_run_error(e).await, + Err(e) => renderer.fail(e), Ok(_) => Ok(()), } } diff --git a/rust/perspective-viewer/src/rust/tasks/reset_all.rs b/rust/perspective-viewer/src/rust/tasks/reset_all.rs index 5714dd2dff..9634529986 100644 --- a/rust/perspective-viewer/src/rust/tasks/reset_all.rs +++ b/rust/perspective-viewer/src/rust/tasks/reset_all.rs @@ -24,7 +24,7 @@ use crate::config::{ }; use crate::presentation::Presentation; use crate::renderer::Renderer; -use crate::session::{OpKind, ResetOptions, Session, StepOutcome}; +use crate::session::{OpKind, ResetOptions, Session}; /// Reset the viewer's `ViewerConfig` to the default. /// @@ -49,12 +49,12 @@ pub fn reset_all( all: bool, ) -> ApiFuture<()> { presentation.set_open_column_settings(None); - let ticket = session.submit(OpKind::Restore { fields: None }, { + let ticket = session.submit(OpKind::Restore { update: None }, { clone!(session, renderer, presentation); move |_ctx| { Box::pin(async move { reset_all_step(&session, &renderer, &presentation, all).await?; - Ok(StepOutcome::Done) + Ok(None) }) } }); diff --git a/rust/perspective-viewer/src/rust/tasks/restore_panel.rs b/rust/perspective-viewer/src/rust/tasks/restore_panel.rs index 28c1850885..ffd1680df3 100644 --- a/rust/perspective-viewer/src/rust/tasks/restore_panel.rs +++ b/rust/perspective-viewer/src/rust/tasks/restore_panel.rs @@ -10,7 +10,7 @@ // ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ // ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ -use std::collections::BTreeSet; +use std::rc::Rc; use perspective_client::clone; use perspective_client::utils::PerspectiveResultExt; @@ -18,9 +18,7 @@ use perspective_client::utils::PerspectiveResultExt; use crate::config::*; use crate::presentation::Presentation; use crate::renderer::Renderer; -use crate::session::{ - BindPlan, MissingTable, OpCtx, OpKind, Session, StepOutcome, probe_table, view_fields, -}; +use crate::session::{BindPlan, MissingTable, OpCtx, OpKind, Session, probe_table}; use crate::tasks::transactional_restore::{Outcome, commit_and_render, prepare}; use crate::tasks::*; use crate::workspace::{PanelId, Workspace}; @@ -81,9 +79,11 @@ pub(crate) enum Rebind { /// Decide a restore's [`BindPlan`], probing the incoming table — without /// touching the session's own binding. +#[allow(clippy::too_many_arguments)] async fn plan_binding( ctx: &OpCtx, session: &Session, + renderer: &Renderer, workspace: &Workspace, table: &TableUpdate, fresh: bool, @@ -98,7 +98,7 @@ async fn plan_binding( .get_table() .is_some_and(|t| t.get_name() == name.as_str()); - if same && !session.is_errored() { + if same && !session.is_errored() && renderer.failure().is_none() { return Ok(Some(BindPlan::Keep)); } @@ -129,56 +129,6 @@ async fn plan_binding( })) } -/// The top-level config keys `update` OVERWRITES, or `None` when it cannot be -/// superseded: `plugin_config` / `columns_config` updates MERGE into their -/// buckets, and a fresh panel's restore is what creates it. -fn restore_fields( - update: &ViewerConfigUpdate, - mode: &RestoreMode, -) -> Option> { - let ViewerConfigUpdate { - version, - plugin, - plugin_config, - columns_config, - settings, - theme, - title, - table, - view_config, - } = update; - - let _ = version; - if matches!(mode, RestoreMode::Fresh) - || matches!(plugin_config, OptionalUpdate::Update(_)) - || matches!(columns_config, OptionalUpdate::Update(_)) - { - return None; - } - - let mut fields = view_fields(view_config); - let mut set = |name: &'static str, present: bool| { - if present { - fields.insert(name); - } - }; - - set("plugin", !matches!(plugin, OptionalUpdate::Missing)); - set( - "plugin_config", - !matches!(plugin_config, OptionalUpdate::Missing), - ); - set( - "columns_config", - !matches!(columns_config, OptionalUpdate::Missing), - ); - set("settings", !matches!(settings, OptionalUpdate::Missing)); - set("theme", !matches!(theme, OptionalUpdate::Missing)); - set("title", !matches!(title, OptionalUpdate::Missing)); - set("table", !matches!(table, OptionalUpdate::Missing)); - Some(fields) -} - /// Apply a [`ViewerConfigUpdate`] to a single panel and re-draw — the one /// pipeline shared by `restorePanel` (an existing panel), `restoreWorkspace`, /// and `addPanel` (both fresh panels). @@ -194,8 +144,12 @@ pub(crate) async fn restore_panel( missing: MissingTable, preamble: Option>>, ) -> ApiResult<()> { - let fields = restore_fields(&update, &mode); - let ticket = session.submit(OpKind::Restore { fields }, { + let update = Rc::new(update); + let kind = OpKind::Restore { + update: (!matches!(mode, RestoreMode::Fresh)).then(|| update.clone()), + }; + + let ticket = session.submit(kind, { clone!(session, renderer, presentation, workspace); move |ctx| { Box::pin(async move { @@ -212,13 +166,13 @@ pub(crate) async fn restore_panel( mode, Rebind::Replace, RunOrigin::Public, - update, + Rc::unwrap_or_clone(update), errors, missing, ) .await?; - Ok(StepOutcome::Done) + Ok(None) }) } }); @@ -248,6 +202,7 @@ pub(crate) async fn restore_panel_step( let Some(plan) = plan_binding( ctx, session, + renderer, workspace, &update.table, fresh, @@ -284,7 +239,7 @@ pub(crate) async fn restore_panel_step( && committed && matches!(errors, RestoreErrors::Publish) { - let _ = session.set_run_error(e.clone()).await; + let _ = renderer.fail(e.clone()); } result?; diff --git a/rust/perspective-viewer/src/rust/tasks/send_column_config.rs b/rust/perspective-viewer/src/rust/tasks/send_column_config.rs index 378ea98e5b..ed71295b59 100644 --- a/rust/perspective-viewer/src/rust/tasks/send_column_config.rs +++ b/rust/perspective-viewer/src/rust/tasks/send_column_config.rs @@ -10,12 +10,13 @@ // ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ // ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ +use futures::FutureExt; use perspective_client::clone; use perspective_js::utils::*; use crate::config::ColumnConfigFieldUpdate; -use crate::renderer::Renderer; -use crate::session::{EditDelta, OpKind, Session, StepOutcome}; +use crate::renderer::{Renderer, StagedEdit}; +use crate::session::{EditDelta, OpKind, Session}; /// Apply a [`ColumnConfigFieldUpdate`] from a column-style sidebar /// control to the active plugin's per-column config bucket on @@ -31,12 +32,14 @@ pub fn send_column_config( update: ColumnConfigFieldUpdate, ) { let column = column_name.to_string(); + let staged = renderer.stage(StagedEdit::ColumnField { + column: column.clone(), + update: update.clone(), + }); + let kind = OpKind::Edit { - delta: EditDelta::ColumnField { - column: column.clone(), - update: update.clone(), - }, - fields: None, + delta: EditDelta::Renderer, + swaps_plugin: false, }; let ticket = session.submit(kind, { @@ -45,32 +48,37 @@ pub fn send_column_config( Box::pin(async move { let view_config = session.committed_view_config().clone(); renderer.update_columns_config_field(&view_config, &session, column, update); - Ok(StepOutcome::Render(Box::pin(async move { - let view_config_snapshot = session.committed_view_config().clone(); - let columns_configs = renderer - .all_columns_configs_materialized(&view_config_snapshot, &session) - .await; + drop(staged); + Ok(Some( + async move { + let view_config_snapshot = session.committed_view_config().clone(); + let columns_configs = renderer + .all_columns_configs_materialized(&view_config_snapshot, &session) + .await; - let plugin_token = - wasm_bindgen::JsValue::from_serde_ext(&renderer.committed_plugin_config()) - .unwrap(); + let plugin_token = wasm_bindgen::JsValue::from_serde_ext( + &renderer.committed_plugin_config(), + ) + .unwrap(); - renderer - .ensure_plugin_selected()? - .restore(&plugin_token, Some(&columns_configs))?; + renderer + .ensure_plugin_selected()? + .restore(&plugin_token, Some(&columns_configs))?; - clone!(session); - renderer - .update_lazy(async move { Ok(session.get_view_with_dimensions()) }) - .await?; + clone!(session); + renderer + .update_lazy(async move { Ok(session.get_view_with_dimensions()) }) + .await?; - renderer - .columns_config_changed - .emit(columns_configs.clone()); + renderer + .columns_config_changed + .emit(columns_configs.clone()); - renderer.column_style_changed.emit(columns_configs); - Ok(()) - }))) + renderer.column_style_changed.emit(columns_configs); + Ok(()) + } + .boxed_local(), + )) }) } }); diff --git a/rust/perspective-viewer/src/rust/tasks/send_plugin_config.rs b/rust/perspective-viewer/src/rust/tasks/send_plugin_config.rs index b4f6b7f86c..b6c5fdae18 100644 --- a/rust/perspective-viewer/src/rust/tasks/send_plugin_config.rs +++ b/rust/perspective-viewer/src/rust/tasks/send_plugin_config.rs @@ -10,12 +10,13 @@ // ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ // ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ +use futures::FutureExt; use perspective_client::clone; use perspective_js::utils::*; use crate::config::ColumnConfigFieldUpdate; -use crate::renderer::Renderer; -use crate::session::{EditDelta, OpKind, Session, StepOutcome}; +use crate::renderer::{Renderer, StagedEdit}; +use crate::session::{EditDelta, OpKind, Session}; /// Apply a [`ColumnConfigFieldUpdate`] from the Plugin-settings tab to /// the active plugin's bucket on [`Renderer`], then re-`restore` the @@ -28,9 +29,10 @@ use crate::session::{EditDelta, OpKind, Session, StepOutcome}; /// /// Column-style updates go through [`super::send_column_config`]. pub fn send_plugin_config(session: &Session, renderer: &Renderer, update: ColumnConfigFieldUpdate) { + let staged = renderer.stage(StagedEdit::PluginField(update.clone())); let kind = OpKind::Edit { - delta: EditDelta::PluginField(update.clone()), - fields: None, + delta: EditDelta::Renderer, + swaps_plugin: false, }; let ticket = session.submit(kind, { @@ -39,13 +41,17 @@ pub fn send_plugin_config(session: &Session, renderer: &Renderer, update: Column Box::pin(async move { let view_config = session.committed_view_config().clone(); let changed = renderer.update_plugin_config_field(&view_config, update); - Ok(StepOutcome::Render(Box::pin(async move { - if changed { - deliver_plugin_config(&session, &renderer).await?; - } + drop(staged); + Ok(Some( + async move { + if changed { + deliver_plugin_config(&session, &renderer).await?; + } - Ok(()) - }))) + Ok(()) + } + .boxed_local(), + )) }) } }); diff --git a/rust/perspective-viewer/src/rust/tasks/set_edit_mode.rs b/rust/perspective-viewer/src/rust/tasks/set_edit_mode.rs index 24fec8c10d..171b37e217 100644 --- a/rust/perspective-viewer/src/rust/tasks/set_edit_mode.rs +++ b/rust/perspective-viewer/src/rust/tasks/set_edit_mode.rs @@ -10,12 +10,13 @@ // ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ // ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ +use futures::FutureExt; use perspective_client::clone; use perspective_js::utils::*; use crate::config::*; -use crate::renderer::Renderer; -use crate::session::{EditDelta, OpKind, Session, StepOutcome}; +use crate::renderer::{Renderer, StagedEdit}; +use crate::session::{EditDelta, OpKind, Session}; /// Set the active plugin's `edit_mode`, persisting it in the [`Renderer`]'s /// plugin bucket and re-`restore`+rendering (the same merged-token path as @@ -33,9 +34,10 @@ pub fn set_edit_mode(session: &Session, renderer: &Renderer, mode: &str) { serde_json::Value::String(mode.to_owned()), ); + let staged = renderer.stage(StagedEdit::PluginConfig(map.clone())); let kind = OpKind::Edit { - delta: EditDelta::PluginConfig(map.clone()), - fields: None, + delta: EditDelta::Renderer, + swaps_plugin: false, }; let ticket = session.submit(kind, { @@ -47,14 +49,18 @@ pub fn set_edit_mode(session: &Session, renderer: &Renderer, mode: &str) { .update_plugin_config(&view_config, OptionalUpdate::Update(map)) .unwrap_or_default(); - Ok(StepOutcome::Render(Box::pin(async move { - if changed { - super::send_plugin_config::deliver_plugin_config(&session, &renderer) - .await?; - } + drop(staged); + Ok(Some( + async move { + if changed { + super::send_plugin_config::deliver_plugin_config(&session, &renderer) + .await?; + } - Ok(()) - }))) + Ok(()) + } + .boxed_local(), + )) }) } }); diff --git a/rust/perspective-viewer/src/rust/tasks/table_lifecycle.rs b/rust/perspective-viewer/src/rust/tasks/table_lifecycle.rs index eddd6a5f85..3b8ca11ae8 100644 --- a/rust/perspective-viewer/src/rust/tasks/table_lifecycle.rs +++ b/rust/perspective-viewer/src/rust/tasks/table_lifecycle.rs @@ -114,7 +114,7 @@ pub(crate) async fn sweep_table_bindings( async fn suspend_panel(panel: &Panel) -> ApiResult<()> { clone!(panel.session, panel.renderer); let ticket = panel.session.submit( - crate::session::OpKind::Restore { fields: None }, + crate::session::OpKind::Restore { update: None }, move |_ctx| { Box::pin(async move { renderer @@ -131,7 +131,7 @@ async fn suspend_panel(panel: &Panel) -> ApiResult<()> { }) .await?; - Ok(crate::session::StepOutcome::Done) + Ok(None) }) }, ); @@ -150,11 +150,11 @@ async fn bind_pending( ) -> ApiResult<()> { clone!(panel.session, panel.renderer, workspace, presentation); let ticket = panel.session.submit( - crate::session::OpKind::Restore { fields: None }, + crate::session::OpKind::Restore { update: None }, move |ctx| { Box::pin(async move { let Some(name) = session.pending_table() else { - return Ok(crate::session::StepOutcome::Done); + return Ok(None); }; let update = ViewerConfigUpdate { @@ -182,11 +182,11 @@ async fn bind_pending( if let Err(e) = &result && session.pending_table().is_some() { - let _ = session.set_run_error(e.clone()).await; + let _ = renderer.fail(e.clone()); } result?; - Ok(crate::session::StepOutcome::Done) + Ok(None) }) }, ); diff --git a/rust/perspective-viewer/src/rust/tasks/transactional_restore.rs b/rust/perspective-viewer/src/rust/tasks/transactional_restore.rs index f4c71ed40d..cd732cc7f2 100644 --- a/rust/perspective-viewer/src/rust/tasks/transactional_restore.rs +++ b/rust/perspective-viewer/src/rust/tasks/transactional_restore.rs @@ -23,11 +23,10 @@ use super::pipeline::{Committed, RunCommit, RunOrigin, RunSpec, locked_run, run_ use crate::config::{OptionalUpdate, SettingsUpdate, ThemeUpdate, ViewerConfigUpdate}; use crate::presentation::Presentation; use crate::renderer::{ - Renderer, ValidatedColumnsConfig, ValidatedPluginConfig, apply_columns_config_to, - apply_plugin_config_to, + PluginRef, PreparedRenderer, Renderer, ValidatedColumnsConfig, ValidatedPluginConfig, }; use crate::session::{ - BindPlan, BindingEffects, OverlayClause, PluginRef, PreparedView, Session, ViewDefaults, + BindPlan, BindingEffects, OverlayClause, PreparedView, Session, ViewDefaults, }; /// A restore that has passed every check and only awaits [`commit`]. @@ -235,7 +234,7 @@ pub(crate) async fn commit_and_render_locked( .await } -/// Replace the panel's state with a [`Prepared`] restore, in one swap. +/// Commit a [`Prepared`] restore to both engines, with no `await` between. fn commit( session: &Session, renderer: &Renderer, @@ -253,36 +252,20 @@ fn commit( let had_plugin = renderer.active_plugin().is_some(); let activate = plugin.is_some(); - let mut plugin_config_changed = false; - let mut columns_config_changed = false; - let binding = session.commit_view(view, |mut next| { - if let Some(plugin) = plugin { - next = next.with_plugin(plugin); - } - - let mut bucket = next.bucket(&target_name); - plugin_config_changed = apply_plugin_config_to(&mut bucket, plugin_config); - columns_config_changed = apply_columns_config_to(&mut bucket, columns_config); - if plugin_config_changed || columns_config_changed { - next = next.with_bucket(&target_name, bucket); - } - - if let Some(theme) = theme { - next = next.with_theme(theme); - } - - if let Some(title) = title { - next = next.with_title(title); - } - - next + let binding = session.commit_view(view, title); + let changed = renderer.commit(PreparedRenderer { + plugin, + target_name, + plugin_config, + columns_config, + theme, }); let committed = Committed { activate, plugin_swapped: activate && had_plugin, - plugin_config_changed, - columns_config_changed, + plugin_config_changed: changed.plugin_config, + columns_config_changed: changed.columns_config, }; (committed, binding) diff --git a/rust/perspective-viewer/src/rust/tasks/update_theme.rs b/rust/perspective-viewer/src/rust/tasks/update_theme.rs index 46177f695e..15f53e9d7f 100644 --- a/rust/perspective-viewer/src/rust/tasks/update_theme.rs +++ b/rust/perspective-viewer/src/rust/tasks/update_theme.rs @@ -13,26 +13,62 @@ //! Theme reset / set task. use futures::future::join_all; +use perspective_client::clone; use perspective_js::utils::*; use crate::presentation::Presentation; -use crate::renderer::Renderer; +use crate::renderer::{Renderer, StagedEdit}; +use crate::session::{EditDelta, OpKind, Session}; use crate::workspace::Workspace; /// Give `renderer` the concrete registry default IF it has none yet — the /// cold-boot case, where the panel was created before the registry first /// parsed and `active_theme_name_sync` had nothing to resolve. Only ever /// FILLS IN, never overwrites, so it cannot repaint a themed panel. -pub(crate) async fn seed_panel_theme(presentation: &Presentation, renderer: &Renderer) { +pub(crate) async fn seed_panel_theme( + presentation: &Presentation, + session: &Session, + renderer: &Renderer, +) { if renderer.theme().is_none() { - renderer.set_theme(presentation.get_default_theme_name().await); + submit_theme( + session, + renderer, + presentation.get_default_theme_name().await, + ); } } +/// Submit a theme pick as a UI edit, visible to the UI from this call. +pub fn submit_theme(session: &Session, renderer: &Renderer, theme: Option) { + let staged = renderer.stage(StagedEdit::Theme(theme.clone())); + let kind = OpKind::Edit { + delta: EditDelta::Renderer, + swaps_plugin: false, + }; + + let _ticket = session.submit(kind, { + clone!(renderer); + move |_ctx| { + Box::pin(async move { + renderer.commit_theme(theme); + drop(staged); + Ok(None) + }) + } + }); +} + +/// [`submit_theme`] plus a synchronous [`Renderer::stamp_theme`]. +pub fn submit_theme_stamped(session: &Session, renderer: &Renderer, theme: Option) { + submit_theme(session, renderer, theme); + renderer.stamp_theme(None); +} + /// [`seed_panel_theme`] for every panel. pub(crate) async fn seed_default_themes(presentation: &Presentation, workspace: &Workspace) { for panel in workspace.panels() { - seed_panel_theme(presentation, &panel.renderer).await; + seed_panel_theme(presentation, &panel.session, &panel.renderer).await; } } @@ -55,6 +91,7 @@ pub(crate) async fn seed_default_themes(presentation: &Presentation, workspace: /// stamped at the plugin's last capture): the picked panel when the value /// is genuinely new, and never a panel that has yet to first-paint. pub fn update_theme( + session: &Session, renderer: &Renderer, presentation: &Presentation, workspace: &Workspace, @@ -65,12 +102,13 @@ pub fn update_theme( // cascade styles. Only "reset to default" has to await the registry, and // it records the resolved NAME, never an empty theme. if let Some(name) = &theme { - renderer.set_theme_stamped(Some(name.clone())); + submit_theme_stamped(session, renderer, Some(name.clone())); } let presentation = presentation.clone(); let workspace = workspace.clone(); let renderer = renderer.clone(); + let session = session.clone(); ApiFuture::spawn(async move { match theme { Some(name) => { @@ -78,7 +116,8 @@ pub fn update_theme( }, None => { presentation.reset_theme().await?; - renderer.set_theme_stamped(presentation.get_default_theme_name().await); + let default = presentation.get_default_theme_name().await; + submit_theme_stamped(&session, &renderer, default); }, } diff --git a/rust/perspective-viewer/src/rust/ui/containers/split_panel.rs b/rust/perspective-viewer/src/rust/ui/containers/split_panel.rs index d1e755fc8e..4ea0285ee6 100644 --- a/rust/perspective-viewer/src/rust/ui/containers/split_panel.rs +++ b/rust/perspective-viewer/src/rust/ui/containers/split_panel.rs @@ -496,44 +496,3 @@ impl ResizingState { Ok(self.body_style.set_property("cursor", &self.cursor)?) } } - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn html_if_empty_branch_is_not_literally_empty() { - let via_if: Html = html! { if false {
} else { <> } }; - assert_ne!(via_if, html! { <> }); - } - - #[test] - fn empty_fragment_is_empty() { - assert!(is_empty_html(&html! { <> })); - } - - #[test] - fn html_if_empty_branch_is_semantically_empty() { - assert!(is_empty_html(&html! { if false {
} else { <> } })); - } - - #[test] - fn nested_empty_fragments_are_empty() { - assert!(is_empty_html(&html! { <><><> })); - } - - #[test] - fn tag_is_not_empty() { - assert!(!is_empty_html(&html! {
})); - } - - #[test] - fn fragment_containing_a_tag_is_not_empty() { - assert!(!is_empty_html(&html! { <>
})); - } - - #[test] - fn text_is_not_empty() { - assert!(!is_empty_html(&html! { { "" } })); - } -} diff --git a/rust/perspective-viewer/src/rust/ui/form/mirrored_textarea.rs b/rust/perspective-viewer/src/rust/ui/form/mirrored_textarea.rs index 1ec3c5585d..c01748392a 100644 --- a/rust/perspective-viewer/src/rust/ui/form/mirrored_textarea.rs +++ b/rust/perspective-viewer/src/rust/ui/form/mirrored_textarea.rs @@ -98,15 +98,3 @@ pub fn MirroredTextarea(props: &MirroredTextareaProps) -> Html {
} } - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn mirror_never_compares_equal() { - let mirror = Mirror(Html::default()); - assert!(mirror != mirror.clone()); - assert!(Mirror(Html::default()) != Mirror(Html::default())); - } -} diff --git a/rust/perspective-viewer/src/rust/ui/form/multi_stop_gradient_selector.rs b/rust/perspective-viewer/src/rust/ui/form/multi_stop_gradient_selector.rs index e97a6980a8..41bf1f0678 100644 --- a/rust/perspective-viewer/src/rust/ui/form/multi_stop_gradient_selector.rs +++ b/rust/perspective-viewer/src/rust/ui/form/multi_stop_gradient_selector.rs @@ -384,92 +384,3 @@ pub fn multi_stop_gradient_selector(props: &MultiStopGradientProps) -> Html { } } - -#[cfg(test)] -mod tests { - use super::*; - - fn stop(color: &str, offset: f64) -> GradientStopSpec { - GradientStopSpec { - color: color.to_owned(), - offset, - } - } - - #[test] - fn sample_clamps_outside_stop_range() { - let stops = vec![stop("#000000", 0.25), stop("#ff0000", 0.75)]; - assert_eq!(sample_gradient_hex(&stops, 0.0), "#000000"); - assert_eq!(sample_gradient_hex(&stops, 1.0), "#ff0000"); - } - - #[test] - fn sample_interpolates_linearly() { - let stops = vec![stop("#000000", 0.0), stop("#ff0000", 1.0)]; - assert_eq!(sample_gradient_hex(&stops, 0.5), "#800000"); - - let stops = vec![ - stop("#000000", 0.0), - stop("#ffffff", 0.5), - stop("#ff0000", 1.0), - ]; - - assert_eq!(sample_gradient_hex(&stops, 0.25), "#808080"); - assert_eq!(sample_gradient_hex(&stops, 0.75), "#ff8080"); - } - - #[test] - fn track_positions_pad_the_bar_edges() { - assert_eq!(track_position(0.0), "calc(8px + (100% - 16px) * 0.0000)"); - assert_eq!(track_position(1.0), "calc(8px + (100% - 16px) * 1.0000)"); - assert_eq!(track_position(0.333), "calc(8px + (100% - 16px) * 0.3330)"); - } - - #[test] - fn gradient_interpolates_on_padded_track() { - let stops = vec![ - stop("#333333", 1.0), - stop("#111111", 0.0), - stop("#222222", 0.5), - ]; - - assert_eq!( - css_gradient(&stops), - format!( - "background:linear-gradient(to right, #111111 {}, #222222 {}, #333333 {})", - track_position(0.0), - track_position(0.5), - track_position(1.0) - ) - ); - } - - #[test] - fn equalize_centers_between_neighbors_or_edges() { - let stops = vec![ - stop("#111111", 0.2), - stop("#222222", 0.25), - stop("#333333", 0.4), - ]; - - assert_eq!(equalized_offset(&stops, 1), (0.2 + 0.4) / 2.0); - - assert_eq!(equalized_offset(&stops, 0), 0.125); - assert_eq!(equalized_offset(&stops, 2), (0.25 + 1.0) / 2.0); - - let single = vec![stop("#111111", 0.9)]; - assert_eq!(equalized_offset(&single, 0), 0.5); - } - - #[test] - fn sample_degenerate_cases() { - assert_eq!(sample_gradient_hex(&[], 0.5), "#000000"); - let stops = vec![stop("#123456", 0.5)]; - assert_eq!(sample_gradient_hex(&stops, 0.1), "#123456"); - assert_eq!(sample_gradient_hex(&stops, 0.9), "#123456"); - - let stops = vec![stop("red", 0.0), stop("blue", 1.0)]; - assert_eq!(sample_gradient_hex(&stops, 0.25), "red"); - assert_eq!(sample_gradient_hex(&stops, 0.75), "blue"); - } -} diff --git a/rust/perspective-viewer/src/rust/ui/form/palette_selector.rs b/rust/perspective-viewer/src/rust/ui/form/palette_selector.rs index 4255900c1f..41c808b953 100644 --- a/rust/perspective-viewer/src/rust/ui/form/palette_selector.rs +++ b/rust/perspective-viewer/src/rust/ui/form/palette_selector.rs @@ -386,54 +386,3 @@ pub fn palette_selector(props: &PaletteProps) -> Html { } } - -#[cfg(test)] -mod tests { - use super::*; - - fn v(items: &[&str]) -> Vec { - items.iter().map(|x| x.to_string()).collect() - } - - #[test] - fn insertion_index_splits_at_centers_and_resolves_rows() { - let centers = [(12.0, 12.0), (40.0, 12.0), (68.0, 12.0), (12.0, 40.0)]; - assert_eq!(insertion_index(¢ers, 0.0, 12.0), 0); - assert_eq!(insertion_index(¢ers, 13.0, 12.0), 1); - assert_eq!(insertion_index(¢ers, 39.0, 12.0), 1); - assert_eq!(insertion_index(¢ers, 41.0, 12.0), 2); - assert_eq!(insertion_index(¢ers, 90.0, 12.0), 3); - assert_eq!(insertion_index(¢ers, 10.0, 41.0), 3); - assert_eq!(insertion_index(¢ers, 30.0, 41.0), 4); - assert_eq!(insertion_index(&[], 5.0, 5.0), 0); - } - - #[test] - fn slot_from_display_maps_previewed_slots_back_to_original_order() { - assert_eq!(slot_from_display(3, 2, 0), 3); - assert_eq!(slot_from_display(2, 2, 0), 3); - assert_eq!(slot_from_display(1, 2, 0), 2); - assert_eq!(slot_from_display(0, 2, 0), 0); - assert_eq!(slot_from_display(0, 0, 2), 0); - assert_eq!(slot_from_display(1, 0, 2), 0); - assert_eq!(slot_from_display(2, 0, 2), 1); - assert_eq!(slot_from_display(3, 0, 2), 2); - assert_eq!(slot_from_display(4, 0, 2), 4); - assert_eq!(slot_from_display(1, 1, 1), 1); - assert_eq!(slot_from_display(2, 1, 1), 1); - assert_eq!(slot_from_display(3, 1, 1), 3); - } - - #[test] - fn reordered_moves_and_rejects_no_ops() { - let values = v(&["a", "b", "c", "d"]); - assert_eq!(reordered(&values, 0, 4), Some(v(&["b", "c", "d", "a"]))); - assert_eq!(reordered(&values, 3, 0), Some(v(&["d", "a", "b", "c"]))); - assert_eq!(reordered(&values, 0, 2), Some(v(&["b", "a", "c", "d"]))); - assert_eq!(reordered(&values, 2, 1), Some(v(&["a", "c", "b", "d"]))); - assert_eq!(reordered(&values, 1, 1), None); - assert_eq!(reordered(&values, 1, 2), None); - assert_eq!(reordered(&values, 4, 0), None); - assert_eq!(reordered(&values, 0, 5), None); - } -} diff --git a/rust/perspective-viewer/src/rust/utils/browser/mod.rs b/rust/perspective-viewer/src/rust/utils/browser/mod.rs index d18f127e76..f0bba3a979 100644 --- a/rust/perspective-viewer/src/rust/utils/browser/mod.rs +++ b/rust/perspective-viewer/src/rust/utils/browser/mod.rs @@ -18,9 +18,6 @@ mod dragdrop; mod request_animation_frame; mod selection; -#[cfg(test)] -mod tests; - pub use self::blob::*; pub use self::download::*; pub use self::dragdrop::*; diff --git a/rust/perspective-viewer/src/rust/utils/browser/tests/debounce.rs b/rust/perspective-viewer/src/rust/utils/browser/tests/debounce.rs deleted file mode 100644 index 6b210b42d5..0000000000 --- a/rust/perspective-viewer/src/rust/utils/browser/tests/debounce.rs +++ /dev/null @@ -1,163 +0,0 @@ -// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ -// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ -// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ -// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ -// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ -// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ -// ┃ Copyright (c) 2017, the Perspective Authors. ┃ -// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ -// ┃ This file is part of the Perspective library, distributed under the terms ┃ -// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ -// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ - -use std::cell::Cell; -use std::rc::Rc; - -use futures::channel::oneshot::*; -use futures::future::join_all; -use wasm_bindgen_futures::spawn_local; -use wasm_bindgen_test::*; - -use super::super::request_animation_frame::set_timeout; -use crate::utils::debounce::*; - -#[wasm_bindgen_test] -pub async fn test_lock() { - let debounce_mutex = DebounceMutex::default(); - let cell = Rc::new(Cell::new(0)); - let (sender, receiver) = channel::(); - spawn_local({ - let cell = cell.clone(); - let debounce_mutex = debounce_mutex.clone(); - async move { - debounce_mutex - .lock(async { - cell.set(1); - set_timeout(10).await.unwrap(); - cell.set(2); - }) - .await - } - }); - - spawn_local({ - let cell = cell.clone(); - let debounce_mutex = debounce_mutex.clone(); - async move { - debounce_mutex - .lock(async { - for _ in 0..10 { - set_timeout(1).await.unwrap(); - if cell.get() == 1 { - sender.send(false).unwrap(); - return; - } - } - sender.send(cell.get() == 2).unwrap(); - }) - .await - } - }); - - assert!(receiver.await.unwrap()); -} - -#[wasm_bindgen_test] -pub async fn test_lock_seq() { - let debounce_mutex = DebounceMutex::default(); - let cell: Rc> = Rc::new(Cell::new(0)); - - let tasks = (0..10) - .map(|_| { - let cell = cell.clone(); - let debounce_mutex = debounce_mutex.clone(); - async move { - debounce_mutex - .lock(async { - set_timeout(10).await.unwrap(); - cell.set(cell.get() + 1); - }) - .await; - } - }) - .collect::>(); - - assert_eq!(join_all(tasks).await.len(), 10); - assert_eq!(cell.get(), 10); -} - -/// A dedicated [`DebounceSlot`]'s task must RUN even when the default slot -/// has a parked runner — coalescing is scoped per slot. Pre-slot (one -/// shared parked flag), `t4` would resolve via `t2`'s settle without its -/// closure ever running: the resize-eaten-by-parked-update bug. -#[wasm_bindgen_test] -pub async fn test_slot_isolated_from_default_coalescing() { - let debounce_mutex = DebounceMutex::default(); - let slot = debounce_mutex.slot(); - let defaults: Rc> = Rc::new(Cell::new(0)); - let slots: Rc> = Rc::new(Cell::new(0)); - - let t1 = debounce_mutex.lock(async { - set_timeout(10).await.unwrap(); - }); - - let t2 = { - let defaults = defaults.clone(); - debounce_mutex.debounce(async move { - defaults.set(defaults.get() + 1); - Ok(()) - }) - }; - - let t3 = { - let defaults = defaults.clone(); - debounce_mutex.debounce(async move { - defaults.set(defaults.get() + 100); - Ok(()) - }) - }; - - let t4 = { - let slots = slots.clone(); - slot.debounce_with(|_| async move { - slots.set(slots.get() + 1); - Ok(()) - }) - }; - - let (_, r2, r3, r4) = futures::join!(t1, t2, t3, t4); - r2.unwrap(); - r3.unwrap(); - r4.unwrap(); - - // `t3` coalesced onto the parked `t2`; `t4` ran its own closure. - assert_eq!(defaults.get(), 1); - assert_eq!(slots.get(), 1); -} - -#[wasm_bindgen_test] -pub async fn test_debounce_seq() { - let debounce_mutex = DebounceMutex::default(); - let cell: Rc> = Rc::new(Cell::new(0)); - - let tasks = (0..10) - .map(|_| { - let cell = cell.clone(); - let debounce_mutex = debounce_mutex.clone(); - async move { - debounce_mutex - .debounce(async { - set_timeout(10).await.unwrap(); - cell.set(cell.get() + 1); - Ok(()) - }) - .await - .unwrap(); - } - }) - .collect::>(); - - assert_eq!(join_all(tasks).await.len(), 10); - assert!(cell.get() < 10); - assert!(cell.get() > 0); -} diff --git a/rust/perspective-viewer/src/rust/utils/css.rs b/rust/perspective-viewer/src/rust/utils/css.rs index 55cd436104..7ba460f50e 100644 --- a/rust/perspective-viewer/src/rust/utils/css.rs +++ b/rust/perspective-viewer/src/rust/utils/css.rs @@ -523,363 +523,3 @@ pub fn resolve_css_refs( dropped } - -#[cfg(test)] -mod tests { - use serde_json::json; - - use super::*; - - fn stop(color: &str, offset: f64) -> GradientStopSpec { - GradientStopSpec { - color: color.to_owned(), - offset, - } - } - - #[test] - fn colors_normalize_to_lowercase_hex() { - assert_eq!(normalize_css_color("#ABC").unwrap(), "#aabbcc"); - assert_eq!(normalize_css_color("#abcd").unwrap(), "#aabbcc"); - assert_eq!(normalize_css_color(" #FF7F0E ").unwrap(), "#ff7f0e"); - assert_eq!(normalize_css_color("#ff7f0e80").unwrap(), "#ff7f0e"); - assert_eq!(normalize_css_color("rgb(255, 127, 14)").unwrap(), "#ff7f0e"); - assert_eq!( - normalize_css_color("rgba(255,127,14,0.5)").unwrap(), - "#ff7f0e" - ); - assert_eq!( - normalize_css_color("rgb(255 127 14 / 50%)").unwrap(), - "#ff7f0e" - ); - assert_eq!( - normalize_css_color("rgb(100%, 0%, 50%)").unwrap(), - "#ff0080" - ); - assert_eq!(normalize_css_color("rgb(300, -5, 14)").unwrap(), "#ff000e"); - assert!(normalize_css_color("red").is_err()); - assert!(normalize_css_color("#12345").is_err()); - assert!(normalize_css_color("hsl(0, 100%, 50%)").is_err()); - assert!(normalize_css_color("#gggggg").is_err()); - assert!(normalize_css_color("").is_err()); - } - - #[test] - fn var_refs_are_whole_value_and_namespaced() { - assert_eq!( - parse_var_ref("var(--psp-user--gradient-1)"), - Some("--psp-user--gradient-1") - ); - assert_eq!( - parse_var_ref(" var( --psp-user--color-hot ) "), - Some("--psp-user--color-hot") - ); - assert_eq!(parse_var_ref("var(--psp-charts--gradient)"), None); - assert_eq!(parse_var_ref("var(--psp-user--)"), None); - assert_eq!(parse_var_ref("var(--psp-user--gradient-1, red)"), None); - assert_eq!(parse_var_ref("#ff0000"), None); - assert_eq!( - parse_var_ref("linear-gradient(var(--psp-user--color-1), #fff)"), - None - ); - } - - #[test] - fn kind_is_carried_by_the_reference_prefix() { - assert_eq!( - CssKind::of_var("--psp-user--gradient-1"), - Some(CssKind::Gradient) - ); - assert_eq!( - CssKind::of_var("--psp-user--palette-warm"), - Some(CssKind::Palette) - ); - assert_eq!(CssKind::of_var("--psp-user--color-1"), Some(CssKind::Color)); - assert_eq!(CssKind::of_var("--psp-user--other-1"), None); - assert_eq!( - CssKind::Palette.short_name("--psp-user--palette-warm"), - "warm" - ); - - assert_eq!( - CssGradient::parse("var(--psp-user--gradient-1)").unwrap(), - CssGradient::Var("--psp-user--gradient-1".to_owned()) - ); - assert!(CssGradient::parse("var(--psp-user--palette-1)").is_err()); - assert!(CssPalette::parse("var(--psp-user--gradient-1)").is_err()); - assert!(CssColor::parse("var(--psp-user--palette-1)").is_err()); - assert_eq!( - CssColor::parse("var(--psp-user--color-1)") - .unwrap() - .to_css(), - "var(--psp-user--color-1)" - ); - - assert_eq!( - canonicalize_css_palette("var(--psp-user--palette-1)").unwrap(), - "var(--psp-user--palette-1)" - ); - } - - #[test] - fn gradient_round_trip_is_canonical_and_idempotent() { - let src = "linear-gradient(to right, #0366d6 0%, #ffffff 33.3%, #ff7f0e 100%)"; - let canon = canonicalize_css_gradient(src).unwrap(); - assert_eq!(canon, src); - assert_eq!(canonicalize_css_gradient(&canon).unwrap(), canon); - - let CssGradient::Literal(stops) = CssGradient::parse(src).unwrap() else { - panic!("expected literal"); - }; - - assert_eq!(stops, vec![ - stop("#0366d6", 0.0), - stop("#ffffff", 0.333), - stop("#ff7f0e", 1.0), - ]); - - let css = gradient_to_css(&[ - stop("#222222", 1.7), - stop("#111111", 1.0 / 3.0), - stop("#000000", -0.5), - ]); - - assert_eq!( - css, - "linear-gradient(to right, #000000 0%, #111111 33.3%, #222222 100%)" - ); - - assert_eq!(format_percent(0.375), "37.5%"); - assert_eq!(format_percent(0.1), "10%"); - assert_eq!(format_percent(1.0), "100%"); - } - - #[test] - fn gradient_accepts_loose_input() { - assert_eq!( - canonicalize_css_gradient("linear-gradient(#0366d6 0%, #ff7f0e 100%)").unwrap(), - "linear-gradient(to right, #0366d6 0%, #ff7f0e 100%)" - ); - - assert_eq!( - canonicalize_css_gradient("LINEAR-GRADIENT(90deg, RGB(3, 102, 214), #FFF, #ff7f0e)") - .unwrap(), - "linear-gradient(to right, #0366d6 0%, #ffffff 50%, #ff7f0e 100%)" - ); - - assert_eq!( - canonicalize_css_gradient( - "linear-gradient(to bottom, #000 10%, #111, #222, #333 70%, #444)" - ) - .unwrap(), - "linear-gradient(to right, #000000 10%, #111111 30%, #222222 50%, #333333 70%, \ - #444444 100%)" - ); - - assert_eq!( - canonicalize_css_gradient("linear-gradient(#fff 120%, #000 -5%)").unwrap(), - "linear-gradient(to right, #000000 0%, #ffffff 100%)" - ); - } - - #[test] - fn gradient_rejects_malformed_input() { - assert!(canonicalize_css_gradient("#ff0000").is_err()); - assert!(canonicalize_css_gradient("linear-gradient(#ff0000)").is_err()); - assert!(canonicalize_css_gradient("linear-gradient(#ff0000 0px, #fff 10px)").is_err()); - assert!(canonicalize_css_gradient("linear-gradient(red, blue)").is_err()); - assert!(canonicalize_css_gradient("radial-gradient(#000, #fff)").is_err()); - assert!(canonicalize_css_gradient("linear-gradient(#000, , #fff)").is_err()); - assert!(canonicalize_css_gradient("").is_err()); - } - - #[test] - fn palette_round_trip_and_rules() { - let src = "linear-gradient(to right, #0366d6, #ff7f0e, #2ca02c)"; - assert_eq!(canonicalize_css_palette(src).unwrap(), src); - assert_eq!( - CssPalette::parse(src).unwrap(), - CssPalette::Literal(vec![ - "#0366d6".to_owned(), - "#ff7f0e".to_owned(), - "#2ca02c".to_owned() - ]) - ); - - assert_eq!( - canonicalize_css_palette("linear-gradient(90deg, rgb(3,102,214), #FFF)").unwrap(), - "linear-gradient(to right, #0366d6, #ffffff)" - ); - - assert_eq!( - canonicalize_css_palette("linear-gradient(#abc)").unwrap(), - "linear-gradient(to right, #aabbcc)" - ); - - assert_eq!( - palette_to_css(&["#000000".to_owned(), "#ffffff".to_owned()]), - "linear-gradient(to right, #000000, #ffffff)" - ); - } - - #[test] - fn palette_rejects_positions() { - let gradient = gradient_to_css(&[stop("#000000", 0.0), stop("#ffffff", 1.0)]); - assert!(CssPalette::parse(&gradient).is_err()); - assert!(canonicalize_css_palette("linear-gradient(#000, #fff 50%)").is_err()); - assert!(canonicalize_css_palette("linear-gradient()").is_err()); - assert!(canonicalize_css_palette("#000000").is_err()); - - let palette = palette_to_css(&["#000000".to_owned(), "#ffffff".to_owned()]); - assert_eq!( - canonicalize_css_gradient(&palette).unwrap(), - "linear-gradient(to right, #000000 0%, #ffffff 100%)" - ); - } - - #[test] - fn color_canonicalizes_literals_and_passes_refs() { - assert_eq!(canonicalize_css_color("#ABC").unwrap(), "#aabbcc"); - assert_eq!( - canonicalize_css_color("var(--psp-user--color-hot)").unwrap(), - "var(--psp-user--color-hot)" - ); - assert!(canonicalize_css_color("var(--psp-user--gradient-1)").is_err()); - assert!(canonicalize_css_color("linear-gradient(#000, #fff)").is_err()); - } - - #[test] - fn palette_set_unions_restored_and_in_use_with_stable_names() { - let mut restored = BTreeMap::new(); - restored.insert( - "--psp-user--gradient-heat".to_owned(), - "linear-gradient(to right, #000000 0%, #ffffff 100%)".to_owned(), - ); - restored.insert("--psp-user--color-unused".to_owned(), "#123456".to_owned()); - let host = vec![NamedValue { - name: "--psp-user--palette-1".to_owned(), - value: "linear-gradient(to right, #ff0000, #00ff00)".to_owned(), - }]; - let taken = - |name: &str| name == "--psp-user--gradient-1" || name == "--psp-user--palette-1"; - let in_use = vec![ - ( - CssKind::Gradient, - "linear-gradient(to right, #000000 0%, #ffffff 100%)".to_owned(), - ), - ( - CssKind::Gradient, - "linear-gradient(to right, #111111 0%, #222222 100%)".to_owned(), - ), - ( - CssKind::Gradient, - "linear-gradient(to right, #111111 0%, #222222 100%)".to_owned(), - ), - ( - CssKind::Palette, - "linear-gradient(to right, #ff0000, #00ff00)".to_owned(), - ), - (CssKind::Color, "#abcdef".to_owned()), - ]; - - let set = assign_palette_names(&restored, &host, &in_use, &taken); - assert_eq!( - set.iter() - .map(|(k, v)| (k.as_str(), v.as_str())) - .collect::>(), - vec![ - ("--psp-user--color-1", "#abcdef"), - ("--psp-user--color-unused", "#123456"), - ( - "--psp-user--gradient-2", - "linear-gradient(to right, #111111 0%, #222222 100%)" - ), - ( - "--psp-user--gradient-heat", - "linear-gradient(to right, #000000 0%, #ffffff 100%)" - ), - ( - "--psp-user--palette-1", - "linear-gradient(to right, #ff0000, #00ff00)" - ), - ] - ); - - assert_eq!( - palette_name_for( - &set, - CssKind::Gradient, - "linear-gradient(to right, #111111 0%, #222222 100%)" - ) - .as_deref(), - Some("--psp-user--gradient-2") - ); - assert_eq!(palette_name_for(&set, CssKind::Gradient, "#abcdef"), None); - - assert_eq!(assign_palette_names(&set, &host, &in_use, &taken), set); - } - - #[test] - fn resolver_inlines_known_refs_and_drops_the_rest() { - let host = |name: &str| -> Option { - match name { - "--psp-user--gradient-1" => Some("linear-gradient(#000, #fff)".to_owned()), - "--psp-user--palette-warm" => Some("linear-gradient(#f00, #ff0)".to_owned()), - "--psp-user--color-hot" => Some("rgb(255, 0, 0)".to_owned()), - "--psp-user--palette-bad" => Some("linear-gradient(#000 0%, #fff 100%)".to_owned()), - "--psp-user--color-empty" => Some(" ".to_owned()), - _ => None, - } - }; - - let mut entry = json!({ - "gradient": "var(--psp-user--gradient-1)", - "palette": "var(--psp-user--palette-warm)", - "color": "var(--psp-user--color-hot)", - "bad_palette": "var(--psp-user--palette-bad)", - "missing": "var(--psp-user--gradient-missing)", - "empty": "var(--psp-user--color-empty)", - "unknown_kind": "var(--psp-user--other-1)", - "literal": "#123456", - "number": 3, - }) - .as_object() - .unwrap() - .clone(); - - let mut dropped = resolve_css_refs(&mut entry, &host); - dropped.sort(); - assert_eq!(dropped, vec![ - ( - "bad_palette".to_owned(), - "--psp-user--palette-bad".to_owned() - ), - ("empty".to_owned(), "--psp-user--color-empty".to_owned()), - ( - "missing".to_owned(), - "--psp-user--gradient-missing".to_owned() - ), - ("unknown_kind".to_owned(), "--psp-user--other-1".to_owned()), - ]); - - assert_eq!( - entry, - json!({ - "gradient": "linear-gradient(to right, #000000 0%, #ffffff 100%)", - "palette": "linear-gradient(to right, #ff0000, #ffff00)", - "color": "#ff0000", - "literal": "#123456", - "number": 3, - }) - .as_object() - .unwrap() - .clone() - ); - - assert!(entry.values().all(|v| { - v.as_str() - .map(|s| parse_var_ref(s).is_none()) - .unwrap_or(true) - })); - } -} diff --git a/rust/perspective-viewer/src/rust/utils/local_fonts.rs b/rust/perspective-viewer/src/rust/utils/local_fonts.rs index 1528ab919a..cf49350517 100644 --- a/rust/perspective-viewer/src/rust/utils/local_fonts.rs +++ b/rust/perspective-viewer/src/rust/utils/local_fonts.rs @@ -357,39 +357,3 @@ fn measure_width(ctx: &web_sys::CanvasRenderingContext2d, family: &str) -> f64 { .map(|x| x.width()) .unwrap_or_default() } - -#[cfg(test)] -mod tests { - use super::*; - - fn owned(xs: &[&str]) -> Vec { - xs.iter().map(|x| (*x).to_owned()).collect() - } - - #[test] - fn generic_families_lead_and_current_appends_when_missing() { - let options = compose_font_family_options("Zapf Chancery", &owned(&["Arial", "Menlo"])); - assert_eq!( - options, - owned(&[ - "inherit", - "monospace", - "sans-serif", - "serif", - "system-ui", - "Arial", - "Menlo", - "Zapf Chancery" - ]) - ); - } - - #[test] - fn current_is_not_duplicated() { - let options = compose_font_family_options("Arial", &owned(&["Arial"])); - assert_eq!(options.iter().filter(|x| *x == "Arial").count(), 1); - - let options = compose_font_family_options("inherit", &[]); - assert_eq!(options, owned(&GENERIC_FONT_FAMILIES)); - } -} diff --git a/rust/perspective-viewer/src/rust/utils/mod.rs b/rust/perspective-viewer/src/rust/utils/mod.rs index 6e803f6ab4..502eba2cae 100644 --- a/rust/perspective-viewer/src/rust/utils/mod.rs +++ b/rust/perspective-viewer/src/rust/utils/mod.rs @@ -34,9 +34,6 @@ mod pubsub; mod spawn; mod weak_scope; -#[cfg(test)] -mod tests; - pub use browser::*; pub use completion::*; pub use css::*; diff --git a/rust/perspective-viewer/src/rust/utils/tests/pubsub.rs b/rust/perspective-viewer/src/rust/utils/tests/pubsub.rs deleted file mode 100644 index 0a256e06b7..0000000000 --- a/rust/perspective-viewer/src/rust/utils/tests/pubsub.rs +++ /dev/null @@ -1,95 +0,0 @@ -// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ -// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ -// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ -// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ -// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ -// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ -// ┃ Copyright (c) 2017, the Perspective Authors. ┃ -// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ -// ┃ This file is part of the Perspective library, distributed under the terms ┃ -// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ -// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ - -use std::cell::RefCell; -use std::rc::Rc; - -use wasm_bindgen_test::*; - -use super::super::pubsub::*; - -#[wasm_bindgen_test] -pub async fn test_pub_sub() { - let pubsub: PubSub = PubSub::default(); - let called: Rc>> = Rc::new(RefCell::new(None)); - let callback = { - let called = called.clone(); - move |x| { - *called.borrow_mut() = Some(x); - } - }; - - let sub = pubsub.add_listener(callback); - pubsub.emit(1); - assert_eq!(*called.borrow(), Some(1)); - drop(sub); -} - -#[wasm_bindgen_test] -pub async fn test_pub_sub_multiple() { - let pubsub: PubSub = PubSub::default(); - let called1: Rc>> = Rc::new(RefCell::new(None)); - let called2: Rc>> = Rc::new(RefCell::new(None)); - let callback1 = { - let called = called1.clone(); - move |x| { - *called.borrow_mut() = Some(x); - } - }; - - let callback2 = { - let called = called2.clone(); - move |x| { - *called.borrow_mut() = Some(x); - } - }; - - // Must be in scope or subscriptions will drop immediately! - let sub1 = pubsub.add_listener(callback1); - let sub2 = pubsub.add_listener(callback2); - - pubsub.emit(1); - assert_eq!(*called1.borrow(), Some(1)); - assert_eq!(*called2.borrow(), Some(1)); - - drop(sub1); - drop(sub2); -} - -#[wasm_bindgen_test] -pub async fn test_pub_sub_multiple_drop_first() { - let pubsub: PubSub = PubSub::default(); - let called1: Rc>> = Rc::new(RefCell::new(None)); - let called2: Rc>> = Rc::new(RefCell::new(None)); - let callback1 = { - let called = called1.clone(); - move |x| { - *called.borrow_mut() = Some(x); - } - }; - - let callback2 = { - let called = called2.clone(); - move |x| { - *called.borrow_mut() = Some(x); - } - }; - - let _ = pubsub.add_listener(callback1); - let sub = pubsub.add_listener(callback2); - - pubsub.emit(1); - assert_eq!(*called1.borrow(), None); - assert_eq!(*called2.borrow(), Some(1)); - - drop(sub); -} diff --git a/rust/perspective-viewer/src/rust/workspace.rs b/rust/perspective-viewer/src/rust/workspace.rs index 7467e37927..24449de9cd 100644 --- a/rust/perspective-viewer/src/rust/workspace.rs +++ b/rust/perspective-viewer/src/rust/workspace.rs @@ -1038,135 +1038,3 @@ impl Workspace { } } } - -#[cfg(test)] -mod tests { - use perspective_client::config::FilterTerm; - - use super::*; - - fn f(col: &str, term: &str) -> Filter { - Filter::new(col, "==", FilterTerm::Scalar(term.into())) - } - - fn p(id: &str) -> PanelId { - PanelId::from(id) - } - - #[test] - fn flatten_orders_and_dedups_across_buckets() { - let mut s = GlobalFilterSet::default(); - assert!(s.set_restored(vec![f("a", "1"), f("b", "2")])); - // A non-empty contribution drops the restored bucket entirely. - assert!(s.set_contribution(&p("x"), vec![f("b", "2"), f("c", "3")])); - assert_eq!(s.flatten(), vec![f("b", "2"), f("c", "3")]); - assert!(s.set_contribution(&p("y"), vec![f("c", "3"), f("d", "4")])); - assert_eq!(s.flatten(), vec![f("b", "2"), f("c", "3"), f("d", "4")]); - } - - #[test] - fn set_contribution_replaces_not_accumulates() { - let mut s = GlobalFilterSet::default(); - assert!(s.set_contribution(&p("x"), vec![f("a", "1")])); - assert!(s.set_contribution(&p("x"), vec![f("a", "2")])); - assert_eq!(s.flatten(), vec![f("a", "2")]); - // Re-selecting the same value reports no visible change. - assert!(!s.set_contribution(&p("x"), vec![f("a", "2")])); - // Empty = deselect: the entry is removed. - assert!(s.set_contribution(&p("x"), Vec::::new())); - assert_eq!(s.flatten(), Vec::::new()); - } - - #[test] - fn contributions_are_per_panel() { - let mut s = GlobalFilterSet::default(); - s.set_contribution(&p("x"), vec![f("a", "1")]); - s.set_contribution(&p("y"), vec![f("b", "2")]); - // Clearing one master's contribution leaves the other's intact. - assert!(s.set_contribution(&p("x"), Vec::::new())); - assert_eq!(s.flatten(), vec![f("b", "2")]); - } - - #[test] - fn remove_clause_removes_from_every_bucket_and_reports_owners() { - let mut s = GlobalFilterSet::default(); - s.set_contribution(&p("x"), vec![f("a", "1"), f("b", "2")]); - s.set_contribution(&p("y"), vec![f("a", "1")]); - // flatten = [a==1, b==2]; removing index 0 must drop BOTH copies of - // a==1 (owners x AND y), and the now-empty "y" entry with it. - let (changed, owners) = s.remove_clause(0); - assert!(changed); - assert_eq!(owners, vec![p("x"), p("y")]); - assert_eq!(s.flatten(), vec![f("b", "2")]); - // Out-of-range is a no-op. - assert_eq!(s.remove_clause(5), (false, Vec::new())); - } - - #[test] - fn restored_bucket_survives_deselect_but_not_selection() { - let mut s = GlobalFilterSet::default(); - s.set_restored(vec![f("a", "1")]); - // A deselect (empty contribution) does NOT drop the restored bucket. - assert!(!s.set_contribution(&p("x"), Vec::::new())); - assert_eq!(s.flatten(), vec![f("a", "1")]); - // A real selection replaces it. - assert!(s.set_contribution(&p("x"), vec![f("b", "2")])); - assert_eq!(s.flatten(), vec![f("b", "2")]); - assert!(s.restored.is_empty()); - } - - #[test] - fn clear_drops_everything_and_reports_owners() { - let mut s = GlobalFilterSet::default(); - s.set_restored(vec![f("a", "1")]); - s.set_contribution(&p("x"), vec![f("b", "2")]); - let (changed, owners) = s.clear(); - assert!(changed); - assert_eq!(owners, vec![p("x")]); - assert_eq!(s.flatten(), Vec::::new()); - assert_eq!(s.clear(), (false, Vec::new())); - } - - #[test] - fn flush_state_spawns_exactly_once_per_cycle() { - let (state, spawn) = FlushState::Idle.mark(true, false); - assert!(spawn); - assert_eq!(state, FlushState::Queued { - layout: true, - active: false - }); - - // Subsequent marks merge without re-spawning. - let (state, spawn) = state.mark(false, true); - assert!(!spawn); - assert_eq!(state, FlushState::Queued { - layout: true, - active: true - }); - - let (state, spawn) = state.mark(true, true); - assert!(!spawn); - assert_eq!(state, FlushState::Queued { - layout: true, - active: true - }); - } - - #[test] - fn flush_state_ignores_empty_marks() { - // A no-event mark neither queues nor spawns — `Queued{false,false}` - // is unconstructible through `mark`. - let (state, spawn) = FlushState::Idle.mark(false, false); - assert!(!spawn); - assert_eq!(state, FlushState::Idle); - - let queued = FlushState::Queued { - layout: true, - active: false, - }; - - let (state, spawn) = queued.mark(false, false); - assert!(!spawn); - assert_eq!(state, queued); - } -} diff --git a/rust/perspective-viewer/src/themes/intl.css b/rust/perspective-viewer/src/themes/intl.css index fc2afa4ac2..7ded20d041 100644 --- a/rust/perspective-viewer/src/themes/intl.css +++ b/rust/perspective-viewer/src/themes/intl.css @@ -74,6 +74,8 @@ perspective-viewer { /* Column Settings */ --psp-label--color--content: "Color"; + --psp-label--palette--content: "Palette"; + --psp-label--gradient--content: "Gradient"; --psp-label--timezone--content: "Timezone"; --psp-label--date-style--content: "Date Style"; --psp-label--time-style--content: "Time Style"; diff --git a/rust/perspective-viewer/src/themes/intl/ar.css b/rust/perspective-viewer/src/themes/intl/ar.css index fd6bfb740d..6365117a7b 100644 --- a/rust/perspective-viewer/src/themes/intl/ar.css +++ b/rust/perspective-viewer/src/themes/intl/ar.css @@ -69,6 +69,8 @@ perspective-viewer { --psp-label--scroll-lock-alt-toggle--content: "تمرير محاذى"; --psp-label--color--content: "اللون"; + --psp-label--palette--content: "لوحة الألوان"; + --psp-label--gradient--content: "تدرج"; --psp-label--timezone--content: "المنطقة الزمنية"; --psp-label--date-style--content: "نمط التاريخ"; --psp-label--time-style--content: "نمط الوقت"; diff --git a/rust/perspective-viewer/src/themes/intl/de.css b/rust/perspective-viewer/src/themes/intl/de.css index 454e27ad95..d58cd2f2e3 100644 --- a/rust/perspective-viewer/src/themes/intl/de.css +++ b/rust/perspective-viewer/src/themes/intl/de.css @@ -74,6 +74,8 @@ perspective-viewer { /* Column Settings */ --psp-label--color--content: "Farbe"; + --psp-label--palette--content: "Palette"; + --psp-label--gradient--content: "Gradient"; --psp-label--timezone--content: "Zeitzone"; --psp-label--date-style--content: "Datumsstil"; --psp-label--time-style--content: "Zeitstil"; diff --git a/rust/perspective-viewer/src/themes/intl/es.css b/rust/perspective-viewer/src/themes/intl/es.css index 8d12d5a34e..496eed3258 100644 --- a/rust/perspective-viewer/src/themes/intl/es.css +++ b/rust/perspective-viewer/src/themes/intl/es.css @@ -74,6 +74,8 @@ perspective-viewer { /* Column Settings */ --psp-label--color--content: "Color"; + --psp-label--palette--content: "Paleta"; + --psp-label--gradient--content: "Gradiente"; --psp-label--timezone--content: "Zona horaria"; --psp-label--date-style--content: "Estilo de fecha"; --psp-label--time-style--content: "Estilo de hora"; diff --git a/rust/perspective-viewer/src/themes/intl/fr.css b/rust/perspective-viewer/src/themes/intl/fr.css index 3aae851c02..15d2be4339 100644 --- a/rust/perspective-viewer/src/themes/intl/fr.css +++ b/rust/perspective-viewer/src/themes/intl/fr.css @@ -74,6 +74,8 @@ perspective-viewer { /* Column Settings */ --psp-label--color--content: "Couleur"; + --psp-label--palette--content: "Palette"; + --psp-label--gradient--content: "Dégradé"; --psp-label--timezone--content: "Fuseau horaire"; --psp-label--date-style--content: "Style de date"; --psp-label--time-style--content: "Style d'heure"; diff --git a/rust/perspective-viewer/src/themes/intl/ja.css b/rust/perspective-viewer/src/themes/intl/ja.css index d786cbb21c..9d98ec8ccf 100644 --- a/rust/perspective-viewer/src/themes/intl/ja.css +++ b/rust/perspective-viewer/src/themes/intl/ja.css @@ -75,6 +75,8 @@ perspective-viewer { /* Column Settings */ --psp-label--color--content: "色"; + --psp-label--palette--content: "パレット"; + --psp-label--gradient--content: "グラデーション"; --psp-label--timezone--content: "タイムゾーン"; --psp-label--date-style--content: "日付スタイル"; --psp-label--time-style--content: "時間スタイル"; diff --git a/rust/perspective-viewer/src/themes/intl/pt.css b/rust/perspective-viewer/src/themes/intl/pt.css index c101b50633..ba4a817c04 100644 --- a/rust/perspective-viewer/src/themes/intl/pt.css +++ b/rust/perspective-viewer/src/themes/intl/pt.css @@ -74,6 +74,8 @@ perspective-viewer { /* Column Settings */ --psp-label--color--content: "Cor"; + --psp-label--palette--content: "Paleta"; + --psp-label--gradient--content: "Gradiente"; --psp-label--timezone--content: "Fuso horário"; --psp-label--date-style--content: "Estilo de data"; --psp-label--time-style--content: "Estilo de tempo"; diff --git a/rust/perspective-viewer/src/themes/intl/zh.css b/rust/perspective-viewer/src/themes/intl/zh.css index 326d91a3f7..8026ff65e2 100644 --- a/rust/perspective-viewer/src/themes/intl/zh.css +++ b/rust/perspective-viewer/src/themes/intl/zh.css @@ -74,6 +74,8 @@ perspective-viewer { /* Column Settings */ --psp-label--color--content: "颜色"; + --psp-label--palette--content: "调色板"; + --psp-label--gradient--content: "渐变"; --psp-label--timezone--content: "时区"; --psp-label--date-style--content: "日期样式"; --psp-label--time-style--content: "时间风格"; diff --git a/rust/perspective-viewer/test/js/agent/chat_panel.spec.ts b/rust/perspective-viewer/test/js/agent/chat_panel.spec.ts index e65d419311..814c20e838 100644 --- a/rust/perspective-viewer/test/js/agent/chat_panel.spec.ts +++ b/rust/perspective-viewer/test/js/agent/chat_panel.spec.ts @@ -34,6 +34,10 @@ test.beforeEach(async ({ page }) => { chat: { completions: { create: async (request) => { + (window["__REQUESTS__"] ??= []).push( + JSON.parse(JSON.stringify(request)), + ); + const step = script.shift(); if (!step) { throw new Error("Fake engine script exhausted"); @@ -272,6 +276,142 @@ test.describe("llm-agent chat panel", () => { ).toBeLessThan(24); }); + test("reasoning streamed under either field name folds into one block", async ({ + page, + }) => { + await configure(page, [ + { + stream: [ + { role: "assistant" }, + { reasoning_content: "th" }, + { reasoning: "ink" }, + { content: "Hel" }, + { content: "lo", reasoning: "" }, + ], + }, + ]); + + await page.locator("perspective-viewer #chat_tabbar_tab").click(); + const input = page.locator("perspective-viewer #chat_input"); + await input.fill("Think"); + await input.press("Enter"); + + const message = page.locator( + "perspective-viewer .chat-assistant:not(.chat-pending):not(.chat-streaming)", + ); + + await expect(message).toContainText("Hello"); + await expect( + page.locator("perspective-viewer .chat-reasoning-body"), + ).toHaveText("think"); + }); + + test("interleaved tool-call fragments assemble by index and both calls run", async ({ + page, + }) => { + const call = (index, fields) => ({ + tool_calls: [{ index, ...fields }], + }); + + await configure(page, [ + { + stream: [ + call(0, { + id: "c0", + function: { name: "get_schema", arguments: "" }, + }), + call(1, { + id: "c1", + function: { + name: "set_view_config", + arguments: '{"config":{"group_by":["Sta', + }, + }), + call(1, { function: { arguments: 'te"]}}' } }), + ], + }, + { text: "Grouped." }, + ]); + + await page.locator("perspective-viewer #chat_tabbar_tab").click(); + const input = page.locator("perspective-viewer #chat_input"); + await input.fill("Group by state"); + await input.press("Enter"); + + await expect( + page.locator( + "perspective-viewer .chat-assistant:not(.chat-pending):not(.chat-streaming)", + ), + ).toHaveText("Grouped.", { timeout: 10000 }); + + const config = await page.evaluate(async () => { + return await document.querySelector("perspective-viewer").save(); + }); + + expect(config.group_by).toEqual(["State"]); + const requests = await page.evaluate(() => window["__REQUESTS__"]); + const assistant = requests[1].messages.find( + (x) => x.role === "assistant", + ); + + expect( + assistant.tool_calls.map((x) => [ + x.id, + x.type, + x.function.name, + x.function.arguments, + ]), + ).toEqual([ + ["c0", "function", "get_schema", ""], + [ + "c1", + "function", + "set_view_config", + '{"config":{"group_by":["State"]}}', + ], + ]); + }); + + for (const field of ["reasoning_content", "reasoning"]) { + test(`${field} is never sent back to the model`, async ({ page }) => { + await configure(page, [ + { + stream: [ + { [field]: "secret thought" }, + { + tool_calls: [ + { + index: 0, + id: "c0", + function: { + name: "get_schema", + arguments: "{}", + }, + }, + ], + }, + ], + }, + { text: "Done." }, + ]); + + await page.locator("perspective-viewer #chat_tabbar_tab").click(); + const input = page.locator("perspective-viewer #chat_input"); + await input.fill("Think then act"); + await input.press("Enter"); + + await expect( + page.locator( + "perspective-viewer .chat-assistant:not(.chat-pending):not(.chat-streaming)", + ), + ).toHaveText("Done.", { timeout: 10000 }); + + const requests = await page.evaluate(() => window["__REQUESTS__"]); + expect(requests).toHaveLength(2); + expect(JSON.stringify(requests[1])).not.toContain("secret thought"); + }); + } + test("a failed tool call renders an error chip with the failure in its tooltip", async ({ page, }) => { diff --git a/rust/perspective-viewer/test/js/agent/markdown_fixtures.spec.ts b/rust/perspective-viewer/test/js/agent/markdown_fixtures.spec.ts index e68e5c6cf1..ae648880e8 100644 --- a/rust/perspective-viewer/test/js/agent/markdown_fixtures.spec.ts +++ b/rust/perspective-viewer/test/js/agent/markdown_fixtures.spec.ts @@ -72,7 +72,134 @@ async function render(page, text) { return message; } +const INLINE_FIXTURES = { + "strong and emphasis": [ + "**Done!** Key *stats* here.", + "

Done! Key stats here.

", + ], + "snake_case identifiers are not emphasis": [ + "call get_schema then set_view_config", + "

call get_schema then set_view_config

", + ], + "emphasis nests inside strong": [ + "**bold with *ital* inside**", + "

bold with ital inside

", + ], + "unpaired delimiters stay literal": [ + "2 * 3 * 4 and a ~ tilde", + "

2 * 3 * 4 and a ~ tilde

", + ], + "code spans and strikethrough": [ + "run `pnpm test` and ~~skip~~ this", + "

run pnpm test and skip this

", + ], + "raw HTML is literal text": [ + "", + "

<script>alert(1)</script>

", + ], + "known entities decode and unknown ones stay literal": [ + "a & b A &unknown; c", + "

a & b A &unknown; c

", + ], +}; + +const BLOCK_FIXTURES = { + "heading, rule and fenced code": [ + "## Stats\n\n---\n\n```sql\nSELECT 1;\n```", + "

Stats

" + + '
SELECT 1;\n
', + ], + "a code block and a nested list inside an item": [ + "- first\n- second:\n ```\n x = 1\n ```\n - inner", + "
  • first
  • second:" + + '
    x = 1\n
    ' + + "
    • inner
", + ], + "an ordered list keeps its start": [ + "3. three\n4. four", + '
  1. three
  2. four
', + ], + "a table aligns its columns and keeps an escaped pipe in code": [ + "| Region | Sales |\n| :--- | ---: |\n| West | `a\\|b` |", + "
" + + '' + + '' + + "" + + '' + + '' + + "
RegionSales
Westa|b
", + ], + "dashes under a paragraph are a rule, not a table": [ + "Title\n---", + "

Title

", + ], + "blockquote, soft break and hard break": [ + "> quoted\n> more\n\nafter \nbreak", + "

quoted more

after
break

", + ], +}; + test.describe("llm-agent markdown fixtures", () => { + for (const [name, [markdown, html]] of Object.entries({ + ...INLINE_FIXTURES, + ...BLOCK_FIXTURES, + })) { + test(name, async ({ page }) => { + const message = await render(page, markdown); + expect(await message.evaluate((x) => x.innerHTML)).toBe(html); + }); + } + + test("raw HTML creates no elements", async ({ page }) => { + const message = await render(page, ""); + await expect(message.locator("script")).toHaveCount(0); + await expect(message).toHaveText(""); + }); + + test("links open externally and images render as alt text with a link", async ({ + page, + }) => { + const message = await render( + page, + '[safe](https://example.com "hi") ' + + "![leak](https://example.com/x.png) ", + ); + + await expect(message.locator("img")).toHaveCount(0); + await expect(message.locator(".chat-md-image")).toHaveText("leak"); + const anchors = await message.locator("a").evaluateAll((xs) => + xs.map((x) => ({ + href: x.getAttribute("href"), + title: x.getAttribute("title"), + target: x.getAttribute("target"), + rel: x.getAttribute("rel"), + text: x.textContent, + })), + ); + + const external = { target: "_blank", rel: "noopener noreferrer" }; + expect(anchors).toEqual([ + { + href: "https://example.com", + title: "hi", + text: "safe", + ...external, + }, + { + href: "https://example.com/x.png", + title: null, + text: " (image)", + ...external, + }, + { + href: "https://a.co", + title: null, + text: "https://a.co", + ...external, + }, + ]); + }); + test("headings, nested lists and code blocks inside items", async ({ page, }) => { diff --git a/rust/perspective-viewer/test/js/column_settings/format_defaults.spec.ts b/rust/perspective-viewer/test/js/column_settings/format_defaults.spec.ts index bcfb7db9c1..0bcdb0c1ad 100644 --- a/rust/perspective-viewer/test/js/column_settings/format_defaults.spec.ts +++ b/rust/perspective-viewer/test/js/column_settings/format_defaults.spec.ts @@ -106,6 +106,30 @@ test.describe("Number format defaults", () => { await expect(sig.locator("input.parameter-max")).toHaveValue("5"); }); + test("choosing percent style serializes its fraction digits", async ({ + page, + }) => { + const view = new PageView(page); + await view.restore({ + settings: true, + plugin: "Debug Styled", + columns: ["Sales"], + }); + + await openStyleTab(view, "Sales"); + await field(view, "style").locator("select").selectOption("Percent"); + const config = (await view.save()) as any; + expect(config.columns_config).toEqual({ + Sales: { + number_format: { + style: "percent", + minimumFractionDigits: 2, + maximumFractionDigits: 2, + }, + }, + }); + }); + test("unrelated edits serialize sparsely under an override", async ({ page, }) => { diff --git a/rust/perspective-viewer/test/js/column_settings/pivot_column.spec.ts b/rust/perspective-viewer/test/js/column_settings/pivot_column.spec.ts new file mode 100644 index 0000000000..c910b320e8 --- /dev/null +++ b/rust/perspective-viewer/test/js/column_settings/pivot_column.spec.ts @@ -0,0 +1,101 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +import { PageView, test, expect } from "../helpers.ts"; +import type { Page } from "@playwright/test"; + +test.describe("Pivot column settings", () => { + test.beforeEach(async ({ page }) => { + await page.goto( + "/rust/perspective-viewer/test/html/superstore-debug.html", + ); + + await page.evaluate(async () => { + while (!window["__TEST_PERSPECTIVE_READY__"]) { + await new Promise((x) => setTimeout(x, 10)); + } + }); + }); + + async function restore(page: Page, config: Record) { + await page.evaluate(async (config) => { + const viewer = document.querySelector("perspective-viewer"); + await viewer!.restore({ settings: true, ...config }); + }, config); + } + + function pill(view: PageView, axis: "group_by" | "split_by") { + return view.settingsPanel.container + .locator(`#${axis} .pivot-column`) + .first(); + } + + for (const axis of ["group_by", "split_by"] as const) { + test(`${axis} pill opens the Style tab for a date column`, async ({ + page, + }) => { + const view = new PageView(page); + await restore(page, { columns: ["Sales"], [axis]: ["Order Date"] }); + await pill(view, axis).locator(".expression-edit-button").click(); + + const sidebar = view.columnSettingsSidebar; + await expect(sidebar.container).toBeVisible(); + await expect(sidebar.nameInput).toHaveValue("Order Date"); + await expect.poll(() => sidebar.getSelectedTab()).toBe("Style"); + }); + + test(`${axis} pill has no button when the type has no controls`, async ({ + page, + }) => { + const view = new PageView(page); + await restore(page, { columns: ["Sales"], [axis]: ["State"] }); + await expect(pill(view, axis)).toBeVisible(); + await expect( + pill(view, axis).locator(".expression-edit-button"), + ).toHaveCount(0); + }); + + test(`${axis} pill indicator clears when the sidebar closes`, async ({ + page, + }) => { + const view = new PageView(page); + await restore(page, { columns: ["Sales"], [axis]: ["Order Date"] }); + + const button = pill(view, axis).locator(".expression-edit-button"); + await button.click(); + await expect(button).toHaveClass(/is-editing/); + + await view.columnSettingsSidebar.closeBtn.click(); + await expect(view.columnSettingsSidebar.container).toBeHidden(); + await expect(button).not.toHaveClass(/is-editing/); + }); + + test(`${axis} pill configures the aggregate type when also active`, async ({ + page, + }) => { + const view = new PageView(page); + await restore(page, { + columns: ["State"], + group_by: ["City"], + [axis]: ["State"], + aggregates: { State: "count" }, + }); + + await pill(view, axis).locator(".expression-edit-button").click(); + + const sidebar = view.columnSettingsSidebar; + await expect(sidebar.container).toBeVisible(); + await expect(sidebar.nameInput).toHaveValue("State"); + await expect(sidebar.typeIcon).toHaveClass(/integer/); + }); + } +}); diff --git a/rust/perspective-viewer/test/js/multi_panel/global_filter.spec.ts b/rust/perspective-viewer/test/js/multi_panel/global_filter.spec.ts index 2d3a3598a8..2f7199c27a 100644 --- a/rust/perspective-viewer/test/js/multi_panel/global_filter.spec.ts +++ b/rust/perspective-viewer/test/js/multi_panel/global_filter.spec.ts @@ -555,6 +555,63 @@ test.describe("Global filters: chips + lifecycle", () => { }); }); +test.describe("Global filters: shared and restored clauses", () => { + test("a clause contributed by two masters renders one chip whose × removes both", async ({ + page, + }) => { + await restore(page, { + ...TRI_CONFIG, + masters: ["one", "two"], + active: "one", + }); + + const m1 = await id_by_title(page, "One"); + const m2 = await id_by_title(page, "Two"); + const detail = await id_by_title(page, "Three"); + const baseline = await num_rows(page, detail); + for (const panel of [m1, m2]) { + await dispatch_select(page, { + panel, + selected: true, + insertFilters: [["Region", "==", "East"]], + }); + } + + await wait_rows_below(page, detail, baseline); + await expect + .poll(async () => (await save(page)).global_filters) + .toEqual([["Region", "==", "East"]]); + + const chips = page + .locator("perspective-viewer") + .locator("#global_filter_bar .global-filter-chip"); + + await expect(chips).toHaveCount(1); + await chips.first().locator(".global-filter-chip-remove").click(); + await wait_rows(page, detail, baseline); + expect((await save(page)).global_filters ?? []).toEqual([]); + }); + + test("a deselect keeps restored filters", async ({ page }) => { + await restore(page, { + ...SPLIT_CONFIG, + masters: ["one"], + global_filters: [["State", "==", "Texas"]], + }); + + const master = await id_by_title(page, "One"); + const detail = await id_by_title(page, "Two"); + const texas_rows = await num_rows(page, detail); + expect(texas_rows).toBeLessThan(await num_rows(page, master)); + await dispatch_select(page, { panel: master, selected: false }); + await page.waitForTimeout(100); + expect(await num_rows(page, detail)).toBe(texas_rows); + expect((await save(page)).global_filters).toEqual([ + ["State", "==", "Texas"], + ]); + }); +}); + test.describe("Global filters: persistence", () => { test("masters + filters round-trip; restored masters are immune; the next selection replaces the restored set", async ({ page, diff --git a/rust/perspective-viewer/test/js/stability/export.spec.ts b/rust/perspective-viewer/test/js/stability/export.spec.ts index 81621df124..5d850c105c 100644 --- a/rust/perspective-viewer/test/js/stability/export.spec.ts +++ b/rust/perspective-viewer/test/js/stability/export.spec.ts @@ -51,6 +51,6 @@ test.describe("Single-Threaded Engine", () => { }); // Superstore as a CSV - expect(value).toEqual(821); + expect(value).toEqual(808); }); }); diff --git a/rust/perspective-viewer/test/js/viewer_api/save_restore.spec.ts b/rust/perspective-viewer/test/js/viewer_api/save_restore.spec.ts index 7e4bea3e16..4be4823b87 100644 --- a/rust/perspective-viewer/test/js/viewer_api/save_restore.spec.ts +++ b/rust/perspective-viewer/test/js/viewer_api/save_restore.spec.ts @@ -133,4 +133,37 @@ test.describe("Save/Restore", async () => { const contents = await get_contents(page); await compareContentsToSnapshot(contents); }); + + test("restore > a plugin without a schema round-trips plugin_config and columns_config verbatim", async ({ + page, + }) => { + const config = await page.evaluate(async () => { + const viewer = document.querySelector("perspective-viewer"); + await viewer.restore({ + plugin: "Debug", + plugin_config: { fg_mode: "series", ghost: 1 }, + columns_config: { Sales: { anything: "goes" } }, + }); + + return await viewer.save(); + }); + + expect(config.plugin_config).toEqual({ fg_mode: "series", ghost: 1 }); + expect(config.columns_config).toEqual({ + Sales: { anything: "goes" }, + }); + }); + + test("restore > plugin_config null clears what was restored", async ({ + page, + }) => { + const config = await page.evaluate(async () => { + const viewer = document.querySelector("perspective-viewer"); + await viewer.restore({ plugin_config: { ghost: 1 } }); + await viewer.restore({ plugin_config: null }); + return await viewer.save(); + }); + + expect(config.plugin_config).toEqual({}); + }); }); diff --git a/rust/perspective-viewer/test/js/viewer_config/expressions.spec.ts b/rust/perspective-viewer/test/js/viewer_config/expressions.spec.ts index 2c4d4772c8..89f00d76d3 100644 --- a/rust/perspective-viewer/test/js/viewer_config/expressions.spec.ts +++ b/rust/perspective-viewer/test/js/viewer_config/expressions.spec.ts @@ -47,7 +47,76 @@ test.beforeEach(async ({ page }) => { }); }); +const HIGHLIGHT_FIXTURES = { + "numbers, operators and column names": [ + '"Sales" + 123', + [ + ["column", '"Sales"'], + ["whitespace", " "], + ["operator", "+"], + ["whitespace", " "], + ["literal", "123"], + ], + ], + "function symbols and string literals": [ + "concat('a', \"State\")", + [ + ["symbol", "concat"], + ["operator", "("], + ["literal", "'a'"], + ["operator", ","], + ["whitespace", " "], + ["column", '"State"'], + ["operator", ")"], + ], + ], + "a comment ends at the line break": [ + "// Title\n1 + 2", + [ + ["comment", "// Title"], + ["br"], + ["literal", "1"], + ["whitespace", " "], + ["operator", "+"], + ["whitespace", " "], + ["literal", "2"], + ], + ], + "an escaped character stays inside its string": [ + "'test\\/'", + [["literal", "'test\\/'"]], + ], +}; + test.describe("Expressions", () => { + for (const [name, [expr, tokens]] of Object.entries(HIGHLIGHT_FIXTURES)) { + test(`editor > highlights ${name}`, async ({ page }) => { + await page.evaluate(openSidebarAndScrollToBottom); + await page.locator("perspective-viewer #add-expression").click(); + const container = page.locator( + "perspective-viewer #editor-container", + ); + + await container.locator("textarea").fill(expr as string); + await expect + .poll(() => + container.locator("#content").evaluate((content) => + Array.from(content.children) + .filter( + (x) => + x.tagName === "BR" || x.textContent !== "", + ) + .map((x) => + x.tagName === "BR" + ? ["br"] + : [x.className, x.textContent], + ), + ), + ) + .toEqual(tokens); + }); + } + test("editor > opens on add-column button click", async ({ page }) => { await page.evaluate(openSidebarAndScrollToBottom); diff --git a/tools/bench/basic_suite.mjs b/tools/bench/basic_suite.mjs index e702feb95e..a6e829d4c5 100644 --- a/tools/bench/basic_suite.mjs +++ b/tools/bench/basic_suite.mjs @@ -22,6 +22,17 @@ const __dirname = url.fileURLToPath(new URL(".", import.meta.url)).slice(0, -1); const _require = createRequire(import.meta.url); +/** + * Hide Memory64 from the client's feature probe so every version loads wasm32. + */ +if (process.env.PSP_BENCH_WASM32) { + const validate = WebAssembly.validate; + WebAssembly.validate = (bytes) => + bytes.length === 13 && bytes[11] === 0x04 && bytes[12] === 0x01 + ? false + : validate(bytes); +} + /** * We use the `dependencies` of this package for the benchmark candidate * module list, so that we only need specify the dependencies and benchmark @@ -72,5 +83,7 @@ perspective_bench.suite( await all_benchmarks.to_data_suite(client, metadata); await all_benchmarks.join_suite(client, metadata); await all_benchmarks.window_suite(client, metadata); + await all_benchmarks.table_view_suite(client, metadata); + await all_benchmarks.update_view_suite(client, metadata); }, ); diff --git a/tools/bench/cross_platform_suite.mjs b/tools/bench/cross_platform_suite.mjs index 1cf95bcc27..da99801355 100644 --- a/tools/bench/cross_platform_suite.mjs +++ b/tools/bench/cross_platform_suite.mjs @@ -13,6 +13,8 @@ import { benchmark } from "./src/js/benchmark.mjs"; import { check_version_gte, + keyed_superstore_uid, + new_keyed_superstore_table, new_superstore_table, } from "./src/js/superstore.mjs"; @@ -147,6 +149,181 @@ export async function window_suite(perspective, metadata) { }); } +const TABLE_VIEW_CONFIGS = { + "": { columns: ["uid", "Region", "Sales", "Profit"] }, + filter: { + columns: ["uid", "Region", "Sales", "Profit"], + filter: [["Sales", ">", 100]], + }, + group_by: { + group_by: ["Product Name"], + columns: ["Sales", "Profit"], + aggregates: { Sales: "sum", Profit: "avg" }, + }, + "group_by: unique": { + group_by: ["uid"], + columns: ["Sales", "Profit"], + aggregates: { Sales: "sum", Profit: "avg" }, + }, + "group_by, split_by": { + group_by: ["Product Name"], + split_by: ["Region"], + columns: ["Sales", "Profit"], + aggregates: { Sales: "sum", Profit: "avg" }, + }, +}; + +const UPDATE_VIEW_CONFIGS = { + "group_by, sum": { + group_by: ["State"], + columns: ["Sales", "Profit"], + aggregates: { Sales: "sum", Profit: "sum" }, + }, + "group_by, expression sum": { + group_by: ["State"], + columns: ["expr"], + expressions: { expr: '"Sales" + 100' }, + aggregates: { expr: "sum" }, + }, + "group_by, sum abs": { + group_by: ["State"], + columns: ["Sales"], + aggregates: { Sales: "sum abs" }, + }, + "group_by split_by, sum": { + group_by: ["State"], + split_by: ["Category"], + columns: ["Sales"], + aggregates: { Sales: "sum" }, + }, +}; + +/** + * Measures pivoted view maintenance under single-row partial updates. + */ +export async function update_view_suite(perspective, metadata) { + if (!check_version_gte(metadata.version, "3.0.0")) { + return; + } + + async function before_all() { + return { + arrow: await new_keyed_superstore_table(perspective, metadata), + }; + } + + for (const [label, config] of Object.entries(UPDATE_VIEW_CONFIGS)) { + await benchmark({ + name: `table.update(row) with view({${label}})`, + before_all, + metadata, + async before({ arrow }) { + const table = await perspective.table(arrow.slice(), { + index: "uid", + }); + + const view = await table.view(config); + await view.to_columns(); + const state = { table, view, tick: 0 }; + await view.on_update(() => state.resolve?.()); + return state; + }, + async after(_, { table, view }) { + await view.delete(); + await table.delete(); + }, + async test(_, state) { + const tick = state.tick++; + const updated = new Promise((x) => (state.resolve = x)); + await state.table.update([ + { + uid: keyed_superstore_uid(tick), + Sales: 100 + (tick % 7), + }, + ]); + + await updated; + }, + }); + } +} + +export async function table_view_suite(perspective, metadata) { + if (!check_version_gte(metadata.version, "3.0.0")) { + return; + } + + async function before_all() { + return { + arrow: await new_keyed_superstore_table(perspective, metadata), + }; + } + + async function make_source({ arrow }, config) { + const table = await perspective.table(arrow.slice(), { + index: "uid", + }); + + const view = await table.view(config); + return { table, view }; + } + + for (const [label, config] of Object.entries(TABLE_VIEW_CONFIGS)) { + const name = label === "" ? "view" : `view({${label}})`; + await benchmark({ + name: `.table(${name})`, + before_all, + metadata, + async before(state) { + return await make_source(state, config); + }, + async after(_, { table, view }, derived) { + await derived.delete(); + await view.delete(); + await table.delete(); + }, + async test(_, { view }) { + const derived = await perspective.table(view); + await derived.size(); + return derived; + }, + }); + + await benchmark({ + name: `table.update(rows) with .table(${name})`, + before_all, + metadata, + async before(state) { + const source = await make_source(state, config); + const derived = await perspective.table(source.view); + const child = await derived.view(); + await child.to_columns(); + const state2 = { ...source, derived, child, tick: 0 }; + await child.on_update(() => state2.resolve?.()); + return state2; + }, + async after(_, { table, view, derived, child }) { + await child.delete(); + await derived.delete(); + await view.delete(); + await table.delete(); + }, + async test(_, source) { + const tick = source.tick++; + const rows = Array.from({ length: 100 }, (_, i) => ({ + uid: keyed_superstore_uid(tick * 100 + i), + Sales: 1000 + tick + i, + })); + + const updated = new Promise((x) => (source.resolve = x)); + await source.table.update(rows); + await updated; + await source.child.to_columns(); + }, + }); + } +} + export async function to_data_suite(perspective, metadata) { async function before_all() { const table = await perspective.table( diff --git a/tools/bench/package.json b/tools/bench/package.json index fc5e57047d..4737dbb602 100644 --- a/tools/bench/package.json +++ b/tools/bench/package.json @@ -14,10 +14,10 @@ "url": "https://github.com/perspective-dev/perspective/packages/bench" }, "scripts": { - "bench_js": "node --experimental-wasm-memory64 basic_suite.mjs", - "bench_windows": "node --experimental-wasm-memory64 windows_suite.mjs", - "bench_python": "node --experimental-wasm-memory64 python_suite.mjs", - "bench_charts": "node --experimental-wasm-memory64 charts_suite.mjs" + "bench_js": "node basic_suite.mjs", + "bench_windows": "node windows_suite.mjs", + "bench_python": "node python_suite.mjs", + "bench_charts": "node charts_suite.mjs" }, "author": "", "license": "Apache-2.0", diff --git a/tools/bench/python_suite.mjs b/tools/bench/python_suite.mjs index 6d0e83758a..58c3761b91 100644 --- a/tools/bench/python_suite.mjs +++ b/tools/bench/python_suite.mjs @@ -82,6 +82,7 @@ perspective_bench.suite( await all_benchmarks.to_data_suite(client, metadata); await all_benchmarks.join_suite(client, metadata); await all_benchmarks.window_suite(client, metadata); + await all_benchmarks.table_view_suite(client, metadata); }, python.start, python.stop, diff --git a/tools/bench/src/js/benchmark.mjs b/tools/bench/src/js/benchmark.mjs index e77684b162..c03f248d91 100644 --- a/tools/bench/src/js/benchmark.mjs +++ b/tools/bench/src/js/benchmark.mjs @@ -160,8 +160,13 @@ async function benchmark_node_version(version, benchmarks_table) { const suite_path = path.join(process.argv[1]); let stats = []; const worker = cp.fork(suite_path, { - execArgv: ["--expose-gc", "--experimental-wasm-memory64"], - env: { BENCH_FLAG: "1" }, + execArgv: ["--expose-gc"], + env: { + BENCH_FLAG: "1", + ...(process.env.PSP_BENCH_WASM32 && { + PSP_BENCH_WASM32: process.env.PSP_BENCH_WASM32, + }), + }, }); let cont; diff --git a/tools/bench/src/js/superstore.mjs b/tools/bench/src/js/superstore.mjs index 4a8ee5b410..2bb42811bf 100644 --- a/tools/bench/src/js/superstore.mjs +++ b/tools/bench/src/js/superstore.mjs @@ -92,6 +92,66 @@ export async function new_superstore_table(perspective, metadata) { return arrow.slice(); } +const SUPERSTORE_ROWS = 9994; +const KEY_STRIDE = 10000; +const KEYED = new Map(); + +/** + * The key of the `n`th row of `new_keyed_superstore_table`. + * @param {number} n + * @returns + */ +export function keyed_superstore_uid(n) { + const total = SUPERSTORE_ROWS * Math.max(SUPERSTORE_COPIES, 1); + const k = n % total; + return ( + Math.floor(k / SUPERSTORE_ROWS) * KEY_STRIDE + (k % SUPERSTORE_ROWS) + 1 + ); +} + +/** + * Load the replicated Superstore data set with a `uid` column that is unique + * across copies, for benchmarks which need an `index`. + * @param {*} metadata + * @returns + */ +export async function new_keyed_superstore_table(perspective, metadata) { + const base = check_version_gte(metadata.version, "2.5.0") + ? SUPERSTORE_FEATHER + : SUPERSTORE_ARROW; + + const cached = KEYED.get(base); + if (cached !== undefined) { + return cached.slice(); + } + + const source = await perspective.table(base.slice()); + const columns = await source.columns(); + let keyed; + for (let i = 0; i < Math.max(SUPERSTORE_COPIES, 1); i++) { + const view = await source.view({ + columns: [...columns, "uid"], + expressions: { uid: `integer("Row ID" + ${i * KEY_STRIDE})` }, + }); + + const arrow = await view.to_arrow(); + await view.delete(); + if (keyed === undefined) { + keyed = await perspective.table(arrow); + } else { + await keyed.update(arrow); + } + } + + const view = await keyed.view(); + const arrow = await view.to_arrow(); + await view.delete(); + await keyed.delete(); + await source.delete(); + KEYED.set(base, arrow); + return arrow.slice(); +} + /** * Check whether a version string e.g. "v1.2.3" is greater or equal to another * version string, which must be of the same length/have the same number of diff --git a/tools/scripts/clean.mjs b/tools/scripts/clean.mjs index d841c2405e..cf93750de9 100644 --- a/tools/scripts/clean.mjs +++ b/tools/scripts/clean.mjs @@ -34,7 +34,7 @@ for (const pkg of PACKAGES) { if (JS_PKGS.length > 0 || RUST_PKGS.length > 0) { console.log(`-- Cleaning ${JS_PKGS.join(", ")} via pnpm`); const flags = JS_PKGS.concat(RUST_PKGS) - .map((x) => `--filter @perspective-dev/${x} --filter ${x} --if-present`) + .map((x) => `--filter @perspective-dev/${x} --filter ${x}`) .join(" "); execSync(`pnpm run ${flags} clean`, { stdio: "inherit" }); diff --git a/tools/scripts/install_emsdk.mjs b/tools/scripts/install_emsdk.mjs index 015b26f811..94a820e7b1 100644 --- a/tools/scripts/install_emsdk.mjs +++ b/tools/scripts/install_emsdk.mjs @@ -20,9 +20,7 @@ import "zx/globals"; const pkg = getWorkspacePackageJson(); -const emscripten = process.env.PSP_PYODIDE - ? pkg.pyodide_emscripten - : pkg.emscripten; +const emscripten = pkg.emscripten; dotenv.config({ path: "./.perspectiverc", quiet: true });