From bf9a0f804f4aa436ddec7aa157b480d13f464040 Mon Sep 17 00:00:00 2001 From: zzylol Date: Mon, 19 Oct 2026 04:10:25 +0000 Subject: [PATCH 1/2] feat!: install the Planner's InstallablePlan; depend on asap-planner and asap-executor only Re-pin ASAPPlanner to main f0dc1825 and use its handoff API. - The control plane calls `asap_planner::plan_installable` once and writes the returned `InstallablePlan` as plan.json. The backend's own pane-chain expansion, Stage 2/3 plumbing and query-time reference rebuild are gone; tests use `InstallablePlan::query_time_reference`. - The data plane reads `stored_outputs` (built_at, pane window, retention_ms) and `query_time_raw_windows`, and selects every pane's and raw window's rows with `asap_executor::RawSampleSelector`. It no longer uses `frontier_from_timing` or `promql_rows`. - The SDS keeps its own identity (sub-DAG hash, offset-only TimeShifts transparent) and keeps each definition for the largest `retention_ms` among its outputs, measured from the newest evaluation that can read it, instead of "largest offset plus two pane widths". - Among Planner crates the workspace depends on `asap-planner` and `asap-executor` only; IR types come through `asap_executor::planner`. - `--plan-selection-settings` on the control plane and `--execution-memory-limit-mib` on the data plane, which the benefit harness passes. --- Cargo.lock | 2383 ++++++++++++++++- Cargo.toml | 12 +- README.md | 28 +- control_plane/Cargo.toml | 6 +- control_plane/src/lib.rs | 225 +- control_plane/src/main.rs | 13 +- control_plane/tests/plan_workload.rs | 141 +- data_plane/Cargo.toml | 4 +- data_plane/src/engine.rs | 10 +- data_plane/src/lib.rs | 11 +- data_plane/src/main.rs | 5 + data_plane/src/precompute.rs | 211 +- data_plane/src/query.rs | 197 +- data_plane/src/sds.rs | 236 +- data_plane/tests/answers.rs | 9 +- data_plane/tests/common/mod.rs | 25 +- data_plane/tests/e2e_examples.rs | 61 +- data_plane/tests/kept.rs | 39 +- data_plane/tests/precompute.rs | 34 +- data_plane/tests/query.rs | 10 +- data_plane/tests/range.rs | 37 +- data_plane/tests/sds.rs | 139 +- scripts/ci/README.md | 2 +- .../test_sync_planner.cpython-310.pyc | Bin 0 -> 3741 bytes scripts/ci/test_sync_planner.py | 9 +- scripts/sync-asapplanner-main.sh | 9 +- 26 files changed, 2834 insertions(+), 1022 deletions(-) create mode 100644 scripts/ci/__pycache__/test_sync_planner.cpython-310.pyc diff --git a/Cargo.lock b/Cargo.lock index 83ad041b..f794f62e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,27 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "const-random", + "getrandom 0.3.4", + "once_cell", + "serde", + "version_check", + "zerocopy", +] + [[package]] name = "aho-corasick" version = "1.1.5" @@ -12,10 +33,25 @@ dependencies = [ ] [[package]] -name = "android-tzdata" -version = "0.1.1" +name = "alloc-no-stdlib" +version = "2.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" +checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" + +[[package]] +name = "alloc-stdlib" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e76a019e91224d279006ff972f1e984179a6e9feb050adba6ce8274aef23195" +dependencies = [ + "alloc-no-stdlib", +] + +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" [[package]] name = "android_system_properties" @@ -82,10 +118,249 @@ version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" +[[package]] +name = "ar_archive_writer" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73cd58deff2140a0a8eae87e417bd01db68a33e148aa93d1e8cd837e55e312b6" +dependencies = [ + "object", +] + +[[package]] +name = "arrayvec" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" + +[[package]] +name = "arrow" +version = "58.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cfdd0833e32a9874d2b55089333ad310c0be208aafa277385ce2461dec90be3" +dependencies = [ + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-csv", + "arrow-data", + "arrow-ipc", + "arrow-json", + "arrow-ord", + "arrow-row", + "arrow-schema", + "arrow-select", + "arrow-string", +] + +[[package]] +name = "arrow-arith" +version = "58.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a41203398f0eaa6f7ec8e62c0da742a21abf282c148fc157f6c35c90e29981a" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "chrono", + "num-traits", +] + +[[package]] +name = "arrow-array" +version = "58.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae33dad492b7df00a217563a7b0ef2874df68a0deea1b1a3acf628152f7f7a69" +dependencies = [ + "ahash", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "chrono", + "chrono-tz", + "half", + "hashbrown 0.17.1", + "num-complex", + "num-integer", + "num-traits", +] + +[[package]] +name = "arrow-buffer" +version = "58.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9552f96391c005e6ab449fa941420935e7e062489b12b8b1b08879b2163f5b5" +dependencies = [ + "bytes", + "half", + "num-bigint", + "num-traits", +] + +[[package]] +name = "arrow-cast" +version = "58.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a8a327c9649f30d8406995f27642b68df354713cca3baaaf100f076f18d5f34" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-ord", + "arrow-schema", + "arrow-select", + "atoi", + "base64", + "chrono", + "comfy-table", + "half", + "lexical-core", + "num-traits", + "ryu", +] + +[[package]] +name = "arrow-csv" +version = "58.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af0dd6d90d1955e9f9a014c1e563ee8aeffc21909085d25623e1da44d96eca26" +dependencies = [ + "arrow-array", + "arrow-cast", + "arrow-schema", + "chrono", + "csv", + "csv-core", + "regex", +] + +[[package]] +name = "arrow-data" +version = "58.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b24852db04738907e06c04ea61e42fe7fda962a34513022dc0d0e754fb7976b" +dependencies = [ + "arrow-buffer", + "arrow-schema", + "half", + "num-integer", + "num-traits", +] + +[[package]] +name = "arrow-ipc" +version = "58.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29a908a11fcfb3fb2f6730f4ac15e367bc644e419155e96238f68cf3adde572b" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "flatbuffers", + "lz4_flex", + "zstd 0.13.3", +] + +[[package]] +name = "arrow-json" +version = "58.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8a96aed3931c076adee39ec2a40d8219fc7f09e79bcdaca1df16272993e1e14" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-ord", + "arrow-schema", + "arrow-select", + "chrono", + "half", + "indexmap", + "itoa", + "lexical-core", + "memchr", + "num-traits", + "ryu", + "serde_core", + "serde_json", + "simdutf8", +] + +[[package]] +name = "arrow-ord" +version = "58.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63a083ec750f5c043f02946b4baf05fcdbb55f4560a3277055caca5cc99f3eb0" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", +] + +[[package]] +name = "arrow-row" +version = "58.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "514ba0ef0d4c5896202dae736251ce415abb43a950bed570fb7981b8716c0e4c" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "half", +] + +[[package]] +name = "arrow-schema" +version = "58.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21ca356ad6425cecb6eb7b28e4f659f1ee7880fbb1a16127de7dd62901efee9e" +dependencies = [ + "serde_core", + "serde_json", +] + +[[package]] +name = "arrow-select" +version = "58.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c58da39eb3d8350ad4a549e5c2bc49284dac554016c69829310350f1731b0aad" +dependencies = [ + "ahash", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "num-traits", +] + +[[package]] +name = "arrow-string" +version = "58.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6789b388467525e3271326b6b4915666ecfdf5142aef09779445c954b67543c" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "memchr", + "num-traits", + "regex", + "regex-syntax", +] + [[package]] name = "asap-executor" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=58999df294fc9b0326341c1f20be62ecc8c2975e#58999df294fc9b0326341c1f20be62ecc8c2975e" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=f0dc1825805f4cbb8866b3baa15bed4319c4e1ab#f0dc1825805f4cbb8866b3baa15bed4319c4e1ab" dependencies = [ "asap-types", "asap_sketchlib", @@ -101,27 +376,50 @@ dependencies = [ [[package]] name = "asap-frontend-common" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=58999df294fc9b0326341c1f20be62ecc8c2975e#58999df294fc9b0326341c1f20be62ecc8c2975e" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=f0dc1825805f4cbb8866b3baa15bed4319c4e1ab#f0dc1825805f4cbb8866b3baa15bed4319c4e1ab" dependencies = [ "asap-types", "serde", "thiserror 2.0.20", ] +[[package]] +name = "asap-frontend-metricsql" +version = "0.1.0" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=f0dc1825805f4cbb8866b3baa15bed4319c4e1ab#f0dc1825805f4cbb8866b3baa15bed4319c4e1ab" +dependencies = [ + "asap-frontend-common", + "asap-types", + "metricsql_parser", + "thiserror 2.0.20", +] + [[package]] name = "asap-frontend-promql" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=58999df294fc9b0326341c1f20be62ecc8c2975e#58999df294fc9b0326341c1f20be62ecc8c2975e" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=f0dc1825805f4cbb8866b3baa15bed4319c4e1ab#f0dc1825805f4cbb8866b3baa15bed4319c4e1ab" dependencies = [ "asap-frontend-common", "asap-types", "promql-parser", ] +[[package]] +name = "asap-frontend-sql" +version = "0.1.0" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=f0dc1825805f4cbb8866b3baa15bed4319c4e1ab#f0dc1825805f4cbb8866b3baa15bed4319c4e1ab" +dependencies = [ + "asap-frontend-common", + "asap-sql-function-catalog", + "asap-types", + "datafusion", + "serde_json", +] + [[package]] name = "asap-logical-asap-optimization" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=58999df294fc9b0326341c1f20be62ecc8c2975e#58999df294fc9b0326341c1f20be62ecc8c2975e" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=f0dc1825805f4cbb8866b3baa15bed4319c4e1ab#f0dc1825805f4cbb8866b3baa15bed4319c4e1ab" dependencies = [ "asap-types", "asap_sketchlib", @@ -133,7 +431,7 @@ dependencies = [ [[package]] name = "asap-physical-asap-optimization" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=58999df294fc9b0326341c1f20be62ecc8c2975e#58999df294fc9b0326341c1f20be62ecc8c2975e" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=f0dc1825805f4cbb8866b3baa15bed4319c4e1ab#f0dc1825805f4cbb8866b3baa15bed4319c4e1ab" dependencies = [ "asap-logical-asap-optimization", "asap-types", @@ -143,7 +441,7 @@ dependencies = [ [[package]] name = "asap-plan-selection" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=58999df294fc9b0326341c1f20be62ecc8c2975e#58999df294fc9b0326341c1f20be62ecc8c2975e" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=f0dc1825805f4cbb8866b3baa15bed4319c4e1ab#f0dc1825805f4cbb8866b3baa15bed4319c4e1ab" dependencies = [ "asap-logical-asap-optimization", "asap-physical-asap-optimization", @@ -154,10 +452,32 @@ dependencies = [ "thiserror 2.0.20", ] +[[package]] +name = "asap-planner" +version = "0.1.0" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=f0dc1825805f4cbb8866b3baa15bed4319c4e1ab#f0dc1825805f4cbb8866b3baa15bed4319c4e1ab" +dependencies = [ + "asap-executor", + "asap-frontend-metricsql", + "asap-frontend-promql", + "asap-frontend-sql", + "asap-logical-asap-optimization", + "asap-plan-selection", + "asap-types", + "serde", + "serde_json", + "thiserror 2.0.20", +] + +[[package]] +name = "asap-sql-function-catalog" +version = "0.1.0" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=f0dc1825805f4cbb8866b3baa15bed4319c4e1ab#f0dc1825805f4cbb8866b3baa15bed4319c4e1ab" + [[package]] name = "asap-types" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=58999df294fc9b0326341c1f20be62ecc8c2975e#58999df294fc9b0326341c1f20be62ecc8c2975e" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=f0dc1825805f4cbb8866b3baa15bed4319c4e1ab#f0dc1825805f4cbb8866b3baa15bed4319c4e1ab" dependencies = [ "serde", "serde_json", @@ -182,6 +502,18 @@ dependencies = [ "xxhash-rust", ] +[[package]] +name = "async-compression" +version = "0.4.51" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d5755640120b24961f8231e05892e4a4f7e0d04e8aeca161508c9c397946970" +dependencies = [ + "compression-codecs", + "compression-core", + "pin-project-lite", + "tokio", +] + [[package]] name = "async-trait" version = "0.1.92" @@ -193,6 +525,15 @@ dependencies = [ "syn 3.0.5", ] +[[package]] +name = "atoi" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +dependencies = [ + "num-traits", +] + [[package]] name = "atomic-waker" version = "1.1.2" @@ -266,6 +607,25 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "beef" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a8241f3ebb85c056b509d4327ad0358fbbba6ffb340bf388f26350aeda225b1" + +[[package]] +name = "bigdecimal" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fa3f3d8cbf4dffcfe4991de61d012bef509a409ecbe9dd41049bfe32b4d4653" +dependencies = [ + "autocfg", + "libm", + "num-bigint", + "num-integer", + "num-traits", +] + [[package]] name = "bincode" version = "1.3.3" @@ -281,6 +641,28 @@ version = "2.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" +[[package]] +name = "blake2" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" +dependencies = [ + "digest 0.10.7", +] + +[[package]] +name = "blake3" +version = "1.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d9e454fc11f76977dc803893aff6304ed33d6a26efae8696573bea74baa27ae" +dependencies = [ + "arrayvec", + "cc", + "cfg-if", + "constant_time_eq", + "cpufeatures 0.3.1", +] + [[package]] name = "block-buffer" version = "0.10.4" @@ -290,18 +672,63 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "brotli" +version = "8.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cc91aac060a7a1e25823bdccbfb6af1875b88f17c6daac97894eed8207166b3" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", +] + +[[package]] +name = "brotli-decompressor" +version = "5.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a32acac15fe1967bc3986b2a6347dffc965602354ea6f450ad07e8bfd253583" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", +] + [[package]] name = "bumpalo" version = "3.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + [[package]] name = "bytes" version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" +[[package]] +name = "bzip2" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3a53fac24f34a81bc9954b5d6cfce0c21e18ec6959f44f56e8e90e4bb7c346c" +dependencies = [ + "libbz2-rs-sys", +] + [[package]] name = "cactus" version = "1.0.7" @@ -315,6 +742,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "005ec2760ca554fae18df7a11195552ec576cd665632a881bc011d5bb2fd4d80" dependencies = [ "find-msvc-tools", + "jobserver", + "libc", "shlex", ] @@ -357,16 +786,26 @@ dependencies = [ [[package]] name = "chrono" -version = "0.4.39" +version = "0.4.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e36cc9d416881d2e24f9a963be5fb1cd90966419ac844274161d10488b3e825" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" dependencies = [ - "android-tzdata", "iana-time-zone", "js-sys", "num-traits", + "serde", "wasm-bindgen", - "windows-targets", + "windows-link", +] + +[[package]] +name = "chrono-tz" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6139a8597ed92cf816dfb33f5dd6cf0bb93a6adc938f11039f371bc5bcd26c3" +dependencies = [ + "chrono", + "phf 0.12.1", ] [[package]] @@ -416,81 +855,864 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" [[package]] -name = "control_plane" -version = "0.1.0" +name = "comfy-table" +version = "7.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "958c5d6ecf1f214b4c2bbbbf6ab9523a864bd136dcf71a7e8904799acfe1ad47" dependencies = [ - "anyhow", - "asap-frontend-promql", - "asap-logical-asap-optimization", - "asap-plan-selection", - "asap-types", - "clap", - "serde", - "serde_json", - "thiserror 1.0.69", - "tracing", - "tracing-subscriber", + "unicode-segmentation", + "unicode-width", ] [[package]] -name = "core-foundation-sys" -version = "0.8.7" +name = "compression-codecs" +version = "0.4.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" - -[[package]] +checksum = "cae36683101b2899f48bd31f8db9839437356188e3b1ff23096e73dab09ba8fe" +dependencies = [ + "bzip2", + "compression-core", + "flate2", + "liblzma", + "memchr", + "zstd 0.14.3", + "zstd-safe 8.2.0", +] + +[[package]] +name = "compression-core" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ebdda89edc534bf8d5dd2857562633b6495557e5846552ade994b99ed13c36a" + +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + +[[package]] +name = "const-random" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359" +dependencies = [ + "const-random-macro", +] + +[[package]] +name = "const-random-macro" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" +dependencies = [ + "getrandom 0.2.17", + "once_cell", + "tiny-keccak", +] + +[[package]] +name = "constant_time_eq" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" + +[[package]] +name = "control_plane" +version = "0.1.0" +dependencies = [ + "anyhow", + "asap-planner", + "clap", + "futures", + "serde", + "serde_json", + "thiserror 1.0.69", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] name = "cpufeatures" version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "cpufeatures" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01a7799fd6b852db0e61728dde9a204c423b44d689dbd432522543614b490e78" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "csv" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52cd9d68cf7efc6ddfaaee42e7288d3a99d613d4b50f76ce9827ae0c6e14f938" +dependencies = [ + "csv-core", + "itoa", + "ryu", + "serde_core", +] + +[[package]] +name = "csv-core" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "704a3c26996a80471189265814dbc2c257598b96b8a7feae2d31ace646bb9782" +dependencies = [ + "memchr", +] + +[[package]] +name = "dashmap" +version = "6.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6361d5c062261c78a176addb82d4c821ae42bed6089de0e12603cd25de2059c" +dependencies = [ + "cfg-if", + "crossbeam-utils", + "hashbrown 0.14.5", + "lock_api", + "once_cell", + "parking_lot_core", +] + +[[package]] +name = "data_plane" +version = "0.1.0" +dependencies = [ + "anyhow", + "asap-executor", + "asap-planner", + "axum", + "chrono", + "clap", + "control_plane", + "form_urlencoded", + "futures", + "http-body-util", + "prost", + "reqwest", + "serde", + "serde_json", + "sha2 0.10.9", + "snap", + "thiserror 1.0.69", + "tokio", + "tower", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "datafusion" +version = "54.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "754ef4e8f073922a26f5b23133b9db4829342362b09be0bc94309cf261c2f098" +dependencies = [ + "arrow", + "arrow-schema", + "async-trait", + "bzip2", + "chrono", + "datafusion-catalog", + "datafusion-catalog-listing", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-datasource-arrow", + "datafusion-datasource-csv", + "datafusion-datasource-json", + "datafusion-datasource-parquet", + "datafusion-execution", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-functions", + "datafusion-functions-aggregate", + "datafusion-functions-nested", + "datafusion-functions-table", + "datafusion-functions-window", + "datafusion-optimizer", + "datafusion-physical-expr", + "datafusion-physical-expr-adapter", + "datafusion-physical-expr-common", + "datafusion-physical-optimizer", + "datafusion-physical-plan", + "datafusion-session", + "datafusion-sql", + "flate2", + "futures", + "indexmap", + "itertools 0.14.0", + "liblzma", + "log", + "object_store", + "parking_lot", + "parquet", + "sqlparser", + "tempfile", + "tokio", + "url", + "uuid", + "zstd 0.13.3", +] + +[[package]] +name = "datafusion-catalog" +version = "54.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06afd1e38dd27bbb1258685a1fc6524df6aff4e07b25b393a47de59635178d99" +dependencies = [ + "arrow", + "async-trait", + "dashmap", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr", + "datafusion-physical-plan", + "datafusion-session", + "futures", + "itertools 0.14.0", + "log", + "object_store", + "parking_lot", + "tokio", +] + +[[package]] +name = "datafusion-catalog-listing" +version = "54.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0668fb32c12065ec242be0e5b4bc62bd7a06a0be3ecd83791ef877e4be67e02" +dependencies = [ + "arrow", + "async-trait", + "datafusion-catalog", + "datafusion-common", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr", + "datafusion-physical-expr-adapter", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "futures", + "itertools 0.14.0", + "log", + "object_store", +] + +[[package]] +name = "datafusion-common" +version = "54.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca43b263cdff57042cfa8fb817fb3469f4878933380dccff25f5e793580abbf9" +dependencies = [ + "arrow", + "arrow-ipc", + "arrow-schema", + "chrono", + "foldhash 0.2.0", + "half", + "hashbrown 0.17.1", + "indexmap", + "itertools 0.14.0", + "libc", + "log", + "object_store", + "parquet", + "recursive", + "sqlparser", + "tokio", + "uuid", + "web-time", +] + +[[package]] +name = "datafusion-common-runtime" +version = "54.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f0ba2b864792bdca4d76c59a1de0ab6e1b61946596b9936888dbd6360035f2" +dependencies = [ + "futures", + "log", + "tokio", +] + +[[package]] +name = "datafusion-datasource" +version = "54.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b840a8bce0bcbf5afad02946d438591e7c373f7afccaf3d874c04485772514dd" +dependencies = [ + "arrow", + "async-compression", + "async-trait", + "bytes", + "bzip2", + "chrono", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr", + "datafusion-physical-expr-adapter", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-session", + "flate2", + "futures", + "glob", + "itertools 0.14.0", + "liblzma", + "log", + "object_store", + "parking_lot", + "rand 0.9.5", + "tokio", + "tokio-util", + "url", + "zstd 0.13.3", +] + +[[package]] +name = "datafusion-datasource-arrow" +version = "54.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a24cc0b9cf6e367f27f27406eff13abf48a11b72446aaa40b3105c0ded5c17d9" +dependencies = [ + "arrow", + "arrow-ipc", + "async-trait", + "bytes", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-session", + "futures", + "itertools 0.14.0", + "object_store", + "tokio", +] + +[[package]] +name = "datafusion-datasource-csv" +version = "54.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1abe56b2a7a2d1d6de5117dd1a203181e267f28529faa5da546947621b697d7" +dependencies = [ + "arrow", + "async-trait", + "bytes", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-session", + "futures", + "object_store", + "regex", + "tokio", +] + +[[package]] +name = "datafusion-datasource-json" +version = "54.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3e467f0611ad7bdd5aad17c63c9bb6182d04e5282e5496d897ea2b49c024ba" +dependencies = [ + "arrow", + "async-trait", + "bytes", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-session", + "futures", + "object_store", + "tokio", + "tokio-stream", +] + +[[package]] +name = "datafusion-datasource-parquet" +version = "54.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cc35b92cd560082155e80d9c826929c852d3c51543f4affd3a51c464a0aab3a" +dependencies = [ + "arrow", + "async-trait", + "bytes", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-functions", + "datafusion-functions-aggregate-common", + "datafusion-physical-expr", + "datafusion-physical-expr-adapter", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-pruning", + "datafusion-session", + "futures", + "itertools 0.14.0", + "log", + "object_store", + "parking_lot", + "parquet", + "tokio", +] + +[[package]] +name = "datafusion-doc" +version = "54.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d69bb69d8769e34f76839c960dbde24c1ac0c885a79b6c3c2287bdc56ec67891" + +[[package]] +name = "datafusion-execution" +version = "54.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8eac0a09bc8d263f52025cad9e001da4d8138d633fa288edda4d06b1772eae6" +dependencies = [ + "arrow", + "arrow-buffer", + "async-trait", + "dashmap", + "datafusion-common", + "datafusion-expr", + "datafusion-physical-expr-common", + "futures", + "log", + "object_store", + "parking_lot", + "rand 0.9.5", + "tempfile", + "url", +] + +[[package]] +name = "datafusion-expr" +version = "54.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eeb14d374767ee0fc62dc79a5ba8bcf8a63c14e993c7d992d0e63adfa23d77d3" +dependencies = [ + "arrow", + "arrow-schema", + "async-trait", + "chrono", + "datafusion-common", + "datafusion-doc", + "datafusion-expr-common", + "datafusion-functions-aggregate-common", + "datafusion-functions-window-common", + "datafusion-physical-expr-common", + "indexmap", + "itertools 0.14.0", + "recursive", + "serde_json", + "sqlparser", +] + +[[package]] +name = "datafusion-expr-common" +version = "54.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7b19a8c95522bee8cbb313d74263b85e355d2b52f42e67ef5694bf5de9e9356" +dependencies = [ + "arrow", + "datafusion-common", + "indexmap", + "itertools 0.14.0", +] + +[[package]] +name = "datafusion-functions" +version = "54.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f64c983bbbdcb729d921a2b2ac3375598719b5cc0c30345ad664936f3176fc7" +dependencies = [ + "arrow", + "arrow-buffer", + "base64", + "blake2", + "blake3", + "chrono", + "chrono-tz", + "datafusion-common", + "datafusion-doc", + "datafusion-execution", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-macros", + "datafusion-physical-expr-common", + "hex", + "itertools 0.14.0", + "log", + "md-5", + "memchr", + "num-traits", + "rand 0.9.5", + "regex", + "sha2 0.11.1", + "uuid", +] + +[[package]] +name = "datafusion-functions-aggregate" +version = "54.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89bc17041e424a47ed062f43df24d84aab8b57c4c3221e5c1a5eef46d6c5718b" +dependencies = [ + "arrow", + "datafusion-common", + "datafusion-doc", + "datafusion-execution", + "datafusion-expr", + "datafusion-functions-aggregate-common", + "datafusion-macros", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "foldhash 0.2.0", + "half", + "log", + "num-traits", +] + +[[package]] +name = "datafusion-functions-aggregate-common" +version = "54.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97dd2a9e865c6108059f5b37b77934f84b50bfb108f837bd0e5c9536e03f0545" +dependencies = [ + "arrow", + "datafusion-common", + "datafusion-expr-common", + "datafusion-physical-expr-common", +] + +[[package]] +name = "datafusion-functions-nested" +version = "54.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75f0bdfeef16d96417b9632ef855645376b242e9006a126dfd0bedfc54a93f5f" +dependencies = [ + "arrow", + "arrow-ord", + "datafusion-common", + "datafusion-doc", + "datafusion-execution", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-functions", + "datafusion-functions-aggregate", + "datafusion-functions-aggregate-common", + "datafusion-macros", + "datafusion-physical-expr-common", + "hashbrown 0.17.1", + "itertools 0.14.0", + "itoa", + "log", + "memchr", +] + +[[package]] +name = "datafusion-functions-table" +version = "54.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4e4941673c917819616877e9993da4503e4f4739812be0bc32c5356184c6383" +dependencies = [ + "arrow", + "async-trait", + "datafusion-catalog", + "datafusion-common", + "datafusion-expr", + "datafusion-physical-expr", + "datafusion-physical-plan", + "parking_lot", +] + +[[package]] +name = "datafusion-functions-window" +version = "54.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12dd2e16c12b84b6f6b41b19f55b366dd1c46876bb35b86896c6349067379e8d" +dependencies = [ + "arrow", + "datafusion-common", + "datafusion-doc", + "datafusion-expr", + "datafusion-functions-window-common", + "datafusion-macros", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "log", +] + +[[package]] +name = "datafusion-functions-window-common" +version = "54.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cdc5e4b6f8b6ef823cc1c761f85088ad4c884fe8df64df3cbcc6b2b84698441" +dependencies = [ + "datafusion-common", + "datafusion-physical-expr-common", +] + +[[package]] +name = "datafusion-macros" +version = "54.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a3614234dd93578c92428cb4f408e020874f0d2b7e6c90c928d9d28b5df2ceb" +dependencies = [ + "datafusion-doc", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "datafusion-optimizer" +version = "54.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0635620b050b81bb92764e99250868f654e2cd5ad1bece413283b3f73c83179" dependencies = [ - "libc", + "arrow", + "chrono", + "datafusion-common", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-physical-expr", + "indexmap", + "itertools 0.14.0", + "log", + "recursive", + "regex", + "regex-syntax", ] [[package]] -name = "cpufeatures" -version = "0.3.1" +name = "datafusion-physical-expr" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" +checksum = "8cabf7a86eb70b816729e33c81bf7767c936ee1226f607a114f5dac2decac8d0" dependencies = [ - "libc", + "arrow", + "datafusion-common", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-functions-aggregate-common", + "datafusion-physical-expr-common", + "half", + "hashbrown 0.17.1", + "indexmap", + "itertools 0.14.0", + "parking_lot", + "petgraph", + "recursive", + "tokio", ] [[package]] -name = "crypto-common" -version = "0.1.7" +name = "datafusion-physical-expr-adapter" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +checksum = "de222e04f7e6744501555a54ab0abe26bfdfebee380af79a9bdc175704246859" dependencies = [ - "generic-array", - "typenum", + "arrow", + "datafusion-common", + "datafusion-expr", + "datafusion-functions", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "itertools 0.14.0", ] [[package]] -name = "data_plane" -version = "0.1.0" +name = "datafusion-physical-expr-common" +version = "54.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72d0d0057fc5a502d45c870cb6d47c66eb7bdd5edb1bd71ad6f3f724975ac2a8" dependencies = [ - "anyhow", - "asap-executor", - "asap-types", - "axum", + "arrow", "chrono", - "clap", - "control_plane", - "form_urlencoded", + "datafusion-common", + "datafusion-expr-common", + "hashbrown 0.17.1", + "indexmap", + "itertools 0.14.0", + "parking_lot", + "pin-project", +] + +[[package]] +name = "datafusion-physical-optimizer" +version = "54.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86046eed10950c5f9aaed9acfd148e9bd2e1dfdfe4f9aef607d1447b271e4183" +dependencies = [ + "arrow", + "datafusion-common", + "datafusion-execution", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-pruning", + "itertools 0.14.0", + "recursive", +] + +[[package]] +name = "datafusion-physical-plan" +version = "54.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bc84da934c903407ba297971ebcc020c4c1a38aafd765d6c144c76eff3fa6a1" +dependencies = [ + "arrow", + "arrow-data", + "arrow-ipc", + "arrow-ord", + "arrow-schema", + "async-trait", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-execution", + "datafusion-expr", + "datafusion-functions", + "datafusion-functions-aggregate-common", + "datafusion-functions-window-common", + "datafusion-physical-expr", + "datafusion-physical-expr-common", "futures", - "http-body-util", - "prost", - "reqwest", - "serde", - "serde_json", - "sha2", - "snap", - "thiserror 1.0.69", + "half", + "hashbrown 0.17.1", + "indexmap", + "itertools 0.14.0", + "log", + "num-traits", + "parking_lot", + "pin-project-lite", "tokio", - "tower", - "tracing", - "tracing-subscriber", +] + +[[package]] +name = "datafusion-pruning" +version = "54.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb63eeac6de19be40f487b65dd84e546195f783c5a9928618e0c4f2a3569b0d7" +dependencies = [ + "arrow", + "datafusion-common", + "datafusion-datasource", + "datafusion-expr-common", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "log", +] + +[[package]] +name = "datafusion-session" +version = "54.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f961d209177f91bd014db5cbb2c33b7d28a2597b9003e77f17aeb712964315a" +dependencies = [ + "async-trait", + "datafusion-common", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-plan", + "parking_lot", +] + +[[package]] +name = "datafusion-sql" +version = "54.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1d71cb454da682b2af7488e1fc1ddd72ee1b28f19297b8ccad73f0a21ee9a69" +dependencies = [ + "arrow", + "bigdecimal", + "chrono", + "datafusion-common", + "datafusion-expr", + "datafusion-functions-nested", + "indexmap", + "log", + "recursive", + "regex", + "sqlparser", ] [[package]] @@ -505,8 +1727,20 @@ version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ - "block-buffer", - "crypto-common", + "block-buffer 0.10.4", + "crypto-common 0.1.7", + "subtle", +] + +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer 0.12.1", + "const-oid", + "crypto-common 0.2.2", ] [[package]] @@ -526,6 +1760,15 @@ version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34" +[[package]] +name = "enquote" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06c36cb11dbde389f4096111698d8b567c0720e3452fd5ac3e6b4e47e1939932" +dependencies = [ + "thiserror 1.0.69", +] + [[package]] name = "equivalent" version = "1.0.2" @@ -542,6 +1785,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + [[package]] name = "filetime" version = "0.2.29" @@ -558,12 +1807,51 @@ version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3e0f1c7c3a72c66fd80abe965175f7523475c0489a87d3ff9d6e8c87d87a9d2d" +[[package]] +name = "fixedbitset" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" + +[[package]] +name = "flatbuffers" +version = "25.12.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35f6839d7b3b98adde531effaf34f0c2badc6f4735d26fe74709d8e513a96ef3" +dependencies = [ + "bitflags", + "rustc_version", +] + +[[package]] +name = "flate2" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb" +dependencies = [ + "crc32fast", + "miniz_oxide", + "zlib-rs", +] + [[package]] name = "fnv" version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + [[package]] name = "form_urlencoded" version = "1.2.2" @@ -719,11 +2007,49 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "glob" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b" + +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "num-traits", + "zerocopy", +] + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "foldhash 0.1.5", +] + [[package]] name = "hashbrown" version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", +] [[package]] name = "heck" @@ -731,6 +2057,12 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + [[package]] name = "http" version = "1.5.0" @@ -776,6 +2108,21 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" +[[package]] +name = "humantime" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15cdd26707701c53297e2fa6afb323d55fbc1d0810c3aec078ae3ef0424c3c15" + +[[package]] +name = "hybrid-array" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27f864f10dfb56725ce5ce5472bc52252c8f93a4ab86327122cebf62c5f59a17" +dependencies = [ + "typenum", +] + [[package]] name = "hyper" version = "1.11.1" @@ -944,85 +2291,205 @@ dependencies = [ ] [[package]] -name = "idna" -version = "1.1.0" +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indexmap" +version = "2.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", +] + +[[package]] +name = "integer-encoding" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" + +[[package]] +name = "ipnet" +version = "2.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "791930b43c0d5973160d90a8f3894509f2b273430f5c5c73b668636d0287c5c0" + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jobserver" +version = "0.1.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" +dependencies = [ + "getrandom 0.4.3", + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.105" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce57d20d1ea864ce2ac172ab472d409214f4fd359f0b2a2775abdf522e2af99e" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "lexical-core" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d8d125a277f807e55a77304455eb7b1cb52f2b18c143b60e766c120bd64a594" +dependencies = [ + "lexical-parse-float", + "lexical-parse-integer", + "lexical-util", + "lexical-write-float", + "lexical-write-integer", +] + +[[package]] +name = "lexical-parse-float" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +checksum = "52a9f232fbd6f550bc0137dcb5f99ab674071ac2d690ac69704593cb4abbea56" dependencies = [ - "idna_adapter", - "smallvec", - "utf8_iter", + "lexical-parse-integer", + "lexical-util", ] [[package]] -name = "idna_adapter" -version = "1.2.2" +name = "lexical-parse-integer" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +checksum = "9a7a039f8fb9c19c996cd7b2fcce303c1b2874fe1aca544edc85c4a5f8489b34" dependencies = [ - "icu_normalizer", - "icu_properties", + "lexical-util", ] [[package]] -name = "indexmap" -version = "2.14.2" +name = "lexical-util" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" +checksum = "2604dd126bb14f13fb5d1bd6a66155079cb9fa655b37f875b3a742c705dbed17" + +[[package]] +name = "lexical-write-float" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50c438c87c013188d415fbabbb1dceb44249ab81664efbd31b14ae55dabb6361" dependencies = [ - "equivalent", - "hashbrown", + "lexical-util", + "lexical-write-integer", ] [[package]] -name = "ipnet" -version = "2.12.2" +name = "lexical-write-integer" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "791930b43c0d5973160d90a8f3894509f2b273430f5c5c73b668636d0287c5c0" +checksum = "409851a618475d2d5796377cad353802345cba92c867d9fbcde9cf4eac4e14df" +dependencies = [ + "lexical-util", +] [[package]] -name = "is_terminal_polyfill" -version = "1.70.2" +name = "libbz2-rs-sys" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" +checksum = "34b357333733e8260735ba5894eb928c02ecc69c78715f01a8019e7fa7f2db4c" [[package]] -name = "itertools" -version = "0.10.5" +name = "libc" +version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" -dependencies = [ - "either", -] +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] -name = "itoa" -version = "1.0.18" +name = "liblzma" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +checksum = "aa980ea6744074a5b5e0ac6b164aca94fc970121869b3e78ae12a2b0097dbea7" +dependencies = [ + "liblzma-sys", +] [[package]] -name = "js-sys" -version = "0.3.105" +name = "liblzma-sys" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce57d20d1ea864ce2ac172ab472d409214f4fd359f0b2a2775abdf522e2af99e" +checksum = "7c7e3581f367a78d7b7e7ae948d023310556f0cfc13156c2e4e00e25616492b9" dependencies = [ - "cfg-if", - "futures-util", - "wasm-bindgen", + "cc", + "libc", + "pkg-config", ] [[package]] -name = "lazy_static" -version = "1.5.0" +name = "libm" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" [[package]] -name = "libc" -version = "0.2.189" +name = "linux-raw-sys" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" [[package]] name = "litemap" @@ -1045,6 +2512,39 @@ version = "0.4.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" +[[package]] +name = "logos" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7251356ef8cb7aec833ddf598c6cb24d17b689d20b993f9d11a3d764e34e6458" +dependencies = [ + "logos-derive", +] + +[[package]] +name = "logos-codegen" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59f80069600c0d66734f5ff52cc42f2dabd6b29d205f333d61fd7832e9e9963f" +dependencies = [ + "beef", + "fnv", + "lazy_static", + "proc-macro2", + "quote", + "regex-syntax", + "syn 2.0.119", +] + +[[package]] +name = "logos-derive" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24fb722b06a9dc12adb0963ed585f19fc61dc5413e6a9be9422ef92c091e731d" +dependencies = [ + "logos-codegen", +] + [[package]] name = "lrlex" version = "0.13.10" @@ -1105,6 +2605,15 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" +[[package]] +name = "lz4_flex" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ef0d4ed8669f8f8826eb00dc878084aa8f253506c4fd5e8f58f5bce72ddb97e" +dependencies = [ + "twox-hash", +] + [[package]] name = "matchers" version = "0.2.0" @@ -1120,18 +2629,69 @@ version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" +[[package]] +name = "md-5" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69b6441f590336821bb897fb28fc622898ccceb1d6cea3fde5ea86b090c4de98" +dependencies = [ + "cfg-if", + "digest 0.11.3", +] + [[package]] name = "memchr" version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" +[[package]] +name = "metricsql_common" +version = "0.1.0" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=f0dc1825805f4cbb8866b3baa15bed4319c4e1ab#f0dc1825805f4cbb8866b3baa15bed4319c4e1ab" +dependencies = [ + "chrono", +] + +[[package]] +name = "metricsql_parser" +version = "0.1.0" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=f0dc1825805f4cbb8866b3baa15bed4319c4e1ab#f0dc1825805f4cbb8866b3baa15bed4319c4e1ab" +dependencies = [ + "ahash", + "chrono", + "enquote", + "logos", + "logos-derive", + "metricsql_common", + "num-traits", + "phf 0.11.3", + "regex", + "scopeguard", + "serde", + "strum", + "strum_macros", + "thiserror 1.0.69", + "tinyvec", + "xxhash-rust", +] + [[package]] name = "mime" version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +[[package]] +name = "miniz_oxide" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" +dependencies = [ + "adler2", + "simd-adler32", +] + [[package]] name = "mio" version = "1.2.3" @@ -1152,6 +2712,25 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "num-bigint" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + [[package]] name = "num-conv" version = "0.2.2" @@ -1159,73 +2738,261 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" [[package]] -name = "num-traits" -version = "0.2.19" +name = "num-integer" +version = "0.1.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", + "libm", +] + +[[package]] +name = "num_threads" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c7398b9c8b70908f6371f47ed36737907c87c52af34c268fed0bf0ceb92ead9" +dependencies = [ + "libc", +] + +[[package]] +name = "object" +version = "0.39.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e5a6c098c7a3b6547378093f5cc30bc54fd361ce711e05293a5cc589562739b" +dependencies = [ + "memchr", +] + +[[package]] +name = "object_store" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "622acbc9100d3c10e2ee15804b0caa40e55c933d5aa53814cd520805b7958a49" +dependencies = [ + "async-trait", + "bytes", + "chrono", + "futures-channel", + "futures-core", + "futures-util", + "http", + "humantime", + "itertools 0.14.0", + "parking_lot", + "percent-encoding", + "thiserror 2.0.20", + "tokio", + "tracing", + "url", + "walkdir", + "wasm-bindgen-futures", + "web-time", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "ordered-float" +version = "2.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68f19d67e5a2795c94e73e0bb1cc1a7edeb2e28efd39e2e1c9b7a40c1108b11c" +dependencies = [ + "num-traits", +] + +[[package]] +name = "packedvec" +version = "1.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a69e0a534dd2e6aefce319af62a0aa0066a76bdfcec0201dfe02df226bc9ec70" +dependencies = [ + "num-traits", + "serde", +] + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "parquet" +version = "58.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d298093b2dec60289dce0684c986d0f7679e9dd15771c2c65406e1aaf604a704" +dependencies = [ + "ahash", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-ipc", + "arrow-schema", + "arrow-select", + "base64", + "brotli", + "bytes", + "chrono", + "flate2", + "futures", + "half", + "hashbrown 0.17.1", + "lz4_flex", + "num-bigint", + "num-integer", + "num-traits", + "object_store", + "paste", + "seq-macro", + "simdutf8", + "snap", + "thrift", + "tokio", + "twox-hash", + "zstd 0.13.3", +] + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "petgraph" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455" +dependencies = [ + "fixedbitset", + "hashbrown 0.15.5", + "indexmap", + "serde", +] + +[[package]] +name = "phf" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078" dependencies = [ - "autocfg", + "phf_macros", + "phf_shared 0.11.3", ] [[package]] -name = "num_threads" -version = "0.1.7" +name = "phf" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c7398b9c8b70908f6371f47ed36737907c87c52af34c268fed0bf0ceb92ead9" +checksum = "913273894cec178f401a31ec4b656318d95473527be05c0752cc41cdc32be8b7" dependencies = [ - "libc", + "phf_shared 0.12.1", ] [[package]] -name = "once_cell" -version = "1.21.4" +name = "phf_generator" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" +dependencies = [ + "phf_shared 0.11.3", + "rand 0.8.8", +] [[package]] -name = "once_cell_polyfill" -version = "1.70.2" +name = "phf_macros" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" +checksum = "f84ac04429c13a7ff43785d75ad27569f2951ce0ffd30a3321230db2fc727216" +dependencies = [ + "phf_generator", + "phf_shared 0.11.3", + "proc-macro2", + "quote", + "syn 2.0.119", +] [[package]] -name = "packedvec" -version = "1.2.5" +name = "phf_shared" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a69e0a534dd2e6aefce319af62a0aa0066a76bdfcec0201dfe02df226bc9ec70" +checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5" dependencies = [ - "num-traits", - "serde", + "siphasher", ] [[package]] -name = "parking_lot" -version = "0.12.5" +name = "phf_shared" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +checksum = "06005508882fb681fd97892ecff4b7fd0fee13ef1aa569f8695dae7ab9099981" dependencies = [ - "lock_api", - "parking_lot_core", + "siphasher", ] [[package]] -name = "parking_lot_core" -version = "0.9.12" +name = "pin-project" +version = "1.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +checksum = "2466b2336ed02bcdca6b294417127b90ec92038d1d5c4fbeac971a922e0e0924" dependencies = [ - "cfg-if", - "libc", - "redox_syscall", - "smallvec", - "windows-link", + "pin-project-internal", ] [[package]] -name = "percent-encoding" -version = "2.3.2" +name = "pin-project-internal" +version = "1.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" +checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] [[package]] name = "pin-project-lite" @@ -1233,6 +3000,12 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" +[[package]] +name = "pkg-config" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" + [[package]] name = "potential_utf" version = "0.1.6" @@ -1296,12 +3069,22 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" dependencies = [ "anyhow", - "itertools", + "itertools 0.10.5", "proc-macro2", "quote", "syn 2.0.119", ] +[[package]] +name = "psm" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4dcd034599e63b970727f70d79e02d62390a4a84f7c6b827c27c46d5ac3fa622" +dependencies = [ + "ar_archive_writer", + "cc", +] + [[package]] name = "quinn" version = "0.11.11" @@ -1379,6 +3162,15 @@ version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" +[[package]] +name = "rand" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e058c7de0b26af77780c769414d6257830bb240f3c38477dbc2c16e5f54d6d4c" +dependencies = [ + "rand_core 0.6.4", +] + [[package]] name = "rand" version = "0.9.5" @@ -1410,6 +3202,12 @@ dependencies = [ "rand_core 0.9.5", ] +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" + [[package]] name = "rand_core" version = "0.9.5" @@ -1434,6 +3232,26 @@ dependencies = [ "rand_core 0.10.1", ] +[[package]] +name = "recursive" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0786a43debb760f491b1bc0269fe5e84155353c67482b9e60d0cfb596054b43e" +dependencies = [ + "recursive-proc-macro-impl", + "stacker", +] + +[[package]] +name = "recursive-proc-macro-impl" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76009fbe0614077fc1a2ce255e3a1881a2e3a3527097d5dc6d8212c585e7e38b" +dependencies = [ + "quote", + "syn 2.0.119", +] + [[package]] name = "redox_syscall" version = "0.5.18" @@ -1549,6 +3367,28 @@ version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "rustix" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.52.0", +] + [[package]] name = "rustls" version = "0.23.44" @@ -1596,12 +3436,33 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "scopeguard" version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + +[[package]] +name = "seq-macro" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc" + [[package]] name = "serde" version = "1.0.229" @@ -1695,7 +3556,18 @@ checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", "cpufeatures 0.2.17", - "digest", + "digest 0.10.7", +] + +[[package]] +name = "sha2" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d7069beb7d6ac7b9acd1039986e73443f24234f41074da099d6f994ac9ad19" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.1", + "digest 0.11.3", ] [[package]] @@ -1723,6 +3595,24 @@ dependencies = [ "libc", ] +[[package]] +name = "simd-adler32" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + +[[package]] +name = "siphasher" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33f4fe9184a62d842c9ef383018f3306d8ba224fd9d836f56d7288308847c256" + [[package]] name = "slab" version = "0.4.12" @@ -1763,12 +3653,47 @@ dependencies = [ "vob 3.0.6", ] +[[package]] +name = "sqlparser" +version = "0.62.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c6d1b651dc4edf07eead2a0c6c78016ce971bc2c10da5266861b13f25e7cec" +dependencies = [ + "log", + "recursive", + "sqlparser_derive", +] + +[[package]] +name = "sqlparser_derive" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6dd45d8fc1c79299bfbb7190e42ccbbdf6a5f52e4a6ad98d92357ea965bd289" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "stable_deref_trait" version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" +[[package]] +name = "stacker" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "707f49d46706bacf8a2b00d51dace3f9de527c13eec3778f570c411f89e69967" +dependencies = [ + "cc", + "cfg-if", + "libc", + "psm", + "windows-sys 0.61.2", +] + [[package]] name = "static_assertions" version = "1.1.0" @@ -1781,6 +3706,28 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +[[package]] +name = "strum" +version = "0.26.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" +dependencies = [ + "strum_macros", +] + +[[package]] +name = "strum_macros" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "rustversion", + "syn 2.0.119", +] + [[package]] name = "subtle" version = "2.6.1" @@ -1829,6 +3776,19 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.3.4", + "once_cell", + "rustix", + "windows-sys 0.52.0", +] + [[package]] name = "thiserror" version = "1.0.69" @@ -1878,6 +3838,17 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "thrift" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e54bc85fc7faa8bc175c4bab5b92ba8d9a3ce893d0e9f42cc455c8ab16a9e09" +dependencies = [ + "byteorder", + "integer-encoding", + "ordered-float", +] + [[package]] name = "time" version = "0.3.55" @@ -1910,6 +3881,15 @@ dependencies = [ "time-core", ] +[[package]] +name = "tiny-keccak" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" +dependencies = [ + "crunchy", +] + [[package]] name = "tinystr" version = "0.8.4" @@ -1973,6 +3953,31 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-stream" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3d06f0b082ba57c26b79407372e57cf2a1e28124f78e9479fe80322cf53420b" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", + "tokio-util", +] + +[[package]] +name = "tokio-util" +version = "0.7.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e464cf451ba96ebfc6f9b6542f17ee8b8956e33f1e40d9690624e59d7a7f8a4b" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + [[package]] name = "tower" version = "0.5.3" @@ -2108,6 +4113,12 @@ version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +[[package]] +name = "unicode-segmentation" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" + [[package]] name = "unicode-width" version = "0.2.2" @@ -2144,6 +4155,17 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" +[[package]] +name = "uuid" +version = "1.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cc1186384beb7dd8eedea376413fd654937285ea6c9cfbb928dc3043ea4b606" +dependencies = [ + "getrandom 0.4.3", + "js-sys", + "wasm-bindgen", +] + [[package]] name = "valuable" version = "0.1.1" @@ -2187,6 +4209,16 @@ dependencies = [ "serde", ] +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + [[package]] name = "want" version = "0.3.1" @@ -2295,6 +4327,15 @@ dependencies = [ "rustls-pki-types", ] +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.52.0", +] + [[package]] name = "windows-core" version = "0.62.2" @@ -2557,8 +4598,60 @@ dependencies = [ "syn 3.0.5", ] +[[package]] +name = "zlib-rs" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b268e58e7c693d7c271f93ffc4ba3b380412554231c85bf61ca7af91042a4112" + [[package]] name = "zmij" version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe 7.3.0", +] + +[[package]] +name = "zstd" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db3ae9bf90af6e40197bef767eeefc46ad6397420fe873830bd85e5fbbfb3c70" +dependencies = [ + "zstd-safe 8.2.0", +] + +[[package]] +name = "zstd-safe" +version = "7.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64d80649ab6db9d9f6f9c80a40becd948eda4714a0a5ac8c4d157a32231c7882" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-safe" +version = "8.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ded520f16b29e1a9b771c722221b2b0a0398c2316d9bb3c42b458a5414d7434b" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.1.2+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "481ada9c543b2323cbb7bbdbb5249177fbf0188ed959ed2c15e4b22421c84ed3" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/Cargo.toml b/Cargo.toml index d87b4fe1..d2633f34 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,13 +7,11 @@ edition = "2021" version = "0.1.0" [workspace.dependencies] -# Keep every Planner crate on the same immutable revision. Alias upstream -# asap-types as planner-types to name its origin at use sites. -planner-types = { package = "asap-types", git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "58999df294fc9b0326341c1f20be62ecc8c2975e" } -asap-frontend-promql = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "58999df294fc9b0326341c1f20be62ecc8c2975e" } -asap-logical-asap-optimization = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "58999df294fc9b0326341c1f20be62ecc8c2975e" } -asap-plan-selection = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "58999df294fc9b0326341c1f20be62ecc8c2975e" } -asap-executor = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "58999df294fc9b0326341c1f20be62ecc8c2975e" } +# The backend depends on two Planner crates, at one immutable revision: +# `asap-planner` to plan and `asap-executor` to execute. Planner IR types +# are reached through `asap_executor::planner`. +asap-planner = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "f0dc1825805f4cbb8866b3baa15bed4319c4e1ab" } +asap-executor = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "f0dc1825805f4cbb8866b3baa15bed4319c4e1ab" } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" diff --git a/README.md b/README.md index 186be56a..f31039f8 100644 --- a/README.md +++ b/README.md @@ -11,15 +11,17 @@ Following the Planner's the Planner decides everything about a plan: which summaries to use, how to share them, what to materialize and which plan is cheapest. The backend supplies its deployment inputs (`DeploymentCapabilities`, and Stage 3's -`PlanSelectionSettings`), then executes the one selected `PhysicalASAPDAG` as -given. +`PlanSelectionSettings`), calls `asap_planner::plan_installable` once, and +executes the returned `InstallablePlan` as given. Among the Planner's crates +it depends only on `asap-planner` (to plan) and `asap-executor` (to execute). ```text PlanningWorkload (JSON) DeploymentCapabilities + PlanSelectionSettings \ / - control_plane: ASAPPlanner stages 0-3 + control_plane: asap_planner::plan_installable | - plan.json: selected PhysicalASAPDAG + query per root + plan.json: InstallablePlan (selected DAG, root per query, + stored outputs with pane and retention, raw windows) | data_plane: Remote Write in, Prometheus query API out ``` @@ -51,7 +53,14 @@ it. It is kept in memory, by definition and pane end, once the pane is closed by the precompute engine's rule (its end at or before the newest sample minus `--allowed-lateness-ms`), and later evaluations reuse it; a pane read before it closes is computed again by later evaluations. Kept panes are -evicted like stored ones, measured from the newest evaluation. Anything else is forwarded +evicted like stored ones, measured from the newest evaluation. + +Each stored definition is kept for the largest `retention_ms` the plan +states among its outputs: a pane that starts more than that before the +newest evaluation is evicted. The Planner's retention covers the range +evaluations the workload declares (`range_evaluation`), so a range query +reaching further back than declared reads an evicted pane and is +forwarded. Anything else is forwarded to Prometheus (`x-asap-execution: forwarded`); a planned query that could not be answered also says why in `x-asap-forward-reason`. @@ -79,7 +88,9 @@ cargo run -p data_plane -- --plan plan.json --prometheus-url http://localhost:90 `DeploymentCapabilities`: every summary the Planner can build, raw data kept in Prometheus, and query-time results kept across evaluations. For example `{"raw_data_retained": false}` makes the Planner build every input at -ingestion time. `--measured-operation-costs costs.json` gives Stage 3 the +ingestion time. `--plan-selection-settings settings.json` replaces Stage 3's +`PlanSelectionSettings` (cost model and unit prices, resource speed, measured +operation costs). `--measured-operation-costs costs.json` gives Stage 3 the deployment's measured sketch operation costs; otherwise it prices every node with cost model 1 and illustrative unit costs. @@ -91,8 +102,9 @@ which the Planner keeps query-time panes across evaluations) as Planner `PlanningWorkload`s. `data_plane/tests/e2e_examples.rs` plans each one, runs the `data_plane` binary against a Prometheus holding generated samples, Remote Writes the same samples, and checks every planned query -against the Planner executor running the selected plan with nothing -materialized. +against the Planner executor running the plan's query-time reference +(`InstallablePlan::query_time_reference`: the same plan with every node at +query time). ## Development diff --git a/control_plane/Cargo.toml b/control_plane/Cargo.toml index c9de2d4f..b20fb2c1 100644 --- a/control_plane/Cargo.toml +++ b/control_plane/Cargo.toml @@ -4,10 +4,8 @@ version.workspace = true edition.workspace = true [dependencies] -planner-types.workspace = true -asap-frontend-promql.workspace = true -asap-logical-asap-optimization.workspace = true -asap-plan-selection.workspace = true +asap-planner.workspace = true +futures = "0.3" serde.workspace = true serde_json.workspace = true thiserror.workspace = true diff --git a/control_plane/src/lib.rs b/control_plane/src/lib.rs index 15e8a4da..619a6e73 100644 --- a/control_plane/src/lib.rs +++ b/control_plane/src/lib.rs @@ -1,37 +1,17 @@ //! Backend control plane: asks ASAPPlanner for the workload's plan and //! writes it for the data plane to install. //! -//! The Planner runs every planning stage, including plan selection. This -//! crate only supplies the deployment inputs (what the data plane can execute -//! and keep, and how Stage 3 prices candidates) and records which query each -//! root of the selected `PhysicalASAPDAG` answers. +//! The Planner runs every planning stage, including plan selection, and +//! returns the selected DAG ready to install ([`InstallablePlan`]). This +//! crate only supplies the deployment inputs: what the data plane can +//! execute and keep, and how Stage 3 prices candidates. use std::path::Path; -use std::rc::Rc; -use asap_logical_asap_optimization::DeploymentCapabilities; -use asap_plan_selection::{ - resize_for_failure_probability, run_stage1_to_stage3, with_confirmed_weights, - PlanSelectionSettings, PlanningModels, Stage1ToStage3Run, +use asap_planner::{ + plan_installable, DeploymentCapabilities, FrontendInput, InstallablePlan, + PlanSelectionSettings, PlanningModels, PlanningWorkload, UserInput, }; -use planner_types::ir::operator::pane_chain::expand_pane_chains; -use planner_types::ir::physical_export::{compile_physical_asap_workload, PhysicalASAPDAGDocument}; -use planner_types::ir::schema::schema_support::with_promql_series_identity; -use planner_types::ir::QueryRoot; -use planner_types::workload::{DataWorkload, PlanningWorkload, QueryLanguage, QueryWorkloadEntry}; -use serde::{Deserialize, Serialize}; - -/// The plan the data plane installs: the Planner's selected physical DAG and -/// the query each of its roots answers. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct InstalledPlan { - /// PromQL text of each root of `dag`, in root order. - pub queries: Vec, - /// The selected candidate's DAG, with every `PaneChain` expanded into - /// the panes the executor runs. - pub dag: PhysicalASAPDAGDocument, -} /// What the backend gives the Planner about its deployment. The Planner /// selects; these only say what can run and what it costs. @@ -46,181 +26,42 @@ pub struct DeploymentInputs { #[derive(Debug, thiserror::Error)] pub enum PlanError { - #[error("only PromQL workloads are supported, got {0:?}")] - UnsupportedLanguage(QueryLanguage), - #[error("workload: {0}")] - Workload(String), - #[error("query {0} lowers to a scalar, which the data plane cannot answer yet")] - ScalarRoot(usize), - #[error("PromQL lowering: {0}")] - Lowering(String), #[error("planning: {0}")] - Planning(String), - #[error("invalid physical DAG: {0}")] - InvalidDag(String), - #[error("the DAG has {roots} roots for {queries} queries")] - RootCount { roots: usize, queries: usize }, + Planning(#[from] asap_planner::PlanError), #[error("{path}: {message}")] File { path: String, message: String }, } -impl InstalledPlan { - pub fn validate(&self) -> Result<(), PlanError> { - self.dag - .validate() - .map_err(|e| PlanError::InvalidDag(e.to_string()))?; - let roots = self.dag.dag.roots.len(); - if roots != self.queries.len() { - return Err(PlanError::RootCount { - roots, - queries: self.queries.len(), - }); - } - Ok(()) - } - - /// Reads and validates a plan written by [`InstalledPlan::save`]. - pub fn load(path: &Path) -> Result { - let file_error = |message: String| PlanError::File { - path: path.display().to_string(), - message, - }; - let text = std::fs::read_to_string(path).map_err(|e| file_error(e.to_string()))?; - let plan: Self = serde_json::from_str(&text).map_err(|e| file_error(e.to_string()))?; - plan.validate()?; - Ok(plan) - } - - pub fn save(&self, path: &Path) -> Result<(), PlanError> { - let text = serde_json::to_string_pretty(self).expect("plans serialize"); - std::fs::write(path, text).map_err(|e| PlanError::File { - path: path.display().to_string(), - message: e.to_string(), - }) - } - - /// The installed plan for `roots`, the timed roots of one candidate in - /// query order. - fn from_roots(queries: Vec, roots: &[QueryRoot]) -> Result { - let roots = roots - .iter() - .enumerate() - .map(|(i, root)| match root { - // The executor runs a pane chain as its panes. - QueryRoot::Operator(node) => expand_pane_chains(node) - .map(QueryRoot::Operator) - .map_err(|e| PlanError::InvalidDag(e.to_string())), - QueryRoot::Scalar(_) => Err(PlanError::ScalarRoot(i)), - }) - .collect::, _>>()?; - let dag = compile_physical_asap_workload(&roots) - .map_err(|e| PlanError::InvalidDag(e.to_string()))?; - let plan = Self { - queries, - dag: PhysicalASAPDAGDocument::new(dag), - }; - plan.validate()?; - Ok(plan) - } -} - -/// One planned workload: what Stages 1–3 returned and the inputs they read. -struct Planned { - queries: Vec, - entries: Vec, - data: DataWorkload, - run: Stage1ToStage3Run, -} - -/// Lowers `workload` (Stage 0) and runs Stages 1–3 under `inputs`. -fn plan(workload: &PlanningWorkload, inputs: &DeploymentInputs) -> Result { - let language = &workload.query_workload.language; - if *language != QueryLanguage::PromQL { - return Err(PlanError::UnsupportedLanguage(language.clone())); - } - workload - .validate() - .map_err(|e| PlanError::Workload(e.to_string()))?; - let roots = asap_frontend_promql::lower_promql_query_workload(workload) - .map_err(|e| PlanError::Lowering(e.to_string()))? - .into_iter() - .enumerate() - .map(|(i, root)| match root { - // Rows carry each series' full identity, the representation - // per-series state needs at runtime, as the Planner's own pass - // prepares PromQL roots. - QueryRoot::Operator(node) => { - let node = with_promql_series_identity(&node).unwrap_or_else(|_| Rc::clone(&node)); - Ok((i, QueryRoot::Operator(node))) - } - QueryRoot::Scalar(_) => Err(PlanError::ScalarRoot(i)), - }) - .collect::, _>>()?; - let entries: Vec = workload.query_workload.entries().collect(); - let data = workload.data_workload.clone().unwrap_or_default(); - let models = PlanningModels::builtin() - .with_deployment_capabilities(&inputs.deployment_capabilities) - .with_plan_selection(&inputs.plan_selection); - let run = run_stage1_to_stage3(roots, &entries, &data, models) - .map_err(|e| PlanError::Planning(e.to_string()))?; - Ok(Planned { - queries: entries.iter().map(|e| e.query.0.clone()).collect(), - entries, - data, - run, - }) -} - -/// Plans `workload` with the Planner's stage pipeline (Stages 0–3) and -/// returns the plan Stage 3 selected. The Planner rejects every candidate -/// that needs more than `inputs.deployment_capabilities`. +/// Plans `workload` with the Planner (frontends and Stages 1–3) and returns +/// the plan Stage 3 selected, ready to install. Only PromQL workloads are +/// planned: the Planner refuses another language for the PromQL frontend. +/// It rejects every candidate that needs more than +/// `inputs.deployment_capabilities`, and refuses a plan the data plane could +/// not run. pub fn plan_workload( workload: &PlanningWorkload, inputs: &DeploymentInputs, -) -> Result { - let planned = plan(workload, inputs)?; - let selection = planned - .run - .stage3_plan_selection - .map_err(|e| PlanError::Planning(e.to_string()))?; - tracing::info!(candidate = %selection.selected, "selected physical plan"); - InstalledPlan::from_roots(planned.queries, &selection.plan) +) -> Result { + let models = PlanningModels::builtin() + .with_deployment_capabilities(&inputs.deployment_capabilities) + .with_plan_selection(&inputs.plan_selection); + let input = UserInput::new(workload, FrontendInput::Promql { histograms: None }, models); + // Only the SQL frontend awaits anything; PromQL planning completes on + // first poll. + let plan = futures::executor::block_on(plan_installable(input))?; + if let Some(selection) = &plan.selection { + tracing::info!(candidate = %selection.selected_candidate_id, "selected physical plan"); + } + Ok(plan) } -/// The selected plan with nothing materialized: the Stage 2 candidate that -/// implements the same logical candidate with every summary built at query -/// time, sized as Stage 3 sizes the selected one. Its answers are what the -/// installed plan's must equal, so tests use it as the reference. -pub fn plan_selected_at_query_time( - workload: &PlanningWorkload, - inputs: &DeploymentInputs, -) -> Result { - let planned = plan(workload, inputs)?; - let run = &planned.run; - let missing = |what: &str| PlanError::Planning(format!("no {what} in the planner's run")); - let selected = run - .selected() - .ok_or_else(|| missing("selected candidate"))?; - // Stage 2 numbers `L`'s candidates `P` (nothing materialized), - // then `P-m`. - let number = selected - .from_logical - .strip_prefix('L') - .ok_or_else(|| missing("logical candidate number"))?; - let unmaterialized = run - .stage2_physical_asap_optimization - .iter() - .find(|p| p.id == format!("P{number}")) - .ok_or_else(|| missing("unmaterialized candidate"))?; - let targets: Vec<_> = planned - .entries - .iter() - .map(|e| e.requirements.accuracy.clone()) - .collect(); - let sized = resize_for_failure_probability(&unmaterialized.roots, &targets, &planned.data) - .map_err(|e| PlanError::Planning(e.to_string()))?; - let roots = with_confirmed_weights(&sized, &planned.data); - InstalledPlan::from_roots(planned.queries, &roots) +/// Writes `plan` as the plan file the data plane installs. +pub fn save_plan(plan: &InstallablePlan, path: &Path) -> Result<(), PlanError> { + let text = serde_json::to_string_pretty(plan).expect("plans serialize"); + std::fs::write(path, text).map_err(|e| PlanError::File { + path: path.display().to_string(), + message: e.to_string(), + }) } /// What this backend's data plane can execute and keep: every summary the diff --git a/control_plane/src/main.rs b/control_plane/src/main.rs index 8773f318..5f0b5282 100644 --- a/control_plane/src/main.rs +++ b/control_plane/src/main.rs @@ -1,9 +1,9 @@ use std::path::{Path, PathBuf}; use anyhow::Context; +use asap_planner::PlanningWorkload; use clap::Parser; -use control_plane::{plan_workload, DeploymentInputs}; -use planner_types::workload::PlanningWorkload; +use control_plane::{plan_workload, save_plan, DeploymentInputs}; /// Plan a workload with ASAPPlanner and write the selected plan. #[derive(Parser)] @@ -15,6 +15,10 @@ struct Args { /// backend's data plane executes and keeps. #[arg(long)] deployment_capabilities: Option, + /// Planner `PlanSelectionSettings` (JSON): cost model and unit prices, + /// resource speed, measured operation costs. Defaults to the Planner's. + #[arg(long)] + plan_selection_settings: Option, /// Planner `MeasuredOperationCost` list (JSON) that Stage 3 prices /// sketch operations from; by default every node is priced from the /// Planner's formulas. @@ -41,11 +45,14 @@ fn main() -> anyhow::Result<()> { if let Some(path) = &args.deployment_capabilities { inputs.deployment_capabilities = read_json(path)?; } + if let Some(path) = &args.plan_selection_settings { + inputs.plan_selection = read_json(path)?; + } if let Some(path) = &args.measured_operation_costs { inputs.plan_selection.measured_operation_costs = read_json(path)?; } let plan = plan_workload(&workload, &inputs)?; - plan.save(&args.out)?; + save_plan(&plan, &args.out)?; println!( "wrote {} ({} queries, {} nodes)", args.out.display(), diff --git a/control_plane/tests/plan_workload.rs b/control_plane/tests/plan_workload.rs index 3aae8c61..c5f2cd65 100644 --- a/control_plane/tests/plan_workload.rs +++ b/control_plane/tests/plan_workload.rs @@ -1,15 +1,11 @@ use std::path::{Path, PathBuf}; -use control_plane::{ - plan_selected_at_query_time, plan_workload, DeploymentInputs, InstalledPlan, PlanError, -}; -use planner_types::ir::{ASAPOp, Operator}; -use planner_types::physical::Materialization; -use planner_types::workload::{PlanningWorkload, QueryLanguage, SqlDialect}; +use asap_planner::{InstallablePlan, PlanningWorkload, StoredOutputBuild}; +use control_plane::{plan_workload, save_plan, DeploymentInputs, PlanError}; -fn example1() -> PlanningWorkload { - let path = - Path::new(env!("CARGO_MANIFEST_DIR")).join("../examples/planner-layering-example1.json"); +fn example(name: &str) -> PlanningWorkload { + let path = Path::new(env!("CARGO_MANIFEST_DIR")) + .join(format!("../examples/planner-layering-{name}.json")); serde_json::from_str(&std::fs::read_to_string(path).unwrap()).unwrap() } @@ -18,13 +14,17 @@ fn temp_path(name: &str) -> PathBuf { } /// The Planner selects one physical DAG for #509 Example 1, with one root per -/// query, and the plan survives the file the data plane loads. +/// query bound to the query's text, and the plan survives the file the data +/// plane loads. #[test] fn plans_example1_and_round_trips_the_plan() { - let workload = example1(); + let workload = example("example1"); let plan = plan_workload(&workload, &DeploymentInputs::default()).unwrap(); assert_eq!( - plan.queries, + plan.queries + .iter() + .map(|q| q.query.clone()) + .collect::>(), workload .query_workload .entries() @@ -33,103 +33,68 @@ fn plans_example1_and_round_trips_the_plan() { ); assert_eq!(plan.dag.dag.roots.len(), 2); let path = temp_path("example1.json"); - plan.save(&path).unwrap(); - assert_eq!(InstalledPlan::load(&path).unwrap(), plan); + save_plan(&plan, &path).unwrap(); + let loaded: InstallablePlan = + serde_json::from_str(&std::fs::read_to_string(&path).unwrap()).unwrap(); + loaded.validate().unwrap(); + assert_eq!(loaded, plan); std::fs::remove_file(path).unwrap(); } -/// Only PromQL workloads are planned; SQL support comes later. +/// Only PromQL workloads are planned: the Planner refuses a SQL workload +/// for the PromQL frontend. #[test] fn rejects_sql_workloads() { - let mut workload = example1(); - workload.query_workload.language = QueryLanguage::SQL(SqlDialect::ClickhouseSQL); + let mut workload: serde_json::Value = serde_json::to_value(example("example1")).unwrap(); + workload["query_workload"]["language"] = serde_json::json!({"sql": "clickhouse_sql"}); + let workload: PlanningWorkload = serde_json::from_value(workload).unwrap(); assert!(matches!( plan_workload(&workload, &DeploymentInputs::default()), - Err(PlanError::UnsupportedLanguage(_)) + Err(PlanError::Planning(_)) )); } -/// A plan whose queries do not match the DAG's roots is not installed. +/// Without raw data the Planner builds Example 4b's six ten-minute panes at +/// ingestion time and lists them as stored outputs, each with its pane and +/// a retention at least as long as its pane reaches back; its query-time +/// reference stores nothing and answers the same queries. #[test] -fn load_rejects_a_plan_without_a_query_per_root() { - let mut plan = plan_workload(&example1(), &DeploymentInputs::default()).unwrap(); - plan.queries.pop(); - let path = temp_path("mismatched.json"); - plan.save(&path).unwrap(); - assert!(matches!( - InstalledPlan::load(&path), - Err(PlanError::RootCount { - roots: 2, - queries: 1 - }) - )); - std::fs::remove_file(path).unwrap(); -} - -fn example4b() -> PlanningWorkload { - let path = - Path::new(env!("CARGO_MANIFEST_DIR")).join("../examples/planner-layering-example4b.json"); - serde_json::from_str(&std::fs::read_to_string(path).unwrap()).unwrap() -} - -/// Without raw data the Planner builds Example 4b's panes at ingestion time, -/// installed as plain panes (no `PaneChain`); the reference plan of the same -/// selection materializes nothing. -#[test] -fn reference_plan_materializes_nothing() { +fn stores_example4b_panes_and_references_them_at_query_time() { let mut inputs = DeploymentInputs::default(); inputs.deployment_capabilities.raw_data_retained = false; - let installed = plan_workload(&example4b(), &inputs).unwrap(); - let materialized = |plan: &InstalledPlan| { - plan.dag - .dag - .nodes - .iter() - .filter(|n| n.materialization != Materialization::NotMaterialized) - .count() - }; - assert_eq!(materialized(&installed), 6, "six ten-minute panes"); - assert!(installed - .dag - .dag - .nodes + let plan = plan_workload(&example("example4b"), &inputs).unwrap(); + assert_eq!(plan.stored_outputs.len(), 6, "six ten-minute panes"); + let mut offsets: Vec = plan + .stored_outputs .iter() - .all(|n| !matches!(n.payload, Operator::ASAP(ASAPOp::PaneChain { .. })))); - let reference = plan_selected_at_query_time(&example4b(), &inputs).unwrap(); - assert_eq!(materialized(&reference), 0); - assert_eq!(reference.queries, installed.queries); + .map(|output| { + assert_eq!(output.built_at, StoredOutputBuild::IngestionRun); + assert_eq!(output.pane.width_ms, 600_000); + assert!(output.retention_ms >= output.pane.reach_ms()); + output.pane.end_offset_ms + }) + .collect(); + offsets.sort(); + assert_eq!(offsets, (0..6).map(|i| i * 600_000).collect::>()); + assert!(plan.query_time_raw_windows.is_empty()); + let reference = plan.query_time_reference().unwrap(); + assert!(reference.stored_outputs.is_empty()); + assert_eq!(reference.queries, plan.queries); } /// The backend keeps query-time results across evaluations, so for a few /// series sampled every millisecond the Planner selects #509 Example 4's -/// B3: five 1-minute KLL panes built at query time and kept, installed as -/// plain panes. +/// B3: five 1-minute KLL panes built at query time and kept. #[test] fn selects_panes_kept_across_evaluations() { let inputs = DeploymentInputs::default(); assert!(inputs.deployment_capabilities.query_time_results_retained); - let path = - Path::new(env!("CARGO_MANIFEST_DIR")).join("../examples/planner-layering-example4-b3.json"); - let workload: PlanningWorkload = - serde_json::from_str(&std::fs::read_to_string(path).unwrap()).unwrap(); - let plan = plan_workload(&workload, &inputs).unwrap(); - let kept = plan - .dag - .dag - .nodes + let plan = plan_workload(&example("example4-b3"), &inputs).unwrap(); + let kept: Vec<_> = plan + .stored_outputs .iter() - .filter(|n| { - matches!( - n.materialization, - Materialization::MatAtQueryKeptAcrossEvaluations(_) - ) - }) - .inspect(|n| { - assert!(matches!( - n.payload, - Operator::ASAP(ASAPOp::SummaryAgg { .. }) - )) - }) - .count(); - assert_eq!(kept, 5, "five one-minute panes"); + .filter(|output| output.built_at == StoredOutputBuild::QueryRun) + .collect(); + assert_eq!(kept.len(), 5, "five one-minute panes"); + assert!(kept.iter().all(|output| output.pane.width_ms == 60_000)); } diff --git a/data_plane/Cargo.toml b/data_plane/Cargo.toml index 74630e2f..b82d13b7 100644 --- a/data_plane/Cargo.toml +++ b/data_plane/Cargo.toml @@ -4,8 +4,7 @@ version.workspace = true edition.workspace = true [dependencies] -control_plane.workspace = true -planner-types.workspace = true +asap-planner.workspace = true asap-executor.workspace = true sha2 = "0.10" serde.workspace = true @@ -25,6 +24,7 @@ form_urlencoded = "1" chrono = { version = "0.4", default-features = false, features = ["alloc"] } [dev-dependencies] +control_plane.workspace = true tower = { version = "0.5", features = ["util"] } http-body-util = "0.1" serde_json.workspace = true diff --git a/data_plane/src/engine.rs b/data_plane/src/engine.rs index f0f140e6..ba44b400 100644 --- a/data_plane/src/engine.rs +++ b/data_plane/src/engine.rs @@ -5,8 +5,7 @@ use std::collections::BTreeMap; use std::sync::mpsc::{sync_channel, Receiver, SyncSender, TrySendError}; -use asap_executor::physical_planner::frontier_from_timing; -use control_plane::InstalledPlan; +use asap_planner::InstallablePlan; use tokio::sync::oneshot; use crate::precompute::{Precompute, PrecomputeStats, RawSample}; @@ -170,10 +169,9 @@ struct Engine { impl Engine { fn new(plan_json: &str, allowed_lateness_ms: i64) -> anyhow::Result { - let plan: InstalledPlan = serde_json::from_str(plan_json)?; + let plan: InstallablePlan = serde_json::from_str(plan_json)?; plan.validate()?; - let frontier = frontier_from_timing(&plan.dag.dag)?; - let sds = Sds::from_dag(&plan.dag.dag, &frontier)?; + let sds = Sds::from_plan(&plan)?; tracing::info!( definitions = sds.definitions.len(), outputs = sds.outputs.len(), @@ -183,7 +181,7 @@ impl Engine { ); let query = QueryEngine::new(&plan, &sds); Ok(Self { - precompute: Precompute::new(plan.dag.dag, &sds, allowed_lateness_ms)?, + precompute: Precompute::new(&plan, &sds, allowed_lateness_ms)?, query, }) } diff --git a/data_plane/src/lib.rs b/data_plane/src/lib.rs index baaad848..c813c4cb 100644 --- a/data_plane/src/lib.rs +++ b/data_plane/src/lib.rs @@ -1,7 +1,8 @@ //! Backend data plane: receives Prometheus Remote Write samples and serves //! the Prometheus query API for an installed plan. //! -//! The installed plan is ASAPPlanner's selected `PhysicalASAPDAG`. The +//! The installed plan is ASAPPlanner's `InstallablePlan`: its selected +//! physical DAG, the outputs it stores and the raw windows it reads. The //! engine thread builds its ingestion-time outputs pane by pane and answers //! the plan's instant and range queries from them; every other query, and //! any planned query it cannot answer, is forwarded to Prometheus. @@ -17,13 +18,13 @@ use std::collections::BTreeMap; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Arc; +use asap_planner::InstallablePlan; use axum::body::Bytes; use axum::extract::{RawQuery, State}; use axum::http::{header, HeaderMap, HeaderValue, Method, StatusCode}; use axum::response::{IntoResponse, Response}; use axum::routing::{get, post}; use axum::Router; -use control_plane::InstalledPlan; use engine::EngineHandle; use precompute::RawSample; use prometheus::{InstantQuery, RangeQuery}; @@ -50,17 +51,17 @@ pub struct DataPlane { } impl DataPlane { - /// Installs the plan serialized in `plan_json` (an [`InstalledPlan`]). + /// Installs the plan serialized in `plan_json` (an [`InstallablePlan`]). /// Panes close `allowed_lateness_ms` after the newest sample passes them. pub fn new( plan_json: String, prometheus_url: String, allowed_lateness_ms: i64, ) -> anyhow::Result { - let plan: InstalledPlan = serde_json::from_str(&plan_json)?; + let plan: InstallablePlan = serde_json::from_str(&plan_json)?; plan.validate()?; Ok(Self { - queries: plan.queries, + queries: plan.queries.into_iter().map(|q| q.query).collect(), engine: EngineHandle::spawn(plan_json, allowed_lateness_ms)?, prometheus_url: prometheus_url.trim_end_matches('/').to_string(), limits: remote_write::Limits::default(), diff --git a/data_plane/src/main.rs b/data_plane/src/main.rs index c6ff5484..d8a24abf 100644 --- a/data_plane/src/main.rs +++ b/data_plane/src/main.rs @@ -18,6 +18,10 @@ struct Args { /// How long after the newest sample passes a pane's end it is built. #[arg(long, default_value_t = 15_000)] allowed_lateness_ms: i64, + /// Working memory one query or pane build may hold, in MiB (the + /// executor's default is 64); an evaluation over it is forwarded. + #[arg(long, default_value_t = 64)] + execution_memory_limit_mib: usize, } #[tokio::main] @@ -26,6 +30,7 @@ async fn main() -> anyhow::Result<()> { .with_env_filter(tracing_subscriber::EnvFilter::from_default_env()) .init(); let args = Args::parse(); + data_plane::precompute::set_execution_memory_limit(args.execution_memory_limit_mib << 20); let plan = std::fs::read_to_string(&args.plan)?; let state = Arc::new(DataPlane::new( plan, diff --git a/data_plane/src/precompute.rs b/data_plane/src/precompute.rs index 0231fcd6..7f9a10af 100644 --- a/data_plane/src/precompute.rs +++ b/data_plane/src/precompute.rs @@ -1,38 +1,35 @@ //! Precompute engine: builds each stored summary pane once, when the //! watermark passes its end. //! -//! For every [`SummaryDefinition`] the engine buffers the samples of its -//! `Scan`'s metric by tumbling pane. A pane `(end - pane, end]` closes when -//! the watermark (the newest sample time minus the allowed lateness) reaches -//! `end`. Closing it runs the plan's `Scan` over the buffered samples, which -//! applies the label matchers, then the definition's build from its -//! `TimeRange` input up to its output, under ingestion scope. The output is +//! For every [`SummaryDefinition`] built at ingestion time the engine +//! buffers the samples of its pane's metric by tumbling pane. A pane `(end - +//! pane, end]` closes when the watermark (the newest sample time minus the +//! allowed lateness) reaches `end`. Closing it selects the pane's rows with +//! the executor's `RawSampleSelector`, which applies the plan's `Scan` +//! (its label matchers), then runs the definition's build from its +//! `TimeRange` up to its output, under ingestion scope. The output is //! stored and never changes; samples older than a closed pane are dropped. //! The first pane a definition sees may have started before the data plane //! did, so it is never built: queries that need it are not answered from //! stored panes. -use std::collections::{BTreeMap, HashMap}; -use std::sync::Arc; +use std::collections::BTreeMap; +use std::sync::{Arc, OnceLock}; use asap_executor::operators::Operator; -use asap_executor::physical_planner::promql_rows::series_row; -use asap_executor::physical_planner::{ - bind_with_data_sources, compile, CompiledPhysicalDAG, InputContract, Source, -}; +use asap_executor::physical_planner::{compile, CompiledPhysicalDAG, InputContract, Source}; +use asap_executor::plan::PhysicalDAG; use asap_executor::runtime::{Limits, RunContext, Scope}; -use asap_executor::sources::{DataSources, MemorySource}; -use asap_executor::values::{Batch, SchemaRef}; +use asap_executor::values::{Batch, SchemaRef, Value}; +use asap_executor::{RawSampleSelector, Sample}; +use asap_planner::{InstallablePlan, PhysicalASAPDAG}; use futures::executor::block_on; use futures::StreamExt; -use planner_types::ir::operator::operator_properties::Source as ScanSource; -use planner_types::ir::physical_export::PhysicalASAPDAG; -use planner_types::ir::{NonASAPOp, Operator as IrOperator}; -use crate::sds::{DefinitionId, PaneBinding, Sds, SummaryDefinition, SummaryStore}; +use crate::sds::{DefinitionId, Sds, SummaryDefinition, SummaryStore}; /// One raw sample: its series' full label set (including `__name__`), time -/// and value. +/// and value. Samples of one series share their labels. #[derive(Debug, Clone, PartialEq)] pub struct RawSample { pub labels: Arc>, @@ -40,6 +37,17 @@ pub struct RawSample { pub value: f64, } +impl RawSample { + /// The executor's sample, for a `RawSampleSelector`. + pub fn to_sample(&self) -> Sample { + Sample { + labels: (*self.labels).clone(), + timestamp_ms: self.timestamp_ms, + value: self.value, + } + } +} + #[derive(Debug, thiserror::Error)] pub enum PrecomputeError { #[error("stored summary {0:?}: {1}")] @@ -61,75 +69,51 @@ pub struct PrecomputeStats { pub panes_failed: u64, } -/// A time-series `Scan` of the plan: its node, metric and row schema. -#[derive(Debug, Clone)] -pub struct ScanInput { - pub node: u64, - pub metric: String, - pub schema: SchemaRef, -} - /// How the panes of one definition are built from raw samples. The /// precompute engine builds stored panes with it, and the query engine kept /// ones. pub(crate) struct PaneBuild { pub(crate) definition: SummaryDefinition, - pub(crate) scan: ScanInput, + /// Selects one pane's rows from raw samples. + pub(crate) selector: RawSampleSelector, build: CompiledPhysicalDAG, - /// Largest offset any output of the definition reads, plus two panes: - /// panes ending at or before `newest - retention_ms` are no longer read. - pub(crate) retention_ms: i64, } impl PaneBuild { - /// Compiles `definition`'s build from its `TimeRange` up to its builder; - /// `outputs` are the outputs that read it. + /// Compiles `definition`'s build from its `TimeRange` up to its builder. pub(crate) fn new( dag: &PhysicalASAPDAG, definition: &SummaryDefinition, - outputs: &BTreeMap, ) -> Result { - let error = - |message: &str| PrecomputeError::Definition(definition.id.clone(), message.into()); - let scan = scan_below(dag, definition.raw_input) - .ok_or_else(|| error("no time-series Scan below its TimeRange"))?; - let raw_schema = Arc::new(output_schema(dag, definition.raw_input).clone()); - if raw_schema != scan.schema { - return Err(error("TimeRange rows differ from its Scan's")); - } + let selector = RawSampleSelector::new(dag, &definition.window) + .map_err(|e| PrecomputeError::Definition(definition.id.clone(), e.to_string()))?; + let raw_schema = Arc::new(output_schema(dag, definition.raw_input()).clone()); let build = compile( dag, - BTreeMap::from([(definition.raw_input, InputContract::bounded(raw_schema))]), + BTreeMap::from([(definition.raw_input(), InputContract::bounded(raw_schema))]), &[definition.builder], )?; - let max_offset = outputs - .values() - .filter(|(id, _)| id == &definition.id) - .map(|(_, binding)| binding.offset_ms) - .max() - .unwrap_or(0); Ok(Self { definition: definition.clone(), - scan, + selector, build, - retention_ms: max_offset + 2 * definition.pane_ms, }) } - /// Runs the plan's `Scan` over `samples`, then the build over the rows - /// it selects, under `scope`. + /// The pane ending at `end_ms`, built from `samples` under `scope`: + /// the selector keeps the pane's samples that the plan's `Scan` selects, + /// then the build runs over them. pub(crate) fn run( &self, - dag: &PhysicalASAPDAG, - samples: &[RawSample], + samples: &[Sample], + end_ms: i64, scope: Scope, ) -> Result { - let selected = select(dag, &self.scan, samples, scope.clone())?; - let input: Source<'_> = - Box::new(Operator::source(self.scan.schema.clone(), vec![selected])?); + let rows = self.selector.rows(end_ms, samples)?; + let input: Source<'_> = Box::new(Operator::source(rows.schema().clone(), vec![rows])?); let build = self .build - .instantiate(BTreeMap::from([(self.definition.raw_input, input)]))?; + .instantiate(BTreeMap::from([(self.definition.raw_input(), input)]))?; let rows = run(&build, self.build.roots(), scope)?; let schema = Arc::new(self.definition.output_schema.clone()); Ok(Batch::try_new(schema, rows)?) @@ -139,14 +123,13 @@ impl PaneBuild { struct Builder { pane: PaneBuild, /// Samples of open panes, by pane end. - open: BTreeMap>, + open: BTreeMap>, /// End of the newest pane built or skipped; samples in it or before it /// are dropped. `None` until the first sample. built_through: Option, } pub struct Precompute { - dag: PhysicalASAPDAG, builders: Vec, allowed_lateness_ms: i64, newest_sample_ms: Option, @@ -155,9 +138,10 @@ pub struct Precompute { } impl Precompute { - /// Compiles one build per definition of `sds`, a description of `dag`. + /// Compiles one build per ingestion-time definition of `sds`, the + /// stored summaries of `plan`. pub fn new( - dag: PhysicalASAPDAG, + plan: &InstallablePlan, sds: &Sds, allowed_lateness_ms: i64, ) -> Result { @@ -166,14 +150,13 @@ impl Precompute { .values() .map(|definition| { Ok(Builder { - pane: PaneBuild::new(&dag, definition, &sds.outputs)?, + pane: PaneBuild::new(&plan.dag.dag, definition)?, open: BTreeMap::new(), built_through: None, }) }) .collect::>()?; Ok(Self { - dag, builders, allowed_lateness_ms, newest_sample_ms: None, @@ -195,7 +178,7 @@ impl Precompute { let metric = sample.labels.get("__name__").map(String::as_str); let mut accepted = false; for builder in &mut self.builders { - if metric != Some(builder.pane.scan.metric.as_str()) { + if metric != Some(builder.pane.selector.metric()) { continue; } let end = pane_end(sample.timestamp_ms, builder.pane.definition.pane_ms); @@ -205,7 +188,11 @@ impl Precompute { self.stats.dropped += 1; continue; } - builder.open.entry(end).or_default().push(sample.clone()); + builder + .open + .entry(end) + .or_default() + .push(sample.to_sample()); accepted = true; } if accepted { @@ -235,7 +222,7 @@ impl Precompute { window_end_ms: end, revision: 1, }; - match builder.pane.run(&self.dag, &samples, scope) { + match builder.pane.run(&samples, end, scope) { Ok(state) => { self.store.insert(&builder.pane.definition.id, end, state); self.stats.panes_built += 1; @@ -248,9 +235,14 @@ impl Precompute { builder.built_through = Some(end); end += pane_ms; } - let definition = &builder.pane.definition.id; - self.store - .evict_through(definition, watermark - builder.pane.retention_ms); + // The newest evaluation that reads stored panes is at most the + // newest closed pane's end, so retention is measured from it, + // not from the watermark, which may already be past it. + let definition = &builder.pane.definition; + if let Some(newest) = builder.built_through { + self.store + .evict_before(&definition.id, definition.oldest_kept_end_ms(newest)); + } } } } @@ -262,39 +254,32 @@ pub fn pane_end(timestamp_ms: i64, pane_ms: i64) -> i64 { -(-timestamp_ms).div_euclid(pane_ms) * pane_ms } -/// The rows of `samples` the plan's `scan` selects: its label matchers -/// applied by the executor's `Scan`. -pub fn select( - dag: &PhysicalASAPDAG, - scan: &ScanInput, - samples: &[RawSample], - scope: Scope, -) -> Result { - let rows = samples - .iter() - .map(|s| series_row(&scan.schema, &s.labels, s.timestamp_ms, s.value)) - .collect::, _>>()?; - let raw = Batch::try_new(scan.schema.clone(), rows)?; - let mut sources = DataSources::default(); - sources.register( - ScanSource::TimeSeries { - metric: scan.metric.clone(), - }, - Arc::new(MemorySource::new(scan.schema.clone(), vec![raw])?), - )?; - let dag = bind_with_data_sources(dag, BTreeMap::new(), &[scan.node], &sources)?; - let selected = run(&dag, &[scan.node], scope)?; - Batch::try_new(scan.schema.clone(), selected) +/// Working memory one execution may hold, in bytes; the executor's own +/// default (64 MiB) unless [`set_execution_memory_limit`] changed it. +static EXECUTION_MEMORY_LIMIT_BYTES: OnceLock = OnceLock::new(); + +/// Sets the working memory every later execution may hold. An execution +/// over it fails, and its query is forwarded. Only the first call counts. +pub fn set_execution_memory_limit(bytes: usize) { + let _ = EXECUTION_MEMORY_LIMIT_BYTES.set(bytes); +} + +fn limits() -> Limits { + let mut limits = Limits::default(); + if let Some(&bytes) = EXECUTION_MEMORY_LIMIT_BYTES.get() { + limits.max_bytes = bytes; + } + limits } /// Executes `dag`'s single root to completion and returns its rows. pub fn run( - dag: &asap_executor::plan::PhysicalDAG<'_, Batch, SchemaRef>, + dag: &PhysicalDAG<'_, Batch, SchemaRef>, roots: &[u64], scope: Scope, -) -> Result>, asap_executor::Error> { +) -> Result>, asap_executor::Error> { block_on(async { - let context = RunContext::new(scope, Limits::default())?; + let context = RunContext::new(scope, limits())?; let mut output = dag.execute(roots, context)?.remove(0); let mut rows = Vec::new(); while let Some(batch) = output.next().await { @@ -304,35 +289,13 @@ pub fn run( }) } -fn output_schema(dag: &PhysicalASAPDAG, id: u64) -> &planner_types::ir::schema::Schema { +pub(crate) fn output_schema( + dag: &PhysicalASAPDAG, + id: u64, +) -> &asap_executor::planner::ir::schema::Schema { &dag.nodes .iter() .find(|n| n.id as u64 == id) - .expect("SDS names DAG nodes") + .expect("stored outputs and raw windows name DAG nodes") .output_schema } - -/// The time-series `Scan` that `id` reads through single-input operators. -pub fn scan_below(dag: &PhysicalASAPDAG, id: u64) -> Option { - let inputs: HashMap = dag - .edges - .iter() - .map(|e| (e.consumer as u64, e.producer as u64)) - .collect(); - let mut current = id; - loop { - let node = dag.nodes.iter().find(|n| n.id as u64 == current)?; - if let IrOperator::NonASAP(NonASAPOp::Scan { - source: ScanSource::TimeSeries { metric }, - .. - }) = &node.payload - { - return Some(ScanInput { - node: current, - metric: metric.clone(), - schema: Arc::new(node.output_schema.clone()), - }); - } - current = *inputs.get(¤t)?; - } -} diff --git a/data_plane/src/query.rs b/data_plane/src/query.rs index 7786e2f9..8cb75c82 100644 --- a/data_plane/src/query.rs +++ b/data_plane/src/query.rs @@ -2,7 +2,7 @@ //! its root of the plan's DAG over stored panes and raw samples. //! //! Each root is compiled once with three kinds of inputs: -//! - stored outputs (the SDS frontier): bound to the pane of their definition +//! - stored outputs built at ingestion time: bound to the pane of their definition //! ending at `T - offset`, which must be stored; //! - kept outputs (`MatAtQueryKeptAcrossEvaluations`): bound to the kept //! pane of their definition ending at `T - offset`. A missing one is built @@ -10,9 +10,11 @@ //! next evaluations once it is closed (its end at or before the //! precompute engine's watermark); kept panes are evicted like stored //! ones, measured from the newest evaluation; -//! - query-time raw ranges, `TimeRange(range) <- TimeShift(offset)* <- Scan`: -//! bound to the samples in `(T - offset - range, T - offset]`, which the -//! caller fetches, filtered by the plan's `Scan`. +//! - query-time raw windows (`InstallablePlan::query_time_raw_windows`): +//! bound to the samples in `(T - offset - width, T - offset]`, which the +//! caller fetches, selected by the executor's `RawSampleSelector` (the +//! plan's `Scan` matchers, and the latest sample per series for an +//! instant selector). //! //! A range query is one evaluation per step, in time order. Each raw range //! is fetched once for all steps and sliced per step, and each kept @@ -26,18 +28,16 @@ use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::sync::Arc; use asap_executor::operators::Operator; -use asap_executor::physical_planner::promql_rows::{ - decode_series_identity, SERIES_IDENTITY_COLUMN, -}; use asap_executor::physical_planner::{compile, CompiledPhysicalDAG, InputContract, Source}; +use asap_executor::planner::ir::physical_export::RawSampleWindow; +use asap_executor::planner::ir::schema::{DataType, FieldDataType, Schema}; +use asap_executor::planner::ir::{NonASAPOp, Operator as IrOperator}; use asap_executor::runtime::Scope; use asap_executor::values::{Batch, Value}; -use control_plane::InstalledPlan; -use planner_types::ir::physical_export::PhysicalASAPDAG; -use planner_types::ir::schema::{DataType, FieldDataType, Schema}; -use planner_types::ir::{NonASAPOp, Operator as IrOperator}; +use asap_executor::{series_labels_of_output_row, RawSampleSelector, Sample}; +use asap_planner::{InstallablePlan, PhysicalASAPDAG}; -use crate::precompute::{run, scan_below, select, PaneBuild, RawSample, ScanInput}; +use crate::precompute::{output_schema, run, PaneBuild, RawSample}; use crate::sds::{DefinitionId, Sds, SummaryStore}; /// Raw samples a query needs: `metric`'s samples in `(start_ms, end_ms]`. @@ -85,10 +85,10 @@ struct KeptInput { } struct RawInput { + /// The window's `TimeRange`, which its rows feed. node: u64, - scan: ScanInput, - range_ms: i64, - offset_ms: i64, + window: RawSampleWindow, + selector: RawSampleSelector, } struct Program { @@ -119,29 +119,33 @@ pub struct QueryEngine { newest_evaluation_ms: Option, /// How each kept definition's panes are built, or why they cannot be. kept_builds: BTreeMap>, - dag: PhysicalASAPDAG, /// Each query, with its compiled root or why it cannot run. roots: Vec<(String, Result)>, } impl QueryEngine { - pub fn new(plan: &InstalledPlan, sds: &Sds) -> Self { - let dag = plan.dag.dag.clone(); + pub fn new(plan: &InstallablePlan, sds: &Sds) -> Self { + let dag = &plan.dag.dag; let kept_builds: BTreeMap<_, _> = sds .kept_definitions .values() .map(|definition| { - let build = PaneBuild::new(&dag, definition, &sds.kept_outputs); + let build = PaneBuild::new(dag, definition); (definition.id.clone(), build.map_err(|e| e.to_string())) }) .collect(); + let windows: HashMap = plan + .query_time_raw_windows + .iter() + .map(|window| (window.time_range_node_id as u64, window)) + .collect(); let roots = plan .queries .iter() - .zip(&dag.roots) - .map(|(query, root)| { - let program = program(&dag, sds, &kept_builds, *root as u64); - (query.trim().to_string(), program) + .map(|query| { + let root = query.root_node_id as u64; + let program = program(dag, sds, &kept_builds, &windows, root); + (query.query.trim().to_string(), program) }) .collect(); Self { @@ -149,7 +153,6 @@ impl QueryEngine { kept_stats: KeptResultStats::default(), newest_evaluation_ms: None, kept_builds, - dag, roots, } } @@ -171,11 +174,14 @@ impl QueryEngine { let mut ranges: Vec = program .raw .iter() - .map(|input| RawRange { - node: input.node, - metric: input.scan.metric.clone(), - start_ms: time_ms - input.offset_ms - input.range_ms, - end_ms: time_ms - input.offset_ms, + .map(|input| { + let (start_ms, end_ms) = input.window.bounds_ms(time_ms); + RawRange { + node: input.node, + metric: input.selector.metric().to_string(), + start_ms, + end_ms, + } }) .collect(); for input in &program.kept { @@ -184,7 +190,7 @@ impl QueryEngine { let build = self.kept_build(&input.definition); ranges.push(RawRange { node: input.raw_input, - metric: build.scan.metric.clone(), + metric: build.selector.metric().to_string(), start_ms: end_ms - build.definition.pane_ms, end_ms, }); @@ -247,7 +253,7 @@ impl QueryEngine { }; let pane = self .kept_build(&input.definition) - .run(&self.dag, samples, pane_scope) + .run(&samples_of(samples), end_ms, pane_scope) .map_err(|e| Unanswered::Execution(e.to_string()))?; self.kept_stats.built += 1; // Kept only once closed, as the precompute engine closes @@ -263,8 +269,12 @@ impl QueryEngine { } for input in &program.raw { let samples = raw.get(&input.node).map(Vec::as_slice).unwrap_or_default(); - let rows = select(&self.dag, &input.scan, samples, scope.clone()).map_err(failed)?; - let source = Operator::source(input.scan.schema.clone(), vec![rows]).map_err(failed)?; + let (_, end_ms) = input.window.bounds_ms(time_ms); + let rows = input + .selector + .rows(end_ms, &samples_of(samples)) + .map_err(failed)?; + let source = Operator::source(rows.schema().clone(), vec![rows]).map_err(failed)?; sources.insert(input.node, Box::new(source)); } // Evict the kept panes older than each definition's retention @@ -273,8 +283,8 @@ impl QueryEngine { self.newest_evaluation_ms = Some(newest); for (definition, build) in &self.kept_builds { if let Ok(build) = build { - self.kept - .evict_through(definition, newest - build.retention_ms); + let oldest = build.definition.oldest_kept_end_ms(newest); + self.kept.evict_before(definition, oldest); } } let dag = program.compiled.instantiate(sources).map_err(failed)?; @@ -308,9 +318,9 @@ impl QueryEngine { .iter() .map(|input| RawRange { node: input.node, - metric: input.scan.metric.clone(), - start_ms: first - input.offset_ms - input.range_ms, - end_ms: last - input.offset_ms, + metric: input.selector.metric().to_string(), + start_ms: input.window.bounds_ms(first).0, + end_ms: input.window.bounds_ms(last).1, }) .collect(); // The first and last end of each kept definition's missing panes. @@ -327,8 +337,8 @@ impl QueryEngine { for (definition, (first_end_ms, last_end_ms)) in missing { let build = self.kept_build(definition); ranges.push(RawRange { - node: build.definition.raw_input, - metric: build.scan.metric.clone(), + node: build.definition.raw_input(), + metric: build.selector.metric().to_string(), start_ms: first_end_ms - build.definition.pane_ms, end_ms: last_end_ms, }); @@ -359,7 +369,13 @@ impl QueryEngine { let raw_inputs: Vec<(u64, i64, i64)> = program .raw .iter() - .map(|input| (input.node, input.offset_ms, input.range_ms)) + .map(|input| { + ( + input.node, + input.window.end_offset_ms, + input.window.width_ms, + ) + }) .collect(); // Each kept output: its definition and offset, the node its pane's // samples feed, and the node its definition's samples were fetched for. @@ -372,7 +388,7 @@ impl QueryEngine { input.definition.clone(), input.offset_ms, input.raw_input, - definition.raw_input, + definition.raw_input(), definition.pane_ms, ) }) @@ -447,6 +463,11 @@ fn slice(runs: &[&[RawSample]], start_ms: i64, end_ms: i64) -> Vec { samples } +/// `samples` as the executor's samples, for a `RawSampleSelector`. +fn samples_of(samples: &[RawSample]) -> Vec { + samples.iter().map(RawSample::to_sample).collect() +} + /// The stored pane `input` reads at `time_ms`. fn pane<'a>( input: &StoredInput, @@ -468,11 +489,13 @@ fn pane<'a>( }) } -/// Compiles `root` with its stored, kept and raw inputs, or says why it cannot. +/// Compiles `root` with its stored, kept and raw inputs, or says why it +/// cannot. `windows` are the plan's query-time raw windows, by `TimeRange`. fn program( dag: &PhysicalASAPDAG, sds: &Sds, kept_builds: &BTreeMap>, + windows: &HashMap, root: u64, ) -> Result { let nodes: HashMap = dag.nodes.iter().map(|n| (n.id as u64, n)).collect(); @@ -509,19 +532,20 @@ fn program( }); continue; } + if let Some(window) = windows.get(&id) { + let selector = RawSampleSelector::new(dag, window).map_err(|e| e.to_string())?; + raw.push(RawInput { + node: id, + window: (*window).clone(), + selector, + }); + continue; + } match &nodes[&id].payload { - IrOperator::NonASAP(NonASAPOp::TimeRange { range, .. }) => { - raw.push(raw_input( - dag, - &nodes, - &producers, - id, - range.as_millis() as i64, - )?); - continue; - } - IrOperator::NonASAP(NonASAPOp::Scan { .. }) => { - return Err(format!("node {id}: a Scan read without a TimeRange")) + IrOperator::NonASAP(NonASAPOp::TimeRange { .. } | NonASAPOp::Scan { .. }) => { + return Err(format!( + "node {id}: raw samples read outside the plan's raw windows" + )) } _ => pending.extend(producers.get(&id).into_iter().flatten().copied()), } @@ -531,16 +555,11 @@ fn program( .iter() .map(|i| i.node) .chain(kept.iter().map(|i| i.node)) + .chain(raw.iter().map(|i| i.node)) { - let schema = Arc::new(nodes[&node].output_schema.clone()); + let schema = Arc::new(output_schema(dag, node).clone()); inputs.insert(node, InputContract::bounded(schema)); } - for input in &raw { - inputs.insert( - input.node, - InputContract::bounded(input.scan.schema.clone()), - ); - } let compiled = compile(dag, inputs, &[root]).map_err(|e| e.to_string())?; Ok(Program { compiled, @@ -551,47 +570,6 @@ fn program( }) } -/// A query-time `TimeRange`: the `Scan` below it and the `TimeShift` -/// offsets between them. -fn raw_input( - dag: &PhysicalASAPDAG, - nodes: &HashMap, - producers: &HashMap>, - time_range: u64, - range_ms: i64, -) -> Result { - let unsupported = |reason: &str| format!("node {time_range}: {reason}"); - let mut offset_ms = 0; - let mut current = time_range; - loop { - current = match producers.get(¤t).map(Vec::as_slice) { - Some([input]) => *input, - _ => return Err(unsupported("a TimeRange must read one input")), - }; - match &nodes[¤t].payload { - IrOperator::NonASAP(NonASAPOp::TimeShift { shift, .. }) if shift.at.is_none() => { - offset_ms += shift.offset_ms - } - IrOperator::NonASAP(NonASAPOp::Scan { .. }) => break, - _ => { - return Err(unsupported( - "only offset TimeShifts between TimeRange and Scan", - )) - } - } - } - let scan = scan_below(dag, time_range).ok_or_else(|| unsupported("not a time-series Scan"))?; - if Arc::new(nodes[&time_range].output_schema.clone()) != scan.schema { - return Err(unsupported("TimeRange rows differ from its Scan's")); - } - Ok(RawInput { - node: time_range, - scan, - range_ms, - offset_ms, - }) -} - /// PromQL vector samples from result rows: labels from the series identity /// and the other text columns, the value from the one `Float64` column. fn result_samples(schema: &Schema, rows: Vec>) -> Result, Unanswered> { @@ -608,17 +586,8 @@ fn result_samples(schema: &Schema, rows: Vec>) -> Result *v, other => return Err(Unanswered::Execution(format!("value {other:?}"))), diff --git a/data_plane/src/sds.rs b/data_plane/src/sds.rs index 78bcca68..60a542cf 100644 --- a/data_plane/src/sds.rs +++ b/data_plane/src/sds.rs @@ -1,33 +1,27 @@ -//! Stored summaries (SDS): what the data plane stores for a plan's -//! ingestion-time outputs, and how a query finds them again. +//! Stored summaries (SDS): what the data plane stores for a plan's stored +//! outputs, and how a query finds them again. //! -//! The Planner's selected DAG marks each output computed at ingestion time -//! (`MatAtIngestion`) and read at query time: the executor's materialization -//! frontier. The control plane installs the DAG with each `PaneChain` -//! expanded into its panes, so each such output reads one tumbling pane of -//! raw samples: +//! The installed plan lists every output it stores across evaluations +//! (`InstallablePlan::stored_outputs`): when it is built (in an ingestion +//! run, or in the query run that first reads it), the pane of raw samples +//! it is built from, and how long it must stay available. At evaluation +//! time `T` an output's pane covers `(T - end_offset - width, T - +//! end_offset]`, the PromQL range convention. //! -//! ```text -//! frontier node <- ... <- TimeRange(width) <- TimeShift(offset)* <- Scan -//! ``` -//! -//! At evaluation time `T` it covers `(T - offset - width, T - offset]`, the -//! PromQL range convention. -//! Outputs that differ only in `offset` compute the same thing over different -//! panes, so they share one [`SummaryDefinition`]: the data plane builds each -//! pane of a definition once, and every output reads the pane it needs. -//! -//! A `MatAtQueryKeptAcrossEvaluations` output (#509 Example 4's B3) is -//! computed at query time and kept for the next evaluations. It has the same -//! pane shape and is described the same way, in [`Sds::kept_definitions`]: -//! the query engine keeps each of its panes by definition and pane end. +//! Outputs that differ only in their pane's offset compute the same thing +//! over different panes, so they share one [`SummaryDefinition`]: the data +//! plane builds each pane of a definition once, and every output reads the +//! pane it needs. Identity is the SDS's own: the Planner states what each +//! node computes, not which nodes are equal. use std::collections::{BTreeMap, HashMap}; -use planner_types::ir::physical_export::{PhysicalASAPDAG, PhysicalASAPDAGNode}; -use planner_types::ir::schema::Schema; -use planner_types::ir::{NonASAPOp, Operator}; -use planner_types::physical::Materialization; +use asap_executor::planner::ir::physical_export::{ + PhysicalASAPDAG, PhysicalASAPDAGNode, RawSampleWindow, +}; +use asap_executor::planner::ir::schema::Schema; +use asap_executor::planner::ir::{NonASAPOp, Operator}; +use asap_planner::{InstallablePlan, StoredOutputBuild}; use sha2::{Digest, Sha256}; /// Identifies what a stored summary computes, independent of node ids and of @@ -44,9 +38,27 @@ pub struct SummaryDefinition { /// The output that builds a pane; any output of the definition computes /// the same, and this is the first in node order. pub builder: u64, - /// The raw input of the build: the `TimeRange` node fed one pane's samples. - pub raw_input: u64, + /// The builder's pane: its `TimeRange`, fed one pane's samples, and the + /// `Scan` that selects them. + pub window: RawSampleWindow, pub output_schema: Schema, + /// How far back from the newest evaluation time a pane stays available: + /// the largest `retention_ms` the plan states for the definition's + /// outputs. A pane that starts earlier is no longer read. + pub retention_ms: i64, +} + +impl SummaryDefinition { + /// The builder's `TimeRange`, which one pane's samples feed. + pub fn raw_input(&self) -> u64 { + self.window.time_range_node_id as u64 + } + + /// The end of the oldest pane still read when the newest evaluation is + /// at `newest_ms`: panes ending before it are evicted. + pub fn oldest_kept_end_ms(&self, newest_ms: i64) -> i64 { + newest_ms - self.retention_ms + self.pane_ms + } } /// How one output reads its definition: at evaluation time `T`, the pane @@ -64,7 +76,7 @@ pub struct PaneBinding { pub struct Sds { /// Definitions built at ingestion time, by identity. pub definitions: BTreeMap, - /// Every frontier output, with its definition and pane. + /// Every output built at ingestion time, with its definition and pane. pub outputs: BTreeMap, /// Definitions of the outputs computed at query time and kept across /// evaluations, by identity. @@ -77,137 +89,65 @@ pub struct Sds { pub enum SdsError { #[error("node {0} is not in the DAG")] UnknownNode(u64), - #[error("output {node} does not read one pane of one Scan: {reason}")] - UnsupportedShape { node: u64, reason: String }, + #[error("stored output {0}'s retention does not fit in i64 milliseconds")] + RetentionOverflow(u64), } impl Sds { - /// The definitions behind `frontier`, the DAG outputs computed at - /// ingestion time and read at query time, and behind the outputs kept - /// across evaluations (`MatAtQueryKeptAcrossEvaluations`). - pub fn from_dag(dag: &PhysicalASAPDAG, frontier: &[u64]) -> Result { + /// The definitions behind `plan`'s stored outputs: those built in an + /// ingestion run and those built in a query run and kept. + pub fn from_plan(plan: &InstallablePlan) -> Result { + let dag = &plan.dag.dag; let nodes: HashMap = dag.nodes.iter().map(|n| (n.id as u64, n)).collect(); - let mut inputs: HashMap> = HashMap::new(); - for edge in &dag.edges { - inputs - .entry(edge.consumer as u64) - .or_default() - .push(edge.producer as u64); - } + let inputs = inputs_of(dag); let mut sds = Sds::default(); let mut hashes = HashMap::new(); - for &node in frontier { - let (definitions, outputs) = (&mut sds.definitions, &mut sds.outputs); - add_output(node, &nodes, &inputs, &mut hashes, definitions, outputs)?; - } - for node in &dag.nodes { - if let Materialization::MatAtQueryKeptAcrossEvaluations(_) = node.materialization { - let (definitions, outputs) = (&mut sds.kept_definitions, &mut sds.kept_outputs); - add_output( - node.id as u64, - &nodes, - &inputs, - &mut hashes, - definitions, - outputs, - )?; + for output in &plan.stored_outputs { + let node = output.node_id as u64; + let (definitions, outputs) = match output.built_at { + StoredOutputBuild::IngestionRun => (&mut sds.definitions, &mut sds.outputs), + StoredOutputBuild::QueryRun => (&mut sds.kept_definitions, &mut sds.kept_outputs), + }; + let id = DefinitionId(hex(&identity(node, &nodes, &inputs, &mut hashes)?)); + let retention_ms = i64::try_from(output.retention_ms) + .map_err(|_| SdsError::RetentionOverflow(node))?; + let output_schema = nodes[&node].output_schema.clone(); + let definition = definitions + .entry(id.clone()) + .or_insert_with(|| SummaryDefinition { + id: id.clone(), + pane_ms: output.pane.width_ms, + builder: node, + window: output.pane.clone(), + output_schema, + retention_ms, + }); + if node < definition.builder { + definition.builder = node; + definition.window = output.pane.clone(); } + definition.retention_ms = definition.retention_ms.max(retention_ms); + let binding = PaneBinding { + offset_ms: output.pane.end_offset_ms, + raw_input: output.pane.time_range_node_id as u64, + }; + outputs.insert(node, (id, binding)); } Ok(sds) } } -/// Adds `node`, which must read one pane, to `outputs`, and its definition -/// to `definitions` if it is new. -fn add_output( - node: u64, - nodes: &HashMap, - inputs: &HashMap>, - hashes: &mut HashMap, - definitions: &mut BTreeMap, - outputs: &mut BTreeMap, -) -> Result<(), SdsError> { - let (raw_input, pane_ms, offset_ms) = pane_of(node, nodes, inputs)?; - let id = DefinitionId(hex(&identity(node, nodes, inputs, hashes)?)); - let output_schema = nodes[&node].output_schema.clone(); - let definition = definitions - .entry(id.clone()) - .or_insert_with(|| SummaryDefinition { - id: id.clone(), - pane_ms, - builder: node, - raw_input, - output_schema, - }); - if node < definition.builder { - definition.builder = node; - definition.raw_input = raw_input; - } - let binding = PaneBinding { - offset_ms, - raw_input, - }; - outputs.insert(node, (id, binding)); - Ok(()) -} - -/// `(raw input, pane width, offset)` of a frontier output: follow its single -/// inputs down to the `TimeRange`, then sum the `TimeShift`s down to the `Scan`. -fn pane_of( - node: u64, - nodes: &HashMap, - inputs: &HashMap>, -) -> Result<(u64, i64, i64), SdsError> { - let unsupported = |reason: &str| SdsError::UnsupportedShape { - node, - reason: reason.into(), - }; - let only_input = |id: u64| match inputs.get(&id).map(Vec::as_slice) { - Some([input]) => Ok(*input), - _ => Err(unsupported( - "every operator above the Scan must have one input", - )), - }; - let mut current = node; - let (raw_input, pane_ms) = loop { - match relational(nodes, current)? { - Some(NonASAPOp::TimeRange { range, .. }) => { - break (current, range.as_millis() as i64); - } - Some(NonASAPOp::TimeShift { .. }) => { - return Err(unsupported("a TimeShift above the TimeRange")); - } - Some(NonASAPOp::Scan { .. }) => return Err(unsupported("no TimeRange")), - _ => current = only_input(current)?, - } - }; - let mut offset_ms = 0; - current = only_input(raw_input)?; - loop { - match relational(nodes, current)? { - Some(NonASAPOp::TimeShift { shift, .. }) if shift.at.is_none() => { - offset_ms += shift.offset_ms; - current = only_input(current)?; - } - Some(NonASAPOp::Scan { .. }) => return Ok((raw_input, pane_ms, offset_ms)), - _ => { - return Err(unsupported( - "only offset TimeShifts between TimeRange and Scan", - )) - } - } - } -} - -fn relational<'a>( - nodes: &HashMap, - id: u64, -) -> Result>, SdsError> { - match &nodes.get(&id).ok_or(SdsError::UnknownNode(id))?.payload { - Operator::NonASAP(operator) => Ok(Some(operator)), - Operator::ASAP(_) => Ok(None), +/// Each node's inputs, in edge order. +fn inputs_of(dag: &PhysicalASAPDAG) -> HashMap> { + let mut inputs: HashMap> = HashMap::new(); + for edge in &dag.edges { + inputs + .entry(edge.consumer as u64) + .or_default() + .push(edge.producer as u64); } + inputs } /// SHA-256 over a node's payload and output schema, then its inputs' @@ -270,10 +210,10 @@ impl SummaryStore { self.panes.get(&(definition.clone(), end_ms)) } - /// Drops the panes of `definition` that end at or before `end_ms`. - pub fn evict_through(&mut self, definition: &DefinitionId, end_ms: i64) { + /// Drops the panes of `definition` that end before `end_ms`. + pub fn evict_before(&mut self, definition: &DefinitionId, end_ms: i64) { self.panes - .retain(|(id, end), _| id != definition || *end > end_ms); + .retain(|(id, end), _| id != definition || *end >= end_ms); } pub fn len(&self) -> usize { diff --git a/data_plane/tests/answers.rs b/data_plane/tests/answers.rs index d8d30fff..28f7dd63 100644 --- a/data_plane/tests/answers.rs +++ b/data_plane/tests/answers.rs @@ -208,12 +208,15 @@ async fn answers_query_time_plans_from_raw_samples() { assert_eq!(response.body, json!({"forwarded": "up"})); } -/// Example 4b's data plane, with Remote Write samples every 15 s in -/// `[1 min, end)` for jobs a and b. +/// Example 4b's data plane, with its query also answered as range requests +/// over the last 10 minutes by 10 minutes, and Remote Write samples every +/// 15 s in `[1 min, end)` for jobs a and b. async fn example4b(end_ms: i64) -> (Arc, Router) { + let workload = common::with_range_evaluation(&common::workload("example4b"), 600_000, 600_000); + let plan = control_plane::plan_workload(&workload, &common::inputs(false)).unwrap(); let state = Arc::new( DataPlane::new( - common::plan_json("example4b", false), + serde_json::to_string(&plan).unwrap(), fake_prometheus(vec![]).await, 0, ) diff --git a/data_plane/tests/common/mod.rs b/data_plane/tests/common/mod.rs index bf670ba3..d470ca1a 100644 --- a/data_plane/tests/common/mod.rs +++ b/data_plane/tests/common/mod.rs @@ -1,7 +1,7 @@ use std::path::Path; -use control_plane::{plan_workload, DeploymentInputs, InstalledPlan}; -use planner_types::workload::PlanningWorkload; +use asap_planner::{InstallablePlan, PlanningWorkload}; +use control_plane::{plan_workload, DeploymentInputs}; /// The deployment inputs of a backend that keeps raw data or not. pub fn inputs(raw_data_retained: bool) -> DeploymentInputs { @@ -17,10 +17,29 @@ pub fn workload(example: &str) -> PlanningWorkload { serde_json::from_str(&std::fs::read_to_string(path).unwrap()).unwrap() } +/// `workload` with every repeating query answered as a range request of +/// steps `step_ms` apart over `span_ms`, so the Planner keeps its stored +/// outputs for the oldest step too. +#[allow(dead_code)] +pub fn with_range_evaluation( + workload: &PlanningWorkload, + step_ms: u64, + span_ms: u64, +) -> PlanningWorkload { + let mut json = serde_json::to_value(workload).unwrap(); + for query in json["query_workload"]["repeating_queries"] + .as_array_mut() + .unwrap() + { + query["range_evaluation"] = serde_json::json!({"step_ms": step_ms, "span_ms": span_ms}); + } + serde_json::from_value(json).unwrap() +} + /// The selected plan of a #509 example workload, for a deployment that /// keeps raw data or not. #[allow(dead_code)] -pub fn plan(example: &str, raw_data_retained: bool) -> InstalledPlan { +pub fn plan(example: &str, raw_data_retained: bool) -> InstallablePlan { plan_workload(&workload(example), &inputs(raw_data_retained)).unwrap() } diff --git a/data_plane/tests/e2e_examples.rs b/data_plane/tests/e2e_examples.rs index 2c2d1efd..6ea6c20f 100644 --- a/data_plane/tests/e2e_examples.rs +++ b/data_plane/tests/e2e_examples.rs @@ -13,16 +13,16 @@ use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; use std::time::{Duration, Instant}; -use asap_executor::physical_planner::frontier_from_timing; +use asap_executor::planner::workload::PlanningWorkload; +use asap_planner::InstallablePlan; use axum::extract::Query; use axum::routing::get; use axum::{Json, Router}; -use control_plane::{plan_selected_at_query_time, plan_workload, DeploymentInputs, InstalledPlan}; +use control_plane::{plan_workload, save_plan, DeploymentInputs}; use data_plane::precompute::RawSample; use data_plane::query::{QueryEngine, ResultSample}; use data_plane::remote_write::{encode, Label, Sample, TimeSeries, WriteRequest}; use data_plane::sds::{Sds, SummaryStore}; -use planner_types::workload::PlanningWorkload; use serde_json::{json, Value}; const MINUTE: i64 = 60_000; @@ -71,14 +71,14 @@ fn series(metric: &str, labels: &[(&str, &str)]) -> BTreeMap { /// The answer of the Planner executor for `query` at `time_ms` when the /// selected plan runs with nothing materialized: every input is a raw range /// over `all`. -/// `plan` is [`plan_selected_at_query_time`]'s plan. +/// `plan` is the installed plan's `InstallablePlan::query_time_reference`. fn reference( - plan: &InstalledPlan, + plan: &InstallablePlan, query: &str, time_ms: i64, all: &[RawSample], ) -> Vec { - let sds = Sds::from_dag(&plan.dag.dag, &frontier_from_timing(&plan.dag.dag).unwrap()).unwrap(); + let sds = Sds::from_plan(plan).unwrap(); assert!(sds.definitions.is_empty()); let mut engine = QueryEngine::new(plan, &sds); let store = SummaryStore::new(); @@ -186,7 +186,7 @@ impl Drop for TempDir { } } -async fn start(plan: &InstalledPlan, prometheus_url: &str) -> DataPlaneProcess { +async fn start(plan: &InstallablePlan, prometheus_url: &str) -> DataPlaneProcess { let dir = std::env::temp_dir().join(format!( "data-plane-e2e-{}-{}", std::process::id(), @@ -194,7 +194,7 @@ async fn start(plan: &InstalledPlan, prometheus_url: &str) -> DataPlaneProcess { )); std::fs::create_dir_all(&dir).unwrap(); let path = dir.join("plan.json"); - plan.save(&path).unwrap(); + save_plan(plan, &path).unwrap(); let port = std::net::TcpListener::bind("127.0.0.1:0") .unwrap() .local_addr() @@ -360,7 +360,7 @@ async fn planned_range_answer( async fn check_range( url: &str, reads: &AtomicUsize, - at_query_time: &InstalledPlan, + at_query_time: &InstallablePlan, query: &str, (start_ms, end_ms, step): (i64, i64, &str), step_ms: i64, @@ -424,17 +424,16 @@ async fn check_example( ) -> usize { let workload = workload(example); let plan = plan_workload(&workload, inputs).unwrap(); - let at_query_time = plan_selected_at_query_time(&workload, inputs).unwrap(); + let at_query_time = plan.query_time_reference().unwrap(); let all = Arc::new(all); let process = start(&plan, &prometheus(all.clone()).await.0).await; remote_write(&process.url, &all).await; - for query in &plan.queries { + for query in plan.queries.iter().map(|q| &q.query) { let expected = reference(&at_query_time, query, time_ms, &all); let actual = planned_answer(&process.url, query, time_ms).await; assert_same(query, &actual, &expected, tolerance); } - let dag = &plan.dag.dag; - let sds = Sds::from_dag(dag, &frontier_from_timing(dag).unwrap()).unwrap(); + let sds = Sds::from_plan(&plan).unwrap(); sds.definitions.len() } @@ -506,8 +505,8 @@ async fn example4a() { /// Example 3b: a p99 panel over the last 5 minutes, every minute; KLL is /// exact below k. With raw data kept the Planner selects a query-time /// build; without, tumbling panes built from Remote Write. Samples end at -/// the evaluation time, as for a live panel: panes more than the window -/// plus one pane older than the newest sample are evicted. +/// the evaluation time, as for a live panel: panes that start more than the +/// plan's retention before the newest closed pane are evicted. #[tokio::test] async fn example3b() { let series = [series("latency_ms", &[("service", "a")])]; @@ -554,15 +553,14 @@ async fn example4_b3() { let workload = workload("example4-b3"); let inputs = DeploymentInputs::default(); let plan = plan_workload(&workload, &inputs).unwrap(); - let at_query_time = plan_selected_at_query_time(&workload, &inputs).unwrap(); - let dag = &plan.dag.dag; - let sds = Sds::from_dag(dag, &frontier_from_timing(dag).unwrap()).unwrap(); + let at_query_time = plan.query_time_reference().unwrap(); + let sds = Sds::from_plan(&plan).unwrap(); assert_eq!((sds.definitions.len(), sds.kept_definitions.len()), (0, 1)); let all = Arc::new(all); let (prometheus_url, range_reads) = prometheus(all.clone()).await; let process = start(&plan, &prometheus_url).await; remote_write(&process.url, &all).await; - let query = &plan.queries[0]; + let query = &plan.queries[0].query; let mut reads = Vec::new(); for minute in [10, 11, 12] { let before = range_reads.load(Ordering::SeqCst); @@ -592,12 +590,12 @@ async fn example1_range() { let workload = workload("example1"); let inputs = DeploymentInputs::default(); let plan = plan_workload(&workload, &inputs).unwrap(); - let at_query_time = plan_selected_at_query_time(&workload, &inputs).unwrap(); + let at_query_time = plan.query_time_reference().unwrap(); let all = Arc::new(all); let (prometheus_url, reads) = prometheus(all.clone()).await; let process = start(&plan, &prometheus_url).await; remote_write(&process.url, &all).await; - for query in &plan.queries { + for query in plan.queries.iter().map(|q| &q.query) { let range = (3 * MINUTE, 6 * MINUTE, "30s"); let read = check_range( &process.url, @@ -613,8 +611,9 @@ async fn example1_range() { } } -/// Example 4b as a range query on a deployment without raw data: every -/// step reads six stored panes and nothing from Prometheus. +/// Example 4b as a range query on a deployment without raw data, its query +/// declared as range requests over 10 minutes: every step reads six stored +/// panes and nothing from Prometheus. #[tokio::test] async fn example4b_range() { let series = [ @@ -625,15 +624,19 @@ async fn example4b_range() { let all = samples(&series, MINUTE, 101 * MINUTE, 15_000, |i, t| { (i * 1000) as f64 + (t / 15_000) as f64 }); - let workload = workload("example4b"); + let workload = common::with_range_evaluation( + &workload("example4b"), + 10 * MINUTE as u64, + 10 * MINUTE as u64, + ); let inputs = without_raw_data(); let plan = plan_workload(&workload, &inputs).unwrap(); - let at_query_time = plan_selected_at_query_time(&workload, &inputs).unwrap(); + let at_query_time = plan.query_time_reference().unwrap(); let all = Arc::new(all); let (prometheus_url, reads) = prometheus(all.clone()).await; let process = start(&plan, &prometheus_url).await; remote_write(&process.url, &all).await; - let query = &plan.queries[0]; + let query = &plan.queries[0].query; let range = (90 * MINUTE, 100 * MINUTE, "10m"); let read = check_range( &process.url, @@ -653,7 +656,7 @@ async fn example4b_range() { /// pane is built once and reused by later steps, and every step equals the /// instant query and the Planner executor. Samples run to 15 min, so every /// pane read is closed and kept. The same range query again reads one raw -/// range, for the panes ending at 6..8 min: they were evicted, measured from +/// range, for the panes ending at 6..9 min: they were evicted, measured from /// the newest step. #[tokio::test] async fn example4_b3_range() { @@ -667,12 +670,12 @@ async fn example4_b3_range() { let workload = workload("example4-b3"); let inputs = DeploymentInputs::default(); let plan = plan_workload(&workload, &inputs).unwrap(); - let at_query_time = plan_selected_at_query_time(&workload, &inputs).unwrap(); + let at_query_time = plan.query_time_reference().unwrap(); let all = Arc::new(all); let (prometheus_url, reads) = prometheus(all.clone()).await; let process = start(&plan, &prometheus_url).await; remote_write(&process.url, &all).await; - let query = &plan.queries[0]; + let query = &plan.queries[0].query; let range = (10 * MINUTE, 14 * MINUTE, "60"); let read = check_range( &process.url, diff --git a/data_plane/tests/kept.rs b/data_plane/tests/kept.rs index 1c82d46a..e11281a8 100644 --- a/data_plane/tests/kept.rs +++ b/data_plane/tests/kept.rs @@ -8,8 +8,7 @@ mod common; use std::collections::BTreeMap; use std::sync::Arc; -use asap_executor::physical_planner::frontier_from_timing; -use control_plane::{plan_selected_at_query_time, InstalledPlan}; +use asap_planner::InstallablePlan; use data_plane::precompute::RawSample; use data_plane::query::{KeptResultStats, QueryEngine, RawRange, ResultSample}; use data_plane::sds::{Sds, SummaryStore}; @@ -17,9 +16,8 @@ use data_plane::sds::{Sds, SummaryStore}; const MINUTE: i64 = 60_000; const P99: &str = "quantile_over_time(0.99, latency_ms[5m])"; -fn engine(plan: &InstalledPlan) -> QueryEngine { - let dag = &plan.dag.dag; - let sds = Sds::from_dag(dag, &frontier_from_timing(dag).unwrap()).unwrap(); +fn engine(plan: &InstallablePlan) -> QueryEngine { + let sds = Sds::from_plan(plan).unwrap(); assert!(sds.definitions.is_empty(), "nothing is built at ingestion"); QueryEngine::new(plan, &sds) } @@ -95,13 +93,10 @@ fn answer_at_watermark( /// The installed B3 plan and the Planner executor's reference for it: the /// same selection with nothing materialized. -fn plans() -> (InstalledPlan, InstalledPlan) { - let workload = common::workload("example4-b3"); - let inputs = common::inputs(true); - ( - control_plane::plan_workload(&workload, &inputs).unwrap(), - plan_selected_at_query_time(&workload, &inputs).unwrap(), - ) +fn plans() -> (InstallablePlan, InstallablePlan) { + let plan = common::plan("example4-b3", true); + let reference = plan.query_time_reference().unwrap(); + (plan, reference) } /// The first evaluation builds all five kept panes and answers like the @@ -164,14 +159,13 @@ fn kept_panes_answer_like_the_reference_and_are_reused() { ); } -/// A kept pane is dropped once it is older than the largest offset any -/// output reads (4 min) plus two pane widths, measured from the newest -/// evaluation: 6 min. +/// A kept pane is dropped once it starts before the plan's retention (5 +/// min: the oldest output's offset, 4 min, plus its 1-minute pane), measured +/// from the newest evaluation. #[test] fn kept_panes_are_evicted_like_stored_panes() { let (plan, _) = plans(); - let dag = &plan.dag.dag; - let sds = Sds::from_dag(dag, &frontier_from_timing(dag).unwrap()).unwrap(); + let sds = Sds::from_plan(&plan).unwrap(); let definition = sds.kept_definitions.keys().next().unwrap().clone(); let all = samples(); let mut kept = engine(&plan); @@ -188,11 +182,11 @@ fn kept_panes_are_evicted_like_stored_panes() { answer(&mut kept, 11 * MINUTE, &all); assert_eq!( ends(&kept), - vec![6, 7, 8, 9, 10, 11], - "6 is not older than 11 - 6" + vec![7, 8, 9, 10, 11], + "the pane ending at 6 min starts before 11 - 5" ); answer(&mut kept, 12 * MINUTE, &all); - assert_eq!(ends(&kept), vec![7, 8, 9, 10, 11, 12]); + assert_eq!(ends(&kept), vec![8, 9, 10, 11, 12]); // An evaluation far ahead builds its own five panes and drops the rest. answer(&mut kept, 20 * MINUTE, &all); assert_eq!(ends(&kept), vec![16, 17, 18, 19, 20]); @@ -211,8 +205,7 @@ fn kept_panes_are_evicted_like_stored_panes() { #[test] fn panes_are_kept_only_once_closed() { let (plan, reference_plan) = plans(); - let dag = &plan.dag.dag; - let sds = Sds::from_dag(dag, &frontier_from_timing(dag).unwrap()).unwrap(); + let sds = Sds::from_plan(&plan).unwrap(); let definition = sds.kept_definitions.keys().next().unwrap().clone(); let mut all = samples(); let mut kept = engine(&plan); @@ -257,7 +250,7 @@ fn panes_are_kept_only_once_closed() { // Once the watermark reaches 10 min, the pane is kept. let (_, ranges) = answer_at_watermark(&mut kept, 11 * MINUTE, Some(10 * MINUTE), &all); assert_eq!(ranges.len(), 2, "the 10- and 11-minute panes: {ranges:?}"); - assert_eq!(ends(&kept), vec![6, 7, 8, 9, 10]); + assert_eq!(ends(&kept), vec![7, 8, 9, 10], "6 starts before 11 - 5"); // Without any sample there is no watermark, and nothing is kept. let mut fresh = engine(&plan); answer_at_watermark(&mut fresh, 10 * MINUTE, None, &all); diff --git a/data_plane/tests/precompute.rs b/data_plane/tests/precompute.rs index 3827f63d..a09cc538 100644 --- a/data_plane/tests/precompute.rs +++ b/data_plane/tests/precompute.rs @@ -3,19 +3,18 @@ mod common; use std::collections::BTreeMap; use std::sync::Arc; -use asap_executor::physical_planner::frontier_from_timing; -use control_plane::{plan_workload, InstalledPlan}; +use asap_executor::planner::workload::{PlanningWorkload, Query}; +use asap_planner::InstallablePlan; +use control_plane::plan_workload; use data_plane::precompute::{pane_end, Precompute, RawSample}; use data_plane::sds::Sds; -use planner_types::workload::{PlanningWorkload, Query}; const MINUTE: i64 = 60_000; const PANE: i64 = 10 * MINUTE; -fn precompute(plan: InstalledPlan) -> (Precompute, Sds) { - let dag = plan.dag.dag; - let sds = Sds::from_dag(&dag, &frontier_from_timing(&dag).unwrap()).unwrap(); - (Precompute::new(dag, &sds, 0).unwrap(), sds) +fn precompute(plan: InstallablePlan) -> (Precompute, Sds) { + let sds = Sds::from_plan(&plan).unwrap(); + (Precompute::new(&plan, &sds, 0).unwrap(), sds) } fn sample(metric: &str, job: &str, timestamp_ms: i64, value: f64) -> RawSample { @@ -114,3 +113,24 @@ fn panes_are_left_open() { assert_eq!(pane_end(-1, PANE), 0); assert_eq!(pane_end(-PANE, PANE), -PANE); } + +/// A pane stays stored while an output reads it: Example 4b's oldest output +/// reads the pane ending 50 min before the evaluation, so with the watermark +/// at 90 min the pane ending at 40 min is kept and the one at 30 min, which +/// starts before the plan's 60-minute retention, is evicted. +#[test] +fn keeps_panes_for_the_plans_retention() { + let (mut engine, sds) = precompute(common::plan("example4b", false)); + let definition = sds.definitions.values().next().unwrap(); + assert_eq!(definition.retention_ms, 60 * MINUTE); + let id = definition.id.clone(); + engine.ingest(&samples("latency_ms", &["a"], MINUTE, 90 * MINUTE + 1)); + assert!( + engine.store.get(&id, 4 * PANE).is_some(), + "read at offset 50 min" + ); + assert!( + engine.store.get(&id, 3 * PANE).is_none(), + "older than the retention" + ); +} diff --git a/data_plane/tests/query.rs b/data_plane/tests/query.rs index 95b7def2..8f80330b 100644 --- a/data_plane/tests/query.rs +++ b/data_plane/tests/query.rs @@ -3,8 +3,7 @@ mod common; use std::collections::BTreeMap; use std::sync::Arc; -use asap_executor::physical_planner::frontier_from_timing; -use control_plane::InstalledPlan; +use asap_planner::InstallablePlan; use data_plane::precompute::{Precompute, RawSample}; use data_plane::query::{QueryEngine, RawRange, ResultSample, Unanswered}; use data_plane::sds::Sds; @@ -16,11 +15,10 @@ struct Fixture { query: QueryEngine, } -fn fixture(plan: InstalledPlan) -> Fixture { - let dag = &plan.dag.dag; - let sds = Sds::from_dag(dag, &frontier_from_timing(dag).unwrap()).unwrap(); +fn fixture(plan: InstallablePlan) -> Fixture { + let sds = Sds::from_plan(&plan).unwrap(); Fixture { - precompute: Precompute::new(dag.clone(), &sds, 0).unwrap(), + precompute: Precompute::new(&plan, &sds, 0).unwrap(), query: QueryEngine::new(&plan, &sds), } } diff --git a/data_plane/tests/range.rs b/data_plane/tests/range.rs index 10e55dda..311fe0e3 100644 --- a/data_plane/tests/range.rs +++ b/data_plane/tests/range.rs @@ -7,18 +7,16 @@ mod common; use std::collections::BTreeMap; use std::sync::Arc; -use asap_executor::physical_planner::frontier_from_timing; use asap_executor::values::Batch; -use control_plane::{plan_selected_at_query_time, InstalledPlan}; +use asap_planner::InstallablePlan; use data_plane::precompute::{Precompute, RawSample}; use data_plane::query::{KeptResultStats, QueryEngine, RawRange, ResultSample, Unanswered}; use data_plane::sds::{Sds, SummaryStore}; const MINUTE: i64 = 60_000; -fn sds(plan: &InstalledPlan) -> Sds { - let dag = &plan.dag.dag; - Sds::from_dag(dag, &frontier_from_timing(dag).unwrap()).unwrap() +fn sds(plan: &InstallablePlan) -> Sds { + Sds::from_plan(plan).unwrap() } /// Samples every 15 s in `[from, to)` of `metric` for each `service`, with @@ -185,10 +183,13 @@ fn raw_samples_are_fetched_once_and_sliced_per_step() { /// off the pane grid, or reading an evicted pane, forward the whole query. #[test] fn every_step_reads_stored_panes() { - let plan = common::plan("example4b", false); + // Declared as range requests over 10 minutes, so the Planner keeps the + // panes the first step reads. + let workload = common::with_range_evaluation(&common::workload("example4b"), 600_000, 600_000); + let plan = control_plane::plan_workload(&workload, &common::inputs(false)).unwrap(); let query = "quantile_over_time(0.99, latency_ms[60m])"; let sds = sds(&plan); - let mut precompute = Precompute::new(plan.dag.dag.clone(), &sds, 0).unwrap(); + let mut precompute = Precompute::new(&plan, &sds, 0).unwrap(); // Values grow with time, so each hour's p99 differs. let all = samples("latency_ms", &["a", "b"], MINUTE, 101 * MINUTE, |i, t| { (i * 1000) as f64 + (t / 15_000) as f64 @@ -234,13 +235,13 @@ fn every_step_reads_stored_panes() { /// it at the later steps; every step answers like the Planner executor. /// Kept panes are evicted as for instant evaluations, measured from the /// newest step: the same range query again rebuilds the panes ending at -/// 6..8 min, which are older than 14 min minus 6. +/// 6..9 min, which start before 14 min minus the plan's 5-minute retention. #[test] fn kept_panes_are_shared_across_steps() { let workload = common::workload("example4-b3"); let inputs = common::inputs(true); let plan = control_plane::plan_workload(&workload, &inputs).unwrap(); - let reference_plan = plan_selected_at_query_time(&workload, &inputs).unwrap(); + let reference_plan = plan.query_time_reference().unwrap(); let query = "quantile_over_time(0.99, latency_ms[5m])"; let all = samples("latency_ms", &["a", "b"], 0, 30 * MINUTE + 1, |i, t| { (i * 100) as f64 + (t / 15_000 * 25 % 101) as f64 @@ -284,17 +285,18 @@ fn kept_panes_are_shared_across_steps() { .iter() .map(|r| (r.start_ms, r.end_ms)) .collect::>(), - vec![(5 * MINUTE, 8 * MINUTE)], + vec![(5 * MINUTE, 9 * MINUTE)], "only the evicted panes are fetched" ); assert_eq!(again, results); - // Panes 6..8 are older than the newest evaluation's retention, so each - // step reading one rebuilds it: 3 at 10 min, 2 at 11, 1 at 12. + // Panes 6..9 start before the newest evaluation's retention (14 min + // minus 5), so each step reading one rebuilds it: 4 at 10 min, 3 at 11, + // 2 at 12, 1 at 13. assert_eq!( engine.kept_stats, KeptResultStats { - built: 15, - reused: 35 + built: 19, + reused: 31 } ); } @@ -308,7 +310,7 @@ fn open_panes_are_recomputed_at_each_step() { let workload = common::workload("example4-b3"); let inputs = common::inputs(true); let plan = control_plane::plan_workload(&workload, &inputs).unwrap(); - let reference_plan = plan_selected_at_query_time(&workload, &inputs).unwrap(); + let reference_plan = plan.query_time_reference().unwrap(); let query = "quantile_over_time(0.99, latency_ms[5m])"; let all = samples("latency_ms", &["a", "b"], 0, 30 * MINUTE + 1, |i, t| { (i * 100) as f64 + (t / 15_000 * 25 % 101) as f64 @@ -339,6 +341,7 @@ fn open_panes_are_recomputed_at_each_step() { let kept_ends: Vec = (0..=30) .filter(|m| engine.kept.get(&definition, m * MINUTE).is_some()) .collect(); - // 6..8 are evicted, measured from the newest step (14 min). - assert_eq!(kept_ends, vec![9, 10, 11, 12]); + // 6..9 are evicted: they start before the newest step (14 min) minus + // the plan's 5-minute retention. + assert_eq!(kept_ends, vec![10, 11, 12]); } diff --git a/data_plane/tests/sds.rs b/data_plane/tests/sds.rs index 60008c07..037e38a6 100644 --- a/data_plane/tests/sds.rs +++ b/data_plane/tests/sds.rs @@ -1,22 +1,17 @@ mod common; -use asap_executor::physical_planner::frontier_from_timing; +use asap_executor::planner::ir::{NonASAPOp, Operator}; +use asap_planner::{InstallablePlan, StoredOutputBuild}; use common::plan; -use data_plane::sds::{DefinitionId, Sds, SdsError, SummaryStore}; -use planner_types::ir::physical_export::PhysicalASAPDAG; -use planner_types::ir::{NonASAPOp, Operator}; -use planner_types::physical::{Materialization, Medium}; - -fn sds_of(dag: &PhysicalASAPDAG) -> Result { - Sds::from_dag(dag, &frontier_from_timing(dag).unwrap()) -} +use data_plane::sds::{DefinitionId, Sds, SummaryStore}; /// Example 4b's six ingestion-time KLL panes are one definition with a -/// 10-minute pane; each output reads the pane `i * 10min` before evaluation. +/// 10-minute pane; each output reads the pane `i * 10min` before evaluation, +/// and the definition is kept for the largest retention among them. #[test] fn example4b_panes_share_one_definition() { let plan = plan("example4b", false); - let sds = sds_of(&plan.dag.dag).unwrap(); + let sds = Sds::from_plan(&plan).unwrap(); assert_eq!(sds.definitions.len(), 1); let definition = sds.definitions.values().next().unwrap(); assert_eq!(definition.pane_ms, 600_000); @@ -30,12 +25,14 @@ fn example4b_panes_share_one_definition() { .collect(); offsets.sort(); assert_eq!(offsets, (0..6).map(|i| i * 600_000).collect::>()); + let largest = plan.stored_outputs.iter().map(|o| o.retention_ms).max(); + assert_eq!(Some(definition.retention_ms as u64), largest); let raw = plan .dag .dag .nodes .iter() - .find(|n| n.id as u64 == definition.raw_input) + .find(|n| n.id as u64 == definition.raw_input()) .unwrap(); assert!(matches!( raw.payload, @@ -43,67 +40,79 @@ fn example4b_panes_share_one_definition() { )); } +/// A definition is kept for the largest retention among its outputs, not +/// its first output's. +#[test] +fn retention_is_the_largest_among_outputs() { + let mut plan = plan("example4b", false); + let last = plan.stored_outputs.len() - 1; + plan.stored_outputs[last].retention_ms = 7 * 3_600_000; + let sds = Sds::from_plan(&plan).unwrap(); + let definition = sds.definitions.values().next().unwrap(); + assert_eq!(definition.retention_ms, 7 * 3_600_000); +} + /// A plan computed entirely at query time stores nothing. #[test] fn query_time_plans_have_no_definitions() { let plan = plan("example1", true); - assert_eq!(sds_of(&plan.dag.dag).unwrap(), Sds::default()); + assert_eq!(Sds::from_plan(&plan).unwrap(), Sds::default()); } -/// Identity depends on what is computed, not on how nodes are numbered. -#[test] -fn identity_ignores_node_numbering() { - let dag = plan("example4b", false).dag.dag; - let mut renumbered = dag.clone(); +/// `plan` with every node id shifted by 100. +fn renumbered(plan: &InstallablePlan) -> InstallablePlan { + let mut plan = plan.clone(); let renumber = |id: &usize| id + 100; - for node in &mut renumbered.nodes { + let dag = &mut plan.dag.dag; + for node in &mut dag.nodes { node.id = renumber(&node.id); node.payload = node.payload.map_children(renumber); } - for edge in &mut renumbered.edges { + for edge in &mut dag.edges { edge.producer = renumber(&edge.producer); edge.consumer = renumber(&edge.consumer); } - renumbered.roots = renumbered.roots.iter().map(renumber).collect(); + dag.roots = dag.roots.iter().map(renumber).collect(); + for output in &mut plan.stored_outputs { + output.node_id = renumber(&output.node_id); + output.pane.time_range_node_id = renumber(&output.pane.time_range_node_id); + output.pane.scan_node_id = renumber(&output.pane.scan_node_id); + } + plan +} + +/// Identity depends on what is computed, not on how nodes are numbered. +#[test] +fn identity_ignores_node_numbering() { + let plan = plan("example4b", false); let ids = |sds: Sds| sds.definitions.into_keys().collect::>(); assert_eq!( - ids(sds_of(&renumbered).unwrap()), - ids(sds_of(&dag).unwrap()) + ids(Sds::from_plan(&renumbered(&plan)).unwrap()), + ids(Sds::from_plan(&plan).unwrap()) ); } /// The same computation over another metric is another definition. #[test] fn identity_depends_on_the_source() { - let dag = plan("example4b", false).dag.dag; - let other: PhysicalASAPDAG = serde_json::from_str( - &serde_json::to_string(&dag) + let plan = plan("example4b", false); + let other: InstallablePlan = serde_json::from_str( + &serde_json::to_string(&plan) .unwrap() .replace("latency_ms", "other"), ) .unwrap(); assert_ne!( - sds_of(&other).unwrap().definitions.into_keys().next(), - sds_of(&dag).unwrap().definitions.into_keys().next() - ); -} - -/// An ingestion-time output must read a pane of raw samples. -#[test] -fn rejects_outputs_without_a_pane() { - let dag = plan("example4b", false).dag.dag; - let scan = dag - .nodes - .iter() - .find(|n| matches!(n.payload, Operator::NonASAP(NonASAPOp::Scan { .. }))) - .unwrap(); - let scan = scan.id as u64; - assert_eq!( - Sds::from_dag(&dag, &[scan]), - Err(SdsError::UnsupportedShape { - node: scan, - reason: "no TimeRange".into() - }) + Sds::from_plan(&other) + .unwrap() + .definitions + .into_keys() + .next(), + Sds::from_plan(&plan) + .unwrap() + .definitions + .into_keys() + .next() ); } @@ -114,12 +123,11 @@ fn rejects_outputs_without_a_pane() { #[test] fn kept_panes_share_one_definition() { let plan = plan("example4-b3", true); - let dag = &plan.dag.dag; - assert!(dag.nodes.iter().any(|n| matches!( - n.materialization, - Materialization::MatAtQueryKeptAcrossEvaluations(_) - ))); - let sds = sds_of(dag).unwrap(); + assert!(plan + .stored_outputs + .iter() + .any(|o| o.built_at == StoredOutputBuild::QueryRun)); + let sds = Sds::from_plan(&plan).unwrap(); assert!(sds.definitions.is_empty() && sds.outputs.is_empty()); assert_eq!(sds.kept_definitions.len(), 1); let definition = sds.kept_definitions.values().next().unwrap(); @@ -136,27 +144,8 @@ fn kept_panes_share_one_definition() { assert_eq!(offsets, (0..5).map(|i| i * 60_000).collect::>()); } -/// A kept output must read one pane of raw samples, like a stored one. -#[test] -fn rejects_kept_outputs_without_a_pane() { - let mut dag = plan("example4-b3", true).dag.dag; - let scan = dag - .nodes - .iter_mut() - .find(|n| matches!(n.payload, Operator::NonASAP(NonASAPOp::Scan { .. }))) - .unwrap(); - scan.materialization = Materialization::MatAtQueryKeptAcrossEvaluations(Medium::Memory); - let scan = scan.id as u64; - assert_eq!( - sds_of(&dag), - Err(SdsError::UnsupportedShape { - node: scan, - reason: "no TimeRange".into() - }) - ); -} - -/// Panes are found by definition and end, and evicted once no query needs them. +/// Panes are found by definition and end, and evicted once no query needs +/// them. #[test] fn store_finds_and_evicts_panes() { let (a, b) = (DefinitionId("a".into()), DefinitionId("b".into())); @@ -167,7 +156,7 @@ fn store_finds_and_evicts_panes() { store.insert(&b, 60, "b60".to_string()); assert_eq!(store.get(&a, 120).map(String::as_str), Some("a120")); assert_eq!(store.get(&a, 90), None); - store.evict_through(&a, 120); + store.evict_before(&a, 180); assert_eq!(store.get(&a, 120), None); assert_eq!(store.get(&a, 180).map(String::as_str), Some("a180")); assert_eq!(store.get(&b, 60).map(String::as_str), Some("b60")); diff --git a/scripts/ci/README.md b/scripts/ci/README.md index 5243fc2d..5c2b711c 100644 --- a/scripts/ci/README.md +++ b/scripts/ci/README.md @@ -2,7 +2,7 @@ The backend keeps reproducible Planner revisions in `Cargo.toml` and `Cargo.lock`. `Sync ASAPPlanner main` checks upstream main daily (or on manual dispatch), updates -all five Planner dependencies together, and opens or refreshes +both Planner dependencies (`asap-planner`, `asap-executor`) together, and opens or refreshes `automation/sync-asapplanner-main` against backend main. GitHub scheduled runs can be delayed. The existing `ASAP_CI_REPO_TOKEN` secret must have read access to private dependency diff --git a/scripts/ci/__pycache__/test_sync_planner.cpython-310.pyc b/scripts/ci/__pycache__/test_sync_planner.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c9ace5f64b0f73180c140af1a53a4e35d3ada0ae GIT binary patch literal 3741 zcmb7H-*4N-9p{mhNKvwqxIvaI!N6)=7j3cBbzKK!T^iN#k^-CO#ab~4EeM)-wy01f zb9anwO$IhNuNkoCB7pR`w>|D3*~`7|DG&Px{Lu9KJ<4{RxL8Jm$Gg-0{CHw@!HSa|iZp>P*okwZd_fk6p16EYz1wGwk|4J=aD*#leuRvA?Ew>qdY zi#dm6=FW{#joQ}?TDfivy!!^NQs=2b9abIH53NCiEm8N8$vl+qKE> z*7g{WV!?blWWE@NoKauwr;#rr9_P~EjpZ;aq#s5R6CzG`{QI$dP;B`b_g6!{llh$O z#6t3ziOvhygt6Wx%H9a&@CAX=+MlaG!@bzeQ z(`dh=E^d_5Z^Cl=Oh}<_hHv>OBf_m5KPF@<@Ztu956NGRsqt0)fK18PfFZ@yd~SaM zk$!Aa3ud<`cl}@cfA)7`3E$RG%3R#;cA+>dwmMNZ?r!jG#3I?->}_;0SuD173&Q&Z z`>2~+Z~08@mVe-_|HEjT%Itig>@*xR?!r2Ca`a($yiY?Jc6a5<7F)>~-|m)Ri*6Km zCB!B`%z_18oqS)_KG|U@+spZ#CEPS}+Wd=cm}c{(_j2<6*F)PmY2G%E8i@VMVz8qn zU1EDIDrCmjIR-XDepMLvm1=f*P(H zv+5LBK@2FFSfaWWlx$t9;BT~g&-ki#P@Pt$Rj75SgSC&}m#(ahylDkc(>YqZW_)ct zx27gF3WKjsjZr-%Q&Tpk)*+$ZGyA}iOQR;O%Z9vgNOnvZ5!na)dsaDcq2z9Z!rIOC zBTq}DBeQX$EP#9R(GTD4ZpCR=485GksoeHkKN5iPw&)I7NJSUL@nzcbTkrMPHv7TD zwJ%zpszj)?$-VPBmyPeX{5!sGwE|6r$M%3~i!oMv5AXjz*zEWEf81<&ZIfSwsPapY z#Q%lxACcRj*IxfY+v4v)rE-sFR+n_{)&9!f3dOMBwv`p_Qf0C9iE2h+E(^{A@QJ*T z%9dT)9!nPB zP_X^sIHSs@aU_+Q3FW}FG0#%vf;wUmL(T#VQl2Qba-N}6DT^1Ws&1}6{CuO|t|?ov zWLvr8Fpal?;ruR6qtGnvny`p(CXZTcE-$|+Orx8MoR`| zWm2c?JM-os16l)9tmDGug?pnT$WOSXDDzpPhy$GDHH7q zZ{q|Ny(hkcGj_1TDc46|C)>}oDVINrAWV{gvK&TcX#~e^00;z07Cj0e##vHG(8Ieh zoIq;Y=4G-B%)9|?T!t6y`m;B<_7{8V zfnOOdBIbwwI83%PJ}&9i;Z;~rn`phC(u98qwd-y76||crYU^1EIR6E-;2&Y~OGu8% z@S89ZKmf2;xIutnB95m3zK&Z*2R2Thgh@;RozklUngM9A=wMuc^RVF9ly1Rb0uSoT zW|LfjzsaZPL-p1pp1}z*I>G5rP(Yk!c+)iJCN-}a+GI`HK&eSCBTl{ZaVp+^)H6t(QPle|-~Uz=wLvMBN$L zkG7L+*WY5G6XtI`^z&jXiK7sWqoZRFg83;%4R2@b5XTc!%r}An zUAz3HJnQXI7N?p)7l|2h84}m9;fyy;_ZzR`UU2KK z({ObQTnAp=f}njtIRUUI3Ig7OF3QQ_e<5IddAUI}`E{&!i!_!PCG3mWBv?0`UVz*a zhMhP|rQkPp(>^p?4dFDx)$j*cXXA9|tysCAlySI&=!q+k*!ptf68yVjE?e%$4Pw9i E7D=oD`Tzg` literal 0 HcmV?d00001 diff --git a/scripts/ci/test_sync_planner.py b/scripts/ci/test_sync_planner.py index dd46b662..6a8209ba 100644 --- a/scripts/ci/test_sync_planner.py +++ b/scripts/ci/test_sync_planner.py @@ -12,8 +12,7 @@ class PlannerSyncTests(unittest.TestCase): revision = "a" * 40 manifest = ''.join( f'{name} = {{ git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "{"b" * 40}" }}\n' - for name in ['planner-types', 'asap-frontend-promql', 'asap-logical-asap-optimization', - 'asap-plan-selection', 'asap-executor'] + for name in ['asap-planner', 'asap-executor'] ) def run_sync(self, manifest, revision=None, cargo_status=0): @@ -37,12 +36,10 @@ def test_updates_all_dependencies_and_lock_resolution(self): root, result = self.run_sync(self.manifest + other) self.assertEqual(result.returncode, 0, result.stderr) updated = (root / 'Cargo.toml').read_text() - self.assertEqual(updated.count(f'rev = "{self.revision}"'), 5) + self.assertEqual(updated.count(f'rev = "{self.revision}"'), 2) self.assertTrue(updated.endswith(other)) self.assertEqual((root / 'cargo-args').read_text().splitlines(), [ - 'update', '-p', 'asap-types@0.1.0', '-p', 'asap-frontend-promql', - '-p', 'asap-logical-asap-optimization', '-p', 'asap-plan-selection', - '-p', 'asap-executor', '--precise', self.revision]) + 'update', '-p', 'asap-planner', '-p', 'asap-executor', '--precise', self.revision]) def test_invalid_revision_does_not_mutate_manifest(self): """An upstream lookup failure cannot produce a malformed pin.""" diff --git a/scripts/sync-asapplanner-main.sh b/scripts/sync-asapplanner-main.sh index 094ec6cf..c4895e62 100755 --- a/scripts/sync-asapplanner-main.sh +++ b/scripts/sync-asapplanner-main.sh @@ -20,16 +20,13 @@ pattern = re.compile( r'(git = "https://github.com/ProjectASAP/ASAPPlanner", rev = ")[0-9a-f]{40}("\s*})' ) updated, replacements = pattern.subn(rf"\g<1>{revision}\2", text) -if replacements != 5: - raise SystemExit(f"expected five ASAPPlanner dependencies, found {replacements}") +if replacements != 2: + raise SystemExit(f"expected two ASAPPlanner dependencies, found {replacements}") path.write_text(updated) PY cargo update \ - -p asap-types@0.1.0 \ - -p asap-frontend-promql \ - -p asap-logical-asap-optimization \ - -p asap-plan-selection \ + -p asap-planner \ -p asap-executor \ --precise "$planner_rev" From 937bc0833bfcbb791688c95a77b9743d9713f222 Mon Sep 17 00:00:00 2001 From: zzylol Date: Mon, 19 Oct 2026 04:10:33 +0000 Subject: [PATCH 2/2] chore: ignore Python bytecode caches --- .gitignore | 1 + .../test_sync_planner.cpython-310.pyc | Bin 3741 -> 0 bytes 2 files changed, 1 insertion(+) delete mode 100644 scripts/ci/__pycache__/test_sync_planner.cpython-310.pyc diff --git a/.gitignore b/.gitignore index 0461e007..4afcf006 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,4 @@ uuid # Generated evaluation archives stay outside version control. /docs/evaluation/**/*.gz +__pycache__/ diff --git a/scripts/ci/__pycache__/test_sync_planner.cpython-310.pyc b/scripts/ci/__pycache__/test_sync_planner.cpython-310.pyc deleted file mode 100644 index c9ace5f64b0f73180c140af1a53a4e35d3ada0ae..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3741 zcmb7H-*4N-9p{mhNKvwqxIvaI!N6)=7j3cBbzKK!T^iN#k^-CO#ab~4EeM)-wy01f zb9anwO$IhNuNkoCB7pR`w>|D3*~`7|DG&Px{Lu9KJ<4{RxL8Jm$Gg-0{CHw@!HSa|iZp>P*okwZd_fk6p16EYz1wGwk|4J=aD*#leuRvA?Ew>qdY zi#dm6=FW{#joQ}?TDfivy!!^NQs=2b9abIH53NCiEm8N8$vl+qKE> z*7g{WV!?blWWE@NoKauwr;#rr9_P~EjpZ;aq#s5R6CzG`{QI$dP;B`b_g6!{llh$O z#6t3ziOvhygt6Wx%H9a&@CAX=+MlaG!@bzeQ z(`dh=E^d_5Z^Cl=Oh}<_hHv>OBf_m5KPF@<@Ztu956NGRsqt0)fK18PfFZ@yd~SaM zk$!Aa3ud<`cl}@cfA)7`3E$RG%3R#;cA+>dwmMNZ?r!jG#3I?->}_;0SuD173&Q&Z z`>2~+Z~08@mVe-_|HEjT%Itig>@*xR?!r2Ca`a($yiY?Jc6a5<7F)>~-|m)Ri*6Km zCB!B`%z_18oqS)_KG|U@+spZ#CEPS}+Wd=cm}c{(_j2<6*F)PmY2G%E8i@VMVz8qn zU1EDIDrCmjIR-XDepMLvm1=f*P(H zv+5LBK@2FFSfaWWlx$t9;BT~g&-ki#P@Pt$Rj75SgSC&}m#(ahylDkc(>YqZW_)ct zx27gF3WKjsjZr-%Q&Tpk)*+$ZGyA}iOQR;O%Z9vgNOnvZ5!na)dsaDcq2z9Z!rIOC zBTq}DBeQX$EP#9R(GTD4ZpCR=485GksoeHkKN5iPw&)I7NJSUL@nzcbTkrMPHv7TD zwJ%zpszj)?$-VPBmyPeX{5!sGwE|6r$M%3~i!oMv5AXjz*zEWEf81<&ZIfSwsPapY z#Q%lxACcRj*IxfY+v4v)rE-sFR+n_{)&9!f3dOMBwv`p_Qf0C9iE2h+E(^{A@QJ*T z%9dT)9!nPB zP_X^sIHSs@aU_+Q3FW}FG0#%vf;wUmL(T#VQl2Qba-N}6DT^1Ws&1}6{CuO|t|?ov zWLvr8Fpal?;ruR6qtGnvny`p(CXZTcE-$|+Orx8MoR`| zWm2c?JM-os16l)9tmDGug?pnT$WOSXDDzpPhy$GDHH7q zZ{q|Ny(hkcGj_1TDc46|C)>}oDVINrAWV{gvK&TcX#~e^00;z07Cj0e##vHG(8Ieh zoIq;Y=4G-B%)9|?T!t6y`m;B<_7{8V zfnOOdBIbwwI83%PJ}&9i;Z;~rn`phC(u98qwd-y76||crYU^1EIR6E-;2&Y~OGu8% z@S89ZKmf2;xIutnB95m3zK&Z*2R2Thgh@;RozklUngM9A=wMuc^RVF9ly1Rb0uSoT zW|LfjzsaZPL-p1pp1}z*I>G5rP(Yk!c+)iJCN-}a+GI`HK&eSCBTl{ZaVp+^)H6t(QPle|-~Uz=wLvMBN$L zkG7L+*WY5G6XtI`^z&jXiK7sWqoZRFg83;%4R2@b5XTc!%r}An zUAz3HJnQXI7N?p)7l|2h84}m9;fyy;_ZzR`UU2KK z({ObQTnAp=f}njtIRUUI3Ig7OF3QQ_e<5IddAUI}`E{&!i!_!PCG3mWBv?0`UVz*a zhMhP|rQkPp(>^p?4dFDx)$j*cXXA9|tysCAlySI&=!q+k*!ptf68yVjE?e%$4Pw9i E7D=oD`Tzg`