diff --git a/Cargo.lock b/Cargo.lock index 14f3084..1d53439 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -79,19 +79,40 @@ version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3095aaf545942ff5abd46654534f15b03a90fba78299d661e045e5d587222f0d" dependencies = [ - "arrow-arith", - "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-csv", - "arrow-data", - "arrow-ipc", - "arrow-json", - "arrow-ord", - "arrow-row", - "arrow-schema", - "arrow-select", - "arrow-string", + "arrow-arith 55.0.0", + "arrow-array 55.0.0", + "arrow-buffer 55.0.0", + "arrow-cast 55.0.0", + "arrow-csv 55.0.0", + "arrow-data 55.0.0", + "arrow-ipc 55.0.0", + "arrow-json 55.0.0", + "arrow-ord 55.0.0", + "arrow-row 55.0.0", + "arrow-schema 55.0.0", + "arrow-select 55.0.0", + "arrow-string 55.0.0", +] + +[[package]] +name = "arrow" +version = "56.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e833808ff2d94ed40d9379848a950d995043c7fb3e81a30b383f4c6033821cc" +dependencies = [ + "arrow-arith 56.2.0", + "arrow-array 56.2.0", + "arrow-buffer 56.2.0", + "arrow-cast 56.2.0", + "arrow-csv 56.2.0", + "arrow-data 56.2.0", + "arrow-ipc 56.2.0", + "arrow-json 56.2.0", + "arrow-ord 56.2.0", + "arrow-row 56.2.0", + "arrow-schema 56.2.0", + "arrow-select 56.2.0", + "arrow-string 56.2.0", ] [[package]] @@ -100,10 +121,24 @@ version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "00752064ff47cee746e816ddb8450520c3a52cbad1e256f6fa861a35f86c45e7" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", + "arrow-array 55.0.0", + "arrow-buffer 55.0.0", + "arrow-data 55.0.0", + "arrow-schema 55.0.0", + "chrono", + "num", +] + +[[package]] +name = "arrow-arith" +version = "56.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad08897b81588f60ba983e3ca39bda2b179bdd84dced378e7df81a5313802ef8" +dependencies = [ + "arrow-array 56.2.0", + "arrow-buffer 56.2.0", + "arrow-data 56.2.0", + "arrow-schema 56.2.0", "chrono", "num", ] @@ -115,9 +150,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cebfe926794fbc1f49ddd0cdaf898956ca9f6e79541efce62dabccfd81380472" dependencies = [ "ahash", - "arrow-buffer", - "arrow-data", - "arrow-schema", + "arrow-buffer 55.0.0", + "arrow-data 55.0.0", + "arrow-schema 55.0.0", "chrono", "chrono-tz", "half", @@ -125,6 +160,22 @@ dependencies = [ "num", ] +[[package]] +name = "arrow-array" +version = "56.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8548ca7c070d8db9ce7aa43f37393e4bfcf3f2d3681df278490772fd1673d08d" +dependencies = [ + "ahash", + "arrow-buffer 56.2.0", + "arrow-data 56.2.0", + "arrow-schema 56.2.0", + "chrono", + "half", + "hashbrown 0.16.0", + "num", +] + [[package]] name = "arrow-buffer" version = "55.0.0" @@ -136,19 +187,51 @@ dependencies = [ "num", ] +[[package]] +name = "arrow-buffer" +version = "56.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e003216336f70446457e280807a73899dd822feaf02087d31febca1363e2fccc" +dependencies = [ + "bytes", + "half", + "num", +] + [[package]] name = "arrow-cast" version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "335f769c5a218ea823d3760a743feba1ef7857cba114c01399a891c2fff34285" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "arrow-select", + "arrow-array 55.0.0", + "arrow-buffer 55.0.0", + "arrow-data 55.0.0", + "arrow-schema 55.0.0", + "arrow-select 55.0.0", + "atoi", + "base64", + "chrono", + "comfy-table", + "half", + "lexical-core", + "num", + "ryu", +] + +[[package]] +name = "arrow-cast" +version = "56.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "919418a0681298d3a77d1a315f625916cb5678ad0d74b9c60108eb15fd083023" +dependencies = [ + "arrow-array 56.2.0", + "arrow-buffer 56.2.0", + "arrow-data 56.2.0", + "arrow-schema 56.2.0", + "arrow-select 56.2.0", "atoi", - "base64 0.22.1", + "base64", "chrono", "comfy-table", "half", @@ -163,9 +246,9 @@ version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "510db7dfbb4d5761826516cc611d97b3a68835d0ece95b034a052601109c0b1b" dependencies = [ - "arrow-array", - "arrow-cast", - "arrow-schema", + "arrow-array 55.0.0", + "arrow-cast 55.0.0", + "arrow-schema 55.0.0", "chrono", "csv", "csv-core", @@ -173,14 +256,41 @@ dependencies = [ "regex", ] +[[package]] +name = "arrow-csv" +version = "56.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa9bf02705b5cf762b6f764c65f04ae9082c7cfc4e96e0c33548ee3f67012eb" +dependencies = [ + "arrow-array 56.2.0", + "arrow-cast 56.2.0", + "arrow-schema 56.2.0", + "chrono", + "csv", + "csv-core", + "regex", +] + [[package]] name = "arrow-data" version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8affacf3351a24039ea24adab06f316ded523b6f8c3dbe28fbac5f18743451b" dependencies = [ - "arrow-buffer", - "arrow-schema", + "arrow-buffer 55.0.0", + "arrow-schema 55.0.0", + "half", + "num", +] + +[[package]] +name = "arrow-data" +version = "56.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5c64fff1d142f833d78897a772f2e5b55b36cb3e6320376f0961ab0db7bd6d0" +dependencies = [ + "arrow-buffer 56.2.0", + "arrow-schema 56.2.0", "half", "num", ] @@ -191,25 +301,61 @@ version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69880a9e6934d9cba2b8630dd08a3463a91db8693b16b499d54026b6137af284" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", + "arrow-array 55.0.0", + "arrow-buffer 55.0.0", + "arrow-data 55.0.0", + "arrow-schema 55.0.0", "flatbuffers", "lz4_flex", ] +[[package]] +name = "arrow-ipc" +version = "56.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d3594dcddccc7f20fd069bc8e9828ce37220372680ff638c5e00dea427d88f5" +dependencies = [ + "arrow-array 56.2.0", + "arrow-buffer 56.2.0", + "arrow-data 56.2.0", + "arrow-schema 56.2.0", + "arrow-select 56.2.0", + "flatbuffers", +] + [[package]] name = "arrow-json" version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d8dafd17a05449e31e0114d740530e0ada7379d7cb9c338fd65b09a8130960b0" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-data", - "arrow-schema", + "arrow-array 55.0.0", + "arrow-buffer 55.0.0", + "arrow-cast 55.0.0", + "arrow-data 55.0.0", + "arrow-schema 55.0.0", + "chrono", + "half", + "indexmap 2.9.0", + "lexical-core", + "memchr", + "num", + "serde", + "serde_json", + "simdutf8", +] + +[[package]] +name = "arrow-json" +version = "56.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88cf36502b64a127dc659e3b305f1d993a544eab0d48cce704424e62074dc04b" +dependencies = [ + "arrow-array 56.2.0", + "arrow-buffer 56.2.0", + "arrow-cast 56.2.0", + "arrow-data 56.2.0", + "arrow-schema 56.2.0", "chrono", "half", "indexmap 2.9.0", @@ -227,11 +373,24 @@ version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "895644523af4e17502d42c3cb6b27cb820f0cb77954c22d75c23a85247c849e1" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "arrow-select", + "arrow-array 55.0.0", + "arrow-buffer 55.0.0", + "arrow-data 55.0.0", + "arrow-schema 55.0.0", + "arrow-select 55.0.0", +] + +[[package]] +name = "arrow-ord" +version = "56.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c8f82583eb4f8d84d4ee55fd1cb306720cddead7596edce95b50ee418edf66f" +dependencies = [ + "arrow-array 56.2.0", + "arrow-buffer 56.2.0", + "arrow-data 56.2.0", + "arrow-schema 56.2.0", + "arrow-select 56.2.0", ] [[package]] @@ -240,10 +399,23 @@ version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9be8a2a4e5e7d9c822b2b8095ecd77010576d824f654d347817640acfc97d229" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", + "arrow-array 55.0.0", + "arrow-buffer 55.0.0", + "arrow-data 55.0.0", + "arrow-schema 55.0.0", + "half", +] + +[[package]] +name = "arrow-row" +version = "56.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d07ba24522229d9085031df6b94605e0f4b26e099fb7cdeec37abd941a73753" +dependencies = [ + "arrow-array 56.2.0", + "arrow-buffer 56.2.0", + "arrow-data 56.2.0", + "arrow-schema 56.2.0", "half", ] @@ -253,6 +425,12 @@ version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7450c76ab7c5a6805be3440dc2e2096010da58f7cab301fdc996a4ee3ee74e49" +[[package]] +name = "arrow-schema" +version = "56.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3aa9e59c611ebc291c28582077ef25c97f1975383f1479b12f3b9ffee2ffabe" + [[package]] name = "arrow-select" version = "55.0.0" @@ -260,10 +438,24 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aa5f5a93c75f46ef48e4001535e7b6c922eeb0aa20b73cf58d09e13d057490d8" dependencies = [ "ahash", - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", + "arrow-array 55.0.0", + "arrow-buffer 55.0.0", + "arrow-data 55.0.0", + "arrow-schema 55.0.0", + "num", +] + +[[package]] +name = "arrow-select" +version = "56.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c41dbbd1e97bfcaee4fcb30e29105fb2c75e4d82ae4de70b792a5d3f66b2e7a" +dependencies = [ + "ahash", + "arrow-array 56.2.0", + "arrow-buffer 56.2.0", + "arrow-data 56.2.0", + "arrow-schema 56.2.0", "num", ] @@ -273,11 +465,28 @@ version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e7005d858d84b56428ba2a98a107fe88c0132c61793cf6b8232a1f9bfc0452b" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "arrow-select", + "arrow-array 55.0.0", + "arrow-buffer 55.0.0", + "arrow-data 55.0.0", + "arrow-schema 55.0.0", + "arrow-select 55.0.0", + "memchr", + "num", + "regex", + "regex-syntax 0.8.2", +] + +[[package]] +name = "arrow-string" +version = "56.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53f5183c150fbc619eede22b861ea7c0eebed8eaac0333eaa7f6da5205fd504d" +dependencies = [ + "arrow-array 56.2.0", + "arrow-buffer 56.2.0", + "arrow-data 56.2.0", + "arrow-schema 56.2.0", + "arrow-select 56.2.0", "memchr", "num", "regex", @@ -332,6 +541,12 @@ version = "0.15.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ae037714f313c1353189ead58ef9eec30a8e8dc101b2622d461418fd59e28a9" +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + [[package]] name = "autocfg" version = "1.1.0" @@ -340,18 +555,17 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] name = "axum" -version = "0.6.20" +version = "0.7.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b829e4e32b91e643de6eafe82b1d90675f5874230191a4ffbc1b336dec4d6bf" +checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f" dependencies = [ "async-trait", "axum-core", - "bitflags 1.3.2", "bytes", "futures-util", - "http 0.2.12", + "http", "http-body", - "hyper", + "http-body-util", "itoa", "matchit", "memchr", @@ -360,25 +574,28 @@ dependencies = [ "pin-project-lite", "rustversion", "serde", - "sync_wrapper", - "tower 0.4.13", + "sync_wrapper 1.0.2", + "tower 0.5.1", "tower-layer", "tower-service", ] [[package]] name = "axum-core" -version = "0.3.4" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "759fa577a247914fd3f7f76d62972792636412fbfd634cd452f6a385a74d2d2c" +checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199" dependencies = [ "async-trait", "bytes", "futures-util", - "http 0.2.12", + "http", "http-body", + "http-body-util", "mime", + "pin-project-lite", "rustversion", + "sync_wrapper 1.0.2", "tower-layer", "tower-service", ] @@ -398,12 +615,6 @@ dependencies = [ "rustc-demangle", ] -[[package]] -name = "base64" -version = "0.21.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" - [[package]] name = "base64" version = "0.22.1" @@ -528,9 +739,9 @@ dependencies = [ [[package]] name = "comfy-table" -version = "7.1.1" +version = "7.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b34115915337defe99b2aff5c2ce6771e5fbc4079f4b506301f5cf394c8452f7" +checksum = "e0d05af1e006a2407bedef5af410552494ce5be9090444dbbcb57258c1af3d56" dependencies = [ "strum", "strum_macros", @@ -574,9 +785,9 @@ dependencies = [ [[package]] name = "core-foundation" -version = "0.9.4" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" dependencies = [ "core-foundation-sys", "libc", @@ -660,9 +871,9 @@ version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ffe060b978f74ab446be722adb8a274e052e005bf6dfd171caadc3abaad10080" dependencies = [ - "arrow", - "arrow-ipc", - "arrow-schema", + "arrow 55.0.0", + "arrow-ipc 55.0.0", + "arrow-schema 55.0.0", "async-trait", "bytes", "chrono", @@ -689,7 +900,7 @@ dependencies = [ "datafusion-session", "datafusion-sql", "futures", - "itertools 0.14.0", + "itertools", "log", "object_store", "parking_lot", @@ -708,7 +919,7 @@ version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61fe34f401bd03724a1f96d12108144f8cd495a3cdda2bf5e091822fb80b7e66" dependencies = [ - "arrow", + "arrow 55.0.0", "async-trait", "dashmap", "datafusion-common", @@ -721,7 +932,7 @@ dependencies = [ "datafusion-session", "datafusion-sql", "futures", - "itertools 0.14.0", + "itertools", "log", "object_store", "parking_lot", @@ -734,7 +945,7 @@ version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4411b8e3bce5e0fc7521e44f201def2e2d5d1b5f176fb56e8cdc9942c890f00" dependencies = [ - "arrow", + "arrow 55.0.0", "async-trait", "datafusion-catalog", "datafusion-common", @@ -758,9 +969,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0734015d81c8375eb5d4869b7f7ecccc2ee8d6cb81948ef737cd0e7b743bd69c" dependencies = [ "ahash", - "arrow", - "arrow-ipc", - "base64 0.22.1", + "arrow 55.0.0", + "arrow-ipc 55.0.0", + "base64", "half", "hashbrown 0.14.5", "indexmap 2.9.0", @@ -790,7 +1001,7 @@ version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04e602dcdf2f50c2abf297cc2203c73531e6f48b29516af7695d338cf2a778b1" dependencies = [ - "arrow", + "arrow 55.0.0", "async-trait", "bytes", "chrono", @@ -804,7 +1015,7 @@ dependencies = [ "datafusion-session", "futures", "glob", - "itertools 0.14.0", + "itertools", "log", "object_store", "rand 0.8.5", @@ -818,7 +1029,7 @@ version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3bb2253952dc32296ed5b84077cb2e0257fea4be6373e1c376426e17ead4ef6" dependencies = [ - "arrow", + "arrow 55.0.0", "async-trait", "bytes", "datafusion-catalog", @@ -843,7 +1054,7 @@ version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b8c7f47a5d2fe03bfa521ec9bafdb8a5c82de8377f60967c3663f00c8790352" dependencies = [ - "arrow", + "arrow 55.0.0", "async-trait", "bytes", "datafusion-catalog", @@ -874,7 +1085,7 @@ version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06f004d100f49a3658c9da6fb0c3a9b760062d96cd4ad82ccc3b7b69a9fb2f84" dependencies = [ - "arrow", + "arrow 55.0.0", "dashmap", "datafusion-common", "datafusion-expr", @@ -893,7 +1104,7 @@ version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a4e4ce3802609be38eeb607ee72f6fe86c3091460de9dbfae9e18db423b3964" dependencies = [ - "arrow", + "arrow 55.0.0", "chrono", "datafusion-common", "datafusion-doc", @@ -913,10 +1124,10 @@ version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "422ac9cf3b22bbbae8cdf8ceb33039107fde1b5492693168f13bd566b1bcc839" dependencies = [ - "arrow", + "arrow 55.0.0", "datafusion-common", "indexmap 2.9.0", - "itertools 0.14.0", + "itertools", "paste", ] @@ -926,9 +1137,9 @@ version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2ddf0a0a2db5d2918349c978d42d80926c6aa2459cd8a3c533a84ec4bb63479e" dependencies = [ - "arrow", - "arrow-buffer", - "base64 0.22.1", + "arrow 55.0.0", + "arrow-buffer 55.0.0", + "base64", "chrono", "datafusion-common", "datafusion-doc", @@ -937,7 +1148,7 @@ dependencies = [ "datafusion-expr-common", "datafusion-macros", "hex", - "itertools 0.14.0", + "itertools", "log", "rand 0.8.5", "regex", @@ -952,7 +1163,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "408a05dafdc70d05a38a29005b8b15e21b0238734dab1e98483fcb58038c5aba" dependencies = [ "ahash", - "arrow", + "arrow 55.0.0", "datafusion-common", "datafusion-doc", "datafusion-execution", @@ -973,7 +1184,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "756d21da2dd6c9bef97af1504970ff56cbf35d03fbd4ffd62827f02f4d2279d4" dependencies = [ "ahash", - "arrow", + "arrow 55.0.0", "datafusion-common", "datafusion-expr-common", "datafusion-physical-expr-common", @@ -985,7 +1196,7 @@ version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cc9a97220736c8fff1446e936be90d57216c06f28969f9ffd3b72ac93c958c8a" dependencies = [ - "arrow", + "arrow 55.0.0", "async-trait", "datafusion-catalog", "datafusion-common", @@ -1039,13 +1250,13 @@ version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "084d9f979c4b155346d3c34b18f4256e6904ded508e9554d90fed416415c3515" dependencies = [ - "arrow", + "arrow 55.0.0", "chrono", "datafusion-common", "datafusion-expr", "datafusion-physical-expr", "indexmap 2.9.0", - "itertools 0.14.0", + "itertools", "log", "regex", "regex-syntax 0.8.2", @@ -1058,7 +1269,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "64c536062b0076f4e30084065d805f389f9fe38af0ca75bcbac86bc5e9fbab65" dependencies = [ "ahash", - "arrow", + "arrow 55.0.0", "datafusion-common", "datafusion-expr", "datafusion-expr-common", @@ -1067,10 +1278,10 @@ dependencies = [ "half", "hashbrown 0.14.5", "indexmap 2.9.0", - "itertools 0.14.0", + "itertools", "log", "paste", - "petgraph 0.7.1", + "petgraph", ] [[package]] @@ -1080,11 +1291,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8a92b53b3193fac1916a1c5b8e3f4347c526f6822e56b71faa5fb372327a863" dependencies = [ "ahash", - "arrow", + "arrow 55.0.0", "datafusion-common", "datafusion-expr-common", "hashbrown 0.14.5", - "itertools 0.14.0", + "itertools", ] [[package]] @@ -1093,7 +1304,7 @@ version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6fa0a5ac94c7cf3da97bedabd69d6bbca12aef84b9b37e6e9e8c25286511b5e2" dependencies = [ - "arrow", + "arrow 55.0.0", "datafusion-common", "datafusion-execution", "datafusion-expr", @@ -1101,7 +1312,7 @@ dependencies = [ "datafusion-physical-expr", "datafusion-physical-expr-common", "datafusion-physical-plan", - "itertools 0.14.0", + "itertools", "log", ] @@ -1112,9 +1323,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "690c615db468c2e5fe5085b232d8b1c088299a6c63d87fd960a354a71f7acb55" dependencies = [ "ahash", - "arrow", - "arrow-ord", - "arrow-schema", + "arrow 55.0.0", + "arrow-ord 55.0.0", + "arrow-schema 55.0.0", "async-trait", "chrono", "datafusion-common", @@ -1128,7 +1339,7 @@ dependencies = [ "half", "hashbrown 0.14.5", "indexmap 2.9.0", - "itertools 0.14.0", + "itertools", "log", "parking_lot", "pin-project-lite", @@ -1141,7 +1352,7 @@ version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ad229a134c7406c057ece00c8743c0c34b97f4e72f78b475fe17b66c5e14fa4f" dependencies = [ - "arrow", + "arrow 55.0.0", "async-trait", "dashmap", "datafusion-common", @@ -1152,7 +1363,7 @@ dependencies = [ "datafusion-physical-plan", "datafusion-sql", "futures", - "itertools 0.14.0", + "itertools", "log", "object_store", "parking_lot", @@ -1165,7 +1376,7 @@ version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "64f6ab28b72b664c21a27b22a2ff815fd390ed224c26e89a93b5a8154a4e8607" dependencies = [ - "arrow", + "arrow 55.0.0", "bigdecimal", "datafusion-common", "datafusion-expr", @@ -1257,12 +1468,6 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6999dc1837253364c2ebb0704ba97994bd874e8f195d665c50b7548f6ea92764" -[[package]] -name = "fixedbitset" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" - [[package]] name = "fixedbitset" version = "0.5.7" @@ -1422,16 +1627,16 @@ checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" [[package]] name = "h2" -version = "0.3.26" +version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81fe527a889e1532da5c525686d96d4c2e74cdd345badf8dfef9f6b39dd5f5e8" +checksum = "17da50a276f1e01e0ba6c029e47b7100754904ee8a278f886546e98575380785" dependencies = [ + "atomic-waker", "bytes", "fnv", "futures-core", "futures-sink", - "futures-util", - "http 0.2.12", + "http", "indexmap 2.9.0", "slab", "tokio", @@ -1474,6 +1679,12 @@ version = "0.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" +[[package]] +name = "hashbrown" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5419bdc4f6a9207fbeba6d11b604d481addf78ecd10c11ad51e76c2f6482748d" + [[package]] name = "heck" version = "0.4.1" @@ -1494,9 +1705,9 @@ checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" [[package]] name = "http" -version = "0.2.12" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" +checksum = "f4a85d31aea989eead29a3aaf9e1115a180df8282431156e533de47660892565" dependencies = [ "bytes", "fnv", @@ -1504,24 +1715,25 @@ dependencies = [ ] [[package]] -name = "http" -version = "1.3.1" +name = "http-body" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4a85d31aea989eead29a3aaf9e1115a180df8282431156e533de47660892565" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ "bytes", - "fnv", - "itoa", + "http", ] [[package]] -name = "http-body" -version = "0.4.6" +name = "http-body-util" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" dependencies = [ "bytes", - "http 0.2.12", + "futures-core", + "http", + "http-body", "pin-project-lite", ] @@ -1545,38 +1757,57 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "0.14.32" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41dfc780fdec9373c01bae43289ea34c972e40ee3c9f6b3c8801a35f35586ce7" +checksum = "cc2b571658e38e0c01b1fdca3bbbe93c00d3d71693ff2770043f8c29bc7d6f80" dependencies = [ "bytes", "futures-channel", - "futures-core", "futures-util", "h2", - "http 0.2.12", + "http", "http-body", "httparse", "httpdate", "itoa", "pin-project-lite", - "socket2", + "smallvec", "tokio", - "tower-service", - "tracing", "want", ] [[package]] name = "hyper-timeout" -version = "0.4.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" +checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" dependencies = [ "hyper", + "hyper-util", "pin-project-lite", "tokio", - "tokio-io-timeout", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f66d5bd4c6f02bf0542fad85d626775bab9258cf795a4256dcaf3161114d1df" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "http", + "http-body", + "hyper", + "libc", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", ] [[package]] @@ -1762,15 +1993,6 @@ dependencies = [ "serde", ] -[[package]] -name = "itertools" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" -dependencies = [ - "either", -] - [[package]] name = "itertools" version = "0.14.0" @@ -2086,9 +2308,9 @@ dependencies = [ "bytes", "chrono", "futures", - "http 1.3.1", + "http", "humantime", - "itertools 0.14.0", + "itertools", "parking_lot", "percent-encoding", "thiserror 2.0.12", @@ -2160,23 +2382,13 @@ version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" -[[package]] -name = "petgraph" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1d3afd2628e69da2be385eb6f2fd57c8ac7977ceeff6dc166ff1657b0e386a9" -dependencies = [ - "fixedbitset 0.4.2", - "indexmap 2.9.0", -] - [[package]] name = "petgraph" version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" dependencies = [ - "fixedbitset 0.5.7", + "fixedbitset", "indexmap 2.9.0", ] @@ -2281,10 +2493,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2981d5b2f34c84069a39fceca0d36dffeb97db8cadba101e7ea6605c8d42294d" dependencies = [ "ahash", - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", + "arrow-array 55.0.0", + "arrow-buffer 55.0.0", + "arrow-data 55.0.0", + "arrow-schema 55.0.0", "atoi_simd", "bytemuck", "chrono", @@ -2379,7 +2591,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "491f5af321169259d5b1294c9fe8ed89faaeac34b4dec4abcedc0d1b3d11013a" dependencies = [ "ahash", - "base64 0.22.1", + "base64", "bytemuck", "ethnum", "hashbrown 0.14.5", @@ -2467,9 +2679,9 @@ dependencies = [ [[package]] name = "prost" -version = "0.12.6" +version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "deb1435c188b76130da55f17a466d252ff7b1418b2ad3e037d127b94e3411f29" +checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" dependencies = [ "bytes", "prost-derive", @@ -2477,17 +2689,16 @@ dependencies = [ [[package]] name = "prost-build" -version = "0.12.6" +version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22505a5c94da8e3b7c2996394d1c933236c4d743e81a410bcca4e6989fc066a4" +checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" dependencies = [ - "bytes", "heck", - "itertools 0.12.1", + "itertools", "log", "multimap", "once_cell", - "petgraph 0.6.4", + "petgraph", "prettyplease", "prost", "prost-types", @@ -2498,12 +2709,12 @@ dependencies = [ [[package]] name = "prost-derive" -version = "0.12.6" +version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81bddcdb20abf9501610992b6759a4c888aef7d1a7247ef75e2404275ac24af1" +checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" dependencies = [ "anyhow", - "itertools 0.12.1", + "itertools", "proc-macro2", "quote", "syn 2.0.100", @@ -2511,9 +2722,9 @@ dependencies = [ [[package]] name = "prost-types" -version = "0.12.6" +version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9091c90b0a32608e984ff2fa4091273cbdd755d54935c51d520887f4a1dbd5b0" +checksum = "52c2c1bf36ddb1a1c396b3601a3cec27c2462e45f07c386894ec3ccf5332bd16" dependencies = [ "prost", ] @@ -2748,11 +2959,12 @@ dependencies = [ [[package]] name = "rustls" -version = "0.22.4" +version = "0.23.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf4ef73721ac7bcd79b2b315da7779d8fc09718c6b3d2d1b2d94850eb8c18432" +checksum = "7160e3e10bf4535308537f3c4e1641468cd0e485175d6163087c0393c7d46643" dependencies = [ "log", + "once_cell", "ring", "rustls-pki-types", "rustls-webpki", @@ -2762,12 +2974,11 @@ dependencies = [ [[package]] name = "rustls-native-certs" -version = "0.7.3" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5bfb394eeed242e909609f56089eecfe5fda225042e8b171791b9c95f5931e5" +checksum = "7fcff2dd52b58a8d98a70243663a0d234c4e2b79235637849d15913394a247d3" dependencies = [ "openssl-probe", - "rustls-pemfile", "rustls-pki-types", "schannel", "security-framework", @@ -2790,9 +3001,9 @@ checksum = "917ce264624a4b4db1c364dcc35bfca9ded014d0a958cd47ad3e960e988ea51c" [[package]] name = "rustls-webpki" -version = "0.102.8" +version = "0.103.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64ca1bc8749bd4cf37b5ce386cc146580777b4e8572c7b97baf22c83f444bee9" +checksum = "e4a72fe2bcf7a6ac6fd7d0b9e5cb68aeb7d4c0a0271730218b3e92d43b4eb435" dependencies = [ "ring", "rustls-pki-types", @@ -2837,9 +3048,9 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "security-framework" -version = "2.11.1" +version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" +checksum = "271720403f46ca04f7ba6f55d438f8bd878d6b8ca0a1046e8228c4145bcbb316" dependencies = [ "bitflags 2.9.0", "core-foundation", @@ -2936,13 +3147,12 @@ dependencies = [ name = "spark-connect-rs" version = "0.0.2" dependencies = [ - "arrow", - "arrow-ipc", + "arrow 56.2.0", + "arrow-ipc 56.2.0", "chrono", "datafusion", "futures", "futures-util", - "http-body", "polars", "polars-arrow", "prost", @@ -3086,6 +3296,12 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" + [[package]] name = "synstructure" version = "0.13.1" @@ -3191,16 +3407,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "tokio-io-timeout" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30b74022ada614a1b4834de765f9bb43877f910cc8ce4be40e89042c9223a8bf" -dependencies = [ - "pin-project-lite", - "tokio", -] - [[package]] name = "tokio-macros" version = "2.5.0" @@ -3214,12 +3420,11 @@ dependencies = [ [[package]] name = "tokio-rustls" -version = "0.25.0" +version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "775e0c0f0adb3a2f22a00c4745d728b479985fc15ee7ca6a2608388c5569860f" +checksum = "8e727b36a1a0e8b74c376ac2211e40c2c8af09fb4013c60d910495810f008e9b" dependencies = [ "rustls", - "rustls-pki-types", "tokio", ] @@ -3250,26 +3455,28 @@ dependencies = [ [[package]] name = "tonic" -version = "0.11.0" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76c4eb7a4e9ef9d4763600161f12f5070b92a578e1b634db88a6887844c91a13" +checksum = "877c5b330756d856ffcc4553ab34a5684481ade925ecc54bcd1bf02b1d0d4d52" dependencies = [ "async-stream", "async-trait", "axum", - "base64 0.21.7", + "base64", "bytes", "h2", - "http 0.2.12", + "http", "http-body", + "http-body-util", "hyper", "hyper-timeout", + "hyper-util", "percent-encoding", "pin-project", "prost", "rustls-native-certs", "rustls-pemfile", - "rustls-pki-types", + "socket2", "tokio", "tokio-rustls", "tokio-stream", @@ -3281,13 +3488,14 @@ dependencies = [ [[package]] name = "tonic-build" -version = "0.11.0" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be4ef6dd70a610078cb4e338a0f79d06bc759ff1b22d2120c2ff02ae264ba9c2" +checksum = "9557ce109ea773b399c9b9e5dca39294110b74f1f342cb347a80d1fce8c26a11" dependencies = [ "prettyplease", "proc-macro2", "prost-build", + "prost-types", "quote", "syn 2.0.100", ] @@ -3318,6 +3526,10 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2873938d487c3cfb9aed7546dc9f2711d867c9f90c46b889989a2cb84eba6b4f" dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper 0.1.2", "tower-layer", "tower-service", ] @@ -3396,9 +3608,9 @@ checksum = "d4c87d22b6e3f4a18d4d40ef354e97c90fcb14dd91d7dc0aa9d8a1172ebf7202" [[package]] name = "unicode-width" -version = "0.1.11" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85" +checksum = "4a1a07cc7db3810833284e8d372ccdc6da29741639ecc70c9ec107df0fa6154c" [[package]] name = "untrusted" diff --git a/Cargo.toml b/Cargo.toml index bae3d64..104bdb6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,23 +32,22 @@ repository = "https://github.com/sjrusso8/spark-connect-rs" rust-version = "1.81" [workspace.dependencies] -tonic = { version ="0.11", default-features = false } +tonic = { version ="0.12", default-features = false } -tokio = { version = "1.44", default-features = false, features = ["macros"] } -tower = { version = "0.5" } +tokio = { version = "1", default-features = false, features = ["macros"] } +tower = { version = "0" } +futures = { version = "0.3" } futures-util = { version = "0.3" } thiserror = { version = "2.0" } -http-body = { version = "0.4.6" } - -arrow = { version = "55", features = ["prettyprint"] } -arrow-ipc = { version = "55" } +arrow = { version = "56", features = ["prettyprint"] } +arrow-ipc = { version = "56" } serde_json = { version = "1" } -prost = { version = "0.12" } -prost-types = { version = "0.12" } +prost = { version = "0.13" } +prost-types = { version = "0.13" } rand = { version = "0.9" } uuid = { version = "1.16", features = ["v4"] } diff --git a/crates/connect/Cargo.toml b/crates/connect/Cargo.toml index 8384506..26a6d0a 100644 --- a/crates/connect/Cargo.toml +++ b/crates/connect/Cargo.toml @@ -40,11 +40,10 @@ tonic = { workspace = true, default-features = false, optional = true } tower = { workspace = true } tokio = { workspace = true, optional = true } +futures = { workspace = true } futures-util = { workspace = true } thiserror = { workspace = true } -http-body = { workspace = true } - arrow = { workspace = true } arrow-ipc = { workspace = true } @@ -70,7 +69,7 @@ futures = "0.3" tokio = { workspace = true, features = ["rt-multi-thread"] } [build-dependencies] -tonic-build = "0.11" +tonic-build = "0.12" [lib] doctest = false diff --git a/crates/connect/src/client/middleware.rs b/crates/connect/src/client/middleware.rs index 92da805..8f48c9f 100644 --- a/crates/connect/src/client/middleware.rs +++ b/crates/connect/src/client/middleware.rs @@ -23,11 +23,10 @@ use std::str::FromStr; use std::task::{Context, Poll}; use futures_util::future::BoxFuture; -use http_body::combinators::UnsyncBoxBody; +use tonic::body::BoxBody; use tonic::codegen::http::Request; use tonic::codegen::http::{HeaderName, HeaderValue}; - use tower::Service; /// Headers to apply a gRPC request @@ -66,13 +65,9 @@ impl HeadersMiddleware { // TODO! as of now Request is not clone. So the retry logic does not work. // https://github.com/tower-rs/tower/pull/790 -impl Service>> for HeadersMiddleware +impl Service> for HeadersMiddleware where - S: Service>> - + Clone - + Send - + Sync - + 'static, + S: Service> + Clone + Send + Sync + 'static, S::Future: Send + 'static, S::Response: Send + Debug + 'static, S::Error: Debug, @@ -85,10 +80,7 @@ where self.inner.poll_ready(cx).map_err(Into::into) } - fn call( - &mut self, - mut request: Request>, - ) -> Self::Future { + fn call(&mut self, mut request: Request) -> Self::Future { let clone = self.inner.clone(); let mut inner = std::mem::replace(&mut self.inner, clone); diff --git a/crates/connect/src/client/mod.rs b/crates/connect/src/client/mod.rs index 7677d14..c14f151 100644 --- a/crates/connect/src/client/mod.rs +++ b/crates/connect/src/client/mod.rs @@ -177,18 +177,25 @@ where } } - pub async fn execute_and_fetch( + async fn execute_and_raw_stream( &mut self, req: spark::ExecutePlanRequest, - ) -> Result<(), SparkError> { + ) -> Result, tonic::Status> { let mut client = self.stub.write().await; - let mut stream = client.execute_plan(req).await?.into_inner(); + let stream = client.execute_plan(req).await?.into_inner(); drop(client); // clear out any prior responses self.handler = ResponseHandler::default(); + Ok(stream) + } + pub async fn execute_and_fetch( + &mut self, + req: spark::ExecutePlanRequest, + ) -> Result<(), SparkError> { + let mut stream = self.execute_and_raw_stream(req).await?; self.process_stream(&mut stream).await?; if self.use_reattachable_execute && self.handler.result_complete { @@ -554,6 +561,18 @@ where )?) } + #[allow(clippy::wrong_self_convention)] + pub async fn to_raw_stream( + &mut self, + plan: spark::Plan, + ) -> Result, SparkError> { + let mut req = self.execute_plan_request_with_metadata(); + + req.plan = Some(plan); + + Ok(self.execute_and_raw_stream(req).await?) + } + #[allow(clippy::wrong_self_convention)] pub(crate) async fn to_first_value(&mut self, plan: spark::Plan) -> Result { let rows = self.to_arrow(plan).await?; diff --git a/crates/connect/src/dataframe.rs b/crates/connect/src/dataframe.rs index a8f4cd8..65dda92 100644 --- a/crates/connect/src/dataframe.rs +++ b/crates/connect/src/dataframe.rs @@ -206,6 +206,14 @@ impl DataFrame { self.spark_session.client().to_arrow(plan).await } + /// Returns a raw stream of `[ExecutePlanResponse]` + pub async fn into_raw_stream( + self, + ) -> Result, SparkError> { + let plan = self.plan.plan_root(); + Ok(self.spark_session.client().to_raw_stream(plan).await?) + } + /// Retrieves the names of all columns in the [DataFrame] as a `Vec`. /// The order of the column names in the list reflects their order in the [DataFrame]. pub async fn columns(self) -> Result, SparkError> { diff --git a/crates/connect/src/session.rs b/crates/connect/src/session.rs index 94184a7..72926dc 100644 --- a/crates/connect/src/session.rs +++ b/crates/connect/src/session.rs @@ -47,6 +47,7 @@ use tonic::transport::Channel; pub struct SparkSessionBuilder { pub channel_builder: ChannelBuilder, configs: HashMap, + use_reattachable_execute: bool, } /// Default connects a Spark cluster running at `sc://127.0.0.1:15002/` @@ -57,6 +58,7 @@ impl Default for SparkSessionBuilder { Self { channel_builder, configs: HashMap::new(), + use_reattachable_execute: true, } } } @@ -68,6 +70,7 @@ impl SparkSessionBuilder { Self { channel_builder, configs: HashMap::new(), + use_reattachable_execute: true, } } @@ -76,6 +79,7 @@ impl SparkSessionBuilder { Self { channel_builder: config.into(), configs: HashMap::new(), + use_reattachable_execute: true, } } @@ -92,6 +96,12 @@ impl SparkSessionBuilder { self } + /// Sets whether or not to use reattachable execute + pub fn use_reattachable_execute(mut self, use_reattachable_execute: bool) -> Self { + self.use_reattachable_execute = use_reattachable_execute; + self + } + /// Sets a name for the application, which will be shown in the Spark web UI. pub fn app_name(mut self, name: &str) -> Self { self.configs @@ -112,14 +122,15 @@ impl SparkSessionBuilder { let client = SparkConnectServiceClient::new(channel); - let spark_connnect_client = + let mut spark_connect_client = SparkConnectClient::new(Arc::new(RwLock::new(client)), self.channel_builder.clone()); + spark_connect_client.use_reattachable_execute = self.use_reattachable_execute; - let mut rt_config = RunTimeConfig::new(&spark_connnect_client); + let mut rt_config = RunTimeConfig::new(&spark_connect_client); rt_config.set_configs(&self.configs).await?; - Ok(SparkSession::new(spark_connnect_client)) + Ok(SparkSession::new(spark_connect_client)) } /// Attempt to connect to a remote Spark Session