Skip to content

Commit 70b0760

Browse files
dantengskydantengsky
authored
refactor: bump crates arrow* and parquet to version 56 (#18997)
* refactor: bump arrow*/parquet to 56 - upgrade arrow* + parquet crates to v56 - patch iceberg to use arrow 56 as well databendlabs/iceberg-rust#4 - patch orc-rust to use arrow 56 as well datafuse-extras/orc-rust#1 - patch arrow-udf-runtime to use arrow 56 as well datafuse-extras/arrow-udf#1 - bump arrow* from 55 to 56 - bump tonic from 0.12 to 0.13 - bump pyo3 from 0.24.1 to 0.25 - bump pyo3-build-config from 0.24 to 0.25 - bump pyo3-build-config from 0.24.2 to 0.25 - bump pyo3 from 0.24 to 0.25 * fix: tweak logic tests Meta data that marks the parquet writer version has been changed to 'parquet-rs version 56.2.0', thus some cases that expect exact parquet file size need to be tweaked. * fix: tweak logic tests * tweak logic tests * fix: tweak logic tests
1 parent c244e83 commit 70b0760

File tree

24 files changed

+1041
-318
lines changed

24 files changed

+1041
-318
lines changed

‎Cargo.lock‎

Lines changed: 599 additions & 228 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Cargo.toml‎

Lines changed: 23 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -217,18 +217,18 @@ anyerror = { version = "=0.1.13" }
217217
anyhow = { version = "1.0.65" }
218218
apache-avro = { version = "0.17.0", features = ["snappy", "zstandard", "xz", "snappy", "bzip"] }
219219
approx = "0.5.1"
220-
arrow = { version = "55" }
221-
arrow-array = { version = "55" }
222-
arrow-buffer = { version = "55" }
223-
arrow-cast = { version = "55", features = ["prettyprint"] }
224-
arrow-csv = { version = "55" }
225-
arrow-data = { version = "55" }
226-
arrow-flight = { version = "55", features = ["flight-sql-experimental", "tls"] }
227-
arrow-ipc = { version = "55", features = ["lz4", "zstd"] }
228-
arrow-json = { version = "55" }
229-
arrow-ord = { version = "55" }
230-
arrow-schema = { version = "55", features = ["serde"] }
231-
arrow-select = { version = "55" }
220+
arrow = { version = "56" }
221+
arrow-array = { version = "56" }
222+
arrow-buffer = { version = "56" }
223+
arrow-cast = { version = "56", features = ["prettyprint"] }
224+
arrow-csv = { version = "56" }
225+
arrow-data = { version = "56" }
226+
arrow-flight = { version = "56", features = ["flight-sql-experimental", "tls-ring"] }
227+
arrow-ipc = { version = "56", features = ["lz4", "zstd"] }
228+
arrow-json = { version = "56" }
229+
arrow-ord = { version = "56" }
230+
arrow-schema = { version = "56", features = ["serde"] }
231+
arrow-select = { version = "56" }
232232
arrow-udf-runtime = { version = "0.8.0", default-features = false, features = ["javascript", "wasm"] }
233233
async-backtrace = "0.2"
234234
async-channel = "2.3.1"
@@ -330,13 +330,13 @@ hyper-util = { version = "0.1.9", features = ["client", "client-legacy", "tokio"
330330
lru = "0.12"
331331

332332
## in branch dev
333-
iceberg = { version = "0.4.0", git = "https://github.com/databendlabs/iceberg-rust", rev = "1dace26ea25a9b9e2066367cbd3b7badc75dd7f9", features = [
333+
iceberg = { version = "0.4.0", git = "https://github.com/databendlabs/iceberg-rust", rev = "32b1403", features = [
334334
"storage-all",
335335
] }
336-
iceberg-catalog-glue = { version = "0.4.0", git = "https://github.com/databendlabs/iceberg-rust", rev = "1dace26ea25a9b9e2066367cbd3b7badc75dd7f9" }
337-
iceberg-catalog-hms = { version = "0.4.0", git = "https://github.com/databendlabs/iceberg-rust", rev = "1dace26ea25a9b9e2066367cbd3b7badc75dd7f9" }
338-
iceberg-catalog-rest = { version = "0.4.0", git = "https://github.com/databendlabs/iceberg-rust", rev = "1dace26ea25a9b9e2066367cbd3b7badc75dd7f9" }
339-
iceberg-catalog-s3tables = { version = "0.4.0", git = "https://github.com/databendlabs/iceberg-rust", rev = "1dace26ea25a9b9e2066367cbd3b7badc75dd7f9" }
336+
iceberg-catalog-glue = { version = "0.4.0", git = "https://github.com/databendlabs/iceberg-rust", rev = "32b1403" }
337+
iceberg-catalog-hms = { version = "0.4.0", git = "https://github.com/databendlabs/iceberg-rust", rev = "32b1403" }
338+
iceberg-catalog-rest = { version = "0.4.0", git = "https://github.com/databendlabs/iceberg-rust", rev = "32b1403" }
339+
iceberg-catalog-s3tables = { version = "0.4.0", git = "https://github.com/databendlabs/iceberg-rust", rev = "32b1403" }
340340

341341
# Explicitly specify compatible AWS SDK versions
342342
aws-config = "1.5.18"
@@ -415,7 +415,7 @@ ordered-float = { version = "5.1.0", default-features = false }
415415
ordq = "0.2.0"
416416
p256 = "0.13"
417417
parking_lot = "0.12.1"
418-
parquet = { version = "55", features = ["async"] }
418+
parquet = { version = "56", features = ["async"] }
419419
passwords = { version = "3.1.16" }
420420
paste = "1.0.15"
421421
percent-encoding = "2.3.1"
@@ -518,9 +518,9 @@ tokio = { version = "1.35.0", features = ["full"] }
518518
tokio-stream = { version = "0.1.11", features = ["net"] }
519519
tokio-util = { version = "0.7.13" }
520520
toml = { version = "0.8", features = ["parse"] }
521-
tonic = { version = "0.12.3", features = ["transport", "codegen", "prost", "tls-roots", "tls"] }
522-
tonic-build = { version = "0.12.3" }
523-
tonic-reflection = { version = "0.12.3" }
521+
tonic = { version = "0.13", features = ["transport", "codegen", "tls-native-roots"] }
522+
tonic-build = { version = "0.13" }
523+
tonic-reflection = { version = "0.13" }
524524
tower = { version = "0.5.1", features = ["util"] }
525525
tower-service = "0.3.3"
526526
twox-hash = "1.6.3"
@@ -643,15 +643,15 @@ overflow-checks = true
643643
rpath = true
644644

645645
[patch.crates-io]
646-
arrow-udf-runtime = { git = "https://github.com/datafuse-extras/arrow-udf.git", rev = "a442343" }
646+
arrow-udf-runtime = { git = "https://github.com/datafuse-extras/arrow-udf.git", rev = "2480dccf1" }
647647
async-backtrace = { git = "https://github.com/datafuse-extras/async-backtrace.git", rev = "dea4553" }
648648
async-recursion = { git = "https://github.com/datafuse-extras/async-recursion.git", rev = "a353334" }
649649
backtrace = { git = "https://github.com/rust-lang/backtrace-rs.git", rev = "72265be" }
650650
color-eyre = { git = "https://github.com/eyre-rs/eyre.git", rev = "e5d92c3" }
651651
deltalake = { git = "https://github.com/delta-io/delta-rs", rev = "9954bff" }
652652
map-api = { git = "https://github.com/databendlabs/map-api", tag = "v0.4.2" }
653653
openraft = { git = "https://github.com/databendlabs/openraft", tag = "v0.10.0-alpha.11" }
654-
orc-rust = { git = "https://github.com/datafuse-extras/orc-rust", rev = "d82aa6d" }
654+
orc-rust = { git = "https://github.com/datafuse-extras/orc-rust", rev = "fc812ad7010" }
655655
recursive = { git = "https://github.com/datafuse-extras/recursive.git", rev = "16e433a" }
656656
sled = { git = "https://github.com/datafuse-extras/sled", tag = "v0.34.7-datafuse.1" }
657657
state-machine-api = { git = "https://github.com/databendlabs/state-machine-api.git", tag = "v0.3.4" }

‎src/bendpy/Cargo.toml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ publish = { workspace = true }
77
edition = { workspace = true }
88

99
[build-dependencies]
10-
pyo3-build-config = "0.24.2"
10+
pyo3-build-config = "0.25"
1111

1212
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
1313
[lib]
@@ -32,7 +32,7 @@ databend-query = { workspace = true, features = [
3232
"simd",
3333
"disable_initial_exec_tls",
3434
] }
35-
pyo3 = { version = "0.24", features = ["generate-import-lib", "abi3-py312"] }
35+
pyo3 = { version = "0.25", features = ["generate-import-lib", "abi3-py312"] }
3636
serde_json = { workspace = true }
3737
sysinfo = { workspace = true }
3838
tokio = { workspace = true, features = ["macros", "rt", "rt-multi-thread", "sync"] }

‎src/common/cloud_control/Cargo.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ hyper-util = { workspace = true }
1616
prost = { workspace = true }
1717
serde = { workspace = true }
1818
tonic = { workspace = true }
19+
#tonic-prost = { workspace = true }
1920

2021
[build-dependencies]
2122
lenient_semver = { workspace = true }

‎src/query/service/src/servers/flight/flight_service.rs‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -79,8 +79,10 @@ impl FlightService {
7979
builder
8080
};
8181

82-
let incoming = TcpIncoming::new(addr, true, None)
83-
.map_err(|e| ErrorCode::CannotListenerPort(format!("{},{}", e, addr)))?;
82+
let incoming = TcpIncoming::bind(addr)
83+
.map_err(|e| ErrorCode::CannotListenerPort(format!("{},{}", e, addr)))?
84+
.with_nodelay(Some(true))
85+
.with_keepalive(None);
8486
let server = builder
8587
.add_service(
8688
FlightServiceServer::new(flight_api_service)

‎src/query/service/src/servers/flight_sql/flight_sql_server.rs‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -86,8 +86,10 @@ impl FlightSQLServer {
8686
builder
8787
};
8888

89-
let incoming = TcpIncoming::new(addr, true, None)
90-
.map_err(|e| ErrorCode::CannotListenerPort(format!("{},{}", e, addr)))?;
89+
let incoming = TcpIncoming::bind(addr)
90+
.map_err(|e| ErrorCode::CannotListenerPort(format!("{},{}", e, addr)))?
91+
.with_nodelay(Some(true))
92+
.with_keepalive(None);
9193

9294
let server = builder
9395
.add_service(FlightServiceServer::new(flight_sql_service))

‎src/query/service/tests/it/parquet_rs/prune_row_groups.rs‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ use databend_common_catalog::plan::ParquetReadOptions;
1919
use databend_common_expression::FunctionContext;
2020
use databend_common_expression::TableSchema;
2121
use databend_common_storages_parquet::ParquetPruner;
22+
use parquet::file::metadata::ParquetMetaDataReader;
2223

2324
use super::data::make_test_file_rg;
2425
use super::data::Scenario;
@@ -46,8 +47,9 @@ async fn test_impl_batch(args: &[(Scenario, &str, Vec<usize>)], prune: bool) {
4647
let plan = get_data_source_plan(fixture.new_query_ctx().await.unwrap(), &sql)
4748
.await
4849
.unwrap();
49-
#[allow(deprecated)]
50-
let parquet_meta = parquet::file::footer::parse_metadata(file.as_file()).unwrap();
50+
let parquet_meta = ParquetMetaDataReader::new()
51+
.parse_and_finish(file.as_file())
52+
.unwrap();
5153
let schema = TableSchema::try_from(arrow_schema.as_ref()).unwrap();
5254
let leaf_fields = Arc::new(schema.leaf_fields());
5355

‎src/query/storages/delta/Cargo.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ async-backtrace = { workspace = true }
2424
async-trait = { workspace = true }
2525
deltalake = { workspace = true }
2626
fastrace = { workspace = true }
27+
itertools = { workspace = true }
2728
object_store_opendal = { workspace = true }
2829
parquet = { workspace = true }
2930
serde = { workspace = true }

0 commit comments

Comments
 (0)