From f0c1d57c868e60c719eaf5207c6267c681597eaf Mon Sep 17 00:00:00 2001 From: Vladimir Petrzhikovskii Date: Tue, 27 Oct 2026 21:24:28 +0100 Subject: [PATCH] fix(proto): preserve `case_insensitive` of `SimilarTo` across a round trip `SimilarToNode` had no field for `case_insensitive`, so `datafusion-proto` encoded an `Expr::SimilarTo` without it and decoded it as `false`. The physical planner runs a case-insensitive `SimilarTo` as a case-insensitive regex match, so after a round trip the expression matched case-sensitively. `SimilarToNode` gets `bool case_insensitive = 5`. A payload without the field decodes as `false`, as before. --- .../proto-models/proto/datafusion.proto | 1 + .../proto-models/src/generated/pbjson.rs | 18 ++++++++++++++++++ .../proto-models/src/generated/prost.rs | 2 ++ .../proto/src/logical_plan/from_proto.rs | 2 +- datafusion/proto/src/logical_plan/to_proto.rs | 3 ++- .../tests/cases/roundtrip_logical_plan.rs | 14 ++++++++------ .../library-user-guide/upgrading/56.0.0.md | 19 +++++++++++++++++++ 7 files changed, 51 insertions(+), 8 deletions(-) diff --git a/datafusion/proto-models/proto/datafusion.proto b/datafusion/proto-models/proto/datafusion.proto index 6c4ef8f0a4b2f..d906c18d0981d 100644 --- a/datafusion/proto-models/proto/datafusion.proto +++ b/datafusion/proto-models/proto/datafusion.proto @@ -760,6 +760,7 @@ message SimilarToNode { LogicalExprNode expr = 2; LogicalExprNode pattern = 3; string escape_char = 4; + bool case_insensitive = 5; } message CaseNode { diff --git a/datafusion/proto-models/src/generated/pbjson.rs b/datafusion/proto-models/src/generated/pbjson.rs index 725cceeb75970..131f2e124ab5d 100644 --- a/datafusion/proto-models/src/generated/pbjson.rs +++ b/datafusion/proto-models/src/generated/pbjson.rs @@ -27128,6 +27128,9 @@ impl serde::Serialize for SimilarToNode { if !self.escape_char.is_empty() { len += 1; } + if self.case_insensitive { + len += 1; + } let mut struct_ser = serializer.serialize_struct("datafusion.SimilarToNode", len)?; if self.negated { struct_ser.serialize_field("negated", &self.negated)?; @@ -27141,6 +27144,9 @@ impl serde::Serialize for SimilarToNode { if !self.escape_char.is_empty() { struct_ser.serialize_field("escapeChar", &self.escape_char)?; } + if self.case_insensitive { + struct_ser.serialize_field("caseInsensitive", &self.case_insensitive)?; + } struct_ser.end() } } @@ -27156,6 +27162,8 @@ impl<'de> serde::Deserialize<'de> for SimilarToNode { "pattern", "escape_char", "escapeChar", + "case_insensitive", + "caseInsensitive", ]; #[allow(clippy::enum_variant_names)] @@ -27164,6 +27172,7 @@ impl<'de> serde::Deserialize<'de> for SimilarToNode { Expr, Pattern, EscapeChar, + CaseInsensitive, } impl<'de> serde::Deserialize<'de> for GeneratedField { fn deserialize(deserializer: D) -> std::result::Result @@ -27189,6 +27198,7 @@ impl<'de> serde::Deserialize<'de> for SimilarToNode { "expr" => Ok(GeneratedField::Expr), "pattern" => Ok(GeneratedField::Pattern), "escapeChar" | "escape_char" => Ok(GeneratedField::EscapeChar), + "caseInsensitive" | "case_insensitive" => Ok(GeneratedField::CaseInsensitive), _ => Err(serde::de::Error::unknown_field(value, FIELDS)), } } @@ -27212,6 +27222,7 @@ impl<'de> serde::Deserialize<'de> for SimilarToNode { let mut expr__ = None; let mut pattern__ = None; let mut escape_char__ = None; + let mut case_insensitive__ = None; while let Some(k) = map_.next_key()? { match k { GeneratedField::Negated => { @@ -27238,6 +27249,12 @@ impl<'de> serde::Deserialize<'de> for SimilarToNode { } escape_char__ = Some(map_.next_value()?); } + GeneratedField::CaseInsensitive => { + if case_insensitive__.is_some() { + return Err(serde::de::Error::duplicate_field("caseInsensitive")); + } + case_insensitive__ = Some(map_.next_value()?); + } } } Ok(SimilarToNode { @@ -27245,6 +27262,7 @@ impl<'de> serde::Deserialize<'de> for SimilarToNode { expr: expr__, pattern: pattern__, escape_char: escape_char__.unwrap_or_default(), + case_insensitive: case_insensitive__.unwrap_or_default(), }) } } diff --git a/datafusion/proto-models/src/generated/prost.rs b/datafusion/proto-models/src/generated/prost.rs index 1034d0443dbce..5267b5a67778a 100644 --- a/datafusion/proto-models/src/generated/prost.rs +++ b/datafusion/proto-models/src/generated/prost.rs @@ -1202,6 +1202,8 @@ pub struct SimilarToNode { pub pattern: ::core::option::Option<::prost::alloc::boxed::Box>, #[prost(string, tag = "4")] pub escape_char: ::prost::alloc::string::String, + #[prost(bool, tag = "5")] + pub case_insensitive: bool, } #[derive(Clone, PartialEq, ::prost::Message)] pub struct CaseNode { diff --git a/datafusion/proto/src/logical_plan/from_proto.rs b/datafusion/proto/src/logical_plan/from_proto.rs index 2910ef356e5d1..e717fdeb2cac8 100644 --- a/datafusion/proto/src/logical_plan/from_proto.rs +++ b/datafusion/proto/src/logical_plan/from_proto.rs @@ -404,7 +404,7 @@ pub fn parse_expr( codec, )?), parse_escape_char(&like.escape_char)?, - false, + like.case_insensitive, ))), ExprType::Case(case) => { let when_then_expr = case diff --git a/datafusion/proto/src/logical_plan/to_proto.rs b/datafusion/proto/src/logical_plan/to_proto.rs index f72b07871bb1a..f269e9f6c2453 100644 --- a/datafusion/proto/src/logical_plan/to_proto.rs +++ b/datafusion/proto/src/logical_plan/to_proto.rs @@ -156,13 +156,14 @@ pub fn serialize_expr( expr, pattern, escape_char, - case_insensitive: _, + case_insensitive, }) => { let pb = Box::new(protobuf::SimilarToNode { negated: *negated, expr: Some(Box::new(serialize_expr(expr.as_ref(), codec)?)), pattern: Some(Box::new(serialize_expr(pattern.as_ref(), codec)?)), escape_char: escape_char.map(|ch| ch.to_string()).unwrap_or_default(), + case_insensitive: *case_insensitive, }); protobuf::LogicalExprNode { expr_type: Some(ExprType::SimilarTo(pb)), diff --git a/datafusion/proto/tests/cases/roundtrip_logical_plan.rs b/datafusion/proto/tests/cases/roundtrip_logical_plan.rs index d27aaf27a04f1..73ad10dc093b5 100644 --- a/datafusion/proto/tests/cases/roundtrip_logical_plan.rs +++ b/datafusion/proto/tests/cases/roundtrip_logical_plan.rs @@ -2921,21 +2921,23 @@ fn roundtrip_ilike() { #[test] fn roundtrip_similar_to() { - fn similar_to(negated: bool, escape_char: Option) { + fn similar_to(negated: bool, escape_char: Option, case_insensitive: bool) { let test_expr = Expr::SimilarTo(Like::new( negated, Box::new(col("col")), Box::new(lit("[0-9]+")), escape_char, - false, + case_insensitive, )); let ctx = SessionContext::new(); roundtrip_expr_test(test_expr, ctx); } - similar_to(true, Some('X')); - similar_to(false, Some('\\')); - similar_to(true, None); - similar_to(false, None); + similar_to(true, Some('X'), false); + similar_to(false, Some('\\'), false); + similar_to(true, None, false); + similar_to(false, None, false); + similar_to(false, None, true); + similar_to(true, Some('X'), true); } #[test] diff --git a/docs/source/library-user-guide/upgrading/56.0.0.md b/docs/source/library-user-guide/upgrading/56.0.0.md index ba87076b3de10..924d43897b9cb 100644 --- a/docs/source/library-user-guide/upgrading/56.0.0.md +++ b/docs/source/library-user-guide/upgrading/56.0.0.md @@ -614,6 +614,25 @@ The protobuf wire format remains backward compatible. See [PR #25058](https://github.com/apache/datafusion/pull/25058) for details. +### Generated protobuf `SimilarToNode` gained a `case_insensitive` field + +The generated `SimilarToNode` now carries `case_insensitive`, so an +`Expr::SimilarTo` with `case_insensitive: true` keeps the flag across +serialization. Earlier versions encoded and decoded it as `false`. + +Code constructing `SimilarToNode` with a struct literal must set the field: + +```rust,ignore +// Before +SimilarToNode { negated, expr, pattern, escape_char } + +// After +SimilarToNode { negated, expr, pattern, escape_char, case_insensitive: false } +``` + +The protobuf wire format remains backward compatible: a payload without the +field decodes as `case_insensitive: false`. + ### `DefaultStatisticsProvider` is deprecated `datafusion_physical_plan::operator_statistics::DefaultStatisticsProvider` is