diff --git a/datafusion/datasource-arrow/src/source.rs b/datafusion/datasource-arrow/src/source.rs index 30af5a8bee8ab..8cd7e3ff596b4 100644 --- a/datafusion/datasource-arrow/src/source.rs +++ b/datafusion/datasource-arrow/src/source.rs @@ -408,16 +408,15 @@ impl FileSource for ArrowSource { datafusion_physical_plan::apply_expression_roots(self.projection.source.iter(), f) } - /// Emit an `ArrowScan` node wrapping the shared base config and recording - /// which Arrow IPC format (file or stream) this source reads. + /// Emit an `ArrowSourceNode` recording which Arrow IPC format (file or + /// stream) this source reads. #[cfg(feature = "proto")] - fn try_to_proto( + fn try_to_proto_file_source( &self, - base: &FileScanConfig, - ctx: &datafusion_physical_plan::proto::ExecutionPlanEncodeCtx<'_>, - ) -> Result> { + _ctx: &datafusion_physical_plan::proto::ExecutionPlanEncodeCtx<'_>, + ) -> Result> { use datafusion_proto_models::protobuf; - use protobuf::physical_plan_node::PhysicalPlanType; + use protobuf::file_source_node::FileSourceType; // Exhaustive destructure: adding a field to `ArrowSource` without // deciding how it is serialized is a compile error, not a silent @@ -437,23 +436,18 @@ impl FileSource for ArrowSource { ArrowFormat::Stream => protobuf::ArrowIpcFormat::Stream, }; - Ok(Some(protobuf::PhysicalPlanNode { - physical_plan_type: Some(PhysicalPlanType::ArrowScan( - protobuf::ArrowScanExecNode { - base_conf: Some(base.try_to_proto(ctx)?), - format: format as i32, - }, - )), + Ok(Some(protobuf::FileSourceNode { + file_source_type: Some(FileSourceType::Arrow(protobuf::ArrowSourceNode { + format: format as i32, + })), })) } } #[cfg(feature = "proto")] impl ArrowSource { - /// Reconstructs a `DataSourceExec` from a protobuf `ArrowScan`. - /// - /// Payloads encoded before the `format` field existed leave it unset; - /// those decode as the IPC file format, matching the historical behavior. + /// Reconstructs a `DataSourceExec` from a legacy protobuf `ArrowScan`. + #[deprecated(since = "56.0.0", note = "use `try_from_proto_file_source` instead")] pub fn try_from_proto( node: &datafusion_proto_models::protobuf::PhysicalPlanNode, ctx: &datafusion_physical_plan::proto::ExecutionPlanDecodeCtx<'_>, @@ -469,15 +463,31 @@ impl ArrowSource { "PhysicalPlanNode is not an ArrowScan" ); }; - - let protobuf::ArrowScanExecNode { base_conf, format } = scan; - - let base_conf = base_conf.as_ref().ok_or_else(|| { + let base_conf = scan.base_conf.as_ref().ok_or_else(|| { datafusion_common::internal_datafusion_err!( "ArrowScanExecNode is missing required field 'base_conf'" ) })?; + let source_node = protobuf::ArrowSourceNode::from(scan); + let source = Self::try_from_proto_file_source(&source_node, base_conf, ctx)?; + let scan_conf = FileScanConfig::try_from_proto(base_conf, ctx, source)?; + Ok(DataSourceExec::from_data_source(scan_conf)) + } + + /// Reconstructs an `ArrowSource` from its protobuf representation. `base` + /// is the enclosing scan's shared config, used for the table schema. + /// + /// Payloads encoded before the `format` field existed leave it unset; + /// those decode as the IPC file format, matching the historical behavior. + pub fn try_from_proto_file_source( + node: &datafusion_proto_models::protobuf::ArrowSourceNode, + base: &datafusion_proto_models::protobuf::FileScanExecConf, + _ctx: &datafusion_physical_plan::proto::ExecutionPlanDecodeCtx<'_>, + ) -> Result> { + use datafusion_datasource::file_scan_config::FileScanConfig; + use datafusion_proto_models::protobuf; + let protobuf::ArrowSourceNode { format } = node; let format = protobuf::ArrowIpcFormat::try_from(*format).map_err(|_| { datafusion_common::internal_datafusion_err!( "Unknown ArrowIpcFormat: {}", @@ -485,15 +495,13 @@ impl ArrowSource { ) })?; - let table_schema = FileScanConfig::parse_table_schema_from_proto(base_conf)?; - let source = match format { + let table_schema = FileScanConfig::parse_table_schema_from_proto(base)?; + Ok(Arc::new(match format { protobuf::ArrowIpcFormat::Stream => { ArrowSource::new_stream_file_source(table_schema) } protobuf::ArrowIpcFormat::File => ArrowSource::new_file_source(table_schema), - }; - let scan_conf = FileScanConfig::try_from_proto(base_conf, ctx, Arc::new(source))?; - Ok(DataSourceExec::from_data_source(scan_conf)) + })) } } diff --git a/datafusion/datasource-avro/src/source.rs b/datafusion/datasource-avro/src/source.rs index e42a97e80c721..244d691a6d296 100644 --- a/datafusion/datasource-avro/src/source.rs +++ b/datafusion/datasource-avro/src/source.rs @@ -179,15 +179,14 @@ impl FileSource for AvroSource { datafusion_physical_plan::apply_expression_roots(self.projection.source.iter(), f) } - /// Emit an `AvroScan` node wrapping the shared base config. + /// Emit an (empty) `AvroSourceNode`. #[cfg(feature = "proto")] - fn try_to_proto( + fn try_to_proto_file_source( &self, - base: &FileScanConfig, - ctx: &datafusion_physical_plan::proto::ExecutionPlanEncodeCtx<'_>, - ) -> Result> { + _ctx: &datafusion_physical_plan::proto::ExecutionPlanEncodeCtx<'_>, + ) -> Result> { use datafusion_proto_models::protobuf; - use protobuf::physical_plan_node::PhysicalPlanType; + use protobuf::file_source_node::FileSourceType; // Exhaustive destructure: adding a field to `AvroSource` without // deciding how it is serialized is a compile error, not a silent @@ -203,18 +202,16 @@ impl FileSource for AvroSource { metrics: _, } = self; - let node = protobuf::AvroScanExecNode { - base_conf: Some(base.try_to_proto(ctx)?), - }; - Ok(Some(protobuf::PhysicalPlanNode { - physical_plan_type: Some(PhysicalPlanType::AvroScan(node)), + Ok(Some(protobuf::FileSourceNode { + file_source_type: Some(FileSourceType::Avro(protobuf::AvroSourceNode {})), })) } } #[cfg(feature = "proto")] impl AvroSource { - /// Reconstructs a `DataSourceExec` from a protobuf `AvroScan`. + /// Reconstructs a `DataSourceExec` from a legacy protobuf `AvroScan`. + #[deprecated(since = "56.0.0", note = "use `try_from_proto_file_source` instead")] pub fn try_from_proto( node: &datafusion_proto_models::protobuf::PhysicalPlanNode, ctx: &datafusion_physical_plan::proto::ExecutionPlanDecodeCtx<'_>, @@ -229,21 +226,31 @@ impl AvroSource { "PhysicalPlanNode is not an AvroScan" ); }; - - let protobuf::AvroScanExecNode { base_conf } = scan; - - let base_conf = base_conf.as_ref().ok_or_else(|| { + let base_conf = scan.base_conf.as_ref().ok_or_else(|| { datafusion_common::internal_datafusion_err!( "AvroScanExecNode is missing required field 'base_conf'" ) })?; - - let table_schema = FileScanConfig::parse_table_schema_from_proto(base_conf)?; - let source = Arc::new(AvroSource::new(table_schema)); - + let source = Self::try_from_proto_file_source( + &protobuf::AvroSourceNode::from(scan), + base_conf, + ctx, + )?; let conf = FileScanConfig::try_from_proto(base_conf, ctx, source)?; Ok(DataSourceExec::from_data_source(conf)) } + + /// Reconstructs an `AvroSource` from its protobuf representation. `base` + /// is the enclosing scan's shared config, used for the table schema. + pub fn try_from_proto_file_source( + node: &datafusion_proto_models::protobuf::AvroSourceNode, + base: &datafusion_proto_models::protobuf::FileScanExecConf, + _ctx: &datafusion_physical_plan::proto::ExecutionPlanDecodeCtx<'_>, + ) -> Result> { + let datafusion_proto_models::protobuf::AvroSourceNode {} = node; + let table_schema = FileScanConfig::parse_table_schema_from_proto(base)?; + Ok(Arc::new(AvroSource::new(table_schema))) + } } mod private { diff --git a/datafusion/datasource-csv/src/file_format.rs b/datafusion/datasource-csv/src/file_format.rs index 15e24df4bf7c7..6e68b748c1df8 100644 --- a/datafusion/datasource-csv/src/file_format.rs +++ b/datafusion/datasource-csv/src/file_format.rs @@ -853,13 +853,11 @@ impl DataSink for CsvSink { } #[cfg(feature = "proto")] - fn try_to_proto( + fn try_to_proto_sink( &self, - exec: &DataSinkExec, - ctx: &datafusion_physical_plan::proto::ExecutionPlanEncodeCtx<'_>, - ) -> Result> { + _ctx: &datafusion_physical_plan::proto::ExecutionPlanEncodeCtx<'_>, + ) -> Result> { use datafusion_proto_models::protobuf; - use protobuf::physical_plan_node::PhysicalPlanType; // Keep an exhaustive guard in the active hook while centralizing the // field mapping in the exhaustive `TryFrom<&CsvSink>` below. @@ -868,17 +866,10 @@ impl DataSink for CsvSink { writer_options: _, } = self; - let input = ctx.encode_child(exec.input())?; - let sort_order = exec.encode_sort_order(ctx)?; - let sink = protobuf::CsvSink::try_from(self)?; - let node = protobuf::CsvSinkExecNode { - input: Some(Box::new(input)), - sink: Some(sink), - sink_schema: Some(exec.schema().as_ref().try_into()?), - sort_order, - }; - Ok(Some(protobuf::PhysicalPlanNode { - physical_plan_type: Some(PhysicalPlanType::CsvSink(Box::new(node))), + Ok(Some(protobuf::DataSinkNode { + sink_type: Some(protobuf::data_sink_node::SinkType::Csv( + protobuf::CsvSink::try_from(self)?, + )), })) } } @@ -928,7 +919,12 @@ impl TryFrom<&datafusion_proto_models::protobuf::CsvSink> for CsvSink { #[cfg(feature = "proto")] impl CsvSink { - /// Reconstructs a [`DataSinkExec`] containing a `CsvSink` from protobuf. + /// Reconstructs a [`DataSinkExec`] containing a `CsvSink` from a legacy + /// protobuf plan node. + #[deprecated( + since = "56.0.0", + note = "decode `DataSinkNode` with `TryFrom<&protobuf::CsvSink>` instead" + )] pub fn try_from_proto( node: &datafusion_proto_models::protobuf::PhysicalPlanNode, ctx: &datafusion_physical_plan::proto::ExecutionPlanDecodeCtx<'_>, diff --git a/datafusion/datasource-csv/src/source.rs b/datafusion/datasource-csv/src/source.rs index 3288f0e1d08be..16766de8a3c6b 100644 --- a/datafusion/datasource-csv/src/source.rs +++ b/datafusion/datasource-csv/src/source.rs @@ -335,18 +335,16 @@ impl FileSource for CsvSource { datafusion_physical_plan::apply_expression_roots(self.projection.source.iter(), f) } - /// Emit a `CsvScan` node wrapping the shared base config and CSV options. + /// Emit a `CsvSourceNode` carrying the CSV options. #[cfg(feature = "proto")] - fn try_to_proto( + fn try_to_proto_file_source( &self, - base: &FileScanConfig, - ctx: &datafusion_physical_plan::proto::ExecutionPlanEncodeCtx<'_>, - ) -> Result> { + _ctx: &datafusion_physical_plan::proto::ExecutionPlanEncodeCtx<'_>, + ) -> Result> { use datafusion_proto_models::protobuf; - use protobuf::physical_plan_node::PhysicalPlanType; + use protobuf::file_source_node::FileSourceType; - let node = protobuf::CsvScanExecNode { - base_conf: Some(base.try_to_proto(ctx)?), + let node = protobuf::CsvSourceNode { has_header: self.has_header(), delimiter: proto_byte_to_string(self.delimiter(), "delimiter")?, quote: proto_byte_to_string(self.quote(), "quote")?, @@ -354,7 +352,7 @@ impl FileSource for CsvSource { .escape() .map(|escape| { Ok::<_, DataFusionError>( - protobuf::csv_scan_exec_node::OptionalEscape::Escape( + protobuf::csv_source_node::OptionalEscape::Escape( proto_byte_to_string(escape, "escape")?, ), ) @@ -364,7 +362,7 @@ impl FileSource for CsvSource { .comment() .map(|comment| { Ok::<_, DataFusionError>( - protobuf::csv_scan_exec_node::OptionalComment::Comment( + protobuf::csv_source_node::OptionalComment::Comment( proto_byte_to_string(comment, "comment")?, ), ) @@ -374,8 +372,8 @@ impl FileSource for CsvSource { truncate_rows: self.truncate_rows(), terminator: self.terminator().map(|terminator| vec![terminator]), }; - Ok(Some(protobuf::PhysicalPlanNode { - physical_plan_type: Some(PhysicalPlanType::CsvScan(node)), + Ok(Some(protobuf::FileSourceNode { + file_source_type: Some(FileSourceType::Csv(node)), })) } } @@ -606,14 +604,12 @@ fn proto_bytes_to_byte(bytes: &[u8], description: &str) -> Result { #[cfg(feature = "proto")] impl CsvSource { - /// Reconstructs a `DataSourceExec` from a protobuf `CsvScan`. - /// - /// Payloads without a terminator use the default newline terminator. + /// Reconstructs a `DataSourceExec` from a legacy protobuf `CsvScan`. + #[deprecated(since = "56.0.0", note = "use `try_from_proto_file_source` instead")] pub fn try_from_proto( node: &datafusion_proto_models::protobuf::PhysicalPlanNode, ctx: &datafusion_physical_plan::proto::ExecutionPlanDecodeCtx<'_>, ) -> Result> { - use datafusion_common::config::CsvOptions; use datafusion_datasource::file_scan_config::FileScanConfig; use datafusion_datasource::source::DataSourceExec; use datafusion_proto_models::protobuf; @@ -623,51 +619,65 @@ impl CsvSource { else { return datafusion_common::internal_err!("PhysicalPlanNode is not a CsvScan"); }; - let base_conf = scan.base_conf.as_ref().ok_or_else(|| { datafusion_common::internal_datafusion_err!( "CsvScanExecNode is missing required field 'base_conf'" ) })?; + let source_node = protobuf::CsvSourceNode::from(scan); + let source = Self::try_from_proto_file_source(&source_node, base_conf, ctx)?; + let conf = FileScanConfig::try_from_proto(base_conf, ctx, source)?; + Ok(DataSourceExec::from_data_source(conf)) + } - let escape = match &scan.optional_escape { - Some(protobuf::csv_scan_exec_node::OptionalEscape::Escape(escape)) => { + /// Reconstructs a `CsvSource` from its protobuf representation. `base` + /// is the enclosing scan's shared config, used for the table schema. + /// + /// Payloads without a terminator use the default newline terminator. + pub fn try_from_proto_file_source( + node: &datafusion_proto_models::protobuf::CsvSourceNode, + base: &datafusion_proto_models::protobuf::FileScanExecConf, + _ctx: &datafusion_physical_plan::proto::ExecutionPlanDecodeCtx<'_>, + ) -> Result> { + use datafusion_common::config::CsvOptions; + use datafusion_datasource::file_scan_config::FileScanConfig; + use datafusion_proto_models::protobuf; + + let escape = match &node.optional_escape { + Some(protobuf::csv_source_node::OptionalEscape::Escape(escape)) => { Some(proto_str_to_byte(escape, "escape")?) } None => None, }; - let comment = match &scan.optional_comment { - Some(protobuf::csv_scan_exec_node::OptionalComment::Comment(comment)) => { + let comment = match &node.optional_comment { + Some(protobuf::csv_source_node::OptionalComment::Comment(comment)) => { Some(proto_str_to_byte(comment, "comment")?) } None => None, }; - let terminator = scan + let terminator = node .terminator .as_deref() .map(|terminator| proto_bytes_to_byte(terminator, "terminator")) .transpose()?; - let table_schema = FileScanConfig::parse_table_schema_from_proto(base_conf)?; + let table_schema = FileScanConfig::parse_table_schema_from_proto(base)?; let csv_options = CsvOptions { - has_header: Some(scan.has_header), - delimiter: proto_str_to_byte(&scan.delimiter, "delimiter")?, - quote: proto_str_to_byte(&scan.quote, "quote")?, - newlines_in_values: Some(scan.newlines_in_values), - truncated_rows: Some(scan.truncate_rows), + has_header: Some(node.has_header), + delimiter: proto_str_to_byte(&node.delimiter, "delimiter")?, + quote: proto_str_to_byte(&node.quote, "quote")?, + newlines_in_values: Some(node.newlines_in_values), + truncated_rows: Some(node.truncate_rows), ..Default::default() }; - let source = Arc::new( + Ok(Arc::new( CsvSource::new(table_schema) .with_csv_options(csv_options) .with_escape(escape) .with_comment(comment) .with_terminator(terminator), - ); - - let conf = FileScanConfig::try_from_proto(base_conf, ctx, source)?; - Ok(DataSourceExec::from_data_source(conf)) + )) } } diff --git a/datafusion/datasource-json/src/file_format.rs b/datafusion/datasource-json/src/file_format.rs index aef89560a3b5f..7a3609072451f 100644 --- a/datafusion/datasource-json/src/file_format.rs +++ b/datafusion/datasource-json/src/file_format.rs @@ -492,13 +492,11 @@ impl DataSink for JsonSink { } #[cfg(feature = "proto")] - fn try_to_proto( + fn try_to_proto_sink( &self, - exec: &DataSinkExec, - ctx: &datafusion_physical_plan::proto::ExecutionPlanEncodeCtx<'_>, - ) -> Result> { + _ctx: &datafusion_physical_plan::proto::ExecutionPlanEncodeCtx<'_>, + ) -> Result> { use datafusion_proto_models::protobuf; - use protobuf::physical_plan_node::PhysicalPlanType; // Keep an exhaustive guard in the active hook while centralizing the // field mapping in the exhaustive `TryFrom<&JsonSink>` below. @@ -507,17 +505,10 @@ impl DataSink for JsonSink { writer_options: _, } = self; - let input = ctx.encode_child(exec.input())?; - let sort_order = exec.encode_sort_order(ctx)?; - let sink = protobuf::JsonSink::try_from(self)?; - let node = protobuf::JsonSinkExecNode { - input: Some(Box::new(input)), - sink: Some(sink), - sink_schema: Some(exec.schema().as_ref().try_into()?), - sort_order, - }; - Ok(Some(protobuf::PhysicalPlanNode { - physical_plan_type: Some(PhysicalPlanType::JsonSink(Box::new(node))), + Ok(Some(protobuf::DataSinkNode { + sink_type: Some(protobuf::data_sink_node::SinkType::Json( + protobuf::JsonSink::try_from(self)?, + )), })) } } @@ -569,7 +560,12 @@ impl TryFrom<&datafusion_proto_models::protobuf::JsonSink> for JsonSink { #[cfg(feature = "proto")] impl JsonSink { - /// Reconstructs a [`DataSinkExec`] containing a `JsonSink` from protobuf. + /// Reconstructs a [`DataSinkExec`] containing a `JsonSink` from a legacy + /// protobuf plan node. + #[deprecated( + since = "56.0.0", + note = "decode `DataSinkNode` with `TryFrom<&protobuf::JsonSink>` instead" + )] pub fn try_from_proto( node: &datafusion_proto_models::protobuf::PhysicalPlanNode, ctx: &datafusion_physical_plan::proto::ExecutionPlanDecodeCtx<'_>, diff --git a/datafusion/datasource-json/src/source.rs b/datafusion/datasource-json/src/source.rs index f4c7021c86bdd..9724a2ad3860e 100644 --- a/datafusion/datasource-json/src/source.rs +++ b/datafusion/datasource-json/src/source.rs @@ -247,15 +247,14 @@ impl FileSource for JsonSource { datafusion_physical_plan::apply_expression_roots(self.projection.source.iter(), f) } - /// Emit a `JsonScan` node wrapping the shared base config. + /// Emit a `JsonSourceNode` carrying the JSON options. #[cfg(feature = "proto")] - fn try_to_proto( + fn try_to_proto_file_source( &self, - base: &FileScanConfig, - ctx: &datafusion_physical_plan::proto::ExecutionPlanEncodeCtx<'_>, - ) -> Result> { + _ctx: &datafusion_physical_plan::proto::ExecutionPlanEncodeCtx<'_>, + ) -> Result> { use datafusion_proto_models::protobuf; - use protobuf::physical_plan_node::PhysicalPlanType; + use protobuf::file_source_node::FileSourceType; // Exhaustive destructure: adding a field to `JsonSource` without // deciding how it is serialized is a compile error, not a silent @@ -272,25 +271,23 @@ impl FileSource for JsonSource { newline_delimited, } = self; - let node = protobuf::JsonScanExecNode { - base_conf: Some(base.try_to_proto(ctx)?), + let node = protobuf::JsonSourceNode { newline_delimited: if *newline_delimited { None } else { Some(false) }, }; - Ok(Some(protobuf::PhysicalPlanNode { - physical_plan_type: Some(PhysicalPlanType::JsonScan(node)), + Ok(Some(protobuf::FileSourceNode { + file_source_type: Some(FileSourceType::Json(node)), })) } } #[cfg(feature = "proto")] impl JsonSource { - /// Reconstructs a `DataSourceExec` from a protobuf `JsonScan`. - /// - /// Payloads without a mode default to newline-delimited JSON. + /// Reconstructs a `DataSourceExec` from a legacy protobuf `JsonScan`. + #[deprecated(since = "56.0.0", note = "use `try_from_proto_file_source` instead")] pub fn try_from_proto( node: &datafusion_proto_models::protobuf::PhysicalPlanNode, ctx: &datafusion_physical_plan::proto::ExecutionPlanDecodeCtx<'_>, @@ -306,28 +303,35 @@ impl JsonSource { "PhysicalPlanNode is not a JsonScan" ); }; - - // Exhaustive destructure: a new field on `JsonScanExecNode` is a - // compile error here rather than a silently ignored wire field. - let protobuf::JsonScanExecNode { - base_conf, - newline_delimited, - } = scan; - - let base_conf = base_conf.as_ref().ok_or_else(|| { + let base_conf = scan.base_conf.as_ref().ok_or_else(|| { datafusion_common::internal_datafusion_err!( "JsonScanExecNode is missing required field 'base_conf'" ) })?; + let source_node = protobuf::JsonSourceNode::from(scan); + let source = Self::try_from_proto_file_source(&source_node, base_conf, ctx)?; + let conf = FileScanConfig::try_from_proto(base_conf, ctx, source)?; + Ok(DataSourceExec::from_data_source(conf)) + } - let table_schema = FileScanConfig::parse_table_schema_from_proto(base_conf)?; - let source = Arc::new( + /// Reconstructs a `JsonSource` from its protobuf representation. `base` + /// is the enclosing scan's shared config, used for the table schema. + /// + /// Payloads without a mode default to newline-delimited JSON. + pub fn try_from_proto_file_source( + node: &datafusion_proto_models::protobuf::JsonSourceNode, + base: &datafusion_proto_models::protobuf::FileScanExecConf, + _ctx: &datafusion_physical_plan::proto::ExecutionPlanDecodeCtx<'_>, + ) -> Result> { + use datafusion_datasource::file_scan_config::FileScanConfig; + + let datafusion_proto_models::protobuf::JsonSourceNode { newline_delimited } = + node; + let table_schema = FileScanConfig::parse_table_schema_from_proto(base)?; + Ok(Arc::new( JsonSource::new(table_schema) .with_newline_delimited(newline_delimited.unwrap_or(true)), - ); - - let conf = FileScanConfig::try_from_proto(base_conf, ctx, source)?; - Ok(DataSourceExec::from_data_source(conf)) + )) } } diff --git a/datafusion/datasource-parquet/src/sink.rs b/datafusion/datasource-parquet/src/sink.rs index 725b4c74c93f4..805ab5573bf65 100644 --- a/datafusion/datasource-parquet/src/sink.rs +++ b/datafusion/datasource-parquet/src/sink.rs @@ -423,13 +423,11 @@ impl DataSink for ParquetSink { } #[cfg(feature = "proto")] - fn try_to_proto( + fn try_to_proto_sink( &self, - exec: &DataSinkExec, - ctx: &datafusion_physical_plan::proto::ExecutionPlanEncodeCtx<'_>, - ) -> Result> { + _ctx: &datafusion_physical_plan::proto::ExecutionPlanEncodeCtx<'_>, + ) -> Result> { use datafusion_proto_models::protobuf; - use protobuf::physical_plan_node::PhysicalPlanType; // Keep the active hook exhaustive while centralizing field mapping in // the exhaustive `TryFrom<&ParquetSink>` below. @@ -443,17 +441,10 @@ impl DataSink for ParquetSink { metrics: _, } = self; - let input = ctx.encode_child(exec.input())?; - let sort_order = exec.encode_sort_order(ctx)?; - let sink = protobuf::ParquetSink::try_from(self)?; - let node = protobuf::ParquetSinkExecNode { - input: Some(Box::new(input)), - sink: Some(sink), - sink_schema: Some(exec.schema().as_ref().try_into()?), - sort_order, - }; - Ok(Some(protobuf::PhysicalPlanNode { - physical_plan_type: Some(PhysicalPlanType::ParquetSink(Box::new(node))), + Ok(Some(protobuf::DataSinkNode { + sink_type: Some(protobuf::data_sink_node::SinkType::Parquet( + protobuf::ParquetSink::try_from(self)?, + )), })) } } @@ -554,7 +545,12 @@ impl TryFrom<&datafusion_proto_models::protobuf::ParquetSink> for ParquetSink { #[cfg(feature = "proto")] impl ParquetSink { - /// Reconstructs a [`DataSinkExec`] containing a `ParquetSink` from protobuf. + /// Reconstructs a [`DataSinkExec`] containing a `ParquetSink` from a legacy + /// protobuf plan node. + #[deprecated( + since = "56.0.0", + note = "decode `DataSinkNode` with `TryFrom<&protobuf::ParquetSink>` instead" + )] pub fn try_from_proto( node: &datafusion_proto_models::protobuf::PhysicalPlanNode, ctx: &datafusion_physical_plan::proto::ExecutionPlanDecodeCtx<'_>, diff --git a/datafusion/datasource-parquet/src/source.rs b/datafusion/datasource-parquet/src/source.rs index fbb6265847afc..77df3ee8b8d87 100644 --- a/datafusion/datasource-parquet/src/source.rs +++ b/datafusion/datasource-parquet/src/source.rs @@ -1081,18 +1081,17 @@ impl FileSource for ParquetSource { ) } - /// Emit a `ParquetScan` node wrapping the shared base config plus the - /// Parquet-specific pushdown predicate and `TableParquetOptions`. + /// Emit a `ParquetSourceNode` carrying the Parquet-specific pushdown + /// predicate and `TableParquetOptions`. #[cfg(feature = "proto")] - fn try_to_proto( + fn try_to_proto_file_source( &self, - base: &FileScanConfig, ctx: &datafusion_physical_plan::proto::ExecutionPlanEncodeCtx<'_>, ) -> datafusion_common::Result< - Option, + Option, > { use datafusion_proto_models::protobuf; - use protobuf::physical_plan_node::PhysicalPlanType; + use protobuf::file_source_node::FileSourceType; let Self { table_parquet_options, @@ -1140,37 +1139,29 @@ impl FileSource for ParquetSource { .map(|hint| usize_to_wire(hint, "ParquetSource", "metadata_size_hint")) .transpose()?; - let node = protobuf::ParquetScanExecNode { - base_conf: Some(base.try_to_proto(ctx)?), + let node = protobuf::ParquetSourceNode { predicate, parquet_options: Some(table_parquet_options.try_into()?), sort_order_for_reorder, reverse_row_groups: *reverse_row_groups, metadata_size_hint, }; - Ok(Some(protobuf::PhysicalPlanNode { - physical_plan_type: Some(PhysicalPlanType::ParquetScan(node)), + Ok(Some(protobuf::FileSourceNode { + file_source_type: Some(FileSourceType::Parquet(node)), })) } } #[cfg(feature = "proto")] impl ParquetSource { - /// Reconstructs a `DataSourceExec` from a protobuf `ParquetScan`. - /// - /// Rebuilds the reader factory from the decode context because it is not serialized. - /// Encryption factories and crypto options are not serialized or restored; - /// plans that rely on them require custom handling. + /// Reconstructs a `DataSourceExec` from a legacy protobuf `ParquetScan`. + #[deprecated(since = "56.0.0", note = "use `try_from_proto_file_source` instead")] pub fn try_from_proto( node: &datafusion_proto_models::protobuf::PhysicalPlanNode, ctx: &datafusion_physical_plan::proto::ExecutionPlanDecodeCtx<'_>, ) -> datafusion_common::Result> { - use crate::CachedParquetFileReaderFactory; - use arrow::datatypes::Schema; - use datafusion_common::config::TableParquetOptions; use datafusion_datasource::file_scan_config::FileScanConfig; use datafusion_datasource::source::DataSourceExec; - use datafusion_execution::object_store::ObjectStoreUrl; use datafusion_proto_models::protobuf; let Some(protobuf::physical_plan_node::PhysicalPlanType::ParquetScan(scan)) = @@ -1180,21 +1171,43 @@ impl ParquetSource { "PhysicalPlanNode is not a ParquetScan" ); }; + let base_conf = scan.base_conf.as_ref().ok_or_else(|| { + datafusion_common::internal_datafusion_err!( + "ParquetScanExecNode is missing required field 'base_conf'" + ) + })?; + let source_node = protobuf::ParquetSourceNode::from(scan); + let source = Self::try_from_proto_file_source(&source_node, base_conf, ctx)?; + let base_config = FileScanConfig::try_from_proto(base_conf, ctx, source)?; + Ok(DataSourceExec::from_data_source(base_config)) + } + + /// Reconstructs a `ParquetSource` from its protobuf representation. + /// `base` is the enclosing scan's shared config, used for the table + /// schema, the predicate's schema and the object store. + /// + /// Rebuilds the reader factory from the decode context because it is not serialized. + /// Encryption factories and crypto options are not serialized or restored; + /// plans that rely on them require custom handling. + pub fn try_from_proto_file_source( + node: &datafusion_proto_models::protobuf::ParquetSourceNode, + base_conf: &datafusion_proto_models::protobuf::FileScanExecConf, + ctx: &datafusion_physical_plan::proto::ExecutionPlanDecodeCtx<'_>, + ) -> datafusion_common::Result> { + use crate::CachedParquetFileReaderFactory; + use arrow::datatypes::Schema; + use datafusion_common::config::TableParquetOptions; + use datafusion_datasource::file_scan_config::FileScanConfig; + use datafusion_execution::object_store::ObjectStoreUrl; + use datafusion_proto_models::protobuf; - let protobuf::ParquetScanExecNode { - base_conf, + let protobuf::ParquetSourceNode { predicate, parquet_options, sort_order_for_reorder, reverse_row_groups, metadata_size_hint, - } = scan; - - let base_conf = base_conf.as_ref().ok_or_else(|| { - datafusion_common::internal_datafusion_err!( - "ParquetScanExecNode is missing required field 'base_conf'" - ) - })?; + } = node; let schema: Arc = Arc::new( base_conf @@ -1272,9 +1285,7 @@ impl ParquetSource { if let Some(predicate) = predicate { source = source.with_predicate(predicate); } - let base_config = - FileScanConfig::try_from_proto(base_conf, ctx, Arc::new(source))?; - Ok(DataSourceExec::from_data_source(base_config)) + Ok(Arc::new(source)) } } diff --git a/datafusion/datasource/src/file.rs b/datafusion/datasource/src/file.rs index d0a1595692ddf..9073162dd4096 100644 --- a/datafusion/datasource/src/file.rs +++ b/datafusion/datasource/src/file.rs @@ -402,6 +402,10 @@ pub trait FileSource: Any + Send + Sync { /// [`PhysicalPlanNode`]: datafusion_proto_models::protobuf::PhysicalPlanNode /// [`FileScanConfig`]: crate::file_scan_config::FileScanConfig #[cfg(feature = "proto")] + #[deprecated( + since = "56.0.0", + note = "implement `try_to_proto_file_source` instead" + )] fn try_to_proto( &self, _base: &FileScanConfig, @@ -409,6 +413,23 @@ pub trait FileSource: Any + Send + Sync { ) -> Result> { Ok(None) } + + /// Serialize the format-specific part of this file source into a + /// [`FileSourceNode`], if it knows how. The shared [`FileScanConfig`] + /// encodes the format-agnostic base alongside it. + /// + /// * `Ok(None)` (the default) — the caller falls back to the deprecated + /// [`Self::try_to_proto`] hook and then to the extension codec. + /// * `Ok(Some(node))` — fully serialized. + /// + /// [`FileSourceNode`]: datafusion_proto_models::protobuf::FileSourceNode + #[cfg(feature = "proto")] + fn try_to_proto_file_source( + &self, + _ctx: &datafusion_physical_plan::proto::ExecutionPlanEncodeCtx<'_>, + ) -> Result> { + Ok(None) + } } impl dyn FileSource { diff --git a/datafusion/datasource/src/file_scan_config/mod.rs b/datafusion/datasource/src/file_scan_config/mod.rs index c94979a472969..fd388f090548d 100644 --- a/datafusion/datasource/src/file_scan_config/mod.rs +++ b/datafusion/datasource/src/file_scan_config/mod.rs @@ -1272,16 +1272,36 @@ impl DataSource for FileScanConfig { } /// Serialize this file scan by delegating to the concrete - /// [`FileSource`]'s - /// [`try_to_proto`](crate::file::FileSource::try_to_proto) hook, passing - /// `self` as the shared spine it needs to emit the base config. + /// [`FileSource`]'s deprecated [`FileSource::try_to_proto`] hook. #[cfg(feature = "proto")] + #[expect(deprecated)] fn try_to_proto( &self, ctx: &datafusion_physical_plan::proto::ExecutionPlanEncodeCtx<'_>, ) -> Result> { self.file_source().try_to_proto(self, ctx) } + + /// Encode the shared base config alongside the concrete [`FileSource`]'s + /// [`FileSourceNode`](datafusion_proto_models::protobuf::FileSourceNode). + #[cfg(feature = "proto")] + fn try_to_proto_source( + &self, + ctx: &datafusion_physical_plan::proto::ExecutionPlanEncodeCtx<'_>, + ) -> Result> { + use datafusion_proto_models::protobuf; + let Some(source) = self.file_source().try_to_proto_file_source(ctx)? else { + return Ok(None); + }; + Ok(Some(protobuf::DataSourceNode { + source_type: Some(protobuf::data_source_node::SourceType::File( + protobuf::FileScanNode { + base: Some(FileScanConfig::try_to_proto(self, ctx)?), + source: Some(source), + }, + )), + })) + } } /// Separates files that require statistics reasoning from files proven empty. diff --git a/datafusion/datasource/src/memory.rs b/datafusion/datasource/src/memory.rs index 43f0a05f3350b..be450c3c8f90a 100644 --- a/datafusion/datasource/src/memory.rs +++ b/datafusion/datasource/src/memory.rs @@ -271,15 +271,14 @@ impl DataSource for MemorySourceConfig { } /// Serialize this `MemorySourceConfig` as a `MemoryScanExecNode` wrapped - /// in a [`PhysicalPlanNode`]. Byte-compatible with the former central - /// `MemoryScan` arm in `datafusion-proto`. + /// in a [`DataSourceNode`]. /// - /// [`PhysicalPlanNode`]: datafusion_proto_models::protobuf::PhysicalPlanNode + /// [`DataSourceNode`]: datafusion_proto_models::protobuf::DataSourceNode #[cfg(feature = "proto")] - fn try_to_proto( + fn try_to_proto_source( &self, ctx: &datafusion_physical_plan::proto::ExecutionPlanEncodeCtx<'_>, - ) -> Result> { + ) -> Result> { use datafusion_common::utils::usize_to_wire; use datafusion_physical_expr_common::sort_expr::sort_exprs_try_to_proto; use datafusion_proto_models::protobuf; @@ -320,23 +319,19 @@ impl DataSource for MemorySourceConfig { }); } - Ok(Some(protobuf::PhysicalPlanNode { - physical_plan_type: Some( - protobuf::physical_plan_node::PhysicalPlanType::MemoryScan( - protobuf::MemoryScanExecNode { - partitions: proto_partitions, - schema: Some(schema.as_ref().try_into()?), - projection: proto_projection, - sort_information: proto_sort_information, - show_sizes: *show_sizes, - fetch: fetch - .map(|fetch| { - usize_to_wire(fetch, "MemoryScanExecNode", "fetch") - }) - .transpose()?, - }, - ), - ), + Ok(Some(protobuf::DataSourceNode { + source_type: Some(protobuf::data_source_node::SourceType::Memory( + protobuf::MemoryScanExecNode { + partitions: proto_partitions, + schema: Some(schema.as_ref().try_into()?), + projection: proto_projection, + sort_information: proto_sort_information, + show_sizes: *show_sizes, + fetch: fetch + .map(|fetch| usize_to_wire(fetch, "MemoryScanExecNode", "fetch")) + .transpose()?, + }, + )), })) } } @@ -702,22 +697,32 @@ impl MemorySourceConfig { #[cfg(feature = "proto")] impl MemorySourceConfig { /// Reconstruct a [`DataSourceExec`] wrapping a `MemorySourceConfig` from - /// its protobuf representation. Byte-compatible with the former central - /// `MemoryScan` arm in `datafusion-proto`. + /// a legacy `MemoryScan` plan node. + #[deprecated(since = "56.0.0", note = "use `try_from_proto_source` instead")] pub fn try_from_proto( node: &datafusion_proto_models::protobuf::PhysicalPlanNode, ctx: &datafusion_physical_plan::proto::ExecutionPlanDecodeCtx<'_>, ) -> Result> { - use datafusion_common::internal_datafusion_err; - use datafusion_common::utils::usize_from_wire; - use datafusion_physical_expr_common::sort_expr::optional_ordering_try_from_proto; - use datafusion_proto_models::protobuf; - let scan = datafusion_physical_plan::expect_plan_variant!( node, - protobuf::physical_plan_node::PhysicalPlanType::MemoryScan, + datafusion_proto_models::protobuf::physical_plan_node::PhysicalPlanType::MemoryScan, "MemorySourceConfig", ); + Ok(DataSourceExec::from_data_source( + Self::try_from_proto_source(scan, ctx)?, + )) + } + + /// Reconstruct a `MemorySourceConfig` from its protobuf representation, + /// the inverse of [`DataSource::try_to_proto_source`]. + pub fn try_from_proto_source( + scan: &datafusion_proto_models::protobuf::MemoryScanExecNode, + ctx: &datafusion_physical_plan::proto::ExecutionPlanDecodeCtx<'_>, + ) -> Result { + use datafusion_common::internal_datafusion_err; + use datafusion_common::utils::usize_from_wire; + use datafusion_physical_expr_common::sort_expr::optional_ordering_try_from_proto; + use datafusion_proto_models::protobuf; // Destructure exhaustively so a newly added protobuf field must be // handled here instead of being silently ignored. @@ -776,7 +781,7 @@ impl MemorySourceConfig { )?; source.sort_information = decoded_sort_information; - Ok(DataSourceExec::from_data_source(source)) + Ok(source) } } diff --git a/datafusion/datasource/src/sink.rs b/datafusion/datasource/src/sink.rs index 6851c9d7f48cb..8241d5621c15e 100644 --- a/datafusion/datasource/src/sink.rs +++ b/datafusion/datasource/src/sink.rs @@ -80,6 +80,7 @@ pub trait DataSink: Any + DisplayAs + Debug + Send + Sync { /// /// Returning `Ok(None)` lets the caller try its extension codec instead. #[cfg(feature = "proto")] + #[deprecated(since = "56.0.0", note = "implement `try_to_proto_sink` instead")] fn try_to_proto( &self, _exec: &DataSinkExec, @@ -87,6 +88,22 @@ pub trait DataSink: Any + DisplayAs + Debug + Send + Sync { ) -> Result> { Ok(None) } + + /// Serialize this sink into a [`DataSinkNode`], if it knows how. + /// [`DataSinkExec`] encodes its input and sort order alongside it. + /// + /// * `Ok(None)` (the default) — the caller falls back to the deprecated + /// [`Self::try_to_proto`] hook and then to the extension codec. + /// * `Ok(Some(node))` — fully serialized. + /// + /// [`DataSinkNode`]: datafusion_proto_models::protobuf::DataSinkNode + #[cfg(feature = "proto")] + fn try_to_proto_sink( + &self, + _ctx: &datafusion_physical_plan::proto::ExecutionPlanEncodeCtx<'_>, + ) -> Result> { + Ok(None) + } } impl dyn DataSink { @@ -372,13 +389,30 @@ impl ExecutionPlan for DataSinkExec { self.sink.metrics() } - /// Delegates protobuf serialization to the underlying sink. + /// Wraps the [`DataSinkNode`] emitted by the underlying sink in a + /// `DataSinkExecNode`, falling back to the deprecated whole-plan hook. + /// + /// [`DataSinkNode`]: datafusion_proto_models::protobuf::DataSinkNode #[cfg(feature = "proto")] fn try_to_proto( &self, ctx: &datafusion_physical_plan::proto::ExecutionPlanEncodeCtx<'_>, ) -> Result> { - self.sink().try_to_proto(self, ctx) + use datafusion_proto_models::protobuf; + let Some(sink) = self.sink().try_to_proto_sink(ctx)? else { + #[expect(deprecated)] + return self.sink().try_to_proto(self, ctx); + }; + let node = protobuf::DataSinkExecNode { + input: Some(Box::new(ctx.encode_child(self.input())?)), + sink: Some(sink), + sort_order: self.encode_sort_order(ctx)?, + }; + Ok(Some(protobuf::PhysicalPlanNode { + physical_plan_type: Some( + protobuf::physical_plan_node::PhysicalPlanType::DataSink(Box::new(node)), + ), + })) } } diff --git a/datafusion/datasource/src/source.rs b/datafusion/datasource/src/source.rs index 32a300ff31b74..3ad2bd016047b 100644 --- a/datafusion/datasource/src/source.rs +++ b/datafusion/datasource/src/source.rs @@ -304,12 +304,29 @@ pub trait DataSource: Any + Send + Sync + Debug { /// /// [`PhysicalPlanNode`]: datafusion_proto_models::protobuf::PhysicalPlanNode #[cfg(feature = "proto")] + #[deprecated(since = "56.0.0", note = "implement `try_to_proto_source` instead")] fn try_to_proto( &self, _ctx: &datafusion_physical_plan::proto::ExecutionPlanEncodeCtx<'_>, ) -> Result> { Ok(None) } + + /// Serialize this data source into a [`DataSourceNode`], if it knows how. + /// [`DataSourceExec`] wraps the result in its own plan node. + /// + /// * `Ok(None)` (the default) — the caller falls back to the deprecated + /// [`Self::try_to_proto`] hook and then to the extension codec. + /// * `Ok(Some(node))` — fully serialized. + /// + /// [`DataSourceNode`]: datafusion_proto_models::protobuf::DataSourceNode + #[cfg(feature = "proto")] + fn try_to_proto_source( + &self, + _ctx: &datafusion_physical_plan::proto::ExecutionPlanEncodeCtx<'_>, + ) -> Result> { + Ok(None) + } } /// Arguments for [`DataSource::open_with_args`] @@ -607,15 +624,28 @@ impl ExecutionPlan for DataSourceExec { Ok(Arc::new(new_exec)) } - /// Delegates serialization to the wrapped [`DataSource`]. For file scans the - /// concrete [`FileSource`] emits the node via its - /// own `try_to_proto` hook, keeping the format-specific wire logic in the - /// format crate. + /// Wraps the [`DataSourceNode`] emitted by the wrapped [`DataSource`] in a + /// `DataSourceExecNode`, falling back to the deprecated whole-plan hook. + /// + /// [`DataSourceNode`]: datafusion_proto_models::protobuf::DataSourceNode #[cfg(feature = "proto")] fn try_to_proto( &self, ctx: &datafusion_physical_plan::proto::ExecutionPlanEncodeCtx<'_>, ) -> Result> { + if let Some(source) = self.data_source().try_to_proto_source(ctx)? { + use datafusion_proto_models::protobuf; + return Ok(Some(protobuf::PhysicalPlanNode { + physical_plan_type: Some( + protobuf::physical_plan_node::PhysicalPlanType::DataSource( + protobuf::DataSourceExecNode { + source: Some(source), + }, + ), + ), + })); + } + #[expect(deprecated)] self.data_source().try_to_proto(ctx) } } diff --git a/datafusion/proto-models/proto/datafusion.proto b/datafusion/proto-models/proto/datafusion.proto index 6c4ef8f0a4b2f..13a204e2e1329 100644 --- a/datafusion/proto-models/proto/datafusion.proto +++ b/datafusion/proto-models/proto/datafusion.proto @@ -901,7 +901,9 @@ message TableReference { // PhysicalPlanNode is a nested type message PhysicalPlanNode { oneof PhysicalPlanType { + // Deprecated: encoders emit `data_source`; decoders still accept this. ParquetScanExecNode parquet_scan = 1; + // Deprecated: encoders emit `data_source`; decoders still accept this. CsvScanExecNode csv_scan = 2; EmptyExecNode empty = 3; ProjectionExecNode projection = 4; @@ -916,6 +918,7 @@ message PhysicalPlanNode { RepartitionExecNode repartition = 14; WindowAggExecNode window = 15; CrossJoinExecNode cross_join = 16; + // Deprecated: encoders emit `data_source`; decoders still accept this. AvroScanExecNode avro_scan = 17; PhysicalExtensionNode extension = 18; UnionExecNode union = 19; @@ -923,24 +926,32 @@ message PhysicalPlanNode { SortPreservingMergeExecNode sort_preserving_merge = 21; NestedLoopJoinExecNode nested_loop_join = 22; AnalyzeExecNode analyze = 23; + // Deprecated: encoders emit `data_sink`; decoders still accept this. JsonSinkExecNode json_sink = 24; SymmetricHashJoinExecNode symmetric_hash_join = 25; InterleaveExecNode interleave = 26; PlaceholderRowExecNode placeholder_row = 27; + // Deprecated: encoders emit `data_sink`; decoders still accept this. CsvSinkExecNode csv_sink = 28; + // Deprecated: encoders emit `data_sink`; decoders still accept this. ParquetSinkExecNode parquet_sink = 29; UnnestExecNode unnest = 30; + // Deprecated: encoders emit `data_source`; decoders still accept this. JsonScanExecNode json_scan = 31; CooperativeExecNode cooperative = 32; GenerateSeriesNode generate_series = 33; SortMergeJoinExecNode sort_merge_join = 34; + // Deprecated: encoders emit `data_source`; decoders still accept this. MemoryScanExecNode memory_scan = 35; AsyncFuncExecNode async_func = 36; BufferExecNode buffer = 37; + // Deprecated: encoders emit `data_source`; decoders still accept this. ArrowScanExecNode arrow_scan = 38; ScalarSubqueryExecNode scalar_subquery = 39; PiecewiseMergeJoinExecNode piecewise_merge_join = 40; AsOfJoinExecNode as_of_join = 41; + DataSourceExecNode data_source = 42; + DataSinkExecNode data_sink = 43; } } @@ -1882,3 +1893,110 @@ message PhysicalScalarSubqueryExprNode { // Serialized separately from data_type to keep older wire formats valid. map metadata = 4; } + +// A `DataSourceExec`: the plan-level wrapper around a `DataSource`. +message DataSourceExecNode { + DataSourceNode source = 1; +} + +// A `DataSource` implementation. +message DataSourceNode { + oneof source_type { + FileScanNode file = 1; + MemoryScanExecNode memory = 2; + ExtensionDataSourceNode extension = 15; + } +} + +// A `FileScanConfig`: the format-agnostic base plus its `FileSource`. +message FileScanNode { + FileScanExecConf base = 1; + FileSourceNode source = 2; +} + +// A `FileSource` implementation. +message FileSourceNode { + oneof file_source_type { + ParquetSourceNode parquet = 1; + CsvSourceNode csv = 2; + JsonSourceNode json = 3; + AvroSourceNode avro = 4; + ArrowSourceNode arrow = 5; + ExtensionFileSourceNode extension = 15; + } +} + +// Field tags mirror `ParquetScanExecNode` (minus `base_conf`). +message ParquetSourceNode { + reserved 1, 2; + PhysicalExprNode predicate = 3; + datafusion_common.TableParquetOptions parquet_options = 4; + PhysicalSortExprNodeCollection sort_order_for_reorder = 5; + bool reverse_row_groups = 6; + optional uint64 metadata_size_hint = 7; +} + +// Field tags mirror `CsvScanExecNode` (minus `base_conf`). +message CsvSourceNode { + reserved 1; + bool has_header = 2; + string delimiter = 3; + string quote = 4; + oneof optional_escape { + string escape = 5; + } + oneof optional_comment { + string comment = 6; + } + bool newlines_in_values = 7; + bool truncate_rows = 8; + optional bytes terminator = 9; +} + +// Field tags mirror `JsonScanExecNode` (minus `base_conf`). +message JsonSourceNode { + reserved 1; + optional bool newline_delimited = 2; +} + +message AvroSourceNode { + reserved 1; +} + +// Field tags mirror `ArrowScanExecNode` (minus `base_conf`). +message ArrowSourceNode { + reserved 1; + ArrowIpcFormat format = 2; +} + +message ExtensionDataSourceNode { + string name = 1; + bytes payload = 2; +} + +message ExtensionFileSourceNode { + string name = 1; + bytes payload = 2; +} + +// A `DataSinkExec`: the plan-level wrapper around a `DataSink`. +message DataSinkExecNode { + PhysicalPlanNode input = 1; + DataSinkNode sink = 2; + PhysicalSortExprNodeCollection sort_order = 3; +} + +// A `DataSink` implementation. +message DataSinkNode { + oneof sink_type { + JsonSink json = 1; + CsvSink csv = 2; + ParquetSink parquet = 3; + ExtensionDataSinkNode extension = 15; + } +} + +message ExtensionDataSinkNode { + string name = 1; + bytes payload = 2; +} diff --git a/datafusion/proto-models/src/generated/pbjson.rs b/datafusion/proto-models/src/generated/pbjson.rs index 725cceeb75970..5cb7fe5aba546 100644 --- a/datafusion/proto-models/src/generated/pbjson.rs +++ b/datafusion/proto-models/src/generated/pbjson.rs @@ -1667,6 +1667,99 @@ impl<'de> serde::Deserialize<'de> for ArrowScanExecNode { deserializer.deserialize_struct("datafusion.ArrowScanExecNode", FIELDS, GeneratedVisitor) } } +impl serde::Serialize for ArrowSourceNode { + #[allow(deprecated)] + fn serialize(&self, serializer: S) -> std::result::Result + where + S: serde::Serializer, + { + use serde::ser::SerializeStruct; + let mut len = 0; + if self.format != 0 { + len += 1; + } + let mut struct_ser = serializer.serialize_struct("datafusion.ArrowSourceNode", len)?; + if self.format != 0 { + let v = ArrowIpcFormat::try_from(self.format) + .map_err(|_| serde::ser::Error::custom(format!("Invalid variant {}", self.format)))?; + struct_ser.serialize_field("format", &v)?; + } + struct_ser.end() + } +} +impl<'de> serde::Deserialize<'de> for ArrowSourceNode { + #[allow(deprecated)] + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + const FIELDS: &[&str] = &[ + "format", + ]; + + #[allow(clippy::enum_variant_names)] + enum GeneratedField { + Format, + } + impl<'de> serde::Deserialize<'de> for GeneratedField { + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + struct GeneratedVisitor; + + impl serde::de::Visitor<'_> for GeneratedVisitor { + type Value = GeneratedField; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(formatter, "expected one of: {:?}", &FIELDS) + } + + #[allow(unused_variables)] + fn visit_str(self, value: &str) -> std::result::Result + where + E: serde::de::Error, + { + match value { + "format" => Ok(GeneratedField::Format), + _ => Err(serde::de::Error::unknown_field(value, FIELDS)), + } + } + } + deserializer.deserialize_identifier(GeneratedVisitor) + } + } + struct GeneratedVisitor; + impl<'de> serde::de::Visitor<'de> for GeneratedVisitor { + type Value = ArrowSourceNode; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("struct datafusion.ArrowSourceNode") + } + + fn visit_map(self, mut map_: V) -> std::result::Result + where + V: serde::de::MapAccess<'de>, + { + let mut format__ = None; + while let Some(k) = map_.next_key()? { + match k { + GeneratedField::Format => { + if format__.is_some() { + return Err(serde::de::Error::duplicate_field("format")); + } + format__ = Some(map_.next_value::()? as i32); + } + } + } + Ok(ArrowSourceNode { + format: format__.unwrap_or_default(), + }) + } + } + deserializer.deserialize_struct("datafusion.ArrowSourceNode", FIELDS, GeneratedVisitor) + } +} impl serde::Serialize for AsOfJoinExecNode { #[allow(deprecated)] fn serialize(&self, serializer: S) -> std::result::Result @@ -2387,6 +2480,77 @@ impl<'de> serde::Deserialize<'de> for AvroScanExecNode { deserializer.deserialize_struct("datafusion.AvroScanExecNode", FIELDS, GeneratedVisitor) } } +impl serde::Serialize for AvroSourceNode { + #[allow(deprecated)] + fn serialize(&self, serializer: S) -> std::result::Result + where + S: serde::Serializer, + { + use serde::ser::SerializeStruct; + let len = 0; + let struct_ser = serializer.serialize_struct("datafusion.AvroSourceNode", len)?; + struct_ser.end() + } +} +impl<'de> serde::Deserialize<'de> for AvroSourceNode { + #[allow(deprecated)] + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + const FIELDS: &[&str] = &[ + ]; + + #[allow(clippy::enum_variant_names)] + enum GeneratedField { + } + impl<'de> serde::Deserialize<'de> for GeneratedField { + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + struct GeneratedVisitor; + + impl serde::de::Visitor<'_> for GeneratedVisitor { + type Value = GeneratedField; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(formatter, "expected one of: {:?}", &FIELDS) + } + + #[allow(unused_variables)] + fn visit_str(self, value: &str) -> std::result::Result + where + E: serde::de::Error, + { + Err(serde::de::Error::unknown_field(value, FIELDS)) + } + } + deserializer.deserialize_identifier(GeneratedVisitor) + } + } + struct GeneratedVisitor; + impl<'de> serde::de::Visitor<'de> for GeneratedVisitor { + type Value = AvroSourceNode; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("struct datafusion.AvroSourceNode") + } + + fn visit_map(self, mut map_: V) -> std::result::Result + where + V: serde::de::MapAccess<'de>, + { + while map_.next_key::()?.is_some() { + let _ = map_.next_value::()?; + } + Ok(AvroSourceNode { + }) + } + } + deserializer.deserialize_struct("datafusion.AvroSourceNode", FIELDS, GeneratedVisitor) + } +} impl serde::Serialize for BareTableReference { #[allow(deprecated)] fn serialize(&self, serializer: S) -> std::result::Result @@ -5482,7 +5646,7 @@ impl<'de> serde::Deserialize<'de> for CsvSinkExecNode { deserializer.deserialize_struct("datafusion.CsvSinkExecNode", FIELDS, GeneratedVisitor) } } -impl serde::Serialize for CteWorkTableScanNode { +impl serde::Serialize for CsvSourceNode { #[allow(deprecated)] fn serialize(&self, serializer: S) -> std::result::Result where @@ -5490,37 +5654,98 @@ impl serde::Serialize for CteWorkTableScanNode { { use serde::ser::SerializeStruct; let mut len = 0; - if !self.name.is_empty() { + if self.has_header { len += 1; } - if self.schema.is_some() { + if !self.delimiter.is_empty() { len += 1; } - let mut struct_ser = serializer.serialize_struct("datafusion.CteWorkTableScanNode", len)?; - if !self.name.is_empty() { - struct_ser.serialize_field("name", &self.name)?; + if !self.quote.is_empty() { + len += 1; } - if let Some(v) = self.schema.as_ref() { - struct_ser.serialize_field("schema", v)?; + if self.newlines_in_values { + len += 1; + } + if self.truncate_rows { + len += 1; + } + if self.terminator.is_some() { + len += 1; + } + if self.optional_escape.is_some() { + len += 1; + } + if self.optional_comment.is_some() { + len += 1; + } + let mut struct_ser = serializer.serialize_struct("datafusion.CsvSourceNode", len)?; + if self.has_header { + struct_ser.serialize_field("hasHeader", &self.has_header)?; + } + if !self.delimiter.is_empty() { + struct_ser.serialize_field("delimiter", &self.delimiter)?; + } + if !self.quote.is_empty() { + struct_ser.serialize_field("quote", &self.quote)?; + } + if self.newlines_in_values { + struct_ser.serialize_field("newlinesInValues", &self.newlines_in_values)?; + } + if self.truncate_rows { + struct_ser.serialize_field("truncateRows", &self.truncate_rows)?; + } + if let Some(v) = self.terminator.as_ref() { + #[allow(clippy::needless_borrow)] + #[allow(clippy::needless_borrows_for_generic_args)] + struct_ser.serialize_field("terminator", pbjson::private::base64::encode(&v).as_str())?; + } + if let Some(v) = self.optional_escape.as_ref() { + match v { + csv_source_node::OptionalEscape::Escape(v) => { + struct_ser.serialize_field("escape", v)?; + } + } + } + if let Some(v) = self.optional_comment.as_ref() { + match v { + csv_source_node::OptionalComment::Comment(v) => { + struct_ser.serialize_field("comment", v)?; + } + } } struct_ser.end() } } -impl<'de> serde::Deserialize<'de> for CteWorkTableScanNode { +impl<'de> serde::Deserialize<'de> for CsvSourceNode { #[allow(deprecated)] fn deserialize(deserializer: D) -> std::result::Result where D: serde::Deserializer<'de>, { const FIELDS: &[&str] = &[ - "name", - "schema", + "has_header", + "hasHeader", + "delimiter", + "quote", + "newlines_in_values", + "newlinesInValues", + "truncate_rows", + "truncateRows", + "terminator", + "escape", + "comment", ]; #[allow(clippy::enum_variant_names)] enum GeneratedField { - Name, - Schema, + HasHeader, + Delimiter, + Quote, + NewlinesInValues, + TruncateRows, + Terminator, + Escape, + Comment, } impl<'de> serde::Deserialize<'de> for GeneratedField { fn deserialize(deserializer: D) -> std::result::Result @@ -5542,8 +5767,14 @@ impl<'de> serde::Deserialize<'de> for CteWorkTableScanNode { E: serde::de::Error, { match value { - "name" => Ok(GeneratedField::Name), - "schema" => Ok(GeneratedField::Schema), + "hasHeader" | "has_header" => Ok(GeneratedField::HasHeader), + "delimiter" => Ok(GeneratedField::Delimiter), + "quote" => Ok(GeneratedField::Quote), + "newlinesInValues" | "newlines_in_values" => Ok(GeneratedField::NewlinesInValues), + "truncateRows" | "truncate_rows" => Ok(GeneratedField::TruncateRows), + "terminator" => Ok(GeneratedField::Terminator), + "escape" => Ok(GeneratedField::Escape), + "comment" => Ok(GeneratedField::Comment), _ => Err(serde::de::Error::unknown_field(value, FIELDS)), } } @@ -5553,23 +5784,181 @@ impl<'de> serde::Deserialize<'de> for CteWorkTableScanNode { } struct GeneratedVisitor; impl<'de> serde::de::Visitor<'de> for GeneratedVisitor { - type Value = CteWorkTableScanNode; + type Value = CsvSourceNode; fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - formatter.write_str("struct datafusion.CteWorkTableScanNode") + formatter.write_str("struct datafusion.CsvSourceNode") } - fn visit_map(self, mut map_: V) -> std::result::Result + fn visit_map(self, mut map_: V) -> std::result::Result where V: serde::de::MapAccess<'de>, { - let mut name__ = None; - let mut schema__ = None; + let mut has_header__ = None; + let mut delimiter__ = None; + let mut quote__ = None; + let mut newlines_in_values__ = None; + let mut truncate_rows__ = None; + let mut terminator__ = None; + let mut optional_escape__ = None; + let mut optional_comment__ = None; while let Some(k) = map_.next_key()? { match k { - GeneratedField::Name => { - if name__.is_some() { - return Err(serde::de::Error::duplicate_field("name")); + GeneratedField::HasHeader => { + if has_header__.is_some() { + return Err(serde::de::Error::duplicate_field("hasHeader")); + } + has_header__ = Some(map_.next_value()?); + } + GeneratedField::Delimiter => { + if delimiter__.is_some() { + return Err(serde::de::Error::duplicate_field("delimiter")); + } + delimiter__ = Some(map_.next_value()?); + } + GeneratedField::Quote => { + if quote__.is_some() { + return Err(serde::de::Error::duplicate_field("quote")); + } + quote__ = Some(map_.next_value()?); + } + GeneratedField::NewlinesInValues => { + if newlines_in_values__.is_some() { + return Err(serde::de::Error::duplicate_field("newlinesInValues")); + } + newlines_in_values__ = Some(map_.next_value()?); + } + GeneratedField::TruncateRows => { + if truncate_rows__.is_some() { + return Err(serde::de::Error::duplicate_field("truncateRows")); + } + truncate_rows__ = Some(map_.next_value()?); + } + GeneratedField::Terminator => { + if terminator__.is_some() { + return Err(serde::de::Error::duplicate_field("terminator")); + } + terminator__ = + map_.next_value::<::std::option::Option<::pbjson::private::BytesDeserialize<_>>>()?.map(|x| x.0) + ; + } + GeneratedField::Escape => { + if optional_escape__.is_some() { + return Err(serde::de::Error::duplicate_field("escape")); + } + optional_escape__ = map_.next_value::<::std::option::Option<_>>()?.map(csv_source_node::OptionalEscape::Escape); + } + GeneratedField::Comment => { + if optional_comment__.is_some() { + return Err(serde::de::Error::duplicate_field("comment")); + } + optional_comment__ = map_.next_value::<::std::option::Option<_>>()?.map(csv_source_node::OptionalComment::Comment); + } + } + } + Ok(CsvSourceNode { + has_header: has_header__.unwrap_or_default(), + delimiter: delimiter__.unwrap_or_default(), + quote: quote__.unwrap_or_default(), + newlines_in_values: newlines_in_values__.unwrap_or_default(), + truncate_rows: truncate_rows__.unwrap_or_default(), + terminator: terminator__, + optional_escape: optional_escape__, + optional_comment: optional_comment__, + }) + } + } + deserializer.deserialize_struct("datafusion.CsvSourceNode", FIELDS, GeneratedVisitor) + } +} +impl serde::Serialize for CteWorkTableScanNode { + #[allow(deprecated)] + fn serialize(&self, serializer: S) -> std::result::Result + where + S: serde::Serializer, + { + use serde::ser::SerializeStruct; + let mut len = 0; + if !self.name.is_empty() { + len += 1; + } + if self.schema.is_some() { + len += 1; + } + let mut struct_ser = serializer.serialize_struct("datafusion.CteWorkTableScanNode", len)?; + if !self.name.is_empty() { + struct_ser.serialize_field("name", &self.name)?; + } + if let Some(v) = self.schema.as_ref() { + struct_ser.serialize_field("schema", v)?; + } + struct_ser.end() + } +} +impl<'de> serde::Deserialize<'de> for CteWorkTableScanNode { + #[allow(deprecated)] + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + const FIELDS: &[&str] = &[ + "name", + "schema", + ]; + + #[allow(clippy::enum_variant_names)] + enum GeneratedField { + Name, + Schema, + } + impl<'de> serde::Deserialize<'de> for GeneratedField { + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + struct GeneratedVisitor; + + impl serde::de::Visitor<'_> for GeneratedVisitor { + type Value = GeneratedField; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(formatter, "expected one of: {:?}", &FIELDS) + } + + #[allow(unused_variables)] + fn visit_str(self, value: &str) -> std::result::Result + where + E: serde::de::Error, + { + match value { + "name" => Ok(GeneratedField::Name), + "schema" => Ok(GeneratedField::Schema), + _ => Err(serde::de::Error::unknown_field(value, FIELDS)), + } + } + } + deserializer.deserialize_identifier(GeneratedVisitor) + } + } + struct GeneratedVisitor; + impl<'de> serde::de::Visitor<'de> for GeneratedVisitor { + type Value = CteWorkTableScanNode; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("struct datafusion.CteWorkTableScanNode") + } + + fn visit_map(self, mut map_: V) -> std::result::Result + where + V: serde::de::MapAccess<'de>, + { + let mut name__ = None; + let mut schema__ = None; + while let Some(k) = map_.next_key()? { + match k { + GeneratedField::Name => { + if name__.is_some() { + return Err(serde::de::Error::duplicate_field("name")); } name__ = Some(map_.next_value()?); } @@ -5888,78 +6277,7 @@ impl<'de> serde::Deserialize<'de> for CustomTableScanNode { deserializer.deserialize_struct("datafusion.CustomTableScanNode", FIELDS, GeneratedVisitor) } } -impl serde::Serialize for DateUnit { - #[allow(deprecated)] - fn serialize(&self, serializer: S) -> std::result::Result - where - S: serde::Serializer, - { - let variant = match self { - Self::Day => "Day", - Self::DateMillisecond => "DateMillisecond", - }; - serializer.serialize_str(variant) - } -} -impl<'de> serde::Deserialize<'de> for DateUnit { - #[allow(deprecated)] - fn deserialize(deserializer: D) -> std::result::Result - where - D: serde::Deserializer<'de>, - { - const FIELDS: &[&str] = &[ - "Day", - "DateMillisecond", - ]; - - struct GeneratedVisitor; - - impl serde::de::Visitor<'_> for GeneratedVisitor { - type Value = DateUnit; - - fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(formatter, "expected one of: {:?}", &FIELDS) - } - - fn visit_i64(self, v: i64) -> std::result::Result - where - E: serde::de::Error, - { - i32::try_from(v) - .ok() - .and_then(|x| x.try_into().ok()) - .ok_or_else(|| { - serde::de::Error::invalid_value(serde::de::Unexpected::Signed(v), &self) - }) - } - - fn visit_u64(self, v: u64) -> std::result::Result - where - E: serde::de::Error, - { - i32::try_from(v) - .ok() - .and_then(|x| x.try_into().ok()) - .ok_or_else(|| { - serde::de::Error::invalid_value(serde::de::Unexpected::Unsigned(v), &self) - }) - } - - fn visit_str(self, value: &str) -> std::result::Result - where - E: serde::de::Error, - { - match value { - "Day" => Ok(DateUnit::Day), - "DateMillisecond" => Ok(DateUnit::DateMillisecond), - _ => Err(serde::de::Error::unknown_variant(value, FIELDS)), - } - } - } - deserializer.deserialize_any(GeneratedVisitor) - } -} -impl serde::Serialize for DistinctNode { +impl serde::Serialize for DataSinkExecNode { #[allow(deprecated)] fn serialize(&self, serializer: S) -> std::result::Result where @@ -5970,14 +6288,26 @@ impl serde::Serialize for DistinctNode { if self.input.is_some() { len += 1; } - let mut struct_ser = serializer.serialize_struct("datafusion.DistinctNode", len)?; + if self.sink.is_some() { + len += 1; + } + if self.sort_order.is_some() { + len += 1; + } + let mut struct_ser = serializer.serialize_struct("datafusion.DataSinkExecNode", len)?; if let Some(v) = self.input.as_ref() { struct_ser.serialize_field("input", v)?; } + if let Some(v) = self.sink.as_ref() { + struct_ser.serialize_field("sink", v)?; + } + if let Some(v) = self.sort_order.as_ref() { + struct_ser.serialize_field("sortOrder", v)?; + } struct_ser.end() } } -impl<'de> serde::Deserialize<'de> for DistinctNode { +impl<'de> serde::Deserialize<'de> for DataSinkExecNode { #[allow(deprecated)] fn deserialize(deserializer: D) -> std::result::Result where @@ -5985,11 +6315,16 @@ impl<'de> serde::Deserialize<'de> for DistinctNode { { const FIELDS: &[&str] = &[ "input", + "sink", + "sort_order", + "sortOrder", ]; #[allow(clippy::enum_variant_names)] enum GeneratedField { Input, + Sink, + SortOrder, } impl<'de> serde::Deserialize<'de> for GeneratedField { fn deserialize(deserializer: D) -> std::result::Result @@ -6012,6 +6347,8 @@ impl<'de> serde::Deserialize<'de> for DistinctNode { { match value { "input" => Ok(GeneratedField::Input), + "sink" => Ok(GeneratedField::Sink), + "sortOrder" | "sort_order" => Ok(GeneratedField::SortOrder), _ => Err(serde::de::Error::unknown_field(value, FIELDS)), } } @@ -6021,17 +6358,19 @@ impl<'de> serde::Deserialize<'de> for DistinctNode { } struct GeneratedVisitor; impl<'de> serde::de::Visitor<'de> for GeneratedVisitor { - type Value = DistinctNode; + type Value = DataSinkExecNode; fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - formatter.write_str("struct datafusion.DistinctNode") + formatter.write_str("struct datafusion.DataSinkExecNode") } - fn visit_map(self, mut map_: V) -> std::result::Result + fn visit_map(self, mut map_: V) -> std::result::Result where V: serde::de::MapAccess<'de>, { let mut input__ = None; + let mut sink__ = None; + let mut sort_order__ = None; while let Some(k) = map_.next_key()? { match k { GeneratedField::Input => { @@ -6040,17 +6379,31 @@ impl<'de> serde::Deserialize<'de> for DistinctNode { } input__ = map_.next_value()?; } - } - } - Ok(DistinctNode { - input: input__, - }) - } - } - deserializer.deserialize_struct("datafusion.DistinctNode", FIELDS, GeneratedVisitor) + GeneratedField::Sink => { + if sink__.is_some() { + return Err(serde::de::Error::duplicate_field("sink")); + } + sink__ = map_.next_value()?; + } + GeneratedField::SortOrder => { + if sort_order__.is_some() { + return Err(serde::de::Error::duplicate_field("sortOrder")); + } + sort_order__ = map_.next_value()?; + } + } + } + Ok(DataSinkExecNode { + input: input__, + sink: sink__, + sort_order: sort_order__, + }) + } + } + deserializer.deserialize_struct("datafusion.DataSinkExecNode", FIELDS, GeneratedVisitor) } } -impl serde::Serialize for DistinctOnNode { +impl serde::Serialize for DataSinkNode { #[allow(deprecated)] fn serialize(&self, serializer: S) -> std::result::Result where @@ -6058,56 +6411,48 @@ impl serde::Serialize for DistinctOnNode { { use serde::ser::SerializeStruct; let mut len = 0; - if !self.on_expr.is_empty() { - len += 1; - } - if !self.select_expr.is_empty() { - len += 1; - } - if !self.sort_expr.is_empty() { - len += 1; - } - if self.input.is_some() { + if self.sink_type.is_some() { len += 1; } - let mut struct_ser = serializer.serialize_struct("datafusion.DistinctOnNode", len)?; - if !self.on_expr.is_empty() { - struct_ser.serialize_field("onExpr", &self.on_expr)?; - } - if !self.select_expr.is_empty() { - struct_ser.serialize_field("selectExpr", &self.select_expr)?; - } - if !self.sort_expr.is_empty() { - struct_ser.serialize_field("sortExpr", &self.sort_expr)?; - } - if let Some(v) = self.input.as_ref() { - struct_ser.serialize_field("input", v)?; + let mut struct_ser = serializer.serialize_struct("datafusion.DataSinkNode", len)?; + if let Some(v) = self.sink_type.as_ref() { + match v { + data_sink_node::SinkType::Json(v) => { + struct_ser.serialize_field("json", v)?; + } + data_sink_node::SinkType::Csv(v) => { + struct_ser.serialize_field("csv", v)?; + } + data_sink_node::SinkType::Parquet(v) => { + struct_ser.serialize_field("parquet", v)?; + } + data_sink_node::SinkType::Extension(v) => { + struct_ser.serialize_field("extension", v)?; + } + } } struct_ser.end() } } -impl<'de> serde::Deserialize<'de> for DistinctOnNode { +impl<'de> serde::Deserialize<'de> for DataSinkNode { #[allow(deprecated)] fn deserialize(deserializer: D) -> std::result::Result where D: serde::Deserializer<'de>, { const FIELDS: &[&str] = &[ - "on_expr", - "onExpr", - "select_expr", - "selectExpr", - "sort_expr", - "sortExpr", - "input", + "json", + "csv", + "parquet", + "extension", ]; #[allow(clippy::enum_variant_names)] enum GeneratedField { - OnExpr, - SelectExpr, - SortExpr, - Input, + Json, + Csv, + Parquet, + Extension, } impl<'de> serde::Deserialize<'de> for GeneratedField { fn deserialize(deserializer: D) -> std::result::Result @@ -6129,10 +6474,10 @@ impl<'de> serde::Deserialize<'de> for DistinctOnNode { E: serde::de::Error, { match value { - "onExpr" | "on_expr" => Ok(GeneratedField::OnExpr), - "selectExpr" | "select_expr" => Ok(GeneratedField::SelectExpr), - "sortExpr" | "sort_expr" => Ok(GeneratedField::SortExpr), - "input" => Ok(GeneratedField::Input), + "json" => Ok(GeneratedField::Json), + "csv" => Ok(GeneratedField::Csv), + "parquet" => Ok(GeneratedField::Parquet), + "extension" => Ok(GeneratedField::Extension), _ => Err(serde::de::Error::unknown_field(value, FIELDS)), } } @@ -6142,60 +6487,58 @@ impl<'de> serde::Deserialize<'de> for DistinctOnNode { } struct GeneratedVisitor; impl<'de> serde::de::Visitor<'de> for GeneratedVisitor { - type Value = DistinctOnNode; + type Value = DataSinkNode; fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - formatter.write_str("struct datafusion.DistinctOnNode") + formatter.write_str("struct datafusion.DataSinkNode") } - fn visit_map(self, mut map_: V) -> std::result::Result + fn visit_map(self, mut map_: V) -> std::result::Result where V: serde::de::MapAccess<'de>, { - let mut on_expr__ = None; - let mut select_expr__ = None; - let mut sort_expr__ = None; - let mut input__ = None; + let mut sink_type__ = None; while let Some(k) = map_.next_key()? { match k { - GeneratedField::OnExpr => { - if on_expr__.is_some() { - return Err(serde::de::Error::duplicate_field("onExpr")); + GeneratedField::Json => { + if sink_type__.is_some() { + return Err(serde::de::Error::duplicate_field("json")); } - on_expr__ = Some(map_.next_value()?); + sink_type__ = map_.next_value::<::std::option::Option<_>>()?.map(data_sink_node::SinkType::Json) +; } - GeneratedField::SelectExpr => { - if select_expr__.is_some() { - return Err(serde::de::Error::duplicate_field("selectExpr")); + GeneratedField::Csv => { + if sink_type__.is_some() { + return Err(serde::de::Error::duplicate_field("csv")); } - select_expr__ = Some(map_.next_value()?); + sink_type__ = map_.next_value::<::std::option::Option<_>>()?.map(data_sink_node::SinkType::Csv) +; } - GeneratedField::SortExpr => { - if sort_expr__.is_some() { - return Err(serde::de::Error::duplicate_field("sortExpr")); + GeneratedField::Parquet => { + if sink_type__.is_some() { + return Err(serde::de::Error::duplicate_field("parquet")); } - sort_expr__ = Some(map_.next_value()?); + sink_type__ = map_.next_value::<::std::option::Option<_>>()?.map(data_sink_node::SinkType::Parquet) +; } - GeneratedField::Input => { - if input__.is_some() { - return Err(serde::de::Error::duplicate_field("input")); + GeneratedField::Extension => { + if sink_type__.is_some() { + return Err(serde::de::Error::duplicate_field("extension")); } - input__ = map_.next_value()?; + sink_type__ = map_.next_value::<::std::option::Option<_>>()?.map(data_sink_node::SinkType::Extension) +; } } } - Ok(DistinctOnNode { - on_expr: on_expr__.unwrap_or_default(), - select_expr: select_expr__.unwrap_or_default(), - sort_expr: sort_expr__.unwrap_or_default(), - input: input__, + Ok(DataSinkNode { + sink_type: sink_type__, }) } } - deserializer.deserialize_struct("datafusion.DistinctOnNode", FIELDS, GeneratedVisitor) + deserializer.deserialize_struct("datafusion.DataSinkNode", FIELDS, GeneratedVisitor) } } -impl serde::Serialize for DmlNode { +impl serde::Serialize for DataSourceExecNode { #[allow(deprecated)] fn serialize(&self, serializer: S) -> std::result::Result where @@ -6203,66 +6546,29 @@ impl serde::Serialize for DmlNode { { use serde::ser::SerializeStruct; let mut len = 0; - if self.dml_type != 0 { - len += 1; - } - if self.input.is_some() { + if self.source.is_some() { len += 1; } - if self.table_name.is_some() { - len += 1; - } - if self.target.is_some() { - len += 1; - } - if self.merge_into.is_some() { - len += 1; - } - let mut struct_ser = serializer.serialize_struct("datafusion.DmlNode", len)?; - if self.dml_type != 0 { - let v = dml_node::Type::try_from(self.dml_type) - .map_err(|_| serde::ser::Error::custom(format!("Invalid variant {}", self.dml_type)))?; - struct_ser.serialize_field("dmlType", &v)?; - } - if let Some(v) = self.input.as_ref() { - struct_ser.serialize_field("input", v)?; - } - if let Some(v) = self.table_name.as_ref() { - struct_ser.serialize_field("tableName", v)?; - } - if let Some(v) = self.target.as_ref() { - struct_ser.serialize_field("target", v)?; - } - if let Some(v) = self.merge_into.as_ref() { - struct_ser.serialize_field("mergeInto", v)?; + let mut struct_ser = serializer.serialize_struct("datafusion.DataSourceExecNode", len)?; + if let Some(v) = self.source.as_ref() { + struct_ser.serialize_field("source", v)?; } struct_ser.end() } } -impl<'de> serde::Deserialize<'de> for DmlNode { +impl<'de> serde::Deserialize<'de> for DataSourceExecNode { #[allow(deprecated)] fn deserialize(deserializer: D) -> std::result::Result where D: serde::Deserializer<'de>, { const FIELDS: &[&str] = &[ - "dml_type", - "dmlType", - "input", - "table_name", - "tableName", - "target", - "merge_into", - "mergeInto", + "source", ]; #[allow(clippy::enum_variant_names)] enum GeneratedField { - DmlType, - Input, - TableName, - Target, - MergeInto, + Source, } impl<'de> serde::Deserialize<'de> for GeneratedField { fn deserialize(deserializer: D) -> std::result::Result @@ -6284,11 +6590,7 @@ impl<'de> serde::Deserialize<'de> for DmlNode { E: serde::de::Error, { match value { - "dmlType" | "dml_type" => Ok(GeneratedField::DmlType), - "input" => Ok(GeneratedField::Input), - "tableName" | "table_name" => Ok(GeneratedField::TableName), - "target" => Ok(GeneratedField::Target), - "mergeInto" | "merge_into" => Ok(GeneratedField::MergeInto), + "source" => Ok(GeneratedField::Source), _ => Err(serde::de::Error::unknown_field(value, FIELDS)), } } @@ -6298,157 +6600,1063 @@ impl<'de> serde::Deserialize<'de> for DmlNode { } struct GeneratedVisitor; impl<'de> serde::de::Visitor<'de> for GeneratedVisitor { - type Value = DmlNode; + type Value = DataSourceExecNode; fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - formatter.write_str("struct datafusion.DmlNode") + formatter.write_str("struct datafusion.DataSourceExecNode") } - fn visit_map(self, mut map_: V) -> std::result::Result + fn visit_map(self, mut map_: V) -> std::result::Result where V: serde::de::MapAccess<'de>, { - let mut dml_type__ = None; - let mut input__ = None; - let mut table_name__ = None; - let mut target__ = None; - let mut merge_into__ = None; + let mut source__ = None; while let Some(k) = map_.next_key()? { match k { - GeneratedField::DmlType => { - if dml_type__.is_some() { - return Err(serde::de::Error::duplicate_field("dmlType")); - } - dml_type__ = Some(map_.next_value::()? as i32); - } - GeneratedField::Input => { - if input__.is_some() { - return Err(serde::de::Error::duplicate_field("input")); - } - input__ = map_.next_value()?; - } - GeneratedField::TableName => { - if table_name__.is_some() { - return Err(serde::de::Error::duplicate_field("tableName")); - } - table_name__ = map_.next_value()?; - } - GeneratedField::Target => { - if target__.is_some() { - return Err(serde::de::Error::duplicate_field("target")); - } - target__ = map_.next_value()?; - } - GeneratedField::MergeInto => { - if merge_into__.is_some() { - return Err(serde::de::Error::duplicate_field("mergeInto")); + GeneratedField::Source => { + if source__.is_some() { + return Err(serde::de::Error::duplicate_field("source")); } - merge_into__ = map_.next_value()?; + source__ = map_.next_value()?; } } } - Ok(DmlNode { - dml_type: dml_type__.unwrap_or_default(), - input: input__, - table_name: table_name__, - target: target__, - merge_into: merge_into__, + Ok(DataSourceExecNode { + source: source__, }) } } - deserializer.deserialize_struct("datafusion.DmlNode", FIELDS, GeneratedVisitor) + deserializer.deserialize_struct("datafusion.DataSourceExecNode", FIELDS, GeneratedVisitor) } } -impl serde::Serialize for dml_node::Type { +impl serde::Serialize for DataSourceNode { #[allow(deprecated)] fn serialize(&self, serializer: S) -> std::result::Result where S: serde::Serializer, { - let variant = match self { - Self::Update => "UPDATE", - Self::Delete => "DELETE", - Self::Ctas => "CTAS", - Self::InsertAppend => "INSERT_APPEND", - Self::InsertOverwrite => "INSERT_OVERWRITE", - Self::InsertReplace => "INSERT_REPLACE", - Self::Truncate => "TRUNCATE", - Self::MergeInto => "MERGE_INTO", - }; - serializer.serialize_str(variant) + use serde::ser::SerializeStruct; + let mut len = 0; + if self.source_type.is_some() { + len += 1; + } + let mut struct_ser = serializer.serialize_struct("datafusion.DataSourceNode", len)?; + if let Some(v) = self.source_type.as_ref() { + match v { + data_source_node::SourceType::File(v) => { + struct_ser.serialize_field("file", v)?; + } + data_source_node::SourceType::Memory(v) => { + struct_ser.serialize_field("memory", v)?; + } + data_source_node::SourceType::Extension(v) => { + struct_ser.serialize_field("extension", v)?; + } + } + } + struct_ser.end() } } -impl<'de> serde::Deserialize<'de> for dml_node::Type { +impl<'de> serde::Deserialize<'de> for DataSourceNode { #[allow(deprecated)] fn deserialize(deserializer: D) -> std::result::Result where D: serde::Deserializer<'de>, { const FIELDS: &[&str] = &[ - "UPDATE", - "DELETE", - "CTAS", - "INSERT_APPEND", - "INSERT_OVERWRITE", - "INSERT_REPLACE", - "TRUNCATE", - "MERGE_INTO", + "file", + "memory", + "extension", ]; + #[allow(clippy::enum_variant_names)] + enum GeneratedField { + File, + Memory, + Extension, + } + impl<'de> serde::Deserialize<'de> for GeneratedField { + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + struct GeneratedVisitor; + + impl serde::de::Visitor<'_> for GeneratedVisitor { + type Value = GeneratedField; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(formatter, "expected one of: {:?}", &FIELDS) + } + + #[allow(unused_variables)] + fn visit_str(self, value: &str) -> std::result::Result + where + E: serde::de::Error, + { + match value { + "file" => Ok(GeneratedField::File), + "memory" => Ok(GeneratedField::Memory), + "extension" => Ok(GeneratedField::Extension), + _ => Err(serde::de::Error::unknown_field(value, FIELDS)), + } + } + } + deserializer.deserialize_identifier(GeneratedVisitor) + } + } + struct GeneratedVisitor; + impl<'de> serde::de::Visitor<'de> for GeneratedVisitor { + type Value = DataSourceNode; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("struct datafusion.DataSourceNode") + } + + fn visit_map(self, mut map_: V) -> std::result::Result + where + V: serde::de::MapAccess<'de>, + { + let mut source_type__ = None; + while let Some(k) = map_.next_key()? { + match k { + GeneratedField::File => { + if source_type__.is_some() { + return Err(serde::de::Error::duplicate_field("file")); + } + source_type__ = map_.next_value::<::std::option::Option<_>>()?.map(data_source_node::SourceType::File) +; + } + GeneratedField::Memory => { + if source_type__.is_some() { + return Err(serde::de::Error::duplicate_field("memory")); + } + source_type__ = map_.next_value::<::std::option::Option<_>>()?.map(data_source_node::SourceType::Memory) +; + } + GeneratedField::Extension => { + if source_type__.is_some() { + return Err(serde::de::Error::duplicate_field("extension")); + } + source_type__ = map_.next_value::<::std::option::Option<_>>()?.map(data_source_node::SourceType::Extension) +; + } + } + } + Ok(DataSourceNode { + source_type: source_type__, + }) + } + } + deserializer.deserialize_struct("datafusion.DataSourceNode", FIELDS, GeneratedVisitor) + } +} +impl serde::Serialize for DateUnit { + #[allow(deprecated)] + fn serialize(&self, serializer: S) -> std::result::Result + where + S: serde::Serializer, + { + let variant = match self { + Self::Day => "Day", + Self::DateMillisecond => "DateMillisecond", + }; + serializer.serialize_str(variant) + } +} +impl<'de> serde::Deserialize<'de> for DateUnit { + #[allow(deprecated)] + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + const FIELDS: &[&str] = &[ + "Day", + "DateMillisecond", + ]; + + struct GeneratedVisitor; + + impl serde::de::Visitor<'_> for GeneratedVisitor { + type Value = DateUnit; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(formatter, "expected one of: {:?}", &FIELDS) + } + + fn visit_i64(self, v: i64) -> std::result::Result + where + E: serde::de::Error, + { + i32::try_from(v) + .ok() + .and_then(|x| x.try_into().ok()) + .ok_or_else(|| { + serde::de::Error::invalid_value(serde::de::Unexpected::Signed(v), &self) + }) + } + + fn visit_u64(self, v: u64) -> std::result::Result + where + E: serde::de::Error, + { + i32::try_from(v) + .ok() + .and_then(|x| x.try_into().ok()) + .ok_or_else(|| { + serde::de::Error::invalid_value(serde::de::Unexpected::Unsigned(v), &self) + }) + } + + fn visit_str(self, value: &str) -> std::result::Result + where + E: serde::de::Error, + { + match value { + "Day" => Ok(DateUnit::Day), + "DateMillisecond" => Ok(DateUnit::DateMillisecond), + _ => Err(serde::de::Error::unknown_variant(value, FIELDS)), + } + } + } + deserializer.deserialize_any(GeneratedVisitor) + } +} +impl serde::Serialize for DistinctNode { + #[allow(deprecated)] + fn serialize(&self, serializer: S) -> std::result::Result + where + S: serde::Serializer, + { + use serde::ser::SerializeStruct; + let mut len = 0; + if self.input.is_some() { + len += 1; + } + let mut struct_ser = serializer.serialize_struct("datafusion.DistinctNode", len)?; + if let Some(v) = self.input.as_ref() { + struct_ser.serialize_field("input", v)?; + } + struct_ser.end() + } +} +impl<'de> serde::Deserialize<'de> for DistinctNode { + #[allow(deprecated)] + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + const FIELDS: &[&str] = &[ + "input", + ]; + + #[allow(clippy::enum_variant_names)] + enum GeneratedField { + Input, + } + impl<'de> serde::Deserialize<'de> for GeneratedField { + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + struct GeneratedVisitor; + + impl serde::de::Visitor<'_> for GeneratedVisitor { + type Value = GeneratedField; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(formatter, "expected one of: {:?}", &FIELDS) + } + + #[allow(unused_variables)] + fn visit_str(self, value: &str) -> std::result::Result + where + E: serde::de::Error, + { + match value { + "input" => Ok(GeneratedField::Input), + _ => Err(serde::de::Error::unknown_field(value, FIELDS)), + } + } + } + deserializer.deserialize_identifier(GeneratedVisitor) + } + } + struct GeneratedVisitor; + impl<'de> serde::de::Visitor<'de> for GeneratedVisitor { + type Value = DistinctNode; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("struct datafusion.DistinctNode") + } + + fn visit_map(self, mut map_: V) -> std::result::Result + where + V: serde::de::MapAccess<'de>, + { + let mut input__ = None; + while let Some(k) = map_.next_key()? { + match k { + GeneratedField::Input => { + if input__.is_some() { + return Err(serde::de::Error::duplicate_field("input")); + } + input__ = map_.next_value()?; + } + } + } + Ok(DistinctNode { + input: input__, + }) + } + } + deserializer.deserialize_struct("datafusion.DistinctNode", FIELDS, GeneratedVisitor) + } +} +impl serde::Serialize for DistinctOnNode { + #[allow(deprecated)] + fn serialize(&self, serializer: S) -> std::result::Result + where + S: serde::Serializer, + { + use serde::ser::SerializeStruct; + let mut len = 0; + if !self.on_expr.is_empty() { + len += 1; + } + if !self.select_expr.is_empty() { + len += 1; + } + if !self.sort_expr.is_empty() { + len += 1; + } + if self.input.is_some() { + len += 1; + } + let mut struct_ser = serializer.serialize_struct("datafusion.DistinctOnNode", len)?; + if !self.on_expr.is_empty() { + struct_ser.serialize_field("onExpr", &self.on_expr)?; + } + if !self.select_expr.is_empty() { + struct_ser.serialize_field("selectExpr", &self.select_expr)?; + } + if !self.sort_expr.is_empty() { + struct_ser.serialize_field("sortExpr", &self.sort_expr)?; + } + if let Some(v) = self.input.as_ref() { + struct_ser.serialize_field("input", v)?; + } + struct_ser.end() + } +} +impl<'de> serde::Deserialize<'de> for DistinctOnNode { + #[allow(deprecated)] + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + const FIELDS: &[&str] = &[ + "on_expr", + "onExpr", + "select_expr", + "selectExpr", + "sort_expr", + "sortExpr", + "input", + ]; + + #[allow(clippy::enum_variant_names)] + enum GeneratedField { + OnExpr, + SelectExpr, + SortExpr, + Input, + } + impl<'de> serde::Deserialize<'de> for GeneratedField { + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + struct GeneratedVisitor; + + impl serde::de::Visitor<'_> for GeneratedVisitor { + type Value = GeneratedField; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(formatter, "expected one of: {:?}", &FIELDS) + } + + #[allow(unused_variables)] + fn visit_str(self, value: &str) -> std::result::Result + where + E: serde::de::Error, + { + match value { + "onExpr" | "on_expr" => Ok(GeneratedField::OnExpr), + "selectExpr" | "select_expr" => Ok(GeneratedField::SelectExpr), + "sortExpr" | "sort_expr" => Ok(GeneratedField::SortExpr), + "input" => Ok(GeneratedField::Input), + _ => Err(serde::de::Error::unknown_field(value, FIELDS)), + } + } + } + deserializer.deserialize_identifier(GeneratedVisitor) + } + } + struct GeneratedVisitor; + impl<'de> serde::de::Visitor<'de> for GeneratedVisitor { + type Value = DistinctOnNode; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("struct datafusion.DistinctOnNode") + } + + fn visit_map(self, mut map_: V) -> std::result::Result + where + V: serde::de::MapAccess<'de>, + { + let mut on_expr__ = None; + let mut select_expr__ = None; + let mut sort_expr__ = None; + let mut input__ = None; + while let Some(k) = map_.next_key()? { + match k { + GeneratedField::OnExpr => { + if on_expr__.is_some() { + return Err(serde::de::Error::duplicate_field("onExpr")); + } + on_expr__ = Some(map_.next_value()?); + } + GeneratedField::SelectExpr => { + if select_expr__.is_some() { + return Err(serde::de::Error::duplicate_field("selectExpr")); + } + select_expr__ = Some(map_.next_value()?); + } + GeneratedField::SortExpr => { + if sort_expr__.is_some() { + return Err(serde::de::Error::duplicate_field("sortExpr")); + } + sort_expr__ = Some(map_.next_value()?); + } + GeneratedField::Input => { + if input__.is_some() { + return Err(serde::de::Error::duplicate_field("input")); + } + input__ = map_.next_value()?; + } + } + } + Ok(DistinctOnNode { + on_expr: on_expr__.unwrap_or_default(), + select_expr: select_expr__.unwrap_or_default(), + sort_expr: sort_expr__.unwrap_or_default(), + input: input__, + }) + } + } + deserializer.deserialize_struct("datafusion.DistinctOnNode", FIELDS, GeneratedVisitor) + } +} +impl serde::Serialize for DmlNode { + #[allow(deprecated)] + fn serialize(&self, serializer: S) -> std::result::Result + where + S: serde::Serializer, + { + use serde::ser::SerializeStruct; + let mut len = 0; + if self.dml_type != 0 { + len += 1; + } + if self.input.is_some() { + len += 1; + } + if self.table_name.is_some() { + len += 1; + } + if self.target.is_some() { + len += 1; + } + if self.merge_into.is_some() { + len += 1; + } + let mut struct_ser = serializer.serialize_struct("datafusion.DmlNode", len)?; + if self.dml_type != 0 { + let v = dml_node::Type::try_from(self.dml_type) + .map_err(|_| serde::ser::Error::custom(format!("Invalid variant {}", self.dml_type)))?; + struct_ser.serialize_field("dmlType", &v)?; + } + if let Some(v) = self.input.as_ref() { + struct_ser.serialize_field("input", v)?; + } + if let Some(v) = self.table_name.as_ref() { + struct_ser.serialize_field("tableName", v)?; + } + if let Some(v) = self.target.as_ref() { + struct_ser.serialize_field("target", v)?; + } + if let Some(v) = self.merge_into.as_ref() { + struct_ser.serialize_field("mergeInto", v)?; + } + struct_ser.end() + } +} +impl<'de> serde::Deserialize<'de> for DmlNode { + #[allow(deprecated)] + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + const FIELDS: &[&str] = &[ + "dml_type", + "dmlType", + "input", + "table_name", + "tableName", + "target", + "merge_into", + "mergeInto", + ]; + + #[allow(clippy::enum_variant_names)] + enum GeneratedField { + DmlType, + Input, + TableName, + Target, + MergeInto, + } + impl<'de> serde::Deserialize<'de> for GeneratedField { + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + struct GeneratedVisitor; + + impl serde::de::Visitor<'_> for GeneratedVisitor { + type Value = GeneratedField; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(formatter, "expected one of: {:?}", &FIELDS) + } + + #[allow(unused_variables)] + fn visit_str(self, value: &str) -> std::result::Result + where + E: serde::de::Error, + { + match value { + "dmlType" | "dml_type" => Ok(GeneratedField::DmlType), + "input" => Ok(GeneratedField::Input), + "tableName" | "table_name" => Ok(GeneratedField::TableName), + "target" => Ok(GeneratedField::Target), + "mergeInto" | "merge_into" => Ok(GeneratedField::MergeInto), + _ => Err(serde::de::Error::unknown_field(value, FIELDS)), + } + } + } + deserializer.deserialize_identifier(GeneratedVisitor) + } + } + struct GeneratedVisitor; + impl<'de> serde::de::Visitor<'de> for GeneratedVisitor { + type Value = DmlNode; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("struct datafusion.DmlNode") + } + + fn visit_map(self, mut map_: V) -> std::result::Result + where + V: serde::de::MapAccess<'de>, + { + let mut dml_type__ = None; + let mut input__ = None; + let mut table_name__ = None; + let mut target__ = None; + let mut merge_into__ = None; + while let Some(k) = map_.next_key()? { + match k { + GeneratedField::DmlType => { + if dml_type__.is_some() { + return Err(serde::de::Error::duplicate_field("dmlType")); + } + dml_type__ = Some(map_.next_value::()? as i32); + } + GeneratedField::Input => { + if input__.is_some() { + return Err(serde::de::Error::duplicate_field("input")); + } + input__ = map_.next_value()?; + } + GeneratedField::TableName => { + if table_name__.is_some() { + return Err(serde::de::Error::duplicate_field("tableName")); + } + table_name__ = map_.next_value()?; + } + GeneratedField::Target => { + if target__.is_some() { + return Err(serde::de::Error::duplicate_field("target")); + } + target__ = map_.next_value()?; + } + GeneratedField::MergeInto => { + if merge_into__.is_some() { + return Err(serde::de::Error::duplicate_field("mergeInto")); + } + merge_into__ = map_.next_value()?; + } + } + } + Ok(DmlNode { + dml_type: dml_type__.unwrap_or_default(), + input: input__, + table_name: table_name__, + target: target__, + merge_into: merge_into__, + }) + } + } + deserializer.deserialize_struct("datafusion.DmlNode", FIELDS, GeneratedVisitor) + } +} +impl serde::Serialize for dml_node::Type { + #[allow(deprecated)] + fn serialize(&self, serializer: S) -> std::result::Result + where + S: serde::Serializer, + { + let variant = match self { + Self::Update => "UPDATE", + Self::Delete => "DELETE", + Self::Ctas => "CTAS", + Self::InsertAppend => "INSERT_APPEND", + Self::InsertOverwrite => "INSERT_OVERWRITE", + Self::InsertReplace => "INSERT_REPLACE", + Self::Truncate => "TRUNCATE", + Self::MergeInto => "MERGE_INTO", + }; + serializer.serialize_str(variant) + } +} +impl<'de> serde::Deserialize<'de> for dml_node::Type { + #[allow(deprecated)] + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + const FIELDS: &[&str] = &[ + "UPDATE", + "DELETE", + "CTAS", + "INSERT_APPEND", + "INSERT_OVERWRITE", + "INSERT_REPLACE", + "TRUNCATE", + "MERGE_INTO", + ]; + + struct GeneratedVisitor; + + impl serde::de::Visitor<'_> for GeneratedVisitor { + type Value = dml_node::Type; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(formatter, "expected one of: {:?}", &FIELDS) + } + + fn visit_i64(self, v: i64) -> std::result::Result + where + E: serde::de::Error, + { + i32::try_from(v) + .ok() + .and_then(|x| x.try_into().ok()) + .ok_or_else(|| { + serde::de::Error::invalid_value(serde::de::Unexpected::Signed(v), &self) + }) + } + + fn visit_u64(self, v: u64) -> std::result::Result + where + E: serde::de::Error, + { + i32::try_from(v) + .ok() + .and_then(|x| x.try_into().ok()) + .ok_or_else(|| { + serde::de::Error::invalid_value(serde::de::Unexpected::Unsigned(v), &self) + }) + } + + fn visit_str(self, value: &str) -> std::result::Result + where + E: serde::de::Error, + { + match value { + "UPDATE" => Ok(dml_node::Type::Update), + "DELETE" => Ok(dml_node::Type::Delete), + "CTAS" => Ok(dml_node::Type::Ctas), + "INSERT_APPEND" => Ok(dml_node::Type::InsertAppend), + "INSERT_OVERWRITE" => Ok(dml_node::Type::InsertOverwrite), + "INSERT_REPLACE" => Ok(dml_node::Type::InsertReplace), + "TRUNCATE" => Ok(dml_node::Type::Truncate), + "MERGE_INTO" => Ok(dml_node::Type::MergeInto), + _ => Err(serde::de::Error::unknown_variant(value, FIELDS)), + } + } + } + deserializer.deserialize_any(GeneratedVisitor) + } +} +impl serde::Serialize for DropViewNode { + #[allow(deprecated)] + fn serialize(&self, serializer: S) -> std::result::Result + where + S: serde::Serializer, + { + use serde::ser::SerializeStruct; + let mut len = 0; + if self.name.is_some() { + len += 1; + } + if self.if_exists { + len += 1; + } + if self.schema.is_some() { + len += 1; + } + let mut struct_ser = serializer.serialize_struct("datafusion.DropViewNode", len)?; + if let Some(v) = self.name.as_ref() { + struct_ser.serialize_field("name", v)?; + } + if self.if_exists { + struct_ser.serialize_field("ifExists", &self.if_exists)?; + } + if let Some(v) = self.schema.as_ref() { + struct_ser.serialize_field("schema", v)?; + } + struct_ser.end() + } +} +impl<'de> serde::Deserialize<'de> for DropViewNode { + #[allow(deprecated)] + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + const FIELDS: &[&str] = &[ + "name", + "if_exists", + "ifExists", + "schema", + ]; + + #[allow(clippy::enum_variant_names)] + enum GeneratedField { + Name, + IfExists, + Schema, + } + impl<'de> serde::Deserialize<'de> for GeneratedField { + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + struct GeneratedVisitor; + + impl serde::de::Visitor<'_> for GeneratedVisitor { + type Value = GeneratedField; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(formatter, "expected one of: {:?}", &FIELDS) + } + + #[allow(unused_variables)] + fn visit_str(self, value: &str) -> std::result::Result + where + E: serde::de::Error, + { + match value { + "name" => Ok(GeneratedField::Name), + "ifExists" | "if_exists" => Ok(GeneratedField::IfExists), + "schema" => Ok(GeneratedField::Schema), + _ => Err(serde::de::Error::unknown_field(value, FIELDS)), + } + } + } + deserializer.deserialize_identifier(GeneratedVisitor) + } + } + struct GeneratedVisitor; + impl<'de> serde::de::Visitor<'de> for GeneratedVisitor { + type Value = DropViewNode; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("struct datafusion.DropViewNode") + } + + fn visit_map(self, mut map_: V) -> std::result::Result + where + V: serde::de::MapAccess<'de>, + { + let mut name__ = None; + let mut if_exists__ = None; + let mut schema__ = None; + while let Some(k) = map_.next_key()? { + match k { + GeneratedField::Name => { + if name__.is_some() { + return Err(serde::de::Error::duplicate_field("name")); + } + name__ = map_.next_value()?; + } + GeneratedField::IfExists => { + if if_exists__.is_some() { + return Err(serde::de::Error::duplicate_field("ifExists")); + } + if_exists__ = Some(map_.next_value()?); + } + GeneratedField::Schema => { + if schema__.is_some() { + return Err(serde::de::Error::duplicate_field("schema")); + } + schema__ = map_.next_value()?; + } + } + } + Ok(DropViewNode { + name: name__, + if_exists: if_exists__.unwrap_or_default(), + schema: schema__, + }) + } + } + deserializer.deserialize_struct("datafusion.DropViewNode", FIELDS, GeneratedVisitor) + } +} +impl serde::Serialize for EmptyExecNode { + #[allow(deprecated)] + fn serialize(&self, serializer: S) -> std::result::Result + where + S: serde::Serializer, + { + use serde::ser::SerializeStruct; + let mut len = 0; + if self.schema.is_some() { + len += 1; + } + if self.partitions != 0 { + len += 1; + } + let mut struct_ser = serializer.serialize_struct("datafusion.EmptyExecNode", len)?; + if let Some(v) = self.schema.as_ref() { + struct_ser.serialize_field("schema", v)?; + } + if self.partitions != 0 { + struct_ser.serialize_field("partitions", &self.partitions)?; + } + struct_ser.end() + } +} +impl<'de> serde::Deserialize<'de> for EmptyExecNode { + #[allow(deprecated)] + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + const FIELDS: &[&str] = &[ + "schema", + "partitions", + ]; + + #[allow(clippy::enum_variant_names)] + enum GeneratedField { + Schema, + Partitions, + } + impl<'de> serde::Deserialize<'de> for GeneratedField { + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + struct GeneratedVisitor; + + impl serde::de::Visitor<'_> for GeneratedVisitor { + type Value = GeneratedField; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(formatter, "expected one of: {:?}", &FIELDS) + } + + #[allow(unused_variables)] + fn visit_str(self, value: &str) -> std::result::Result + where + E: serde::de::Error, + { + match value { + "schema" => Ok(GeneratedField::Schema), + "partitions" => Ok(GeneratedField::Partitions), + _ => Err(serde::de::Error::unknown_field(value, FIELDS)), + } + } + } + deserializer.deserialize_identifier(GeneratedVisitor) + } + } + struct GeneratedVisitor; + impl<'de> serde::de::Visitor<'de> for GeneratedVisitor { + type Value = EmptyExecNode; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("struct datafusion.EmptyExecNode") + } + + fn visit_map(self, mut map_: V) -> std::result::Result + where + V: serde::de::MapAccess<'de>, + { + let mut schema__ = None; + let mut partitions__ = None; + while let Some(k) = map_.next_key()? { + match k { + GeneratedField::Schema => { + if schema__.is_some() { + return Err(serde::de::Error::duplicate_field("schema")); + } + schema__ = map_.next_value()?; + } + GeneratedField::Partitions => { + if partitions__.is_some() { + return Err(serde::de::Error::duplicate_field("partitions")); + } + partitions__ = + Some(map_.next_value::<::pbjson::private::NumberDeserialize<_>>()?.0) + ; + } + } + } + Ok(EmptyExecNode { + schema: schema__, + partitions: partitions__.unwrap_or_default(), + }) + } + } + deserializer.deserialize_struct("datafusion.EmptyExecNode", FIELDS, GeneratedVisitor) + } +} +impl serde::Serialize for EmptyRelationNode { + #[allow(deprecated)] + fn serialize(&self, serializer: S) -> std::result::Result + where + S: serde::Serializer, + { + use serde::ser::SerializeStruct; + let mut len = 0; + if self.produce_one_row { + len += 1; + } + if self.schema.is_some() { + len += 1; + } + let mut struct_ser = serializer.serialize_struct("datafusion.EmptyRelationNode", len)?; + if self.produce_one_row { + struct_ser.serialize_field("produceOneRow", &self.produce_one_row)?; + } + if let Some(v) = self.schema.as_ref() { + struct_ser.serialize_field("schema", v)?; + } + struct_ser.end() + } +} +impl<'de> serde::Deserialize<'de> for EmptyRelationNode { + #[allow(deprecated)] + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + const FIELDS: &[&str] = &[ + "produce_one_row", + "produceOneRow", + "schema", + ]; + + #[allow(clippy::enum_variant_names)] + enum GeneratedField { + ProduceOneRow, + Schema, + } + impl<'de> serde::Deserialize<'de> for GeneratedField { + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + struct GeneratedVisitor; + + impl serde::de::Visitor<'_> for GeneratedVisitor { + type Value = GeneratedField; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(formatter, "expected one of: {:?}", &FIELDS) + } + + #[allow(unused_variables)] + fn visit_str(self, value: &str) -> std::result::Result + where + E: serde::de::Error, + { + match value { + "produceOneRow" | "produce_one_row" => Ok(GeneratedField::ProduceOneRow), + "schema" => Ok(GeneratedField::Schema), + _ => Err(serde::de::Error::unknown_field(value, FIELDS)), + } + } + } + deserializer.deserialize_identifier(GeneratedVisitor) + } + } struct GeneratedVisitor; - - impl serde::de::Visitor<'_> for GeneratedVisitor { - type Value = dml_node::Type; + impl<'de> serde::de::Visitor<'de> for GeneratedVisitor { + type Value = EmptyRelationNode; fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(formatter, "expected one of: {:?}", &FIELDS) - } - - fn visit_i64(self, v: i64) -> std::result::Result - where - E: serde::de::Error, - { - i32::try_from(v) - .ok() - .and_then(|x| x.try_into().ok()) - .ok_or_else(|| { - serde::de::Error::invalid_value(serde::de::Unexpected::Signed(v), &self) - }) - } - - fn visit_u64(self, v: u64) -> std::result::Result - where - E: serde::de::Error, - { - i32::try_from(v) - .ok() - .and_then(|x| x.try_into().ok()) - .ok_or_else(|| { - serde::de::Error::invalid_value(serde::de::Unexpected::Unsigned(v), &self) - }) + formatter.write_str("struct datafusion.EmptyRelationNode") } - fn visit_str(self, value: &str) -> std::result::Result - where - E: serde::de::Error, + fn visit_map(self, mut map_: V) -> std::result::Result + where + V: serde::de::MapAccess<'de>, { - match value { - "UPDATE" => Ok(dml_node::Type::Update), - "DELETE" => Ok(dml_node::Type::Delete), - "CTAS" => Ok(dml_node::Type::Ctas), - "INSERT_APPEND" => Ok(dml_node::Type::InsertAppend), - "INSERT_OVERWRITE" => Ok(dml_node::Type::InsertOverwrite), - "INSERT_REPLACE" => Ok(dml_node::Type::InsertReplace), - "TRUNCATE" => Ok(dml_node::Type::Truncate), - "MERGE_INTO" => Ok(dml_node::Type::MergeInto), - _ => Err(serde::de::Error::unknown_variant(value, FIELDS)), + let mut produce_one_row__ = None; + let mut schema__ = None; + while let Some(k) = map_.next_key()? { + match k { + GeneratedField::ProduceOneRow => { + if produce_one_row__.is_some() { + return Err(serde::de::Error::duplicate_field("produceOneRow")); + } + produce_one_row__ = Some(map_.next_value()?); + } + GeneratedField::Schema => { + if schema__.is_some() { + return Err(serde::de::Error::duplicate_field("schema")); + } + schema__ = map_.next_value()?; + } + } } + Ok(EmptyRelationNode { + produce_one_row: produce_one_row__.unwrap_or_default(), + schema: schema__, + }) } } - deserializer.deserialize_any(GeneratedVisitor) + deserializer.deserialize_struct("datafusion.EmptyRelationNode", FIELDS, GeneratedVisitor) } } -impl serde::Serialize for DropViewNode { +impl serde::Serialize for EmptyTableScanNode { #[allow(deprecated)] fn serialize(&self, serializer: S) -> std::result::Result where @@ -6456,46 +7664,54 @@ impl serde::Serialize for DropViewNode { { use serde::ser::SerializeStruct; let mut len = 0; - if self.name.is_some() { + if self.table_name.is_some() { len += 1; } - if self.if_exists { + if self.schema.is_some() { len += 1; } - if self.schema.is_some() { + if self.projection.is_some() { len += 1; } - let mut struct_ser = serializer.serialize_struct("datafusion.DropViewNode", len)?; - if let Some(v) = self.name.as_ref() { - struct_ser.serialize_field("name", v)?; + if !self.filters.is_empty() { + len += 1; } - if self.if_exists { - struct_ser.serialize_field("ifExists", &self.if_exists)?; + let mut struct_ser = serializer.serialize_struct("datafusion.EmptyTableScanNode", len)?; + if let Some(v) = self.table_name.as_ref() { + struct_ser.serialize_field("tableName", v)?; } if let Some(v) = self.schema.as_ref() { struct_ser.serialize_field("schema", v)?; } + if let Some(v) = self.projection.as_ref() { + struct_ser.serialize_field("projection", v)?; + } + if !self.filters.is_empty() { + struct_ser.serialize_field("filters", &self.filters)?; + } struct_ser.end() } } -impl<'de> serde::Deserialize<'de> for DropViewNode { +impl<'de> serde::Deserialize<'de> for EmptyTableScanNode { #[allow(deprecated)] fn deserialize(deserializer: D) -> std::result::Result where D: serde::Deserializer<'de>, { const FIELDS: &[&str] = &[ - "name", - "if_exists", - "ifExists", + "table_name", + "tableName", "schema", + "projection", + "filters", ]; #[allow(clippy::enum_variant_names)] enum GeneratedField { - Name, - IfExists, + TableName, Schema, + Projection, + Filters, } impl<'de> serde::Deserialize<'de> for GeneratedField { fn deserialize(deserializer: D) -> std::result::Result @@ -6517,9 +7733,10 @@ impl<'de> serde::Deserialize<'de> for DropViewNode { E: serde::de::Error, { match value { - "name" => Ok(GeneratedField::Name), - "ifExists" | "if_exists" => Ok(GeneratedField::IfExists), + "tableName" | "table_name" => Ok(GeneratedField::TableName), "schema" => Ok(GeneratedField::Schema), + "projection" => Ok(GeneratedField::Projection), + "filters" => Ok(GeneratedField::Filters), _ => Err(serde::de::Error::unknown_field(value, FIELDS)), } } @@ -6529,32 +7746,27 @@ impl<'de> serde::Deserialize<'de> for DropViewNode { } struct GeneratedVisitor; impl<'de> serde::de::Visitor<'de> for GeneratedVisitor { - type Value = DropViewNode; + type Value = EmptyTableScanNode; fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - formatter.write_str("struct datafusion.DropViewNode") + formatter.write_str("struct datafusion.EmptyTableScanNode") } - fn visit_map(self, mut map_: V) -> std::result::Result + fn visit_map(self, mut map_: V) -> std::result::Result where V: serde::de::MapAccess<'de>, { - let mut name__ = None; - let mut if_exists__ = None; + let mut table_name__ = None; let mut schema__ = None; + let mut projection__ = None; + let mut filters__ = None; while let Some(k) = map_.next_key()? { match k { - GeneratedField::Name => { - if name__.is_some() { - return Err(serde::de::Error::duplicate_field("name")); - } - name__ = map_.next_value()?; - } - GeneratedField::IfExists => { - if if_exists__.is_some() { - return Err(serde::de::Error::duplicate_field("ifExists")); + GeneratedField::TableName => { + if table_name__.is_some() { + return Err(serde::de::Error::duplicate_field("tableName")); } - if_exists__ = Some(map_.next_value()?); + table_name__ = map_.next_value()?; } GeneratedField::Schema => { if schema__.is_some() { @@ -6562,19 +7774,32 @@ impl<'de> serde::Deserialize<'de> for DropViewNode { } schema__ = map_.next_value()?; } + GeneratedField::Projection => { + if projection__.is_some() { + return Err(serde::de::Error::duplicate_field("projection")); + } + projection__ = map_.next_value()?; + } + GeneratedField::Filters => { + if filters__.is_some() { + return Err(serde::de::Error::duplicate_field("filters")); + } + filters__ = Some(map_.next_value()?); + } } } - Ok(DropViewNode { - name: name__, - if_exists: if_exists__.unwrap_or_default(), + Ok(EmptyTableScanNode { + table_name: table_name__, schema: schema__, + projection: projection__, + filters: filters__.unwrap_or_default(), }) } } - deserializer.deserialize_struct("datafusion.DropViewNode", FIELDS, GeneratedVisitor) + deserializer.deserialize_struct("datafusion.EmptyTableScanNode", FIELDS, GeneratedVisitor) } } -impl serde::Serialize for EmptyExecNode { +impl serde::Serialize for ExplainExecNode { #[allow(deprecated)] fn serialize(&self, serializer: S) -> std::result::Result where @@ -6585,20 +7810,26 @@ impl serde::Serialize for EmptyExecNode { if self.schema.is_some() { len += 1; } - if self.partitions != 0 { + if !self.stringified_plans.is_empty() { len += 1; } - let mut struct_ser = serializer.serialize_struct("datafusion.EmptyExecNode", len)?; + if self.verbose { + len += 1; + } + let mut struct_ser = serializer.serialize_struct("datafusion.ExplainExecNode", len)?; if let Some(v) = self.schema.as_ref() { struct_ser.serialize_field("schema", v)?; } - if self.partitions != 0 { - struct_ser.serialize_field("partitions", &self.partitions)?; + if !self.stringified_plans.is_empty() { + struct_ser.serialize_field("stringifiedPlans", &self.stringified_plans)?; + } + if self.verbose { + struct_ser.serialize_field("verbose", &self.verbose)?; } struct_ser.end() } } -impl<'de> serde::Deserialize<'de> for EmptyExecNode { +impl<'de> serde::Deserialize<'de> for ExplainExecNode { #[allow(deprecated)] fn deserialize(deserializer: D) -> std::result::Result where @@ -6606,13 +7837,16 @@ impl<'de> serde::Deserialize<'de> for EmptyExecNode { { const FIELDS: &[&str] = &[ "schema", - "partitions", + "stringified_plans", + "stringifiedPlans", + "verbose", ]; #[allow(clippy::enum_variant_names)] enum GeneratedField { Schema, - Partitions, + StringifiedPlans, + Verbose, } impl<'de> serde::Deserialize<'de> for GeneratedField { fn deserialize(deserializer: D) -> std::result::Result @@ -6635,7 +7869,8 @@ impl<'de> serde::Deserialize<'de> for EmptyExecNode { { match value { "schema" => Ok(GeneratedField::Schema), - "partitions" => Ok(GeneratedField::Partitions), + "stringifiedPlans" | "stringified_plans" => Ok(GeneratedField::StringifiedPlans), + "verbose" => Ok(GeneratedField::Verbose), _ => Err(serde::de::Error::unknown_field(value, FIELDS)), } } @@ -6645,18 +7880,19 @@ impl<'de> serde::Deserialize<'de> for EmptyExecNode { } struct GeneratedVisitor; impl<'de> serde::de::Visitor<'de> for GeneratedVisitor { - type Value = EmptyExecNode; + type Value = ExplainExecNode; fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - formatter.write_str("struct datafusion.EmptyExecNode") + formatter.write_str("struct datafusion.ExplainExecNode") } - fn visit_map(self, mut map_: V) -> std::result::Result + fn visit_map(self, mut map_: V) -> std::result::Result where V: serde::de::MapAccess<'de>, { let mut schema__ = None; - let mut partitions__ = None; + let mut stringified_plans__ = None; + let mut verbose__ = None; while let Some(k) = map_.next_key()? { match k { GeneratedField::Schema => { @@ -6665,26 +7901,31 @@ impl<'de> serde::Deserialize<'de> for EmptyExecNode { } schema__ = map_.next_value()?; } - GeneratedField::Partitions => { - if partitions__.is_some() { - return Err(serde::de::Error::duplicate_field("partitions")); + GeneratedField::StringifiedPlans => { + if stringified_plans__.is_some() { + return Err(serde::de::Error::duplicate_field("stringifiedPlans")); } - partitions__ = - Some(map_.next_value::<::pbjson::private::NumberDeserialize<_>>()?.0) - ; + stringified_plans__ = Some(map_.next_value()?); + } + GeneratedField::Verbose => { + if verbose__.is_some() { + return Err(serde::de::Error::duplicate_field("verbose")); + } + verbose__ = Some(map_.next_value()?); } } } - Ok(EmptyExecNode { + Ok(ExplainExecNode { schema: schema__, - partitions: partitions__.unwrap_or_default(), + stringified_plans: stringified_plans__.unwrap_or_default(), + verbose: verbose__.unwrap_or_default(), }) } } - deserializer.deserialize_struct("datafusion.EmptyExecNode", FIELDS, GeneratedVisitor) + deserializer.deserialize_struct("datafusion.ExplainExecNode", FIELDS, GeneratedVisitor) } } -impl serde::Serialize for EmptyRelationNode { +impl serde::Serialize for ExplainNode { #[allow(deprecated)] fn serialize(&self, serializer: S) -> std::result::Result where @@ -6692,38 +7933,56 @@ impl serde::Serialize for EmptyRelationNode { { use serde::ser::SerializeStruct; let mut len = 0; - if self.produce_one_row { + if self.input.is_some() { len += 1; } - if self.schema.is_some() { + if self.verbose { len += 1; } - let mut struct_ser = serializer.serialize_struct("datafusion.EmptyRelationNode", len)?; - if self.produce_one_row { - struct_ser.serialize_field("produceOneRow", &self.produce_one_row)?; + if self.format != 0 { + len += 1; } - if let Some(v) = self.schema.as_ref() { - struct_ser.serialize_field("schema", v)?; + if self.show_statistics.is_some() { + len += 1; + } + let mut struct_ser = serializer.serialize_struct("datafusion.ExplainNode", len)?; + if let Some(v) = self.input.as_ref() { + struct_ser.serialize_field("input", v)?; + } + if self.verbose { + struct_ser.serialize_field("verbose", &self.verbose)?; + } + if self.format != 0 { + let v = super::datafusion_common::ExplainFormat::try_from(self.format) + .map_err(|_| serde::ser::Error::custom(format!("Invalid variant {}", self.format)))?; + struct_ser.serialize_field("format", &v)?; + } + if let Some(v) = self.show_statistics.as_ref() { + struct_ser.serialize_field("showStatistics", v)?; } struct_ser.end() } } -impl<'de> serde::Deserialize<'de> for EmptyRelationNode { +impl<'de> serde::Deserialize<'de> for ExplainNode { #[allow(deprecated)] fn deserialize(deserializer: D) -> std::result::Result where D: serde::Deserializer<'de>, { const FIELDS: &[&str] = &[ - "produce_one_row", - "produceOneRow", - "schema", + "input", + "verbose", + "format", + "show_statistics", + "showStatistics", ]; #[allow(clippy::enum_variant_names)] enum GeneratedField { - ProduceOneRow, - Schema, + Input, + Verbose, + Format, + ShowStatistics, } impl<'de> serde::Deserialize<'de> for GeneratedField { fn deserialize(deserializer: D) -> std::result::Result @@ -6745,8 +8004,10 @@ impl<'de> serde::Deserialize<'de> for EmptyRelationNode { E: serde::de::Error, { match value { - "produceOneRow" | "produce_one_row" => Ok(GeneratedField::ProduceOneRow), - "schema" => Ok(GeneratedField::Schema), + "input" => Ok(GeneratedField::Input), + "verbose" => Ok(GeneratedField::Verbose), + "format" => Ok(GeneratedField::Format), + "showStatistics" | "show_statistics" => Ok(GeneratedField::ShowStatistics), _ => Err(serde::de::Error::unknown_field(value, FIELDS)), } } @@ -6756,44 +8017,60 @@ impl<'de> serde::Deserialize<'de> for EmptyRelationNode { } struct GeneratedVisitor; impl<'de> serde::de::Visitor<'de> for GeneratedVisitor { - type Value = EmptyRelationNode; + type Value = ExplainNode; fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - formatter.write_str("struct datafusion.EmptyRelationNode") + formatter.write_str("struct datafusion.ExplainNode") } - fn visit_map(self, mut map_: V) -> std::result::Result + fn visit_map(self, mut map_: V) -> std::result::Result where V: serde::de::MapAccess<'de>, { - let mut produce_one_row__ = None; - let mut schema__ = None; + let mut input__ = None; + let mut verbose__ = None; + let mut format__ = None; + let mut show_statistics__ = None; while let Some(k) = map_.next_key()? { match k { - GeneratedField::ProduceOneRow => { - if produce_one_row__.is_some() { - return Err(serde::de::Error::duplicate_field("produceOneRow")); + GeneratedField::Input => { + if input__.is_some() { + return Err(serde::de::Error::duplicate_field("input")); } - produce_one_row__ = Some(map_.next_value()?); + input__ = map_.next_value()?; } - GeneratedField::Schema => { - if schema__.is_some() { - return Err(serde::de::Error::duplicate_field("schema")); + GeneratedField::Verbose => { + if verbose__.is_some() { + return Err(serde::de::Error::duplicate_field("verbose")); } - schema__ = map_.next_value()?; + verbose__ = Some(map_.next_value()?); + } + GeneratedField::Format => { + if format__.is_some() { + return Err(serde::de::Error::duplicate_field("format")); + } + format__ = Some(map_.next_value::()? as i32); + } + GeneratedField::ShowStatistics => { + if show_statistics__.is_some() { + return Err(serde::de::Error::duplicate_field("showStatistics")); + } + show_statistics__ = map_.next_value()?; } } } - Ok(EmptyRelationNode { - produce_one_row: produce_one_row__.unwrap_or_default(), - schema: schema__, + Ok(ExplainNode { + input: input__, + verbose: verbose__.unwrap_or_default(), + format: format__.unwrap_or_default(), + show_statistics: show_statistics__, }) } } - deserializer.deserialize_struct("datafusion.EmptyRelationNode", FIELDS, GeneratedVisitor) + deserializer.deserialize_struct("datafusion.ExplainNode", FIELDS, GeneratedVisitor) } } -impl serde::Serialize for EmptyTableScanNode { +impl serde::Serialize for ExtensionDataSinkNode { #[allow(deprecated)] fn serialize(&self, serializer: S) -> std::result::Result where @@ -6801,54 +8078,39 @@ impl serde::Serialize for EmptyTableScanNode { { use serde::ser::SerializeStruct; let mut len = 0; - if self.table_name.is_some() { - len += 1; - } - if self.schema.is_some() { - len += 1; - } - if self.projection.is_some() { + if !self.name.is_empty() { len += 1; } - if !self.filters.is_empty() { + if !self.payload.is_empty() { len += 1; } - let mut struct_ser = serializer.serialize_struct("datafusion.EmptyTableScanNode", len)?; - if let Some(v) = self.table_name.as_ref() { - struct_ser.serialize_field("tableName", v)?; - } - if let Some(v) = self.schema.as_ref() { - struct_ser.serialize_field("schema", v)?; - } - if let Some(v) = self.projection.as_ref() { - struct_ser.serialize_field("projection", v)?; + let mut struct_ser = serializer.serialize_struct("datafusion.ExtensionDataSinkNode", len)?; + if !self.name.is_empty() { + struct_ser.serialize_field("name", &self.name)?; } - if !self.filters.is_empty() { - struct_ser.serialize_field("filters", &self.filters)?; + if !self.payload.is_empty() { + #[allow(clippy::needless_borrow)] + #[allow(clippy::needless_borrows_for_generic_args)] + struct_ser.serialize_field("payload", pbjson::private::base64::encode(&self.payload).as_str())?; } struct_ser.end() } } -impl<'de> serde::Deserialize<'de> for EmptyTableScanNode { +impl<'de> serde::Deserialize<'de> for ExtensionDataSinkNode { #[allow(deprecated)] fn deserialize(deserializer: D) -> std::result::Result where D: serde::Deserializer<'de>, { const FIELDS: &[&str] = &[ - "table_name", - "tableName", - "schema", - "projection", - "filters", + "name", + "payload", ]; #[allow(clippy::enum_variant_names)] enum GeneratedField { - TableName, - Schema, - Projection, - Filters, + Name, + Payload, } impl<'de> serde::Deserialize<'de> for GeneratedField { fn deserialize(deserializer: D) -> std::result::Result @@ -6870,10 +8132,8 @@ impl<'de> serde::Deserialize<'de> for EmptyTableScanNode { E: serde::de::Error, { match value { - "tableName" | "table_name" => Ok(GeneratedField::TableName), - "schema" => Ok(GeneratedField::Schema), - "projection" => Ok(GeneratedField::Projection), - "filters" => Ok(GeneratedField::Filters), + "name" => Ok(GeneratedField::Name), + "payload" => Ok(GeneratedField::Payload), _ => Err(serde::de::Error::unknown_field(value, FIELDS)), } } @@ -6883,60 +8143,46 @@ impl<'de> serde::Deserialize<'de> for EmptyTableScanNode { } struct GeneratedVisitor; impl<'de> serde::de::Visitor<'de> for GeneratedVisitor { - type Value = EmptyTableScanNode; + type Value = ExtensionDataSinkNode; fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - formatter.write_str("struct datafusion.EmptyTableScanNode") + formatter.write_str("struct datafusion.ExtensionDataSinkNode") } - fn visit_map(self, mut map_: V) -> std::result::Result + fn visit_map(self, mut map_: V) -> std::result::Result where V: serde::de::MapAccess<'de>, { - let mut table_name__ = None; - let mut schema__ = None; - let mut projection__ = None; - let mut filters__ = None; + let mut name__ = None; + let mut payload__ = None; while let Some(k) = map_.next_key()? { match k { - GeneratedField::TableName => { - if table_name__.is_some() { - return Err(serde::de::Error::duplicate_field("tableName")); - } - table_name__ = map_.next_value()?; - } - GeneratedField::Schema => { - if schema__.is_some() { - return Err(serde::de::Error::duplicate_field("schema")); - } - schema__ = map_.next_value()?; - } - GeneratedField::Projection => { - if projection__.is_some() { - return Err(serde::de::Error::duplicate_field("projection")); + GeneratedField::Name => { + if name__.is_some() { + return Err(serde::de::Error::duplicate_field("name")); } - projection__ = map_.next_value()?; + name__ = Some(map_.next_value()?); } - GeneratedField::Filters => { - if filters__.is_some() { - return Err(serde::de::Error::duplicate_field("filters")); + GeneratedField::Payload => { + if payload__.is_some() { + return Err(serde::de::Error::duplicate_field("payload")); } - filters__ = Some(map_.next_value()?); + payload__ = + Some(map_.next_value::<::pbjson::private::BytesDeserialize<_>>()?.0) + ; } } } - Ok(EmptyTableScanNode { - table_name: table_name__, - schema: schema__, - projection: projection__, - filters: filters__.unwrap_or_default(), + Ok(ExtensionDataSinkNode { + name: name__.unwrap_or_default(), + payload: payload__.unwrap_or_default(), }) } } - deserializer.deserialize_struct("datafusion.EmptyTableScanNode", FIELDS, GeneratedVisitor) + deserializer.deserialize_struct("datafusion.ExtensionDataSinkNode", FIELDS, GeneratedVisitor) } } -impl serde::Serialize for ExplainExecNode { +impl serde::Serialize for ExtensionDataSourceNode { #[allow(deprecated)] fn serialize(&self, serializer: S) -> std::result::Result where @@ -6944,46 +8190,39 @@ impl serde::Serialize for ExplainExecNode { { use serde::ser::SerializeStruct; let mut len = 0; - if self.schema.is_some() { - len += 1; - } - if !self.stringified_plans.is_empty() { + if !self.name.is_empty() { len += 1; } - if self.verbose { + if !self.payload.is_empty() { len += 1; } - let mut struct_ser = serializer.serialize_struct("datafusion.ExplainExecNode", len)?; - if let Some(v) = self.schema.as_ref() { - struct_ser.serialize_field("schema", v)?; - } - if !self.stringified_plans.is_empty() { - struct_ser.serialize_field("stringifiedPlans", &self.stringified_plans)?; + let mut struct_ser = serializer.serialize_struct("datafusion.ExtensionDataSourceNode", len)?; + if !self.name.is_empty() { + struct_ser.serialize_field("name", &self.name)?; } - if self.verbose { - struct_ser.serialize_field("verbose", &self.verbose)?; + if !self.payload.is_empty() { + #[allow(clippy::needless_borrow)] + #[allow(clippy::needless_borrows_for_generic_args)] + struct_ser.serialize_field("payload", pbjson::private::base64::encode(&self.payload).as_str())?; } struct_ser.end() } } -impl<'de> serde::Deserialize<'de> for ExplainExecNode { +impl<'de> serde::Deserialize<'de> for ExtensionDataSourceNode { #[allow(deprecated)] fn deserialize(deserializer: D) -> std::result::Result where D: serde::Deserializer<'de>, { const FIELDS: &[&str] = &[ - "schema", - "stringified_plans", - "stringifiedPlans", - "verbose", + "name", + "payload", ]; #[allow(clippy::enum_variant_names)] enum GeneratedField { - Schema, - StringifiedPlans, - Verbose, + Name, + Payload, } impl<'de> serde::Deserialize<'de> for GeneratedField { fn deserialize(deserializer: D) -> std::result::Result @@ -7005,9 +8244,8 @@ impl<'de> serde::Deserialize<'de> for ExplainExecNode { E: serde::de::Error, { match value { - "schema" => Ok(GeneratedField::Schema), - "stringifiedPlans" | "stringified_plans" => Ok(GeneratedField::StringifiedPlans), - "verbose" => Ok(GeneratedField::Verbose), + "name" => Ok(GeneratedField::Name), + "payload" => Ok(GeneratedField::Payload), _ => Err(serde::de::Error::unknown_field(value, FIELDS)), } } @@ -7017,52 +8255,46 @@ impl<'de> serde::Deserialize<'de> for ExplainExecNode { } struct GeneratedVisitor; impl<'de> serde::de::Visitor<'de> for GeneratedVisitor { - type Value = ExplainExecNode; + type Value = ExtensionDataSourceNode; fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - formatter.write_str("struct datafusion.ExplainExecNode") + formatter.write_str("struct datafusion.ExtensionDataSourceNode") } - fn visit_map(self, mut map_: V) -> std::result::Result + fn visit_map(self, mut map_: V) -> std::result::Result where V: serde::de::MapAccess<'de>, { - let mut schema__ = None; - let mut stringified_plans__ = None; - let mut verbose__ = None; + let mut name__ = None; + let mut payload__ = None; while let Some(k) = map_.next_key()? { match k { - GeneratedField::Schema => { - if schema__.is_some() { - return Err(serde::de::Error::duplicate_field("schema")); - } - schema__ = map_.next_value()?; - } - GeneratedField::StringifiedPlans => { - if stringified_plans__.is_some() { - return Err(serde::de::Error::duplicate_field("stringifiedPlans")); + GeneratedField::Name => { + if name__.is_some() { + return Err(serde::de::Error::duplicate_field("name")); } - stringified_plans__ = Some(map_.next_value()?); + name__ = Some(map_.next_value()?); } - GeneratedField::Verbose => { - if verbose__.is_some() { - return Err(serde::de::Error::duplicate_field("verbose")); + GeneratedField::Payload => { + if payload__.is_some() { + return Err(serde::de::Error::duplicate_field("payload")); } - verbose__ = Some(map_.next_value()?); + payload__ = + Some(map_.next_value::<::pbjson::private::BytesDeserialize<_>>()?.0) + ; } } } - Ok(ExplainExecNode { - schema: schema__, - stringified_plans: stringified_plans__.unwrap_or_default(), - verbose: verbose__.unwrap_or_default(), + Ok(ExtensionDataSourceNode { + name: name__.unwrap_or_default(), + payload: payload__.unwrap_or_default(), }) } } - deserializer.deserialize_struct("datafusion.ExplainExecNode", FIELDS, GeneratedVisitor) + deserializer.deserialize_struct("datafusion.ExtensionDataSourceNode", FIELDS, GeneratedVisitor) } } -impl serde::Serialize for ExplainNode { +impl serde::Serialize for ExtensionFileSourceNode { #[allow(deprecated)] fn serialize(&self, serializer: S) -> std::result::Result where @@ -7070,56 +8302,39 @@ impl serde::Serialize for ExplainNode { { use serde::ser::SerializeStruct; let mut len = 0; - if self.input.is_some() { - len += 1; - } - if self.verbose { - len += 1; - } - if self.format != 0 { + if !self.name.is_empty() { len += 1; } - if self.show_statistics.is_some() { + if !self.payload.is_empty() { len += 1; } - let mut struct_ser = serializer.serialize_struct("datafusion.ExplainNode", len)?; - if let Some(v) = self.input.as_ref() { - struct_ser.serialize_field("input", v)?; - } - if self.verbose { - struct_ser.serialize_field("verbose", &self.verbose)?; - } - if self.format != 0 { - let v = super::datafusion_common::ExplainFormat::try_from(self.format) - .map_err(|_| serde::ser::Error::custom(format!("Invalid variant {}", self.format)))?; - struct_ser.serialize_field("format", &v)?; + let mut struct_ser = serializer.serialize_struct("datafusion.ExtensionFileSourceNode", len)?; + if !self.name.is_empty() { + struct_ser.serialize_field("name", &self.name)?; } - if let Some(v) = self.show_statistics.as_ref() { - struct_ser.serialize_field("showStatistics", v)?; + if !self.payload.is_empty() { + #[allow(clippy::needless_borrow)] + #[allow(clippy::needless_borrows_for_generic_args)] + struct_ser.serialize_field("payload", pbjson::private::base64::encode(&self.payload).as_str())?; } struct_ser.end() } } -impl<'de> serde::Deserialize<'de> for ExplainNode { +impl<'de> serde::Deserialize<'de> for ExtensionFileSourceNode { #[allow(deprecated)] fn deserialize(deserializer: D) -> std::result::Result where D: serde::Deserializer<'de>, { const FIELDS: &[&str] = &[ - "input", - "verbose", - "format", - "show_statistics", - "showStatistics", + "name", + "payload", ]; - - #[allow(clippy::enum_variant_names)] - enum GeneratedField { - Input, - Verbose, - Format, - ShowStatistics, + + #[allow(clippy::enum_variant_names)] + enum GeneratedField { + Name, + Payload, } impl<'de> serde::Deserialize<'de> for GeneratedField { fn deserialize(deserializer: D) -> std::result::Result @@ -7141,10 +8356,8 @@ impl<'de> serde::Deserialize<'de> for ExplainNode { E: serde::de::Error, { match value { - "input" => Ok(GeneratedField::Input), - "verbose" => Ok(GeneratedField::Verbose), - "format" => Ok(GeneratedField::Format), - "showStatistics" | "show_statistics" => Ok(GeneratedField::ShowStatistics), + "name" => Ok(GeneratedField::Name), + "payload" => Ok(GeneratedField::Payload), _ => Err(serde::de::Error::unknown_field(value, FIELDS)), } } @@ -7154,57 +8367,43 @@ impl<'de> serde::Deserialize<'de> for ExplainNode { } struct GeneratedVisitor; impl<'de> serde::de::Visitor<'de> for GeneratedVisitor { - type Value = ExplainNode; + type Value = ExtensionFileSourceNode; fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - formatter.write_str("struct datafusion.ExplainNode") + formatter.write_str("struct datafusion.ExtensionFileSourceNode") } - fn visit_map(self, mut map_: V) -> std::result::Result + fn visit_map(self, mut map_: V) -> std::result::Result where V: serde::de::MapAccess<'de>, { - let mut input__ = None; - let mut verbose__ = None; - let mut format__ = None; - let mut show_statistics__ = None; + let mut name__ = None; + let mut payload__ = None; while let Some(k) = map_.next_key()? { match k { - GeneratedField::Input => { - if input__.is_some() { - return Err(serde::de::Error::duplicate_field("input")); - } - input__ = map_.next_value()?; - } - GeneratedField::Verbose => { - if verbose__.is_some() { - return Err(serde::de::Error::duplicate_field("verbose")); - } - verbose__ = Some(map_.next_value()?); - } - GeneratedField::Format => { - if format__.is_some() { - return Err(serde::de::Error::duplicate_field("format")); + GeneratedField::Name => { + if name__.is_some() { + return Err(serde::de::Error::duplicate_field("name")); } - format__ = Some(map_.next_value::()? as i32); + name__ = Some(map_.next_value()?); } - GeneratedField::ShowStatistics => { - if show_statistics__.is_some() { - return Err(serde::de::Error::duplicate_field("showStatistics")); + GeneratedField::Payload => { + if payload__.is_some() { + return Err(serde::de::Error::duplicate_field("payload")); } - show_statistics__ = map_.next_value()?; + payload__ = + Some(map_.next_value::<::pbjson::private::BytesDeserialize<_>>()?.0) + ; } } } - Ok(ExplainNode { - input: input__, - verbose: verbose__.unwrap_or_default(), - format: format__.unwrap_or_default(), - show_statistics: show_statistics__, + Ok(ExtensionFileSourceNode { + name: name__.unwrap_or_default(), + payload: payload__.unwrap_or_default(), }) } } - deserializer.deserialize_struct("datafusion.ExplainNode", FIELDS, GeneratedVisitor) + deserializer.deserialize_struct("datafusion.ExtensionFileSourceNode", FIELDS, GeneratedVisitor) } } impl serde::Serialize for FileFormatKind { @@ -8016,6 +9215,114 @@ impl<'de> serde::Deserialize<'de> for FileScanExecConf { deserializer.deserialize_struct("datafusion.FileScanExecConf", FIELDS, GeneratedVisitor) } } +impl serde::Serialize for FileScanNode { + #[allow(deprecated)] + fn serialize(&self, serializer: S) -> std::result::Result + where + S: serde::Serializer, + { + use serde::ser::SerializeStruct; + let mut len = 0; + if self.base.is_some() { + len += 1; + } + if self.source.is_some() { + len += 1; + } + let mut struct_ser = serializer.serialize_struct("datafusion.FileScanNode", len)?; + if let Some(v) = self.base.as_ref() { + struct_ser.serialize_field("base", v)?; + } + if let Some(v) = self.source.as_ref() { + struct_ser.serialize_field("source", v)?; + } + struct_ser.end() + } +} +impl<'de> serde::Deserialize<'de> for FileScanNode { + #[allow(deprecated)] + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + const FIELDS: &[&str] = &[ + "base", + "source", + ]; + + #[allow(clippy::enum_variant_names)] + enum GeneratedField { + Base, + Source, + } + impl<'de> serde::Deserialize<'de> for GeneratedField { + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + struct GeneratedVisitor; + + impl serde::de::Visitor<'_> for GeneratedVisitor { + type Value = GeneratedField; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(formatter, "expected one of: {:?}", &FIELDS) + } + + #[allow(unused_variables)] + fn visit_str(self, value: &str) -> std::result::Result + where + E: serde::de::Error, + { + match value { + "base" => Ok(GeneratedField::Base), + "source" => Ok(GeneratedField::Source), + _ => Err(serde::de::Error::unknown_field(value, FIELDS)), + } + } + } + deserializer.deserialize_identifier(GeneratedVisitor) + } + } + struct GeneratedVisitor; + impl<'de> serde::de::Visitor<'de> for GeneratedVisitor { + type Value = FileScanNode; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("struct datafusion.FileScanNode") + } + + fn visit_map(self, mut map_: V) -> std::result::Result + where + V: serde::de::MapAccess<'de>, + { + let mut base__ = None; + let mut source__ = None; + while let Some(k) = map_.next_key()? { + match k { + GeneratedField::Base => { + if base__.is_some() { + return Err(serde::de::Error::duplicate_field("base")); + } + base__ = map_.next_value()?; + } + GeneratedField::Source => { + if source__.is_some() { + return Err(serde::de::Error::duplicate_field("source")); + } + source__ = map_.next_value()?; + } + } + } + Ok(FileScanNode { + base: base__, + source: source__, + }) + } + } + deserializer.deserialize_struct("datafusion.FileScanNode", FIELDS, GeneratedVisitor) + } +} impl serde::Serialize for FileSinkConfig { #[allow(deprecated)] fn serialize(&self, serializer: S) -> std::result::Result @@ -8253,7 +9560,168 @@ impl<'de> serde::Deserialize<'de> for FileSinkConfig { }) } } - deserializer.deserialize_struct("datafusion.FileSinkConfig", FIELDS, GeneratedVisitor) + deserializer.deserialize_struct("datafusion.FileSinkConfig", FIELDS, GeneratedVisitor) + } +} +impl serde::Serialize for FileSourceNode { + #[allow(deprecated)] + fn serialize(&self, serializer: S) -> std::result::Result + where + S: serde::Serializer, + { + use serde::ser::SerializeStruct; + let mut len = 0; + if self.file_source_type.is_some() { + len += 1; + } + let mut struct_ser = serializer.serialize_struct("datafusion.FileSourceNode", len)?; + if let Some(v) = self.file_source_type.as_ref() { + match v { + file_source_node::FileSourceType::Parquet(v) => { + struct_ser.serialize_field("parquet", v)?; + } + file_source_node::FileSourceType::Csv(v) => { + struct_ser.serialize_field("csv", v)?; + } + file_source_node::FileSourceType::Json(v) => { + struct_ser.serialize_field("json", v)?; + } + file_source_node::FileSourceType::Avro(v) => { + struct_ser.serialize_field("avro", v)?; + } + file_source_node::FileSourceType::Arrow(v) => { + struct_ser.serialize_field("arrow", v)?; + } + file_source_node::FileSourceType::Extension(v) => { + struct_ser.serialize_field("extension", v)?; + } + } + } + struct_ser.end() + } +} +impl<'de> serde::Deserialize<'de> for FileSourceNode { + #[allow(deprecated)] + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + const FIELDS: &[&str] = &[ + "parquet", + "csv", + "json", + "avro", + "arrow", + "extension", + ]; + + #[allow(clippy::enum_variant_names)] + enum GeneratedField { + Parquet, + Csv, + Json, + Avro, + Arrow, + Extension, + } + impl<'de> serde::Deserialize<'de> for GeneratedField { + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + struct GeneratedVisitor; + + impl serde::de::Visitor<'_> for GeneratedVisitor { + type Value = GeneratedField; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(formatter, "expected one of: {:?}", &FIELDS) + } + + #[allow(unused_variables)] + fn visit_str(self, value: &str) -> std::result::Result + where + E: serde::de::Error, + { + match value { + "parquet" => Ok(GeneratedField::Parquet), + "csv" => Ok(GeneratedField::Csv), + "json" => Ok(GeneratedField::Json), + "avro" => Ok(GeneratedField::Avro), + "arrow" => Ok(GeneratedField::Arrow), + "extension" => Ok(GeneratedField::Extension), + _ => Err(serde::de::Error::unknown_field(value, FIELDS)), + } + } + } + deserializer.deserialize_identifier(GeneratedVisitor) + } + } + struct GeneratedVisitor; + impl<'de> serde::de::Visitor<'de> for GeneratedVisitor { + type Value = FileSourceNode; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("struct datafusion.FileSourceNode") + } + + fn visit_map(self, mut map_: V) -> std::result::Result + where + V: serde::de::MapAccess<'de>, + { + let mut file_source_type__ = None; + while let Some(k) = map_.next_key()? { + match k { + GeneratedField::Parquet => { + if file_source_type__.is_some() { + return Err(serde::de::Error::duplicate_field("parquet")); + } + file_source_type__ = map_.next_value::<::std::option::Option<_>>()?.map(file_source_node::FileSourceType::Parquet) +; + } + GeneratedField::Csv => { + if file_source_type__.is_some() { + return Err(serde::de::Error::duplicate_field("csv")); + } + file_source_type__ = map_.next_value::<::std::option::Option<_>>()?.map(file_source_node::FileSourceType::Csv) +; + } + GeneratedField::Json => { + if file_source_type__.is_some() { + return Err(serde::de::Error::duplicate_field("json")); + } + file_source_type__ = map_.next_value::<::std::option::Option<_>>()?.map(file_source_node::FileSourceType::Json) +; + } + GeneratedField::Avro => { + if file_source_type__.is_some() { + return Err(serde::de::Error::duplicate_field("avro")); + } + file_source_type__ = map_.next_value::<::std::option::Option<_>>()?.map(file_source_node::FileSourceType::Avro) +; + } + GeneratedField::Arrow => { + if file_source_type__.is_some() { + return Err(serde::de::Error::duplicate_field("arrow")); + } + file_source_type__ = map_.next_value::<::std::option::Option<_>>()?.map(file_source_node::FileSourceType::Arrow) +; + } + GeneratedField::Extension => { + if file_source_type__.is_some() { + return Err(serde::de::Error::duplicate_field("extension")); + } + file_source_type__ = map_.next_value::<::std::option::Option<_>>()?.map(file_source_node::FileSourceType::Extension) +; + } + } + } + Ok(FileSourceNode { + file_source_type: file_source_type__, + }) + } + } + deserializer.deserialize_struct("datafusion.FileSourceNode", FIELDS, GeneratedVisitor) } } impl serde::Serialize for FilterExecNode { @@ -12333,6 +13801,98 @@ impl<'de> serde::Deserialize<'de> for JsonSinkExecNode { deserializer.deserialize_struct("datafusion.JsonSinkExecNode", FIELDS, GeneratedVisitor) } } +impl serde::Serialize for JsonSourceNode { + #[allow(deprecated)] + fn serialize(&self, serializer: S) -> std::result::Result + where + S: serde::Serializer, + { + use serde::ser::SerializeStruct; + let mut len = 0; + if self.newline_delimited.is_some() { + len += 1; + } + let mut struct_ser = serializer.serialize_struct("datafusion.JsonSourceNode", len)?; + if let Some(v) = self.newline_delimited.as_ref() { + struct_ser.serialize_field("newlineDelimited", v)?; + } + struct_ser.end() + } +} +impl<'de> serde::Deserialize<'de> for JsonSourceNode { + #[allow(deprecated)] + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + const FIELDS: &[&str] = &[ + "newline_delimited", + "newlineDelimited", + ]; + + #[allow(clippy::enum_variant_names)] + enum GeneratedField { + NewlineDelimited, + } + impl<'de> serde::Deserialize<'de> for GeneratedField { + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + struct GeneratedVisitor; + + impl serde::de::Visitor<'_> for GeneratedVisitor { + type Value = GeneratedField; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(formatter, "expected one of: {:?}", &FIELDS) + } + + #[allow(unused_variables)] + fn visit_str(self, value: &str) -> std::result::Result + where + E: serde::de::Error, + { + match value { + "newlineDelimited" | "newline_delimited" => Ok(GeneratedField::NewlineDelimited), + _ => Err(serde::de::Error::unknown_field(value, FIELDS)), + } + } + } + deserializer.deserialize_identifier(GeneratedVisitor) + } + } + struct GeneratedVisitor; + impl<'de> serde::de::Visitor<'de> for GeneratedVisitor { + type Value = JsonSourceNode; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("struct datafusion.JsonSourceNode") + } + + fn visit_map(self, mut map_: V) -> std::result::Result + where + V: serde::de::MapAccess<'de>, + { + let mut newline_delimited__ = None; + while let Some(k) = map_.next_key()? { + match k { + GeneratedField::NewlineDelimited => { + if newline_delimited__.is_some() { + return Err(serde::de::Error::duplicate_field("newlineDelimited")); + } + newline_delimited__ = map_.next_value()?; + } + } + } + Ok(JsonSourceNode { + newline_delimited: newline_delimited__, + }) + } + } + deserializer.deserialize_struct("datafusion.JsonSourceNode", FIELDS, GeneratedVisitor) + } +} impl serde::Serialize for Lambda { #[allow(deprecated)] fn serialize(&self, serializer: S) -> std::result::Result @@ -17576,6 +19136,173 @@ impl<'de> serde::Deserialize<'de> for ParquetSortingColumns { deserializer.deserialize_struct("datafusion.ParquetSortingColumns", FIELDS, GeneratedVisitor) } } +impl serde::Serialize for ParquetSourceNode { + #[allow(deprecated)] + fn serialize(&self, serializer: S) -> std::result::Result + where + S: serde::Serializer, + { + use serde::ser::SerializeStruct; + let mut len = 0; + if self.predicate.is_some() { + len += 1; + } + if self.parquet_options.is_some() { + len += 1; + } + if self.sort_order_for_reorder.is_some() { + len += 1; + } + if self.reverse_row_groups { + len += 1; + } + if self.metadata_size_hint.is_some() { + len += 1; + } + let mut struct_ser = serializer.serialize_struct("datafusion.ParquetSourceNode", len)?; + if let Some(v) = self.predicate.as_ref() { + struct_ser.serialize_field("predicate", v)?; + } + if let Some(v) = self.parquet_options.as_ref() { + struct_ser.serialize_field("parquetOptions", v)?; + } + if let Some(v) = self.sort_order_for_reorder.as_ref() { + struct_ser.serialize_field("sortOrderForReorder", v)?; + } + if self.reverse_row_groups { + struct_ser.serialize_field("reverseRowGroups", &self.reverse_row_groups)?; + } + if let Some(v) = self.metadata_size_hint.as_ref() { + #[allow(clippy::needless_borrow)] + #[allow(clippy::needless_borrows_for_generic_args)] + struct_ser.serialize_field("metadataSizeHint", ToString::to_string(&v).as_str())?; + } + struct_ser.end() + } +} +impl<'de> serde::Deserialize<'de> for ParquetSourceNode { + #[allow(deprecated)] + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + const FIELDS: &[&str] = &[ + "predicate", + "parquet_options", + "parquetOptions", + "sort_order_for_reorder", + "sortOrderForReorder", + "reverse_row_groups", + "reverseRowGroups", + "metadata_size_hint", + "metadataSizeHint", + ]; + + #[allow(clippy::enum_variant_names)] + enum GeneratedField { + Predicate, + ParquetOptions, + SortOrderForReorder, + ReverseRowGroups, + MetadataSizeHint, + } + impl<'de> serde::Deserialize<'de> for GeneratedField { + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + struct GeneratedVisitor; + + impl serde::de::Visitor<'_> for GeneratedVisitor { + type Value = GeneratedField; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(formatter, "expected one of: {:?}", &FIELDS) + } + + #[allow(unused_variables)] + fn visit_str(self, value: &str) -> std::result::Result + where + E: serde::de::Error, + { + match value { + "predicate" => Ok(GeneratedField::Predicate), + "parquetOptions" | "parquet_options" => Ok(GeneratedField::ParquetOptions), + "sortOrderForReorder" | "sort_order_for_reorder" => Ok(GeneratedField::SortOrderForReorder), + "reverseRowGroups" | "reverse_row_groups" => Ok(GeneratedField::ReverseRowGroups), + "metadataSizeHint" | "metadata_size_hint" => Ok(GeneratedField::MetadataSizeHint), + _ => Err(serde::de::Error::unknown_field(value, FIELDS)), + } + } + } + deserializer.deserialize_identifier(GeneratedVisitor) + } + } + struct GeneratedVisitor; + impl<'de> serde::de::Visitor<'de> for GeneratedVisitor { + type Value = ParquetSourceNode; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("struct datafusion.ParquetSourceNode") + } + + fn visit_map(self, mut map_: V) -> std::result::Result + where + V: serde::de::MapAccess<'de>, + { + let mut predicate__ = None; + let mut parquet_options__ = None; + let mut sort_order_for_reorder__ = None; + let mut reverse_row_groups__ = None; + let mut metadata_size_hint__ = None; + while let Some(k) = map_.next_key()? { + match k { + GeneratedField::Predicate => { + if predicate__.is_some() { + return Err(serde::de::Error::duplicate_field("predicate")); + } + predicate__ = map_.next_value()?; + } + GeneratedField::ParquetOptions => { + if parquet_options__.is_some() { + return Err(serde::de::Error::duplicate_field("parquetOptions")); + } + parquet_options__ = map_.next_value()?; + } + GeneratedField::SortOrderForReorder => { + if sort_order_for_reorder__.is_some() { + return Err(serde::de::Error::duplicate_field("sortOrderForReorder")); + } + sort_order_for_reorder__ = map_.next_value()?; + } + GeneratedField::ReverseRowGroups => { + if reverse_row_groups__.is_some() { + return Err(serde::de::Error::duplicate_field("reverseRowGroups")); + } + reverse_row_groups__ = Some(map_.next_value()?); + } + GeneratedField::MetadataSizeHint => { + if metadata_size_hint__.is_some() { + return Err(serde::de::Error::duplicate_field("metadataSizeHint")); + } + metadata_size_hint__ = + map_.next_value::<::std::option::Option<::pbjson::private::NumberDeserialize<_>>>()?.map(|x| x.0) + ; + } + } + } + Ok(ParquetSourceNode { + predicate: predicate__, + parquet_options: parquet_options__, + sort_order_for_reorder: sort_order_for_reorder__, + reverse_row_groups: reverse_row_groups__.unwrap_or_default(), + metadata_size_hint: metadata_size_hint__, + }) + } + } + deserializer.deserialize_struct("datafusion.ParquetSourceNode", FIELDS, GeneratedVisitor) + } +} impl serde::Serialize for PartialTableReference { #[allow(deprecated)] fn serialize(&self, serializer: S) -> std::result::Result @@ -22264,6 +23991,12 @@ impl serde::Serialize for PhysicalPlanNode { physical_plan_node::PhysicalPlanType::AsOfJoin(v) => { struct_ser.serialize_field("asOfJoin", v)?; } + physical_plan_node::PhysicalPlanType::DataSource(v) => { + struct_ser.serialize_field("dataSource", v)?; + } + physical_plan_node::PhysicalPlanType::DataSink(v) => { + struct_ser.serialize_field("dataSink", v)?; + } } } struct_ser.end() @@ -22340,6 +24073,10 @@ impl<'de> serde::Deserialize<'de> for PhysicalPlanNode { "piecewiseMergeJoin", "as_of_join", "asOfJoin", + "data_source", + "dataSource", + "data_sink", + "dataSink", ]; #[allow(clippy::enum_variant_names)] @@ -22384,6 +24121,8 @@ impl<'de> serde::Deserialize<'de> for PhysicalPlanNode { ScalarSubquery, PiecewiseMergeJoin, AsOfJoin, + DataSource, + DataSink, } impl<'de> serde::Deserialize<'de> for GeneratedField { fn deserialize(deserializer: D) -> std::result::Result @@ -22445,6 +24184,8 @@ impl<'de> serde::Deserialize<'de> for PhysicalPlanNode { "scalarSubquery" | "scalar_subquery" => Ok(GeneratedField::ScalarSubquery), "piecewiseMergeJoin" | "piecewise_merge_join" => Ok(GeneratedField::PiecewiseMergeJoin), "asOfJoin" | "as_of_join" => Ok(GeneratedField::AsOfJoin), + "dataSource" | "data_source" => Ok(GeneratedField::DataSource), + "dataSink" | "data_sink" => Ok(GeneratedField::DataSink), _ => Err(serde::de::Error::unknown_field(value, FIELDS)), } } @@ -22745,6 +24486,20 @@ impl<'de> serde::Deserialize<'de> for PhysicalPlanNode { return Err(serde::de::Error::duplicate_field("asOfJoin")); } physical_plan_type__ = map_.next_value::<::std::option::Option<_>>()?.map(physical_plan_node::PhysicalPlanType::AsOfJoin) +; + } + GeneratedField::DataSource => { + if physical_plan_type__.is_some() { + return Err(serde::de::Error::duplicate_field("dataSource")); + } + physical_plan_type__ = map_.next_value::<::std::option::Option<_>>()?.map(physical_plan_node::PhysicalPlanType::DataSource) +; + } + GeneratedField::DataSink => { + if physical_plan_type__.is_some() { + return Err(serde::de::Error::duplicate_field("dataSink")); + } + physical_plan_type__ = map_.next_value::<::std::option::Option<_>>()?.map(physical_plan_node::PhysicalPlanType::DataSink) ; } } diff --git a/datafusion/proto-models/src/generated/prost.rs b/datafusion/proto-models/src/generated/prost.rs index 1034d0443dbce..9658ed38305f7 100644 --- a/datafusion/proto-models/src/generated/prost.rs +++ b/datafusion/proto-models/src/generated/prost.rs @@ -1396,7 +1396,7 @@ pub mod table_reference { pub struct PhysicalPlanNode { #[prost( oneof = "physical_plan_node::PhysicalPlanType", - tags = "1, 2, 3, 4, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41" + tags = "1, 2, 3, 4, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43" )] pub physical_plan_type: ::core::option::Option, } @@ -1404,8 +1404,10 @@ pub struct PhysicalPlanNode { pub mod physical_plan_node { #[derive(Clone, PartialEq, ::prost::Oneof)] pub enum PhysicalPlanType { + /// Deprecated: encoders emit `data_source`; decoders still accept this. #[prost(message, tag = "1")] ParquetScan(super::ParquetScanExecNode), + /// Deprecated: encoders emit `data_source`; decoders still accept this. #[prost(message, tag = "2")] CsvScan(super::CsvScanExecNode), #[prost(message, tag = "3")] @@ -1434,6 +1436,7 @@ pub mod physical_plan_node { Window(::prost::alloc::boxed::Box), #[prost(message, tag = "16")] CrossJoin(::prost::alloc::boxed::Box), + /// Deprecated: encoders emit `data_source`; decoders still accept this. #[prost(message, tag = "17")] AvroScan(super::AvroScanExecNode), #[prost(message, tag = "18")] @@ -1450,6 +1453,7 @@ pub mod physical_plan_node { NestedLoopJoin(::prost::alloc::boxed::Box), #[prost(message, tag = "23")] Analyze(::prost::alloc::boxed::Box), + /// Deprecated: encoders emit `data_sink`; decoders still accept this. #[prost(message, tag = "24")] JsonSink(::prost::alloc::boxed::Box), #[prost(message, tag = "25")] @@ -1458,12 +1462,15 @@ pub mod physical_plan_node { Interleave(super::InterleaveExecNode), #[prost(message, tag = "27")] PlaceholderRow(super::PlaceholderRowExecNode), + /// Deprecated: encoders emit `data_sink`; decoders still accept this. #[prost(message, tag = "28")] CsvSink(::prost::alloc::boxed::Box), + /// Deprecated: encoders emit `data_sink`; decoders still accept this. #[prost(message, tag = "29")] ParquetSink(::prost::alloc::boxed::Box), #[prost(message, tag = "30")] Unnest(::prost::alloc::boxed::Box), + /// Deprecated: encoders emit `data_source`; decoders still accept this. #[prost(message, tag = "31")] JsonScan(super::JsonScanExecNode), #[prost(message, tag = "32")] @@ -1472,12 +1479,14 @@ pub mod physical_plan_node { GenerateSeries(super::GenerateSeriesNode), #[prost(message, tag = "34")] SortMergeJoin(::prost::alloc::boxed::Box), + /// Deprecated: encoders emit `data_source`; decoders still accept this. #[prost(message, tag = "35")] MemoryScan(super::MemoryScanExecNode), #[prost(message, tag = "36")] AsyncFunc(::prost::alloc::boxed::Box), #[prost(message, tag = "37")] Buffer(::prost::alloc::boxed::Box), + /// Deprecated: encoders emit `data_source`; decoders still accept this. #[prost(message, tag = "38")] ArrowScan(super::ArrowScanExecNode), #[prost(message, tag = "39")] @@ -1488,6 +1497,10 @@ pub mod physical_plan_node { ), #[prost(message, tag = "41")] AsOfJoin(::prost::alloc::boxed::Box), + #[prost(message, tag = "42")] + DataSource(super::DataSourceExecNode), + #[prost(message, tag = "43")] + DataSink(::prost::alloc::boxed::Box), } } #[derive(Clone, PartialEq, ::prost::Message)] @@ -2835,6 +2848,176 @@ pub struct PhysicalScalarSubqueryExprNode { ::prost::alloc::string::String, >, } +/// A `DataSourceExec`: the plan-level wrapper around a `DataSource`. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct DataSourceExecNode { + #[prost(message, optional, tag = "1")] + pub source: ::core::option::Option, +} +/// A `DataSource` implementation. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct DataSourceNode { + #[prost(oneof = "data_source_node::SourceType", tags = "1, 2, 15")] + pub source_type: ::core::option::Option, +} +/// Nested message and enum types in `DataSourceNode`. +pub mod data_source_node { + #[derive(Clone, PartialEq, ::prost::Oneof)] + pub enum SourceType { + #[prost(message, tag = "1")] + File(super::FileScanNode), + #[prost(message, tag = "2")] + Memory(super::MemoryScanExecNode), + #[prost(message, tag = "15")] + Extension(super::ExtensionDataSourceNode), + } +} +/// A `FileScanConfig`: the format-agnostic base plus its `FileSource`. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct FileScanNode { + #[prost(message, optional, tag = "1")] + pub base: ::core::option::Option, + #[prost(message, optional, tag = "2")] + pub source: ::core::option::Option, +} +/// A `FileSource` implementation. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct FileSourceNode { + #[prost(oneof = "file_source_node::FileSourceType", tags = "1, 2, 3, 4, 5, 15")] + pub file_source_type: ::core::option::Option, +} +/// Nested message and enum types in `FileSourceNode`. +pub mod file_source_node { + #[derive(Clone, PartialEq, ::prost::Oneof)] + pub enum FileSourceType { + #[prost(message, tag = "1")] + Parquet(super::ParquetSourceNode), + #[prost(message, tag = "2")] + Csv(super::CsvSourceNode), + #[prost(message, tag = "3")] + Json(super::JsonSourceNode), + #[prost(message, tag = "4")] + Avro(super::AvroSourceNode), + #[prost(message, tag = "5")] + Arrow(super::ArrowSourceNode), + #[prost(message, tag = "15")] + Extension(super::ExtensionFileSourceNode), + } +} +/// Field tags mirror `ParquetScanExecNode` (minus `base_conf`). +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ParquetSourceNode { + #[prost(message, optional, tag = "3")] + pub predicate: ::core::option::Option, + #[prost(message, optional, tag = "4")] + pub parquet_options: ::core::option::Option< + super::datafusion_common::TableParquetOptions, + >, + #[prost(message, optional, tag = "5")] + pub sort_order_for_reorder: ::core::option::Option, + #[prost(bool, tag = "6")] + pub reverse_row_groups: bool, + #[prost(uint64, optional, tag = "7")] + pub metadata_size_hint: ::core::option::Option, +} +/// Field tags mirror `CsvScanExecNode` (minus `base_conf`). +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct CsvSourceNode { + #[prost(bool, tag = "2")] + pub has_header: bool, + #[prost(string, tag = "3")] + pub delimiter: ::prost::alloc::string::String, + #[prost(string, tag = "4")] + pub quote: ::prost::alloc::string::String, + #[prost(bool, tag = "7")] + pub newlines_in_values: bool, + #[prost(bool, tag = "8")] + pub truncate_rows: bool, + #[prost(bytes = "vec", optional, tag = "9")] + pub terminator: ::core::option::Option<::prost::alloc::vec::Vec>, + #[prost(oneof = "csv_source_node::OptionalEscape", tags = "5")] + pub optional_escape: ::core::option::Option, + #[prost(oneof = "csv_source_node::OptionalComment", tags = "6")] + pub optional_comment: ::core::option::Option, +} +/// Nested message and enum types in `CsvSourceNode`. +pub mod csv_source_node { + #[derive(Clone, PartialEq, Eq, Hash, ::prost::Oneof)] + pub enum OptionalEscape { + #[prost(string, tag = "5")] + Escape(::prost::alloc::string::String), + } + #[derive(Clone, PartialEq, Eq, Hash, ::prost::Oneof)] + pub enum OptionalComment { + #[prost(string, tag = "6")] + Comment(::prost::alloc::string::String), + } +} +/// Field tags mirror `JsonScanExecNode` (minus `base_conf`). +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct JsonSourceNode { + #[prost(bool, optional, tag = "2")] + pub newline_delimited: ::core::option::Option, +} +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct AvroSourceNode {} +/// Field tags mirror `ArrowScanExecNode` (minus `base_conf`). +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct ArrowSourceNode { + #[prost(enumeration = "ArrowIpcFormat", tag = "2")] + pub format: i32, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct ExtensionDataSourceNode { + #[prost(string, tag = "1")] + pub name: ::prost::alloc::string::String, + #[prost(bytes = "vec", tag = "2")] + pub payload: ::prost::alloc::vec::Vec, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct ExtensionFileSourceNode { + #[prost(string, tag = "1")] + pub name: ::prost::alloc::string::String, + #[prost(bytes = "vec", tag = "2")] + pub payload: ::prost::alloc::vec::Vec, +} +/// A `DataSinkExec`: the plan-level wrapper around a `DataSink`. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct DataSinkExecNode { + #[prost(message, optional, boxed, tag = "1")] + pub input: ::core::option::Option<::prost::alloc::boxed::Box>, + #[prost(message, optional, tag = "2")] + pub sink: ::core::option::Option, + #[prost(message, optional, tag = "3")] + pub sort_order: ::core::option::Option, +} +/// A `DataSink` implementation. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct DataSinkNode { + #[prost(oneof = "data_sink_node::SinkType", tags = "1, 2, 3, 15")] + pub sink_type: ::core::option::Option, +} +/// Nested message and enum types in `DataSinkNode`. +pub mod data_sink_node { + #[derive(Clone, PartialEq, ::prost::Oneof)] + pub enum SinkType { + #[prost(message, tag = "1")] + Json(super::JsonSink), + #[prost(message, tag = "2")] + Csv(super::CsvSink), + #[prost(message, tag = "3")] + Parquet(super::ParquetSink), + #[prost(message, tag = "15")] + Extension(super::ExtensionDataSinkNode), + } +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct ExtensionDataSinkNode { + #[prost(string, tag = "1")] + pub name: ::prost::alloc::string::String, + #[prost(bytes = "vec", tag = "2")] + pub payload: ::prost::alloc::vec::Vec, +} #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] #[repr(i32)] pub enum AsOfMatchOperator { diff --git a/datafusion/proto-models/src/lib.rs b/datafusion/proto-models/src/lib.rs index c953a37fc6f9f..605dad4a867c1 100644 --- a/datafusion/proto-models/src/lib.rs +++ b/datafusion/proto-models/src/lib.rs @@ -47,6 +47,7 @@ pub mod from_proto; pub mod generated; +pub mod physical_plan_compat; pub mod registry; pub mod to_proto; diff --git a/datafusion/proto-models/src/physical_plan_compat.rs b/datafusion/proto-models/src/physical_plan_compat.rs new file mode 100644 index 0000000000000..c14295f1dc1f4 --- /dev/null +++ b/datafusion/proto-models/src/physical_plan_compat.rs @@ -0,0 +1,203 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Conversions from the deprecated flat scan messages (`ParquetScanExecNode`, +//! `CsvScanExecNode`, …) to the format-specific part of the layered +//! `FileSourceNode`. +//! +//! Each conversion maps only the format fields; the shared `base_conf` is +//! handled by the caller. The legacy messages are destructured exhaustively so +//! that adding a field to one of them fails to compile until it is mapped here. + +use crate::protobuf::{ + ArrowScanExecNode, ArrowSourceNode, AvroScanExecNode, AvroSourceNode, + CsvScanExecNode, CsvSourceNode, JsonScanExecNode, JsonSourceNode, + ParquetScanExecNode, ParquetSourceNode, csv_scan_exec_node, csv_source_node, +}; + +impl From<&ParquetScanExecNode> for ParquetSourceNode { + fn from(scan: &ParquetScanExecNode) -> Self { + let ParquetScanExecNode { + base_conf: _, + predicate, + parquet_options, + sort_order_for_reorder, + reverse_row_groups, + metadata_size_hint, + } = scan; + Self { + predicate: predicate.clone(), + parquet_options: parquet_options.clone(), + sort_order_for_reorder: sort_order_for_reorder.clone(), + reverse_row_groups: *reverse_row_groups, + metadata_size_hint: *metadata_size_hint, + } + } +} + +impl From<&CsvScanExecNode> for CsvSourceNode { + fn from(scan: &CsvScanExecNode) -> Self { + let CsvScanExecNode { + base_conf: _, + has_header, + delimiter, + quote, + optional_escape, + optional_comment, + newlines_in_values, + truncate_rows, + terminator, + } = scan; + Self { + has_header: *has_header, + delimiter: delimiter.clone(), + quote: quote.clone(), + optional_escape: optional_escape.as_ref().map( + |csv_scan_exec_node::OptionalEscape::Escape(e)| { + csv_source_node::OptionalEscape::Escape(e.clone()) + }, + ), + optional_comment: optional_comment.as_ref().map( + |csv_scan_exec_node::OptionalComment::Comment(c)| { + csv_source_node::OptionalComment::Comment(c.clone()) + }, + ), + newlines_in_values: *newlines_in_values, + truncate_rows: *truncate_rows, + terminator: terminator.clone(), + } + } +} + +impl From<&JsonScanExecNode> for JsonSourceNode { + fn from(scan: &JsonScanExecNode) -> Self { + let JsonScanExecNode { + base_conf: _, + newline_delimited, + } = scan; + Self { + newline_delimited: *newline_delimited, + } + } +} + +impl From<&AvroScanExecNode> for AvroSourceNode { + fn from(scan: &AvroScanExecNode) -> Self { + let AvroScanExecNode { base_conf: _ } = scan; + Self {} + } +} + +impl From<&ArrowScanExecNode> for ArrowSourceNode { + fn from(scan: &ArrowScanExecNode) -> Self { + let ArrowScanExecNode { + base_conf: _, + format, + } = scan; + Self { format: *format } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::datafusion_common::TableParquetOptions; + use crate::protobuf::{ + FileScanExecConf, PhysicalExprNode, PhysicalSortExprNodeCollection, + }; + + #[test] + fn parquet_scan_maps_format_fields() { + let scan = ParquetScanExecNode { + base_conf: Some(FileScanExecConf::default()), + predicate: Some(PhysicalExprNode::default()), + parquet_options: Some(TableParquetOptions::default()), + sort_order_for_reorder: Some(PhysicalSortExprNodeCollection::default()), + reverse_row_groups: true, + metadata_size_hint: Some(42), + }; + assert_eq!( + ParquetSourceNode::from(&scan), + ParquetSourceNode { + predicate: Some(PhysicalExprNode::default()), + parquet_options: Some(TableParquetOptions::default()), + sort_order_for_reorder: Some(PhysicalSortExprNodeCollection::default()), + reverse_row_groups: true, + metadata_size_hint: Some(42), + } + ); + } + + #[test] + fn csv_scan_maps_format_fields() { + let scan = CsvScanExecNode { + base_conf: Some(FileScanExecConf::default()), + has_header: true, + delimiter: ";".to_string(), + quote: "'".to_string(), + optional_escape: Some(csv_scan_exec_node::OptionalEscape::Escape( + "\\".to_string(), + )), + optional_comment: Some(csv_scan_exec_node::OptionalComment::Comment( + "#".to_string(), + )), + newlines_in_values: true, + truncate_rows: true, + terminator: Some(b"\n".to_vec()), + }; + assert_eq!( + CsvSourceNode::from(&scan), + CsvSourceNode { + has_header: true, + delimiter: ";".to_string(), + quote: "'".to_string(), + optional_escape: Some(csv_source_node::OptionalEscape::Escape( + "\\".to_string() + )), + optional_comment: Some(csv_source_node::OptionalComment::Comment( + "#".to_string() + )), + newlines_in_values: true, + truncate_rows: true, + terminator: Some(b"\n".to_vec()), + } + ); + } + + #[test] + fn json_and_arrow_scans_map_format_fields() { + let json = JsonScanExecNode { + base_conf: None, + newline_delimited: Some(false), + }; + assert_eq!( + JsonSourceNode::from(&json), + JsonSourceNode { + newline_delimited: Some(false) + } + ); + let arrow = ArrowScanExecNode { + base_conf: None, + format: 1, + }; + assert_eq!(ArrowSourceNode::from(&arrow), ArrowSourceNode { format: 1 }); + assert_eq!( + AvroSourceNode::from(&AvroScanExecNode { base_conf: None }), + AvroSourceNode {} + ); + } +} diff --git a/datafusion/proto/src/physical_plan/mod.rs b/datafusion/proto/src/physical_plan/mod.rs index 06e8a42d6f23c..bbe0b456652f0 100644 --- a/datafusion/proto/src/physical_plan/mod.rs +++ b/datafusion/proto/src/physical_plan/mod.rs @@ -26,6 +26,9 @@ use datafusion_catalog::memory::MemorySourceConfig; use datafusion_common::{ DataFusionError, Result, internal_datafusion_err, internal_err, not_impl_err, }; +use datafusion_datasource::file_scan_config::FileScanConfig; +use datafusion_datasource::sink::{DataSink, DataSinkExec}; +use datafusion_datasource::source::DataSourceExec; use datafusion_datasource_arrow::source::ArrowSource; #[cfg(feature = "avro")] use datafusion_datasource_avro::source::AvroSource; @@ -84,7 +87,9 @@ use prost::bytes::BufMut; use crate::physical_plan::from_proto::parse_physical_expr_with_converter; use crate::physical_plan::to_proto::serialize_physical_expr_with_converter; use crate::protobuf::physical_plan_node::PhysicalPlanType; -use crate::protobuf::{self, proto_error}; +use crate::protobuf::{ + self, data_sink_node::SinkType, file_source_node::FileSourceType, proto_error, +}; pub mod from_proto; pub mod to_proto; @@ -1232,37 +1237,71 @@ pub trait PhysicalPlanNodeExt: Sized { PhysicalPlanType::Filter(_) => { FilterExec::try_from_proto(self.node(), &decode_ctx) } - PhysicalPlanType::CsvScan(_) => { - CsvSource::try_from_proto(self.node(), &decode_ctx) - } - PhysicalPlanType::JsonScan(_) => { - JsonSource::try_from_proto(self.node(), &decode_ctx) - } - PhysicalPlanType::ParquetScan(_) => { - #[cfg(feature = "parquet")] - { - ParquetSource::try_from_proto(self.node(), &decode_ctx) - } - #[cfg(not(feature = "parquet"))] - not_impl_err!( - "Unable to process a Parquet PhysicalPlan when the `parquet` feature is not enabled" - ) - } - PhysicalPlanType::AvroScan(_) => { - #[cfg(feature = "avro")] - { - AvroSource::try_from_proto(self.node(), &decode_ctx) - } - #[cfg(not(feature = "avro"))] - panic!( - "Unable to process a Avro PhysicalPlan when `avro` feature is not enabled" - ) - } - PhysicalPlanType::MemoryScan(_) => { - MemorySourceConfig::try_from_proto(self.node(), &decode_ctx) - } - PhysicalPlanType::ArrowScan(_) => { - ArrowSource::try_from_proto(self.node(), &decode_ctx) + PhysicalPlanType::DataSource(node) => decode_data_source(node, &decode_ctx), + PhysicalPlanType::DataSink(node) => decode_data_sink(node, &decode_ctx), + // Deprecated flat variants: decoded through the layered path. + PhysicalPlanType::ParquetScan(n) => decode_file_scan( + required(n.base_conf.as_ref(), "ParquetScanExecNode", "base_conf")?, + &FileSourceType::Parquet(n.into()), + &decode_ctx, + ), + PhysicalPlanType::CsvScan(n) => decode_file_scan( + required(n.base_conf.as_ref(), "CsvScanExecNode", "base_conf")?, + &FileSourceType::Csv(n.into()), + &decode_ctx, + ), + PhysicalPlanType::JsonScan(n) => decode_file_scan( + required(n.base_conf.as_ref(), "JsonScanExecNode", "base_conf")?, + &FileSourceType::Json(n.into()), + &decode_ctx, + ), + PhysicalPlanType::AvroScan(n) => decode_file_scan( + required(n.base_conf.as_ref(), "AvroScanExecNode", "base_conf")?, + &FileSourceType::Avro(n.into()), + &decode_ctx, + ), + PhysicalPlanType::ArrowScan(n) => decode_file_scan( + required(n.base_conf.as_ref(), "ArrowScanExecNode", "base_conf")?, + &FileSourceType::Arrow(n.into()), + &decode_ctx, + ), + PhysicalPlanType::MemoryScan(n) => Ok(DataSourceExec::from_data_source( + MemorySourceConfig::try_from_proto_source(n, &decode_ctx)?, + )), + PhysicalPlanType::JsonSink(n) => build_data_sink( + n.input.as_deref(), + Arc::new(JsonSink::try_from(required( + n.sink.as_ref(), + "JsonSinkExecNode", + "sink", + )?)?), + n.sort_order.as_ref(), + &decode_ctx, + ), + PhysicalPlanType::CsvSink(n) => build_data_sink( + n.input.as_deref(), + Arc::new(CsvSink::try_from(required( + n.sink.as_ref(), + "CsvSinkExecNode", + "sink", + )?)?), + n.sort_order.as_ref(), + &decode_ctx, + ), + #[cfg(feature = "parquet")] + PhysicalPlanType::ParquetSink(n) => build_data_sink( + n.input.as_deref(), + Arc::new(ParquetSink::try_from(required( + n.sink.as_ref(), + "ParquetSinkExecNode", + "sink", + )?)?), + n.sort_order.as_ref(), + &decode_ctx, + ), + #[cfg(not(feature = "parquet"))] + PhysicalPlanType::ParquetSink(_) => { + not_impl_err!("ParquetSink requires the `parquet` feature") } #[expect( deprecated, @@ -1325,20 +1364,6 @@ pub trait PhysicalPlanNodeExt: Sized { PhysicalPlanType::Analyze(_) => { AnalyzeExec::try_from_proto(self.node(), &decode_ctx) } - PhysicalPlanType::JsonSink(_) => { - JsonSink::try_from_proto(self.node(), &decode_ctx) - } - PhysicalPlanType::CsvSink(_) => { - CsvSink::try_from_proto(self.node(), &decode_ctx) - } - PhysicalPlanType::ParquetSink(_) => { - #[cfg(feature = "parquet")] - { - ParquetSink::try_from_proto(self.node(), &decode_ctx) - } - #[cfg(not(feature = "parquet"))] - not_impl_err!("ParquetSink requires the `parquet` feature") - } PhysicalPlanType::Unnest(_) => { UnnestExec::try_from_proto(self.node(), &decode_ctx) } @@ -2162,3 +2187,143 @@ impl ExecutionPlanDecode for ConverterPlanDecoder<'_, '_> { } } } + +/// Borrow a required message field, or fail with a protobuf error naming it. +fn required<'a, T>(field: Option<&'a T>, message: &str, name: &str) -> Result<&'a T> { + field.ok_or_else(|| { + proto_error(format!("{message} is missing required field '{name}'")) + }) +} + +/// Decode a `DataSourceExecNode`, dispatching on the built-in source types. +fn decode_data_source( + node: &protobuf::DataSourceExecNode, + ctx: &ExecutionPlanDecodeCtx<'_>, +) -> Result> { + use protobuf::data_source_node::SourceType; + + let source = node + .source + .as_ref() + .and_then(|source| source.source_type.as_ref()) + .ok_or_else(|| proto_error("DataSourceExecNode is missing its source"))?; + match source { + SourceType::File(scan) => { + let base = scan + .base + .as_ref() + .ok_or_else(|| proto_error("FileScanNode is missing its base"))?; + let file_source = scan + .source + .as_ref() + .and_then(|source| source.file_source_type.as_ref()) + .ok_or_else(|| proto_error("FileScanNode is missing its source"))?; + decode_file_scan(base, file_source, ctx) + } + SourceType::Memory(node) => Ok(DataSourceExec::from_data_source( + MemorySourceConfig::try_from_proto_source(node, ctx)?, + )), + SourceType::Extension(ext) => { + not_impl_err!("No decoder for extension DataSource '{}'", ext.name) + } + } +} + +/// Decode a file scan from its shared config and format-specific source. +fn decode_file_scan( + base: &protobuf::FileScanExecConf, + source: &FileSourceType, + ctx: &ExecutionPlanDecodeCtx<'_>, +) -> Result> { + let file_source = match source { + #[cfg(feature = "parquet")] + FileSourceType::Parquet(node) => { + ParquetSource::try_from_proto_file_source(node, base, ctx)? + } + #[cfg(not(feature = "parquet"))] + FileSourceType::Parquet(_) => { + return not_impl_err!( + "Unable to process a Parquet PhysicalPlan when the `parquet` feature is not enabled" + ); + } + FileSourceType::Csv(node) => { + CsvSource::try_from_proto_file_source(node, base, ctx)? + } + FileSourceType::Json(node) => { + JsonSource::try_from_proto_file_source(node, base, ctx)? + } + #[cfg(feature = "avro")] + FileSourceType::Avro(node) => { + AvroSource::try_from_proto_file_source(node, base, ctx)? + } + #[cfg(not(feature = "avro"))] + FileSourceType::Avro(_) => { + return not_impl_err!( + "Unable to process an Avro PhysicalPlan when the `avro` feature is not enabled" + ); + } + FileSourceType::Arrow(node) => { + ArrowSource::try_from_proto_file_source(node, base, ctx)? + } + FileSourceType::Extension(ext) => { + return not_impl_err!("No decoder for extension FileSource '{}'", ext.name); + } + }; + Ok(DataSourceExec::from_data_source( + FileScanConfig::try_from_proto(base, ctx, file_source)?, + )) +} + +/// Decode a `DataSinkExecNode`, dispatching on the built-in sink types. +fn decode_data_sink( + node: &protobuf::DataSinkExecNode, + ctx: &ExecutionPlanDecodeCtx<'_>, +) -> Result> { + let protobuf::DataSinkExecNode { + input, + sink, + sort_order, + } = node; + // Report a missing input before a missing sink. + let input = required(input.as_deref(), "DataSinkExecNode", "input")?; + let sink = sink + .as_ref() + .and_then(|sink| sink.sink_type.as_ref()) + .ok_or_else(|| proto_error("DataSinkExecNode is missing its sink"))?; + build_data_sink( + Some(input), + decode_sink_type(sink)?, + sort_order.as_ref(), + ctx, + ) +} + +/// Decode one of the built-in sink types. +fn decode_sink_type(sink: &SinkType) -> Result> { + Ok(match sink { + SinkType::Json(sink) => Arc::new(JsonSink::try_from(sink)?), + SinkType::Csv(sink) => Arc::new(CsvSink::try_from(sink)?), + #[cfg(feature = "parquet")] + SinkType::Parquet(sink) => Arc::new(ParquetSink::try_from(sink)?), + #[cfg(not(feature = "parquet"))] + SinkType::Parquet(_) => { + return not_impl_err!("ParquetSink requires the `parquet` feature"); + } + SinkType::Extension(ext) => { + return not_impl_err!("No decoder for extension DataSink '{}'", ext.name); + } + }) +} + +/// Wrap a decoded sink in a `DataSinkExec` over its decoded input. +fn build_data_sink( + input: Option<&protobuf::PhysicalPlanNode>, + sink: Arc, + sort_order: Option<&protobuf::PhysicalSortExprNodeCollection>, + ctx: &ExecutionPlanDecodeCtx<'_>, +) -> Result> { + let input = ctx.decode_required_child(input, "DataSinkExecNode", "input")?; + let sort_order = + DataSinkExec::decode_sort_order(sort_order, ctx, input.schema().as_ref())?; + Ok(Arc::new(DataSinkExec::new(input, sink, sort_order))) +} diff --git a/datafusion/proto/tests/cases/plans/deprecated_data_source_proto_serialization.rs b/datafusion/proto/tests/cases/plans/deprecated_data_source_proto_serialization.rs new file mode 100644 index 0000000000000..7469c8cb4c8d3 --- /dev/null +++ b/datafusion/proto/tests/cases/plans/deprecated_data_source_proto_serialization.rs @@ -0,0 +1,448 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Built-in scans and sinks encode as `data_source` / `data_sink`, and the +//! deprecated flat variants (`csv_scan`, `json_sink`, …) still decode. + +use super::roundtrip_test; +use datafusion::arrow::array::Int32Array; +use datafusion::arrow::datatypes::{DataType, Field, Schema, SchemaRef}; +use datafusion::arrow::record_batch::RecordBatch; +use datafusion::datasource::file_format::csv::CsvSink; +use datafusion::datasource::file_format::json::JsonSink; +use datafusion::datasource::file_format::parquet::ParquetSink; +use datafusion::datasource::listing::{ListingTableUrl, PartitionedFile}; +use datafusion::datasource::memory::MemorySourceConfig; +use datafusion::datasource::object_store::ObjectStoreUrl; +use datafusion::datasource::physical_plan::{ + ArrowSource, CsvSource, FileGroup, FileOutputMode, FileScanConfigBuilder, + FileSinkConfig, FileSource, JsonSource, ParquetSource, +}; +use datafusion::datasource::sink::{DataSink, DataSinkExec}; +use datafusion::datasource::source::DataSourceExec; +use datafusion::physical_plan::ExecutionPlan; +use datafusion::physical_plan::placeholder_row::PlaceholderRowExec; +use datafusion::prelude::SessionContext; +use datafusion_common::Result; +use datafusion_common::config::TableParquetOptions; +use datafusion_common::file_options::csv_writer::CsvWriterOptions; +use datafusion_common::file_options::json_writer::JsonWriterOptions; +use datafusion_common::parsers::CompressionTypeVariant; +use datafusion_datasource::TableSchema; +use datafusion_expr::dml::InsertOp; +use datafusion_proto::physical_plan::{AsExecutionPlan, DefaultPhysicalExtensionCodec}; +use datafusion_proto::protobuf::physical_plan_node::PhysicalPlanType; +use datafusion_proto::protobuf::{ + self, PhysicalPlanNode, data_sink_node::SinkType, data_source_node::SourceType, + file_source_node::FileSourceType, +}; +use prost::Message; +use std::sync::Arc; + +fn schema() -> SchemaRef { + Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)])) +} + +fn file_scan(source: Arc, path: &str) -> Arc { + DataSourceExec::from_data_source( + FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), source) + .with_file_groups(vec![FileGroup::new(vec![PartitionedFile::new( + path.to_string(), + 1024, + )])]) + .build(), + ) +} + +fn sink_exec(sink: Arc) -> Arc { + Arc::new(DataSinkExec::new( + Arc::new(PlaceholderRowExec::new(schema())), + sink, + None, + )) +} + +fn sink_config(extension: &str) -> Result { + Ok(FileSinkConfig { + original_url: String::default(), + object_store_url: ObjectStoreUrl::local_filesystem(), + file_group: FileGroup::new(vec![PartitionedFile::new("/tmp".to_string(), 1)]), + table_paths: vec![ListingTableUrl::parse("file:///")?], + output_schema: schema(), + table_partition_cols: vec![], + insert_op: InsertOp::Append, + keep_partition_by_columns: false, + file_extension: extension.into(), + file_output_mode: FileOutputMode::SingleFile, + }) +} + +/// One plan per built-in scan and sink. +fn built_in_plans() -> Result>> { + let table_schema = TableSchema::from(&schema()); + let batch = + RecordBatch::try_new(schema(), vec![Arc::new(Int32Array::from(vec![1]))])?; + let mut plans = vec![ + file_scan( + Arc::new(ParquetSource::new(table_schema.clone())), + "/f.parquet", + ), + file_scan(Arc::new(CsvSource::new(table_schema.clone())), "/f.csv"), + file_scan(Arc::new(JsonSource::new(table_schema.clone())), "/f.json"), + file_scan( + Arc::new(ArrowSource::new_stream_file_source(table_schema.clone())), + "/f.arrows", + ), + DataSourceExec::from_data_source(MemorySourceConfig::try_new( + &[vec![batch]], + schema(), + None, + )?), + sink_exec(Arc::new(JsonSink::new( + sink_config("json")?, + JsonWriterOptions::new(CompressionTypeVariant::UNCOMPRESSED), + ))), + sink_exec(Arc::new(CsvSink::new( + sink_config("csv")?, + CsvWriterOptions::new( + Default::default(), + CompressionTypeVariant::UNCOMPRESSED, + ), + ))), + sink_exec(Arc::new(ParquetSink::new( + sink_config("parquet")?, + TableParquetOptions::default(), + ))), + ]; + #[cfg(feature = "avro")] + plans.push(file_scan( + Arc::new(datafusion_datasource_avro::source::AvroSource::new( + table_schema, + )), + "/f.avro", + )); + #[cfg(not(feature = "avro"))] + let _ = &mut plans; + Ok(plans) +} + +/// Rewrite the layered encoding into the deprecated flat variant. +fn downgrade(plan: PhysicalPlanType) -> PhysicalPlanType { + match plan { + PhysicalPlanType::DataSource(node) => match node.source.unwrap().source_type { + Some(SourceType::File(scan)) => { + let base_conf = scan.base; + match scan.source.unwrap().file_source_type.unwrap() { + FileSourceType::Parquet(s) => { + PhysicalPlanType::ParquetScan(protobuf::ParquetScanExecNode { + base_conf, + predicate: s.predicate, + parquet_options: s.parquet_options, + sort_order_for_reorder: s.sort_order_for_reorder, + reverse_row_groups: s.reverse_row_groups, + metadata_size_hint: s.metadata_size_hint, + }) + } + FileSourceType::Csv(s) => { + use protobuf::{ + csv_scan_exec_node as old, csv_source_node as new, + }; + PhysicalPlanType::CsvScan(protobuf::CsvScanExecNode { + base_conf, + has_header: s.has_header, + delimiter: s.delimiter, + quote: s.quote, + optional_escape: s.optional_escape.map( + |new::OptionalEscape::Escape(e)| { + old::OptionalEscape::Escape(e) + }, + ), + optional_comment: s.optional_comment.map( + |new::OptionalComment::Comment(c)| { + old::OptionalComment::Comment(c) + }, + ), + newlines_in_values: s.newlines_in_values, + truncate_rows: s.truncate_rows, + terminator: s.terminator, + }) + } + FileSourceType::Json(s) => { + PhysicalPlanType::JsonScan(protobuf::JsonScanExecNode { + base_conf, + newline_delimited: s.newline_delimited, + }) + } + FileSourceType::Avro(_) => { + PhysicalPlanType::AvroScan(protobuf::AvroScanExecNode { + base_conf, + }) + } + FileSourceType::Arrow(s) => { + PhysicalPlanType::ArrowScan(protobuf::ArrowScanExecNode { + base_conf, + format: s.format, + }) + } + FileSourceType::Extension(_) => unreachable!(), + } + } + Some(SourceType::Memory(scan)) => PhysicalPlanType::MemoryScan(scan), + other => unreachable!("{other:?}"), + }, + PhysicalPlanType::DataSink(node) => { + let protobuf::DataSinkExecNode { + input, + sink, + sort_order, + } = *node; + match sink.unwrap().sink_type.unwrap() { + SinkType::Json(sink) => { + PhysicalPlanType::JsonSink(Box::new(protobuf::JsonSinkExecNode { + input, + sink: Some(sink), + sink_schema: None, + sort_order, + })) + } + SinkType::Csv(sink) => { + PhysicalPlanType::CsvSink(Box::new(protobuf::CsvSinkExecNode { + input, + sink: Some(sink), + sink_schema: None, + sort_order, + })) + } + SinkType::Parquet(sink) => PhysicalPlanType::ParquetSink(Box::new( + protobuf::ParquetSinkExecNode { + input, + sink: Some(sink), + sink_schema: None, + sort_order, + }, + )), + SinkType::Extension(_) => unreachable!(), + } + } + other => unreachable!("not a scan or sink: {other:?}"), + } +} + +fn decode(node: &PhysicalPlanNode) -> Result> { + let ctx = SessionContext::new(); + node.try_into_physical_plan( + ctx.task_ctx().as_ref(), + &DefaultPhysicalExtensionCodec {}, + ) +} + +fn encode(plan: Arc) -> Result { + PhysicalPlanNode::try_from_physical_plan(plan, &DefaultPhysicalExtensionCodec {}) +} + +fn decode_err(node: PhysicalPlanNode) -> String { + decode(&node).unwrap_err().to_string() +} + +fn data_source_node(source_type: Option) -> PhysicalPlanNode { + PhysicalPlanNode { + physical_plan_type: Some(PhysicalPlanType::DataSource( + protobuf::DataSourceExecNode { + source: Some(protobuf::DataSourceNode { source_type }), + }, + )), + } +} + +fn file_scan_node( + base: Option, + file_source_type: Option, +) -> PhysicalPlanNode { + data_source_node(Some(SourceType::File(protobuf::FileScanNode { + base, + source: Some(protobuf::FileSourceNode { file_source_type }), + }))) +} + +fn csv_base() -> Result { + let plan = file_scan( + Arc::new(CsvSource::new(TableSchema::from(&schema()))), + "/f.csv", + ); + match encode(plan)?.physical_plan_type { + Some(PhysicalPlanType::DataSource(protobuf::DataSourceExecNode { + source: + Some(protobuf::DataSourceNode { + source_type: Some(SourceType::File(scan)), + }), + })) => Ok(scan.base.unwrap()), + other => unreachable!("{other:?}"), + } +} + +fn sink_node( + input: Option, + sink: Option, +) -> PhysicalPlanNode { + PhysicalPlanNode { + physical_plan_type: Some(PhysicalPlanType::DataSink(Box::new( + protobuf::DataSinkExecNode { + input: input.map(Box::new), + sink, + sort_order: None, + }, + ))), + } +} + +#[test] +fn built_ins_encode_layered_and_legacy_variants_decode() -> Result<()> { + for plan in built_in_plans()? { + roundtrip_test(Arc::clone(&plan))?; + let layered = encode(Arc::clone(&plan))?; + assert!( + matches!( + layered.physical_plan_type, + Some(PhysicalPlanType::DataSource(_) | PhysicalPlanType::DataSink(_)) + ), + "unexpected encoding: {layered:?}" + ); + + // The legacy variant decodes to the same plan and re-encodes in the + // layered form. + let legacy = PhysicalPlanNode { + physical_plan_type: Some(downgrade( + layered.physical_plan_type.clone().unwrap(), + )), + }; + let decoded = decode(&legacy)?; + assert_eq!(format!("{plan:?}"), format!("{decoded:?}")); + assert_eq!( + layered.encode_to_vec(), + encode(decoded)?.encode_to_vec(), + "{legacy:?}" + ); + } + Ok(()) +} + +#[test] +fn extension_nodes_without_decoder_error() -> Result<()> { + let err = decode_err(data_source_node(Some(SourceType::Extension( + protobuf::ExtensionDataSourceNode { + name: "my_source".to_string(), + payload: vec![], + }, + )))); + assert!( + err.contains("No decoder for extension DataSource 'my_source'"), + "{err}" + ); + + let err = decode_err(file_scan_node( + Some(csv_base()?), + Some(FileSourceType::Extension( + protobuf::ExtensionFileSourceNode { + name: "my_format".to_string(), + payload: vec![1, 2, 3], + }, + )), + )); + assert!( + err.contains("No decoder for extension FileSource 'my_format'"), + "{err}" + ); + + let input = encode(Arc::new(PlaceholderRowExec::new(schema())))?; + let err = decode_err(sink_node( + Some(input), + Some(protobuf::DataSinkNode { + sink_type: Some(SinkType::Extension(protobuf::ExtensionDataSinkNode { + name: "my_sink".to_string(), + payload: vec![], + })), + }), + )); + assert!( + err.contains("No decoder for extension DataSink 'my_sink'"), + "{err}" + ); + Ok(()) +} + +#[test] +fn missing_fields_give_clear_errors() -> Result<()> { + let missing_source = PhysicalPlanNode { + physical_plan_type: Some(PhysicalPlanType::DataSource( + protobuf::DataSourceExecNode { source: None }, + )), + }; + assert!( + decode_err(missing_source).contains("DataSourceExecNode is missing its source") + ); + assert!( + decode_err(data_source_node(None)) + .contains("DataSourceExecNode is missing its source") + ); + assert!( + decode_err(file_scan_node( + None, + Some(FileSourceType::Avro(protobuf::AvroSourceNode {})) + )) + .contains("FileScanNode is missing its base") + ); + assert!( + decode_err(file_scan_node(Some(csv_base()?), None)) + .contains("FileScanNode is missing its source") + ); + + let input = encode(Arc::new(PlaceholderRowExec::new(schema())))?; + let err = decode_err(sink_node(Some(input.clone()), None)); + assert!( + err.contains("DataSinkExecNode is missing its sink"), + "{err}" + ); + let err = decode_err(sink_node( + Some(input), + Some(protobuf::DataSinkNode { sink_type: None }), + )); + assert!( + err.contains("DataSinkExecNode is missing its sink"), + "{err}" + ); + let err = decode_err(sink_node(None, None)); + assert!(err.contains("input"), "{err}"); + + // A legacy sink with no sink payload names the missing legacy field. + let input = encode(Arc::new(PlaceholderRowExec::new(schema())))?; + let legacy = PhysicalPlanNode { + physical_plan_type: Some(PhysicalPlanType::JsonSink(Box::new( + protobuf::JsonSinkExecNode { + input: Some(Box::new(input)), + sink: None, + sink_schema: None, + sort_order: None, + }, + ))), + }; + let err = decode_err(legacy); + assert!( + err.contains("JsonSinkExecNode is missing required field 'sink'"), + "{err}" + ); + Ok(()) +} diff --git a/datafusion/proto/tests/cases/plans/mod.rs b/datafusion/proto/tests/cases/plans/mod.rs index 69a6256af9263..baa0758defe19 100644 --- a/datafusion/proto/tests/cases/plans/mod.rs +++ b/datafusion/proto/tests/cases/plans/mod.rs @@ -34,6 +34,7 @@ use datafusion_proto::physical_plan::{ use std::sync::Arc; mod aggregates; +mod deprecated_data_source_proto_serialization; mod dispatch; mod dynamic_filters; mod expr_registry; diff --git a/datafusion/proto/tests/cases/plans/sinks.rs b/datafusion/proto/tests/cases/plans/sinks.rs index 1bc8c9d41e512..d71137031f0d4 100644 --- a/datafusion/proto/tests/cases/plans/sinks.rs +++ b/datafusion/proto/tests/cases/plans/sinks.rs @@ -487,26 +487,28 @@ fn roundtrip_parquet_sink() -> Result<()> { #[cfg(feature = "json")] let roundtripped = super::roundtrip_test_json_and_return(roundtripped, &ctx)?; let node = PhysicalPlanNode::try_from_physical_plan(roundtripped, &codec)?; - let Some(protobuf::physical_plan_node::PhysicalPlanType::ParquetSink(node)) = + let Some(protobuf::physical_plan_node::PhysicalPlanType::DataSink(node)) = node.physical_plan_type else { - panic!("expected ParquetSink node"); + panic!("expected DataSink node"); }; - let actual = node - .sink - .expect("ParquetSinkExecNode should contain a sink") - .sorting_columns - .map(|columns| { - columns - .columns - .into_iter() - .map(|column| SortingColumn { - column_idx: column.column_idx, - descending: column.descending, - nulls_first: column.nulls_first, - }) - .collect::>() - }); + let Some(protobuf::DataSinkNode { + sink_type: Some(protobuf::data_sink_node::SinkType::Parquet(sink)), + }) = node.sink + else { + panic!("expected ParquetSink"); + }; + let actual = sink.sorting_columns.map(|columns| { + columns + .columns + .into_iter() + .map(|column| SortingColumn { + column_idx: column.column_idx, + descending: column.descending, + nulls_first: column.nulls_first, + }) + .collect::>() + }); assert_eq!(actual, sorting_columns); } Ok(()) diff --git a/datafusion/proto/tests/cases/plans/sources.rs b/datafusion/proto/tests/cases/plans/sources.rs index 40821583678d1..2117b3b3d944c 100644 --- a/datafusion/proto/tests/cases/plans/sources.rs +++ b/datafusion/proto/tests/cases/plans/sources.rs @@ -68,6 +68,7 @@ use datafusion_proto::physical_plan::{ }; use datafusion_proto::protobuf; use datafusion_proto::protobuf::PhysicalPlanNode; +use datafusion_proto::protobuf::file_source_node::FileSourceType; use prost::Message; use std::collections::HashMap; use std::fmt::{Display, Formatter}; @@ -152,11 +153,9 @@ fn roundtrip_parquet_exec_with_pruning_predicate() -> Result<()> { )?; #[cfg(feature = "json")] let roundtripped = super::roundtrip_test_json_and_return(roundtripped, &ctx)?; - let node = PhysicalPlanNode::try_from_physical_plan(roundtripped, &codec)?; - let Some(protobuf::physical_plan_node::PhysicalPlanType::ParquetScan(scan)) = - node.physical_plan_type - else { - return internal_err!("Expected ParquetScan node"); + let mut node = PhysicalPlanNode::try_from_physical_plan(roundtripped, &codec)?; + let (_, FileSourceType::Parquet(scan)) = file_scan_parts(&mut node)? else { + return internal_err!("Expected Parquet source"); }; assert_eq!( scan.metadata_size_hint, @@ -205,15 +204,7 @@ fn file_scan_rejects_zero_batch_size() -> Result<()> { DataSourceExec::from_data_source(scan_config), &codec, )?; - let Some(protobuf::physical_plan_node::PhysicalPlanType::ParquetScan(scan)) = - node.physical_plan_type.as_mut() - else { - return internal_err!("Expected ParquetScan node"); - }; - scan.base_conf - .as_mut() - .expect("Parquet scan has a base config") - .batch_size = Some(0); + file_scan_parts(&mut node)?.0.batch_size = Some(0); let ctx = SessionContext::new(); let err = node @@ -377,11 +368,11 @@ fn arrow_scan_without_format_field_decodes_as_file_format() -> Result<()> { let mut node = PhysicalPlanNode::decode(bytes.as_ref()).map_err(|e| { internal_datafusion_err!("Failed to decode PhysicalPlanNode: {e}") })?; - match node.physical_plan_type.as_mut() { - Some(protobuf::physical_plan_node::PhysicalPlanType::ArrowScan(scan)) => { - scan.format = protobuf::ArrowIpcFormat::File as i32; + match file_scan_parts(&mut node)? { + (_, FileSourceType::Arrow(source)) => { + source.format = protobuf::ArrowIpcFormat::File as i32; } - other => return internal_err!("Expected ArrowScan node, got {other:?}"), + other => return internal_err!("Expected Arrow source, got {other:?}"), } let ctx = SessionContext::new(); @@ -434,15 +425,12 @@ fn roundtrip_json_scan_preserves_format_options() -> Result<()> { // Payloads written before these fields existed must keep their historical // defaults: newline-delimited JSON without compression. let mut node = PhysicalPlanNode::try_from_physical_plan(plan, &codec)?; - match node.physical_plan_type.as_mut() { - Some(protobuf::physical_plan_node::PhysicalPlanType::JsonScan(scan)) => { - scan.newline_delimited = None; - scan.base_conf - .as_mut() - .expect("JSON scan has a base config") - .file_compression_type = None; + match file_scan_parts(&mut node)? { + (base, FileSourceType::Json(source)) => { + source.newline_delimited = None; + base.file_compression_type = None; } - other => return internal_err!("Expected JsonScan node, got {other:?}"), + other => return internal_err!("Expected Json source, got {other:?}"), } let decoded = node.try_into_physical_plan(ctx.task_ctx().as_ref(), &codec)?; let file_scan = decoded @@ -586,13 +574,13 @@ fn roundtrip_csv_scan_preserves_format_options() -> Result<()> { assert_eq!(csv_source.escape(), Some(b'\\')); assert_eq!(csv_source.comment(), Some(b'#')); assert_eq!(csv_source.terminator(), Some(0xff)); - let node = + let mut node = PhysicalPlanNode::try_from_physical_plan(Arc::clone(&roundtripped), &codec)?; - match node.physical_plan_type { - Some(protobuf::physical_plan_node::PhysicalPlanType::CsvScan(scan)) => { + match file_scan_parts(&mut node)? { + (_, FileSourceType::Csv(scan)) => { assert_eq!(scan.newlines_in_values, newlines_in_values.unwrap_or(false)); } - other => return internal_err!("Expected CsvScan node, got {other:?}"), + other => return internal_err!("Expected Csv source, got {other:?}"), } assert!(csv_source.truncate_rows()); assert_eq!(file_scan.file_compression_type, FileCompressionType::GZIP); @@ -604,11 +592,11 @@ fn roundtrip_csv_scan_preserves_format_options() -> Result<()> { for invalid_terminator in [vec![], vec![b'\r', b'\n']] { let mut node = PhysicalPlanNode::try_from_physical_plan(Arc::clone(&plan), &codec)?; - match node.physical_plan_type.as_mut() { - Some(protobuf::physical_plan_node::PhysicalPlanType::CsvScan(scan)) => { + match file_scan_parts(&mut node)? { + (_, FileSourceType::Csv(scan)) => { scan.terminator = Some(invalid_terminator); } - other => return internal_err!("Expected CsvScan node, got {other:?}"), + other => return internal_err!("Expected Csv source, got {other:?}"), } let err = node .try_into_physical_plan(ctx.task_ctx().as_ref(), &codec) @@ -1225,3 +1213,28 @@ fn roundtrip_parquet_exec_range_output_partitioning_utf8view_split_points() -> R Ok(()) } + +/// Split an encoded file-scan `DataSourceExec` into its base config and source. +pub(crate) fn file_scan_parts( + node: &mut PhysicalPlanNode, +) -> Result<(&mut protobuf::FileScanExecConf, &mut FileSourceType)> { + use protobuf::data_source_node::SourceType; + match node.physical_plan_type.as_mut() { + Some(protobuf::physical_plan_node::PhysicalPlanType::DataSource( + protobuf::DataSourceExecNode { + source: + Some(protobuf::DataSourceNode { + source_type: + Some(SourceType::File(protobuf::FileScanNode { + base: Some(base), + source: + Some(protobuf::FileSourceNode { + file_source_type: Some(source), + }), + })), + }), + }, + )) => Ok((base, source)), + other => internal_err!("Expected a file scan DataSource node, got {other:?}"), + } +} diff --git a/docs/source/library-user-guide/upgrading/56.0.0.md b/docs/source/library-user-guide/upgrading/56.0.0.md index ba87076b3de10..bc10caf6f53e7 100644 --- a/docs/source/library-user-guide/upgrading/56.0.0.md +++ b/docs/source/library-user-guide/upgrading/56.0.0.md @@ -150,6 +150,39 @@ to obtain the same boolean value: `None` and `Some(false)` mean false, while `Some(true)` means true. CSV reading, repartitioning, and serialization behavior are unchanged. +### Whole-plan `DataSource` / `FileSource` / `DataSink` proto hooks are deprecated + +Scans and sinks now serialize as layered `data_source` / `data_sink` plan +nodes. Each level emits only its own message: + +| Deprecated | Replacement | +| ---------------------------------------------------------- | -------------------------------------------------------------- | +| `DataSource::try_to_proto` → `PhysicalPlanNode` | `DataSource::try_to_proto_source` → `DataSourceNode` | +| `FileSource::try_to_proto(base, ctx)` → `PhysicalPlanNode` | `FileSource::try_to_proto_file_source(ctx)` → `FileSourceNode` | +| `DataSink::try_to_proto(exec, ctx)` → `PhysicalPlanNode` | `DataSink::try_to_proto_sink(ctx)` → `DataSinkNode` | +| `{Csv,Json,Avro,Arrow,Parquet}Source::try_from_proto` | `…Source::try_from_proto_file_source` | +| `MemorySourceConfig::try_from_proto` | `MemorySourceConfig::try_from_proto_source` | +| `{Csv,Json,Parquet}Sink::try_from_proto` | `TryFrom<&protobuf::{Csv,Json,Parquet}Sink>` | + +The deprecated hooks are still called when the new one returns `None`, so +existing implementations keep working. The flat `parquet_scan`, `csv_scan`, +`json_scan`, `avro_scan`, `arrow_scan`, `memory_scan`, `json_sink`, +`csv_sink` and `parquet_sink` variants are no longer emitted but still decode. + +Plans encoded by 56.0.0 use the new `data_source` / `data_sink` variants, which +older readers do not understand: a 55.x process cannot decode scans or sinks +serialized by 56.0.0. Upgrade readers before (or together with) writers in +mixed-version deployments. + +**Migration guide:** implement the new hook and return only your own message; +`FileScanConfig`, `DataSourceExec` and `DataSinkExec` encode the base config, +input and sort order. + +The built-in sources and sinks implement only the new hooks, so their +deprecated `try_to_proto` hooks return `None`. A wrapper that forwards the +deprecated hook to a built-in (for example `inner.try_to_proto(base, ctx)`) +must forward the new hook instead; the compiler warns on the deprecated call. + ### `ParquetAccessPlan::into_overall_row_selection` is deprecated `ParquetAccessPlan::into_overall_row_selection` is deprecated starting in 56.0.0.