fix: Block::to_record_batch fail when a column is array of NULLs. (#18989)

Author: youngsofun

src/query/expression/src/converts/arrow/to.rs

@@ -234,7 +234,7 @@ impl From<&TableField> for Field {
             }
         };
 
-        Field::new(f.name(), ty, f.is_nullable()).with_metadata(metadata)
+        Field::new(f.name(), ty, f.is_nullable_or_null()).with_metadata(metadata)
     }
 }
 
@@ -428,3 +428,25 @@ impl Column {
         (&self).into()
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use databend_common_column::buffer::Buffer;
+
+    use super::*;
+    use crate::types::AnyType;
+    use crate::types::ArrayColumn;
+
+    #[test]
+    fn test_to_record_batch_with_null_array() {
+        let array =
+            ArrayColumn::<AnyType>::new(Column::Null { len: 3 }, Buffer::from(vec![0_u64, 3]));
+        let block = DataBlock::new_from_columns(vec![Column::Array(Box::new(array))]);
+        let schema = DataSchema::new(vec![DataField::new(
+            "arr",
+            DataType::Array(Box::new(DataType::Null)),
+        )]);
+
+        assert!(block.to_record_batch_with_dataschema(&schema).is_ok());
+    }
+}

tests/sqllogictests/suites/stage/unload.test

@@ -34,18 +34,18 @@ create file format if not exists csv_zip type=csv compression=zip;
 
 
 # test csv
-query
+query 
 copy into @unload from ii file_format=(type=csv);
 -----
 ----
 3 12 12
 
-query
+query 
 select right(name, 4), size from list_stage(location=>'@unload');
 ----
 .csv 12
 
-query
+query 
 select $1, $2 from @unload(file_format=>'csv');
 ----
 1 2
@@ -56,46 +56,46 @@ select $1, $2 from @unload(file_format=>'csv');
 statement ok
 remove @unload;
 
-query
+query 
 copy into @unload from ii file_format=(format_name='csv_gzip');
 ----
 3 12 32
 
 statement error
 copy into @unload from ii file_format=(format_name='csv_snappy');
 
-query
+query 
 copy into @unload from ii file_format=(format_name='csv_none');
 ----
 3 12 12
 
-query
+query 
 copy into @unload from ii file_format=(format_name='csv_zip');
 ----
 3 12 136
 
-query
+query 
 select right(name, 7), size from list_stage(location=>'@unload');
 ----
 .csv.gz 32
 000.csv 12
 csv.zip 136
 
-query
+query 
 select $1, $2 from @unload(file_format => 'csv_gzip' pattern => '.*.gz');
 ----
 1 2
 3 4
 5 6
 
-query
+query 
 select $1, $2 from @unload(file_format => 'csv_none' pattern => '.*.csv');
 ----
 1 2
 3 4
 5 6
 
-query
+query 
 select $1, $2 from @unload(file_format => 'csv_zip' pattern => '.*.zip');
 ----
 1 2
@@ -106,37 +106,48 @@ select $1, $2 from @unload(file_format => 'csv_zip' pattern => '.*.zip');
 statement ok
 remove @unload;
 
-query
+query 
 copy into @unload from ii file_format=(format_name='tsv');
 ----
 3 12 12
 
-query
+query 
 select right(name, 4), size from list_stage(location=>'@unload');
 ----
 .tsv 12
 
-query
+query 
 select $1, $2 from @unload(file_format => 'tsv');
 ----
 1 2
 3 4
 5 6
 
-query
+query 
 copy into @unload/a_raw_path.csv from (select 1,2) file_format=(type=csv) single=true include_query_id=false use_raw_path=true detailed_output=true overwrite=true;
 ----
 a_raw_path.csv 4 1
 
-query
+query 
 copy into @unload/a_raw_path.csv from (select 3,4) file_format=(type=csv) single=true include_query_id=false use_raw_path=true detailed_output=true overwrite=true;
 ----
 a_raw_path.csv 4 1
 
-query
+query 
 select $1, $2 from @unload/a_raw_path.csv (file_format => 'csv');
 ----
 3 4
 
 statement error 1006.*file already exists
-copy into @unload/a_raw_path.csv from (select 3,4) file_format=(type=csv) single=true include_query_id=false use_raw_path=true detailed_output=false overwrite=false;
+copy into @unload/a_raw_path.csv from (select 3,4) file_format=(type=csv) single=true include_query_id=false use_raw_path=true detailed_output=false overwrite=false;
+
+
+query 
+copy into @unload/array_of_nulls from (select [NULL, NULL]);
+----
+1 8 548
+
+query 
+select * from @unload/array_of_nulls;
+----
+[NULL,NULL]