fix: Handle null-aware joins correctly in FilterNullJoinKeys when its enabled

datafusion/optimizer/src/filter_null_join_keys.rs

@@ -52,6 +52,7 @@ impl OptimizerRule for FilterNullJoinKeys {
         match plan {
             LogicalPlan::Join(mut join)
                 if !join.on.is_empty()
+                    && !join.null_aware
                     && join.null_equality == NullEquality::NullEqualsNothing =>
             {
                 let (left_preserved, right_preserved) =
@@ -359,4 +360,50 @@ mod tests {
         let t2 = table_scan(Some("t2"), &schema, None)?.build()?;
         Ok((t1, t2))
     }
+
+    #[test]
+    fn null_aware_left_mark_join_keys_not_filtered() -> Result<()> {
+        let (t1, t2) = test_tables()?;
+        let plan = build_null_aware_plan(t1, t2, JoinType::LeftMark)?;
+
+        assert_optimized_plan_equal!(plan, @r"
+        LeftMark Join: t1.id = t2.optional_id null_aware
+          TableScan: t1
+          TableScan: t2
+        ")
+    }
+
+    #[test]
+    fn null_aware_left_anti_join_keys_not_filtered() -> Result<()> {
+        let (t1, t2) = test_tables()?;
+        let plan = build_null_aware_plan(t1, t2, JoinType::LeftAnti)?;
+
+        assert_optimized_plan_equal!(plan, @r"
+        LeftAnti Join: t1.id = t2.optional_id null_aware
+          TableScan: t1
+          TableScan: t2
+        ")
+    }
+
+    /// A join whose nullable right key would get an `IS NOT NULL` filter if it
+    /// were not null-aware.
+    fn build_null_aware_plan(
+        left_table: LogicalPlan,
+        right_table: LogicalPlan,
+        join_type: JoinType,
+    ) -> Result<LogicalPlan> {
+        LogicalPlanBuilder::from(left_table)
+            .join_detailed_with_options(
+                right_table,
+                join_type,
+                (
+                    vec![Column::from_qualified_name("t1.id")],
+                    vec![Column::from_qualified_name("t2.optional_id")],
+                ),
+                None,
+                NullEquality::NullEqualsNothing,
+                true,
+            )?
+            .build()
+    }
 }

datafusion/sqllogictest/test_files/null_aware_anti_join.slt

@@ -70,6 +70,20 @@ query IT rowsort
 SELECT * FROM outer_table WHERE id NOT IN (SELECT id FROM inner_table_with_null);
 ----
 
+# Regression test
+
+statement ok
+set datafusion.optimizer.filter_null_join_keys = true;
+
+# The subquery NULL must reach the join: every row's NOT IN is UNKNOWN or
+# FALSE, so the result stays empty.
+query IT rowsort
+SELECT * FROM outer_table WHERE id NOT IN (SELECT id FROM inner_table_with_null);
+----
+
+statement ok
+reset datafusion.optimizer.filter_null_join_keys;
+
 # Verify the result is empty even though there are rows in outer_table
 # that don't match the non-NULL value (2) in the subquery.
 # This is correct null-aware behavior: if subquery contains NULL, result is unknown.