Skip to content

[SPARK-58544][SQL] Fix vector distance and norm functions returning wrong results from intermediate float overflow #57746

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
SEPURI-SAI-KRISHNA wants to merge 1 commit into
base: master
Choose a base branch
from
Open

[SPARK-58544][SQL] Fix vector distance and norm functions returning wrong results from intermediate float overflow #57746

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
190 changes: 110 additions & 80 deletions ...lyst/src/main/java/org/apache/spark/sql/catalyst/expressions/VectorFunctionImplUtils.java
View file
Open in desktop

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion ...catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/vectorExpressions.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ import org.apache.spark.unsafe.Platform
examples = """
Examples:
> SELECT _FUNC_(array(1.0F, 2.0F, 3.0F), array(4.0F, 5.0F, 6.0F));
0.9746319
0.97463185
""",
since = "4.2.0",
group = "vector_funcs"
Expand Down
42 changes: 42 additions & 0 deletions sql/core/src/test/resources/sql-tests/analyzer-results/vector-distance.sql.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -654,3 +654,45 @@ SELECT vector_l2_distance(
-- !query analysis
Project [vector_l2_distance(array(1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, 14.0, 15.0, 16.0), array(16.0, 15.0, 14.0, 13.0, 12.0, 11.0, 10.0, 9.0, 8.0, 7.0, 6.0, 5.0, 4.0, 3.0, 2.0, 1.0)) AS vector_l2_distance(array(1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, 14.0, 15.0, 16.0), array(16.0, 15.0, 14.0, 13.0, 12.0, 11.0, 10.0, 9.0, 8.0, 7.0, 6.0, 5.0, 4.0, 3.0, 2.0, 1.0))#x]
+- OneRowRelation


-- !query
SELECT vector_cosine_similarity(array(3.0e19F, 4.0e19F), array(3.0e19F, 4.0e19F))
-- !query analysis
Project [vector_cosine_similarity(array(3.0E19, 4.0E19), array(3.0E19, 4.0E19)) AS vector_cosine_similarity(array(3.0E19, 4.0E19), array(3.0E19, 4.0E19))#x]
+- OneRowRelation


-- !query
SELECT vector_cosine_similarity(array(3.0e19F, 4.0e19F), array(-3.0e19F, -4.0e19F))
-- !query analysis
Project [vector_cosine_similarity(array(3.0E19, 4.0E19), array(-3.0E19, -4.0E19)) AS vector_cosine_similarity(array(3.0E19, 4.0E19), array(-3.0E19, -4.0E19))#x]
+- OneRowRelation


-- !query
SELECT vector_inner_product(array(1.0e20F, 1.0e20F), array(1.0e20F, -1.0e20F))
-- !query analysis
Project [vector_inner_product(array(1.0E20, 1.0E20), array(1.0E20, -1.0E20)) AS vector_inner_product(array(1.0E20, 1.0E20), array(1.0E20, -1.0E20))#x]
+- OneRowRelation


-- !query
SELECT vector_l2_distance(array(3.0e19F, 4.0e19F), array(0.0F, 0.0F))
-- !query analysis
Project [vector_l2_distance(array(3.0E19, 4.0E19), array(0.0, 0.0)) AS vector_l2_distance(array(3.0E19, 4.0E19), array(0.0, 0.0))#x]
+- OneRowRelation


-- !query
SELECT vector_cosine_similarity(array(1.0e-23F, 0.0F), array(1.0e-23F, 0.0F))
-- !query analysis
Project [vector_cosine_similarity(array(1.0E-23, 0.0), array(1.0E-23, 0.0)) AS vector_cosine_similarity(array(1.0E-23, 0.0), array(1.0E-23, 0.0))#x]
+- OneRowRelation


-- !query
SELECT vector_cosine_similarity(array(1.0e-23F, 1.0e-23F), array(1.0e-23F, -1.0e-23F))
-- !query analysis
Project [vector_cosine_similarity(array(1.0E-23, 1.0E-23), array(1.0E-23, -1.0E-23)) AS vector_cosine_similarity(array(1.0E-23, 1.0E-23), array(1.0E-23, -1.0E-23))#x]
+- OneRowRelation
49 changes: 49 additions & 0 deletions sql/core/src/test/resources/sql-tests/analyzer-results/vector-norm.sql.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -572,3 +572,52 @@ SELECT vector_norm(
-- !query analysis
Project [vector_norm(array(1.0, 2.0, 3.0, 4.0, 5.0, cast(null as float), 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, 14.0, 15.0, 16.0), 2.0) AS vector_norm(array(1.0, 2.0, 3.0, 4.0, 5.0, CAST(NULL AS FLOAT), 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, 14.0, 15.0, 16.0), 2.0)#x]
+- OneRowRelation


-- !query
SELECT vector_norm(array(3.0e19F, 4.0e19F), 2.0F)
-- !query analysis
Project [vector_norm(array(3.0E19, 4.0E19), 2.0) AS vector_norm(array(3.0E19, 4.0E19), 2.0)#x]
+- OneRowRelation


-- !query
SELECT vector_normalize(array(3.0e19F, 4.0e19F), 2.0F)
-- !query analysis
Project [vector_normalize(array(3.0E19, 4.0E19), 2.0) AS vector_normalize(array(3.0E19, 4.0E19), 2.0)#x]
+- OneRowRelation


-- !query
SELECT vector_norm(array(3.0e38F, 3.0e38F), 1.0F)
-- !query analysis
Project [vector_norm(array(3.0E38, 3.0E38), 1.0) AS vector_norm(array(3.0E38, 3.0E38), 1.0)#x]
+- OneRowRelation


-- !query
SELECT vector_normalize(array(3.0e38F, 3.0e38F), 1.0F)
-- !query analysis
Project [vector_normalize(array(3.0E38, 3.0E38), 1.0) AS vector_normalize(array(3.0E38, 3.0E38), 1.0)#x]
+- OneRowRelation


-- !query
SELECT vector_norm(array(1.0e-23F, 0.0F), 2.0F)
-- !query analysis
Project [vector_norm(array(1.0E-23, 0.0), 2.0) AS vector_norm(array(1.0E-23, 0.0), 2.0)#x]
+- OneRowRelation


-- !query
SELECT vector_normalize(array(1.0e-23F, 0.0F), 2.0F)
-- !query analysis
Project [vector_normalize(array(1.0E-23, 0.0), 2.0) AS vector_normalize(array(1.0E-23, 0.0), 2.0)#x]
+- OneRowRelation


-- !query
SELECT vector_normalize(array(1.0e-23F, 1.0e-23F), 2.0F)
-- !query analysis
Project [vector_normalize(array(1.0E-23, 1.0E-23), 2.0) AS vector_normalize(array(1.0E-23, 1.0E-23), 2.0)#x]
+- OneRowRelation
23 changes: 23 additions & 0 deletions sql/core/src/test/resources/sql-tests/inputs/vector-distance.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -130,3 +130,26 @@ SELECT vector_l2_distance(
array(1.0F, 2.0F, 3.0F, 4.0F, 5.0F, 6.0F, 7.0F, 8.0F, 9.0F, 10.0F, 11.0F, 12.0F, 13.0F, 14.0F, 15.0F, 16.0F),
array(16.0F, 15.0F, 14.0F, 13.0F, 12.0F, 11.0F, 10.0F, 9.0F, 8.0F, 7.0F, 6.0F, 5.0F, 4.0F, 3.0F, 2.0F, 1.0F)
);

-- SPARK-58544: large magnitudes, intermediate sums of squares/products must not overflow the
-- float range

-- vector_cosine_similarity is scale invariant: identical vectors have similarity 1.0
SELECT vector_cosine_similarity(array(3.0e19F, 4.0e19F), array(3.0e19F, 4.0e19F));

-- vector_cosine_similarity: opposite vectors have similarity -1.0
SELECT vector_cosine_similarity(array(3.0e19F, 4.0e19F), array(-3.0e19F, -4.0e19F));

-- vector_inner_product: individual products overflow the float range but cancel out
SELECT vector_inner_product(array(1.0e20F, 1.0e20F), array(1.0e20F, -1.0e20F));

-- vector_l2_distance: sqrt((3e19)^2 + (4e19)^2) = 5e19
SELECT vector_l2_distance(array(3.0e19F, 4.0e19F), array(0.0F, 0.0F));

-- SPARK-58544: small magnitudes, intermediate sums of squares/products must not underflow to zero

-- vector_cosine_similarity of identical tiny vectors is 1.0, not NULL
SELECT vector_cosine_similarity(array(1.0e-23F, 0.0F), array(1.0e-23F, 0.0F));

-- vector_cosine_similarity of orthogonal tiny vectors is 0.0, not NULL
SELECT vector_cosine_similarity(array(1.0e-23F, 1.0e-23F), array(1.0e-23F, -1.0e-23F));
24 changes: 24 additions & 0 deletions sql/core/src/test/resources/sql-tests/inputs/vector-norm.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -153,3 +153,27 @@ SELECT vector_norm(
array(1.0F, 2.0F, 3.0F, 4.0F, 5.0F, CAST(NULL AS FLOAT), 7.0F, 8.0F, 9.0F, 10.0F, 11.0F, 12.0F, 13.0F, 14.0F, 15.0F, 16.0F),
2.0F
);

-- SPARK-58544: large magnitudes, the intermediate sum of squares must not overflow the float range

-- vector_norm: sqrt((3e19)^2 + (4e19)^2) = 5e19
SELECT vector_norm(array(3.0e19F, 4.0e19F), 2.0F);

-- vector_normalize: [3e19, 4e19] normalizes to [0.6, 0.8]
SELECT vector_normalize(array(3.0e19F, 4.0e19F), 2.0F);

-- vector_norm: the L1 norm itself is not representable as a float, so it stays infinite
SELECT vector_norm(array(3.0e38F, 3.0e38F), 1.0F);

-- vector_normalize: normalization still succeeds when the norm exceeds the float range
SELECT vector_normalize(array(3.0e38F, 3.0e38F), 1.0F);

-- SPARK-58544: small magnitudes, the intermediate sum of squares must not underflow to zero

-- vector_norm: the L2 norm of a tiny vector is not zero
SELECT vector_norm(array(1.0e-23F, 0.0F), 2.0F);

-- vector_normalize: normalization is scale invariant, so the result is a unit vector, not NULL
SELECT vector_normalize(array(1.0e-23F, 0.0F), 2.0F);

SELECT vector_normalize(array(1.0e-23F, 1.0e-23F), 2.0F);
50 changes: 49 additions & 1 deletion sql/core/src/test/resources/sql-tests/results/vector-distance.sql.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ SELECT vector_cosine_similarity(array(1.0F, 2.0F, 3.0F), array(4.0F, 5.0F, 6.0F)
-- !query schema
struct<vector_cosine_similarity(array(1.0, 2.0, 3.0), array(4.0, 5.0, 6.0)):float>
-- !query output
0.9746319
0.97463185


-- !query
Expand Down Expand Up @@ -744,3 +744,51 @@ SELECT vector_l2_distance(
struct<vector_l2_distance(array(1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, 14.0, 15.0, 16.0), array(16.0, 15.0, 14.0, 13.0, 12.0, 11.0, 10.0, 9.0, 8.0, 7.0, 6.0, 5.0, 4.0, 3.0, 2.0, 1.0)):float>
-- !query output
36.878178


-- !query
SELECT vector_cosine_similarity(array(3.0e19F, 4.0e19F), array(3.0e19F, 4.0e19F))
-- !query schema
struct<vector_cosine_similarity(array(3.0E19, 4.0E19), array(3.0E19, 4.0E19)):float>
-- !query output
1.0


-- !query
SELECT vector_cosine_similarity(array(3.0e19F, 4.0e19F), array(-3.0e19F, -4.0e19F))
-- !query schema
struct<vector_cosine_similarity(array(3.0E19, 4.0E19), array(-3.0E19, -4.0E19)):float>
-- !query output
-1.0


-- !query
SELECT vector_inner_product(array(1.0e20F, 1.0e20F), array(1.0e20F, -1.0e20F))
-- !query schema
struct<vector_inner_product(array(1.0E20, 1.0E20), array(1.0E20, -1.0E20)):float>
-- !query output
0.0


-- !query
SELECT vector_l2_distance(array(3.0e19F, 4.0e19F), array(0.0F, 0.0F))
-- !query schema
struct<vector_l2_distance(array(3.0E19, 4.0E19), array(0.0, 0.0)):float>
-- !query output
5.0E19


-- !query
SELECT vector_cosine_similarity(array(1.0e-23F, 0.0F), array(1.0e-23F, 0.0F))
-- !query schema
struct<vector_cosine_similarity(array(1.0E-23, 0.0), array(1.0E-23, 0.0)):float>
-- !query output
1.0


-- !query
SELECT vector_cosine_similarity(array(1.0e-23F, 1.0e-23F), array(1.0e-23F, -1.0e-23F))
-- !query schema
struct<vector_cosine_similarity(array(1.0E-23, 1.0E-23), array(1.0E-23, -1.0E-23)):float>
-- !query output
0.0
58 changes: 57 additions & 1 deletion sql/core/src/test/resources/sql-tests/results/vector-norm.sql.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -668,7 +668,7 @@ SELECT vector_normalize(
-- !query schema
struct<vector_normalize(array(1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, 14.0, 15.0, 16.0), 2.0):array<float>>
-- !query output
[0.025854385,0.05170877,0.07756316,0.10341754,0.12927192,0.15512632,0.1809807,0.20683508,0.23268947,0.25854385,0.28439823,0.31025264,0.33610702,0.3619614,0.38781577,0.41367015]
[0.025854385,0.05170877,0.07756315,0.10341754,0.12927192,0.1551263,0.1809807,0.20683508,0.23268946,0.25854385,0.28439823,0.3102526,0.336107,0.3619614,0.38781577,0.41367015]


-- !query
Expand All @@ -680,3 +680,59 @@ SELECT vector_norm(
struct<vector_norm(array(1.0, 2.0, 3.0, 4.0, 5.0, CAST(NULL AS FLOAT), 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, 14.0, 15.0, 16.0), 2.0):float>
-- !query output
NULL


-- !query
SELECT vector_norm(array(3.0e19F, 4.0e19F), 2.0F)
-- !query schema
struct<vector_norm(array(3.0E19, 4.0E19), 2.0):float>
-- !query output
5.0E19


-- !query
SELECT vector_normalize(array(3.0e19F, 4.0e19F), 2.0F)
-- !query schema
struct<vector_normalize(array(3.0E19, 4.0E19), 2.0):array<float>>
-- !query output
[0.6,0.8]


-- !query
SELECT vector_norm(array(3.0e38F, 3.0e38F), 1.0F)
-- !query schema
struct<vector_norm(array(3.0E38, 3.0E38), 1.0):float>
-- !query output
Infinity


-- !query
SELECT vector_normalize(array(3.0e38F, 3.0e38F), 1.0F)
-- !query schema
struct<vector_normalize(array(3.0E38, 3.0E38), 1.0):array<float>>
-- !query output
[0.5,0.5]


-- !query
SELECT vector_norm(array(1.0e-23F, 0.0F), 2.0F)
-- !query schema
struct<vector_norm(array(1.0E-23, 0.0), 2.0):float>
-- !query output
1.0E-23


-- !query
SELECT vector_normalize(array(1.0e-23F, 0.0F), 2.0F)
-- !query schema
struct<vector_normalize(array(1.0E-23, 0.0), 2.0):array<float>>
-- !query output
[1.0,0.0]


-- !query
SELECT vector_normalize(array(1.0e-23F, 1.0e-23F), 2.0F)
-- !query schema
struct<vector_normalize(array(1.0E-23, 1.0E-23), 2.0):array<float>>
-- !query output
[0.70710677,0.70710677]