Fix varargs overload resolution with wildcard types (#24669)

Fixes #24072

When comparing overloaded methods where one is non-varargs with wildcard
types (e.g., `Class[? <: T]`) and another is varargs, the non-varargs
method should be preferred.
Previously, the compiler failed to distinguish these methods , and
results in an ambiguity error.

```scala
def blub[T](a: Class[? <: T]): Unit // m1
def blub[T](a: Class[T], ints: Int*): Unit // m2
blub(classOf[Object]) // m1 should be picked, but fails to resolve
````

The problem is `compare(m1, m2)` returned 0 because:

- (1). `m2` (varargs) is correctly considered "not as good" as `m1`.
- (2). `m1` (non-varargs) was also considered "not as good" as `m2`.
(but `m1` should be as good as `m2`! because `Class[Concrete]` can be
applied to both m1 and m2).

The (2) occurred because `Class[? <: T]` is not a subtype of `Class[T]`
(due to invariance). Consequently, `isApplicableMethodRef(m2, Class[? <:
T])` returned `false` because `isCompatible(Class[? <: T], Class[T])`
returned `false` during the applicability check against the method.

This commit adds special handling in `TestApplication.argOK` to check if
wildcard upper bounds are compatible with their formal types during
overload resolution, in addition to `isCompatible`.

Author: tanishiking

compiler/src/dotty/tools/dotc/typer/Applications.scala

@@ -883,7 +883,29 @@ trait Applications extends Compatibility {
             case SAMType(samMeth, samParent) => argtpe <:< samMeth.toFunctionType(isJava = samParent.classSymbol.is(JavaDefined))
             case _ => false
 
-        isCompatible(argtpe, formal)
+        // For overload resolution, allow wildcard upper bounds to match their bound type.
+        // For example, given:
+        //   def blub[T](a: Class[? <: T]): String = "a"
+        //   def blub[T](a: Class[T], ints: Int*): String = "b"
+        //   blub(classOf[Object])
+        //
+        // The non-varargs overload should be preferred. While Class[? <: T] is not a
+        // subtype of Class[T] (Class is invariant), for overload resolution we consider
+        // Class[? <: T] "applicable" where Class[T] is expected by checking if the
+        // wildcard's upper bound is a subtype of the formal type parameter.
+        def wildcardArgOK =
+          argtpe match
+            case at @ AppliedType(tycon1, args1) if at.hasWildcardArg =>
+              formal match
+                case AppliedType(tycon2, args2)
+                if tycon1 =:= tycon2 && args1.length == args2.length =>
+                  args1.lazyZip(args2).forall {
+                    case (TypeBounds(_, hi), formal) => hi relaxed_<:< formal
+                  }
+                case _ => false
+            case _ => false
+
+        isCompatible(argtpe, formal) || wildcardArgOK
         // Only allow SAM-conversion to PartialFunction if implicit conversions
         // are enabled. This is necessary to avoid ambiguity between an overload
         // taking a PartialFunction and one taking a Function1 because

tests/neg/overload_repeated.scala

@@ -2,5 +2,11 @@ object Test {
   def bar1(x: Any) = 1
   def bar1(x: String*) = 2
 
+  // (b) isn't as good as (a) because of ints
+  // (a) isn't as good as (b) because T in Class[? <: T] may not subtype of Number
+  def bar2[T](a: Class[? <: T]): Int = 1 // (a)
+  def bar2[T <: Number](a: Class[T], ints: Int*): Int = 2 // (b)
+
   assert(bar1("") == 1) // error: ambiguous in Scala 2 and Scala 3
+  assert(bar2(classOf[Integer]) == 1) // error: ambiguous in Scala 2 and Scala 3
 }

tests/run/overload_repeated/A_1.java

@@ -11,13 +11,34 @@ class A_1 {
   public static <T> int foo4(T x) { return 1; }
   public static <T> int foo4(T x, T... y) { return 2; }
 
+  // https://github.com/scala/scala3/issues/24072
+  public static <T> int foo5(Class<? extends T> a) { return 1; }
+  public static <T> int foo5(Class<T> a, int... ints) { return 2; }
+
+  public static <T> int foo6(Class<T> a, int... ints) { return 1; }
+  public static <T> int foo6(int a) { return 2; }
+
+  public static <T extends Number> int foo7(Class<? extends T> a) { return 1; }
+  public static <T extends Number> int foo7(Class<T> a, int... ints) { return 2; }
+
+  public static <T extends Number> int foo8(Class<? extends T> a) { return 1; } // (a)
+  public static <T> int foo8(Class<T> a, int... ints) { return 2; } // (b)
+
+  public static <T> int foo9(Class<? extends T> a) { return 1; } // (a)
+  public static <T extends Number> int foo9(Class<T> a, int... ints) { return 2; } // (b)
+
   public static boolean check() {
     // Java prefers non-varargs to varargs:
     // https://docs.oracle.com/javase/specs/jls/se8/html/jls-15.html#jls-15.12.2
     return
         foo1("") == 1 &&
         foo2("") == 1 &&
         foo3("") == 1 &&
-        foo4("") == 1;
+        foo4("") == 1 &&
+        foo5(Object.class) == 1 &&
+        foo6(Object.class) == 1 &&
+        foo7(Integer.class) == 1 &&
+        foo8(Integer.class) == 1 &&
+        foo9(Integer.class) == 1;
   }
 }

tests/run/overload_repeated/B_2.scala

@@ -13,6 +13,22 @@ object Test {
   def bar4[T](x: T): Int = 1
   def bar4[T](x: T, xs: T*): Int = 2
 
+  // https://github.com/scala/scala3/issues/24072
+  def bar5[T](a: Class[? <: T]): Int = 1
+  def bar5[T](a: Class[T], ints: Int*): Int = 2
+
+  def bar6[T](a: Class[T], ints: Int*): Int = 1
+  def bar6[T](a: Int): Int = 2
+
+  def bar7[T <: Number](a: Class[? <: T]): Int = 1
+  def bar7[T <: Number](a: Class[T], ints: Int*): Int = 2
+
+  def bar8[T <: Number](a: Class[? <: T]): Int = 1 // (a)
+  def bar8[T](a: Class[T], ints: Int*): Int = 2 // (b)
+
+  def bar9[T](a: Class[? <: T]): Int = 1 // (a)
+  def bar9[T <: Number](a: Class[T], ints: Int*): Int = 2 // (b)
+
   def main(args: Array[String]): Unit = {
     // In Java, varargs are always less specific than non-varargs (see
     // https://docs.oracle.com/javase/specs/jls/se8/html/jls-15.html#jls-15.12.2),
@@ -22,11 +38,21 @@ object Test {
     assert(A_1.foo2("") == 1) // Same as in Java and Scala 2
     assert(A_1.foo3("") == 1) // Same as in Java and Scala 2
     assert(A_1.foo4("") == 1) // Same as in Java and Scala 2
+    assert(A_1.foo5(classOf[Object]) == 1) // Same as in Java and Scala 2
+    assert(A_1.foo6(classOf[Object]) == 1) // Same as in Java and Scala 2
+    assert(A_1.foo7(classOf[Integer]) == 1) // Same as in Java and Scala 2
+    assert(A_1.foo8(classOf[Integer]) == 1) // Same as in Java and Scala 2
+    // assert(A_1.foo9(classOf[Integer]) == 1) // Works in Java, ambiguous in Scala 2 and 3
 
     // Same with Scala varargs:
     // assert(bar1("") == 1) // Works in Java, ambiguous in Scala 2 and Dotty
     assert(bar2("") == 1) // same in Scala 2
     assert(bar3("") == 1) // same in Scala 2
     assert(bar4("") == 1) // same in Scala 2
+    assert(bar5(classOf[Object]) == 1) // same in Scala2
+    assert(bar6(classOf[Object]) == 1) // same in Scala2
+    assert(bar7(classOf[Integer]) == 1) // same in Scala2
+    assert(bar8(classOf[Integer]) == 1) // same in Scala2
+    // assert(bar9(classOf[Integer]) == 1) Works in Java, ambiguous in Scala 2 and 3
   }
 }