Try to only instantiate one type variable bounded function types

Author: noti0na1

compiler/src/dotty/tools/dotc/typer/Typer.scala

@@ -1921,21 +1921,39 @@ class Typer(@constructorOnly nestingLevel: Int = 0) extends Namer
         NoType
     }
 
-    def tryToInstantiateInUnion(tp: Type): Unit = tp match
-      case tp: OrType =>
-        tryToInstantiateInUnion(tp.tp1)
-        tryToInstantiateInUnion(tp.tp2)
+    /** Try to instantiate one type variable bounded by function types that appear
+     *  deeply inside `tp`, including union or intersection types.
+     */
+    def tryToInstantiateDeeply(tp: Type): Boolean = tp match
+      case tp: AndOrType =>
+        tryToInstantiateDeeply(tp.tp1)
+        || tryToInstantiateDeeply(tp.tp2)
       case tp: FlexibleType =>
-        tryToInstantiateInUnion(tp.hi)
-      case tp: TypeVar =>
+        tryToInstantiateDeeply(tp.hi)
+      case tp: TypeVar if isConstrainedByFunctionType(tp) =>
+        // Only instantiate if the type variable is constrained by function types
         isFullyDefined(tp, ForceDegree.flipBottom)
-      case _ =>
+      case _ => false
+
+    def isConstrainedByFunctionType(tvar: TypeVar): Boolean =
+      val origin = tvar.origin
+      val bounds = ctx.typerState.constraint.bounds(origin)
+      def containsFunctionType(tp: Type): Boolean = tp.dealias match
+        case tp if defn.isFunctionType(tp) => true
+        case SAMType(_, _) => true
+        case tp: AndOrType =>
+          containsFunctionType(tp.tp1) || containsFunctionType(tp.tp2)
+        case tp: FlexibleType =>
+          containsFunctionType(tp.hi)
+        case _ => false
+      containsFunctionType(bounds.lo) || containsFunctionType(bounds.hi)
 
     if untpd.isFunctionWithUnknownParamType(tree) && !calleeType.exists then
-      // Try to instantiate `pt` when possible, including type variables in union types
-      // to help finding function types. If it does not work the error will be reported
-      // later in `inferredParam`, when we try to infer the parameter type.
-      tryToInstantiateInUnion(pt)
+      // Try to instantiate `pt` when possible, by searching a nested type variable
+      // bounded by function types to help infer parameter types.
+      // If it does not work the error will be reported later in `inferredParam`,
+      // when we try to infer the parameter type.
+      tryToInstantiateDeeply(pt)
 
     val (protoFormals, resultTpt) = decomposeProtoFunction(pt, params.length, tree.srcPos)
 

tests/neg-custom-args/captures/i15923.check

@@ -6,7 +6,7 @@
    |
    |Note that capability lcap cannot be included in outer capture set 's1 of parameter cap.
    |
-   |where:    => refers to a fresh root capability created in anonymous function of type (using lcap: scala.caps.Capability): test2.Cap^{lcap} -> [T] => (op: test2.Cap^{lcap} => T) -> T when instantiating expected result type test2.Cap^{lcap} ->{cap²} [T] => (op: test2.Cap^'s6 ->'s7 T) ->'s8 T of function literal
+   |where:    => refers to a fresh root capability created in anonymous function of type (using lcap: scala.caps.Capability): test2.Cap^{lcap} -> [T] => (op: test2.Cap => T) -> T when instantiating expected result type test2.Cap^{lcap} ->{cap²} [T] => (op: test2.Cap^'s6 ->'s7 T) ->'s8 T of function literal
    |
    | longer explanation available when compiling with `-explain`
 -- [E007] Type Mismatch Error: tests/neg-custom-args/captures/i15923.scala:12:21 ---------------------------------------

tests/pos/infer-function-type-in-union.scala

@@ -29,4 +29,14 @@ def test =
   }
   val o3: MyOption[(String, String) => Boolean] = MyOption {
     (x, y) => x.length > y.length
-  }
+  }
+
+
+class Box[T]
+val box: Box[Unit] = ???
+def ff1[T, U](x: T | U, y: Box[U]): T = ???
+def ff2[T, U](x: T & U): T = ???
+
+def test2 =
+  val a1: Any => Any = ff1(x => x, box)
+  val a2: Any => Any = ff2(x => x)