Fix #5666: Fix normalizing map when testing for seen pairs

When testing for seen pairs we subject both types to a "normalize" type map
that derefereces LayVals. This is tricky but necessary. But we should take
care that the map does not disturbe the types in any other way. In particular
it should not replace a TypeVar by its instance.

Author: odersky

compiler/src/dotty/tools/dotc/core/TypeComparer.scala

@@ -171,6 +171,8 @@ class TypeComparer(initctx: Context) extends ConstraintHandling[AbsentContext] {
             derefCount += 1
             if (derefCount >= DerefLimit) NoType
             else try mapOver(t.ref) finally derefCount -= 1
+          case tp: TypeVar =>
+            tp
           case _ =>
             mapOver(t)
         }

tests/pos/i5666.scala

@@ -0,0 +1,20 @@
+// This file should still compile if Config.LogPrendingSubtypesthreshold is set to 9.
+sealed trait HList
+trait HNil extends HList
+trait HCons[+H, +T] extends HList
+
+trait Concat[L1, L2] { type Out }
+object Concat {
+  implicit def i0[L]:
+    Concat[HNil, L] { type Out = L } = null
+
+  implicit def i1[H, T, L, O]
+    (implicit c: Concat[T, L] { type Out = O }):
+      Concat[HCons[H, T], L] { type Out = HCons[H, O] } = null
+}
+
+object Test {
+  type L1 = HCons[Unit, HNil]
+
+  implicitly[Concat[L1, L1]]
+}