Prioritize TASTy files over classfiles on classpath aggregation

In most cases the TASTy file is chosen over the classfile in a classpath
because they are packaged together. However, for the `scala-library`
(Scala 2 compiled library) and `scala2-library-tasty` (Scala 3 compiled Scala 2 library)
we have the classfiles in one jar and the TASTy files in another jar.
Given that the classpaths order in guaranteed to be deterministic we might
end up with the classfile being loaded first and the TASTy file second.
The aggregator must be capable of choosing the TASTy file over the classfile
in this case as well. The aggregator will only choose the new TASTy over
the old classfile if the TASTy file has no classfile in its classpath. In
other words, we only use this new behaviour for TASTy only classpaths.

This also implies that we can just add the `scala2-library-tasty` as a
dependency in any project to use it. Note that this jar is not published
yet.

Author: nicolasstucki

compiler/src/dotty/tools/dotc/classpath/AggregateClassPath.scala

@@ -102,10 +102,15 @@ case class AggregateClassPath(aggregates: Seq[ClassPath]) extends ClassPath {
     ClassPathEntries(distinctPackages, distinctClassesAndSources)
   }
 
-  /**
-   * Returns only one entry for each name. If there's both a source and a class entry, it
-   * creates an entry containing both of them. If there would be more than one class or source
-   * entries for the same class it always would use the first entry of each type found on a classpath.
+  /** Returns only one entry for each name.
+   *
+   *  If there's both a source and a class entry, it
+   *  creates an entry containing both of them. If there would be more than one class or source
+   *  entries for the same class it always would use the first entry of each type found on a classpath.
+   *
+   *  A TASTy file with no class file entry will be chosen over a class file entry. This can happen if we load
+   *  the Scala 2 library as it has one JAR containing the class files and one JAR containing the TASTy files.
+   *  As classpath orders are not guaranteed to be deterministic we might end up having the TASTy in a later classpath entry.
    */
   private def mergeClassesAndSources(entries: scala.collection.Seq[ClassRepresentation]): Seq[ClassRepresentation] = {
     // based on the implementation from MergedClassPath
@@ -124,6 +129,12 @@ case class AggregateClassPath(aggregates: Seq[ClassPath]) extends ClassPath {
             mergedEntries(index) = BinaryAndSourceFilesEntry(existing, entry)
           case (entry: BinaryFileEntry, existing: SourceFileEntry) =>
             mergedEntries(index) = BinaryAndSourceFilesEntry(entry, existing)
+          case (entry: StandaloneTastyFileEntry, _: ClassFileEntry) =>
+            // Here we do not create a TastyWithClassFileEntry because the TASTy and the classfile
+            // come from different classpaths. These may not have the same TASTy UUID.
+            mergedEntries(index) = entry
+          case (entry: StandaloneTastyFileEntry, BinaryAndSourceFilesEntry(_: ClassFileEntry, sourceEntry)) =>
+            mergedEntries(index) = BinaryAndSourceFilesEntry(entry, sourceEntry)
           case _ =>
       }
       else {

compiler/src/dotty/tools/dotc/classpath/ClassPath.scala

@@ -52,7 +52,8 @@ sealed trait BinaryFileEntry extends ClassRepresentation {
 object BinaryFileEntry {
   def apply(file: AbstractFile): BinaryFileEntry =
     if file.isTasty then
-      TastyWithClassFileEntry(file)
+      if file.resolveSiblingWithExtension("class") != null then TastyWithClassFileEntry(file)
+      else StandaloneTastyFileEntry(file)
     else
       ClassFileEntry(file)
 }
@@ -67,6 +68,11 @@ private[dotty] final case class TastyWithClassFileEntry(file: AbstractFile) exte
   def binary: Option[AbstractFile] = Some(file)
 }
 
+/** A TASTy file that does not have an associated class file */
+private[dotty] final case class StandaloneTastyFileEntry(file: AbstractFile) extends BinaryFileEntry {
+  def binary: Option[AbstractFile] = Some(file)
+}
+
 private[dotty] final case class SourceFileEntry(file: AbstractFile) extends ClassRepresentation {
   final def fileName: String = file.name
   def name: String = FileUtils.stripSourceExtension(file.name)

compiler/src/dotty/tools/io/AbstractFile.scala

@@ -258,6 +258,9 @@ abstract class AbstractFile extends Iterable[AbstractFile] {
   final def resolveSibling(name: String): AbstractFile | Null =
     container.lookupName(name, directory = false)
 
+  final def resolveSiblingWithExtension(extension: String): AbstractFile | Null =
+    resolveSibling(name.stripSuffix(this.extension) + extension)
+
   private def fileOrSubdirectoryNamed(name: String, isDir: Boolean): AbstractFile =
     lookupName(name, isDir) match {
       case null =>

project/Build.scala

@@ -785,24 +785,21 @@ object Build {
           else if (debugFromTasty) "dotty.tools.dotc.fromtasty.Debug"
           else "dotty.tools.dotc.Main"
 
-        val scala2LibraryTasty = scala2Library.value match {
-          case Scala2LibraryJar => Seq.empty
+        var extraClasspath = Seq(scalaLib, dottyLib)
+
+        scala2Library.value match {
+          case Scala2LibraryJar =>
           case Scala2LibraryTasty =>
             jars.get("scala2-library-tasty") match {
-              case Some(jar) => Seq(jar)
-              case None =>
-                log.warn("Scala2LibraryTasty is ignored on non-bootstrapped compiler")
-                Seq.empty
-            }
+              case Some(jar) => extraClasspath :+= jar
+              case None => log.warn("Scala2LibraryTasty is ignored on non-bootstrapped compiler")
+            };
           case Scala2LibraryCCTasty =>
             jars.get("scala2-library-cc-tasty") match {
-              case Some(jar) => Seq(jar)
-              case None =>
-                log.warn("Scala2LibraryCCTasty is ignored on non-bootstrapped compiler")
-                Seq.empty
+              case Some(jar) => extraClasspath :+= jar
+              case None => log.warn("Scala2LibraryCCTasty is ignored on non-bootstrapped compiler")
             }
         }
-        var extraClasspath = scala2LibraryTasty ++ Seq(scalaLib, dottyLib)
 
         if (decompile && !args.contains("-classpath"))
           extraClasspath ++= Seq(".")
@@ -1270,13 +1267,6 @@ object Build {
       Compile / compile / fullClasspath ~= {
         _.filterNot(file => file.data.getName == s"scala-library-$stdlibBootstrappedVersion.jar")
       },
-      Compile / compile / dependencyClasspath := {
-        // make sure that the scala2-library (tasty of `scala2-library-tasty`) is listed before the scala-library (classfiles)
-        val (bootstrappedLib, otherLibs) =
-          (Compile / compile / dependencyClasspath).value
-            .partition(_.data.getName == s"scala2-library-${dottyVersion}.jar")
-        bootstrappedLib ++ otherLibs
-      },
     )
 
   lazy val `scala3-sbt-bridge` = project.in(file("sbt-bridge/src")).
@@ -1923,6 +1913,7 @@ object Build {
         (`scala3-interfaces` / publishLocalBin),
         (`scala3-compiler-bootstrapped` / publishLocalBin),
         (`scala3-library-bootstrapped` / publishLocalBin),
+        (`scala2-library-tasty` / publishLocal),
         (`scala3-library-bootstrappedJS` / publishLocalBin),
         (`tasty-core-bootstrapped` / publishLocalBin),
         (`scala3-staging` / publishLocalBin),

sbt-test/sbt-dotty/scala2-library-tasty/build.sbt

@@ -0,0 +1,4 @@
+scalaVersion := sys.props("plugin.scalaVersion")
+
+libraryDependencies += "org.scala-lang" %% "scala2-library-tasty" % scalaVersion.value
+scalacOptions += "-Yscala2-unpickler:never" // check that we do not load symbol from the Scala 2 library classfiles (use TASTy)

sbt-test/sbt-dotty/scala2-library-tasty/src/main/scala/hello/Hello.scala

@@ -0,0 +1,4 @@
+package hello
+
+@main def hello: Unit =
+  println(Some("Hello world!")) // load Some form the Scala 2 library TASTy

sbt-test/sbt-dotty/scala2-library-tasty/test

@@ -0,0 +1 @@
+> run