Merge pull request #3237 from scala-steward-org/topic/multiple-non-local-repos-files

Allow multiple and non-local repos files

Author: fthomas

docs/help.md

@@ -5,7 +5,7 @@ All command line arguments for the `scala-steward` application.
 ```
 Usage:
     scala-steward validate-repo-config
-    scala-steward --workspace <file> --repos-file <file> [--git-author-name <string>] --git-author-email <string> [--git-author-signing-key <string>] --git-ask-pass <file> [--sign-commits] [--forge-type <forge-type>] [--forge-api-host <uri>] --forge-login <string> [--do-not-fork] [--add-labels] [--ignore-opts-files] [--env-var <name=value>]... [--process-timeout <duration>] [--whitelist <string>]... [--read-only <string>]... [--enable-sandbox | --disable-sandbox] [--max-buffer-size <integer>] [--repo-config <uri>]... [--disable-default-repo-config] [--scalafix-migrations <uri>]... [--disable-default-scalafix-migrations] [--artifact-migrations <uri>]... [--disable-default-artifact-migrations] [--cache-ttl <duration>] [--bitbucket-use-default-reviewers] [--bitbucket-server-use-default-reviewers] [--gitlab-merge-when-pipeline-succeeds] [--gitlab-required-reviewers <integer>] [--gitlab-remove-source-branch] [--azure-repos-organization <string>] [--github-app-id <integer> --github-app-key-file <file>] [--url-checker-test-url <uri>]... [--default-maven-repo <string>] [--refresh-backoff-period <duration>]
+    scala-steward --workspace <file> --repos-file <uri> [--repos-file <uri>]... [--git-author-name <string>] --git-author-email <string> [--git-author-signing-key <string>] --git-ask-pass <file> [--sign-commits] [--forge-type <forge-type>] [--forge-api-host <uri>] --forge-login <string> [--do-not-fork] [--add-labels] [--ignore-opts-files] [--env-var <name=value>]... [--process-timeout <duration>] [--whitelist <string>]... [--read-only <string>]... [--enable-sandbox | --disable-sandbox] [--max-buffer-size <integer>] [--repo-config <uri>]... [--disable-default-repo-config] [--scalafix-migrations <uri>]... [--disable-default-scalafix-migrations] [--artifact-migrations <uri>]... [--disable-default-artifact-migrations] [--cache-ttl <duration>] [--bitbucket-use-default-reviewers] [--bitbucket-server-use-default-reviewers] [--gitlab-merge-when-pipeline-succeeds] [--gitlab-required-reviewers <integer>] [--gitlab-remove-source-branch] [--azure-repos-organization <string>] [--github-app-id <integer> --github-app-key-file <file>] [--url-checker-test-url <uri>]... [--default-maven-repo <string>] [--refresh-backoff-period <duration>]
 
 
 
@@ -14,8 +14,8 @@ Options and flags:
         Display this help text.
     --workspace <file>
         Location for cache and temporary files
-    --repos-file <file>
-        A markdown formatted file with a repository list
+    --repos-file <uri>
+        A markdown formatted file with a repository list (can be used multiple times)
     --git-author-name <string>
         Git "user.name"; default: Scala Steward
     --git-author-email <string>

modules/core/src/main/scala/org/scalasteward/core/application/Cli.scala

@@ -75,8 +75,8 @@ object Cli {
   private val workspace: Opts[File] =
     option[File]("workspace", "Location for cache and temporary files")
 
-  private val reposFile: Opts[File] =
-    option[File]("repos-file", "A markdown formatted file with a repository list")
+  private val reposFiles: Opts[Nel[Uri]] =
+    options[Uri]("repos-file", s"A markdown formatted file with a repository list $multiple")
 
   private val gitAuthorName: Opts[String] = {
     val default = "Scala Steward"
@@ -348,7 +348,7 @@ object Cli {
 
   private val configOpts: Opts[Config] = (
     workspace,
-    reposFile,
+    reposFiles,
     gitCfg,
     forgeCfg,
     ignoreOptsFiles,

modules/core/src/main/scala/org/scalasteward/core/application/Config.scala

@@ -53,7 +53,7 @@ import scala.concurrent.duration.FiniteDuration
   */
 final case class Config(
     workspace: File,
-    reposFile: File,
+    reposFiles: Nel[Uri],
     gitCfg: GitCfg,
     forgeCfg: ForgeCfg,
     ignoreOptsFiles: Boolean,

modules/core/src/main/scala/org/scalasteward/core/application/Context.scala

@@ -75,6 +75,7 @@ final class Context[F[_]](implicit
     val refreshErrorAlg: RefreshErrorAlg[F],
     val repoCacheAlg: RepoCacheAlg[F],
     val repoConfigAlg: RepoConfigAlg[F],
+    val reposFilesLoader: ReposFilesLoader[F],
     val sbtAlg: SbtAlg[F],
     val scalaCliAlg: ScalaCliAlg[F],
     val scalafixMigrationsFinder: ScalafixMigrationsFinder,
@@ -234,6 +235,7 @@ object Context {
       implicit val editAlg: EditAlg[F] = new EditAlg[F]
       implicit val nurtureAlg: NurtureAlg[F] = new NurtureAlg[F](config.forgeCfg)
       implicit val pruningAlg: PruningAlg[F] = new PruningAlg[F]
+      implicit val reposFilesLoader: ReposFilesLoader[F] = new ReposFilesLoader[F]()
       implicit val gitHubAppApiAlg: GitHubAppApiAlg[F] =
         new GitHubAppApiAlg[F](config.forgeCfg.apiHost)
       implicit val stewardAlg: StewardAlg[F] = new StewardAlg[F](config)

modules/core/src/main/scala/org/scalasteward/core/application/ReposFilesLoader.scala

@@ -0,0 +1,50 @@
+/*
+ * Copyright 2018-2023 Scala Steward contributors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.scalasteward.core.application
+
+import cats.effect.Sync
+import cats.syntax.all._
+import fs2.Stream
+import org.http4s.Uri
+import org.scalasteward.core.data.Repo
+import org.scalasteward.core.io.FileAlg
+import org.scalasteward.core.util.Nel
+import org.typelevel.log4cats.Logger
+
+final class ReposFilesLoader[F[_]](implicit
+    fileAlg: FileAlg[F],
+    logger: Logger[F],
+    F: Sync[F]
+) {
+  def loadAll(reposFiles: Nel[Uri]): Stream[F, Repo] =
+    Stream.emits(reposFiles.toList).evalMap(loadRepos).flatMap(Stream.emits)
+
+  private def loadRepos(reposFile: Uri): F[List[Repo]] =
+    for {
+      _ <- logger.debug(s"Loading repos from $reposFile")
+      content <- fileAlg.readUri(reposFile)
+      repos <- Stream.fromIterator(content.linesIterator, 4096).mapFilter(Repo.parse).compile.toList
+      _ <-
+        if (repos.nonEmpty) F.unit
+        else {
+          val msg = s"No repos found in ${reposFile.renderString}. " +
+            s"Check the formatting of that file. " +
+            s"""The format is "- $$owner/$$repo" or "- $$owner/$$repo:$$branch"."""
+          logger.warn(msg)
+        }
+    } yield repos
+}

modules/core/src/main/scala/org/scalasteward/core/application/StewardAlg.scala

@@ -16,7 +16,6 @@
 
 package org.scalasteward.core.application
 
-import better.files.File
 import cats.effect.{ExitCode, Sync}
 import cats.syntax.all._
 import fs2.Stream
@@ -43,16 +42,11 @@ final class StewardAlg[F[_]](config: Config)(implicit
     nurtureAlg: NurtureAlg[F],
     pruningAlg: PruningAlg[F],
     repoCacheAlg: RepoCacheAlg[F],
+    reposFilesLoader: ReposFilesLoader[F],
     selfCheckAlg: SelfCheckAlg[F],
     workspaceAlg: WorkspaceAlg[F],
     F: Sync[F]
 ) {
-  private def readRepos(reposFile: File): Stream[F, Repo] =
-    Stream
-      .eval(fileAlg.readFile(reposFile).map(_.getOrElse("")))
-      .flatMap(content => Stream.fromIterator(content.linesIterator, 1024))
-      .mapFilter(Repo.parse)
-
   private def getGitHubAppRepos(githubApp: GitHubApp): Stream[F, Repo] =
     Stream.evals[F, List, Repo] {
       for {
@@ -95,22 +89,16 @@ final class StewardAlg[F[_]](config: Config)(implicit
         _ <- workspaceAlg.removeAnyRunSpecificFiles
         exitCode <-
           (config.githubApp.map(getGitHubAppRepos).getOrElse(Stream.empty) ++
-            readRepos(config.reposFile))
+            reposFilesLoader.loadAll(config.reposFiles))
             .evalMap(repo => steward(repo).map(_.bimap(repo -> _, _ => repo)))
             .compile
             .toList
-            .flatMap {
-              case Nil =>
-                val msg = "No repos specified. " +
-                  s"Check the formatting of ${config.reposFile.pathAsString}. " +
-                  s"""The format is "- $$owner/$$repo" or "- $$owner/$$repo:$$branch"."""
-                logger.warn(msg).as(ExitCode.Success)
-              case results =>
-                val runResults = RunResults(results)
-                for {
-                  summaryFile <- workspaceAlg.runSummaryFile
-                  _ <- fileAlg.writeFile(summaryFile, runResults.markdownSummary)
-                } yield runResults.exitCode
+            .flatMap { results =>
+              val runResults = RunResults(results)
+              for {
+                summaryFile <- workspaceAlg.runSummaryFile
+                _ <- fileAlg.writeFile(summaryFile, runResults.markdownSummary)
+              } yield runResults.exitCode
             }
       } yield exitCode
     }

modules/core/src/test/scala/org/scalasteward/core/application/CliTest.scala

@@ -9,6 +9,7 @@ import org.scalasteward.core.application.Cli.ParseResult._
 import org.scalasteward.core.application.Config.StewardUsage
 import org.scalasteward.core.forge.ForgeType
 import org.scalasteward.core.forge.github.GitHubApp
+import org.scalasteward.core.util.Nel
 import scala.concurrent.duration._
 
 class CliTest extends FunSuite {
@@ -38,7 +39,7 @@ class CliTest extends FunSuite {
     )
 
     assertEquals(obtained.workspace, File("a"))
-    assertEquals(obtained.reposFile, File("b"))
+    assertEquals(obtained.reposFiles, Nel.one(uri"b"))
     assertEquals(obtained.gitCfg.gitAuthor.email, "d")
     assertEquals(obtained.gitCfg.gitAskPass, File("f"))
     assertEquals(obtained.forgeCfg.tpe, ForgeType.GitLab)
@@ -84,7 +85,7 @@ class CliTest extends FunSuite {
 
     assert(!obtained.processCfg.sandboxCfg.enableSandbox)
     assertEquals(obtained.workspace, File("a"))
-    assertEquals(obtained.reposFile, File("b"))
+    assertEquals(obtained.reposFiles, Nel.one(uri"b"))
     assertEquals(obtained.gitCfg.gitAuthor.email, "d")
     assertEquals(obtained.gitCfg.gitAskPass, File("f"))
     assertEquals(obtained.forgeCfg.login, "e")

modules/core/src/test/scala/org/scalasteward/core/application/ReposFilesLoaderTest.scala

@@ -0,0 +1,31 @@
+package org.scalasteward.core.application
+
+import munit.CatsEffectSuite
+import org.scalasteward.core.data.Repo
+import org.scalasteward.core.git.Branch
+import org.scalasteward.core.mock.MockContext.context.reposFilesLoader
+import org.scalasteward.core.mock.{MockConfig, MockState}
+import org.scalasteward.core.util.Nel
+
+class ReposFilesLoaderTest extends CatsEffectSuite {
+  test("non-empty repos file") {
+    val initialState = MockState.empty.addUris(MockConfig.reposFile -> "- a/b\n- c/d:e")
+    val obtained =
+      reposFilesLoader.loadAll(Nel.one(MockConfig.reposFile)).compile.toList.runA(initialState)
+    assertIO(obtained, List(Repo("a", "b"), Repo("c", "d", Some(Branch("e")))))
+  }
+
+  test("malformed repos file") {
+    val initialState = MockState.empty.addUris(MockConfig.reposFile -> " - a/b")
+    val obtained =
+      reposFilesLoader.loadAll(Nel.one(MockConfig.reposFile)).compile.toList.runA(initialState)
+    assertIO(obtained, List.empty)
+  }
+
+  test("non-existing repos file") {
+    val initialState = MockState.empty
+    val obtained =
+      reposFilesLoader.loadAll(Nel.one(MockConfig.reposFile)).compile.toList.runA(initialState)
+    assertIO(obtained.attempt.map(_.isLeft), true)
+  }
+}

modules/core/src/test/scala/org/scalasteward/core/application/StewardAlgTest.scala

@@ -3,11 +3,11 @@ package org.scalasteward.core.application
 import cats.effect.ExitCode
 import munit.CatsEffectSuite
 import org.scalasteward.core.mock.MockContext.context.stewardAlg
-import org.scalasteward.core.mock.MockState
+import org.scalasteward.core.mock.{MockConfig, MockState}
 
 class StewardAlgTest extends CatsEffectSuite {
   test("runF") {
-    val exitCode = stewardAlg.runF.runA(MockState.empty)
+    val exitCode = stewardAlg.runF.runA(MockState.empty.addUris(MockConfig.reposFile -> ""))
     assertIO(exitCode, ExitCode.Success)
   }
 }

modules/core/src/test/scala/org/scalasteward/core/mock/MockConfig.scala

@@ -1,15 +1,17 @@
 package org.scalasteward.core.mock
 
 import better.files.File
+import org.http4s.Uri
 import org.scalasteward.core.application.Cli.ParseResult.Success
 import org.scalasteward.core.application.{Cli, Config}
 
 object MockConfig {
   val mockRoot: File = File.temp / "scala-steward"
+  val reposFile: Uri = Uri.unsafeFromString((mockRoot / "repos.md").pathAsString)
   mockRoot.delete(true) // Ensure folder is cleared of previous test files
   private val args: List[String] = List(
     s"--workspace=$mockRoot/workspace",
-    s"--repos-file=$mockRoot/repos.md",
+    s"--repos-file=$reposFile",
     "--git-author-name=Bot Doe",
     "--git-author-email=bot@example.org",
     s"--git-ask-pass=$mockRoot/askpass.sh",