Support links in using file directive (implements #1328)

modules/build/src/main/scala/scala/build/Build.scala

@@ -267,7 +267,8 @@ object Build {
       ),
       logger,
       options.suppressWarningOptions,
-      options.internal.exclude
+      options.internal.exclude,
+      download = options.downloader
     )
 
   private def build(

modules/build/src/main/scala/scala/build/CrossSources.scala

@@ -1,6 +1,7 @@
 package scala.build
 
-import java.io.File
+import coursier.cache.FileCache
+import coursier.util.{Artifact, Task}
 
 import scala.build.CollectionOps.*
 import scala.build.EitherCps.{either, value}
@@ -11,7 +12,8 @@ import scala.build.errors.{
   CompositeBuildException,
   ExcludeDefinitionError,
   MalformedDirectiveError,
-  Severity
+  Severity,
+  UsingFileFromUriError
 }
 import scala.build.input.ElementsUtils.*
 import scala.build.input.*
@@ -161,7 +163,8 @@ object CrossSources {
     logger: Logger,
     suppressWarningOptions: SuppressWarningOptions,
     exclude: Seq[Positioned[String]] = Nil,
-    maybeRecoverOnError: BuildException => Option[BuildException] = e => Some(e)
+    maybeRecoverOnError: BuildException => Option[BuildException] = e => Some(e),
+    download: BuildOptions.Download = BuildOptions.Download.notSupported
   )(using ScalaCliInvokeData): Either[BuildException, (CrossSources, Inputs)] = either {
 
     def preprocessSources(elems: Seq[SingleElement])
@@ -209,8 +212,9 @@ object CrossSources {
         .flatMap(_.options)
         .flatMap(_.internal.extraSourceFiles)
         .distinct
-    val inputsElemFromDirectives: Seq[SingleFile] =
-      value(resolveInputsFromSources(sourcesFromDirectives, inputs.enableMarkdown))
+
+    val inputsElemFromDirectives: Seq[SingleElement] =
+      value(resolveInputsFromSources(sourcesFromDirectives, inputs.enableMarkdown, download))
     val preprocessedSourcesFromDirectives: Seq[PreprocessedSource] =
       value(preprocessSources(inputsElemFromDirectives.pipe(elements =>
         value(excludeSources(elements, inputs.workspace, allExclude))
@@ -403,7 +407,32 @@ object CrossSources {
     fromInputs ++ fromSources ++ fromSourcesWithRequirements
   }
 
-  private def resolveInputsFromSources(sources: Seq[Positioned[os.Path]], enableMarkdown: Boolean) =
+  private def downloadFile(download: BuildOptions.Download)(pUri: Positioned[java.net.URI]) =
+    download(pUri.value.toString).left.map(
+      new UsingFileFromUriError(pUri.value, pUri.positions, _)
+    ).map(content =>
+      Seq(Virtual(pUri.value.toString, content))
+    )
+
+  type CodeFile = os.Path | java.net.URI
+
+  private def resolveInputsFromSources(
+    sources: Seq[Positioned[CodeFile]],
+    enableMarkdown: Boolean,
+    download: BuildOptions.Download
+  ) =
+    val links = sources.collect {
+      case Positioned(pos, value: java.net.URI) => Positioned(pos, value)
+    }
+    val paths = sources.collect {
+      case Positioned(pos, value: os.Path) => Positioned(pos, value)
+    }
+
+    (resolveInputsFromPath(paths, enableMarkdown) ++ links.map(downloadFile(download))).sequence
+      .left.map(CompositeBuildException(_))
+      .map(_.flatten)
+
+  private def resolveInputsFromPath(sources: Seq[Positioned[os.Path]], enableMarkdown: Boolean) =
     sources.map { source =>
       val sourcePath   = source.value
       lazy val dir     = sourcePath / os.up
@@ -424,9 +453,7 @@ object CrossSources {
           else s"$sourcePath: not found path defined in using directive."
         Left(new MalformedDirectiveError(msg, source.positions))
       }
-    }.sequence
-      .left.map(CompositeBuildException(_))
-      .map(_.flatten)
+    }
 
   /** Filters out the sources from the input sequence based on the provided 'exclude' patterns. The
     * exclude patterns can be absolute paths, relative paths, or glob patterns.

modules/build/src/main/scala/scala/build/bsp/BspImpl.scala

@@ -113,7 +113,8 @@ final class BspImpl(
         logger = persistentLogger,
         suppressWarningOptions = buildOptions.suppressWarningOptions,
         exclude = buildOptions.internal.exclude,
-        maybeRecoverOnError = maybeRecoverOnError(Scope.Main)
+        maybeRecoverOnError = maybeRecoverOnError(Scope.Main),
+        download = buildOptions.downloader
       ).left.map((_, Scope.Main))
     }
 

modules/build/src/main/scala/scala/build/input/Inputs.scala

@@ -11,7 +11,7 @@ import scala.build.errors.{BuildException, InputsException, WorkspaceError}
 import scala.build.input.ElementsUtils.*
 import scala.build.internal.Constants
 import scala.build.internal.zip.WrappedZipInputStream
-import scala.build.options.Scope
+import scala.build.options.{BuildOptions, Scope}
 import scala.build.preprocessing.SheBang.isShebangScript
 import scala.util.matching.Regex
 import scala.util.{Properties, Try}
@@ -225,7 +225,7 @@ object Inputs {
   def validateArgs(
     args: Seq[String],
     cwd: os.Path,
-    download: String => Either[String, Array[Byte]],
+    download: BuildOptions.Download,
     stdinOpt: => Option[Array[Byte]],
     acceptFds: Boolean,
     enableMarkdown: Boolean
@@ -423,7 +423,7 @@ object Inputs {
     args: Seq[String],
     cwd: os.Path,
     defaultInputs: () => Option[Inputs] = () => None,
-    download: String => Either[String, Array[Byte]] = _ => Left("URL not supported"),
+    download: BuildOptions.Download = BuildOptions.Download.notSupported,
     stdinOpt: => Option[Array[Byte]] = None,
     scriptSnippetList: List[String] = List.empty,
     scalaSnippetList: List[String] = List.empty,

modules/cli/src/main/scala/scala/cli/commands/bsp/Bsp.scala

@@ -109,7 +109,8 @@ object Bsp extends ScalaCommand[BspOptions] {
             ),
             persistentLogger,
             baseOptions.suppressWarningOptions,
-            baseOptions.internal.exclude
+            baseOptions.internal.exclude,
+            download = baseOptions.downloader
           )
 
           val (allInputs, finalBuildOptions) = {

modules/cli/src/main/scala/scala/cli/commands/dependencyupdate/DependencyUpdate.scala

@@ -45,7 +45,8 @@ object DependencyUpdate extends ScalaCommand[DependencyUpdateOptions] {
         ),
         logger,
         buildOptions.suppressWarningOptions,
-        buildOptions.internal.exclude
+        buildOptions.internal.exclude,
+        download = buildOptions.downloader
       ).orExit(logger)
 
     val sharedOptions = crossSources.sharedOptions(buildOptions)

modules/cli/src/main/scala/scala/cli/commands/export0/Export.scala

@@ -50,7 +50,8 @@ object Export extends ScalaCommand[ExportOptions] {
         ),
         logger,
         buildOptions.suppressWarningOptions,
-        buildOptions.internal.exclude
+        buildOptions.internal.exclude,
+        download = buildOptions.downloader
       )
     }
 

modules/cli/src/main/scala/scala/cli/commands/fix/BuiltInRules.scala

@@ -176,7 +176,8 @@ object BuiltInRules extends CommandHelpers {
       ),
       logger = logger,
       suppressWarningOptions = SuppressWarningOptions.suppressAll,
-      exclude = buildOptions.internal.exclude
+      exclude = buildOptions.internal.exclude,
+      download = buildOptions.downloader
     ).orExit(logger)
 
     val sharedOptions = crossSources.sharedOptions(buildOptions)

modules/cli/src/main/scala/scala/cli/commands/publish/PublishSetup.scala

@@ -89,7 +89,8 @@ object PublishSetup extends ScalaCommand[PublishSetupOptions] {
         ),
         logger,
         cliBuildOptions.suppressWarningOptions,
-        cliBuildOptions.internal.exclude
+        cliBuildOptions.internal.exclude,
+        download = cliBuildOptions.downloader
       ).orExit(logger)
 
       val crossSourcesSharedOptions = crossSources.sharedOptions(cliBuildOptions)

modules/cli/src/main/scala/scala/cli/commands/setupide/SetupIde.scala

@@ -43,7 +43,8 @@ object SetupIde extends ScalaCommand[SetupIdeOptions] {
           ),
           logger,
           options.suppressWarningOptions,
-          options.internal.exclude
+          options.internal.exclude,
+          download = options.downloader
         )
       }
 

modules/cli/src/main/scala/scala/cli/commands/shared/SharedOptions.scala

@@ -29,7 +29,6 @@ import scala.build.interactive.Interactive.{InteractiveAsk, InteractiveNop}
 import scala.build.internal.util.WarningMessages
 import scala.build.internal.{Constants, FetchExternalBinary, OsLibc, Util}
 import scala.build.internals.ConsoleUtils.ScalaCliConsole
-import scala.build.options.ScalaVersionUtil.fileWithTtl0
 import scala.build.options.{BuildOptions, ComputeVersion, Platform, ScalacOpt, ShadowingSeq}
 import scala.build.preprocessing.directives.ClasspathUtils.*
 import scala.build.preprocessing.directives.Toolkit.maxScalaNativeWarningMsg
@@ -658,7 +657,7 @@ final case class SharedOptions(
     Inputs.validateArgs(
       args,
       Os.pwd,
-      SharedOptions.downloadInputs(coursierCache),
+      BuildOptions.Download.changing(coursierCache),
       SharedOptions.readStdin(logger = logger),
       !Properties.isWin,
       enableMarkdown = true
@@ -675,15 +674,6 @@ object SharedOptions {
   implicit lazy val help: Help[SharedOptions]                = Help.derive
   implicit lazy val jsonCodec: JsonValueCodec[SharedOptions] = JsonCodecMaker.make
 
-  private def downloadInputs(cache: FileCache[Task]): String => Either[String, Array[Byte]] = {
-    url =>
-      val artifact = Artifact(url).withChanging(true)
-      cache.fileWithTtl0(artifact)
-        .left
-        .map(_.describe)
-        .map(f => os.read.bytes(os.Path(f, Os.pwd)))
-  }
-
   /** [[Inputs]] builder, handy when you don't have a [[SharedOptions]] instance at hand */
   def inputs(
     args: Seq[String],
@@ -715,7 +705,7 @@ object SharedOptions {
       args,
       Os.pwd,
       defaultInputs = defaultInputs,
-      download = downloadInputs(cache),
+      download = BuildOptions.Download.changing(cache),
       stdinOpt = readStdin(logger = logger),
       scriptSnippetList = scriptSnippetList,
       scalaSnippetList = scalaSnippetList,

modules/directives/src/main/scala/scala/build/errors/UsingFileFromUriError.scala

@@ -0,0 +1,11 @@
+package scala.build.errors
+
+import java.net.URI
+
+import scala.build.Position
+
+final class UsingFileFromUriError(uri: URI, positions: Seq[Position], description: String)
+    extends BuildException(
+      message = s"Error using file from $uri - $description",
+      positions = positions
+    )

modules/directives/src/main/scala/scala/build/preprocessing/directives/Sources.scala

@@ -11,6 +11,9 @@ import scala.util.Try
 
 @DirectiveGroupName("Custom sources")
 @DirectiveExamples("//> using file utils.scala")
+@DirectiveExamples(
+  "//> using file https://raw.githubusercontent.com/softwaremill/sttp/refs/heads/master/examples/src/main/scala/sttp/client4/examples/json/GetAndParseJsonCatsEffectCirce.scala"
+)
 @DirectiveUsage(
   "`//> using file `_path_ | `//> using files `_path1_ _path2_ …",
   """`//> using file` _path_
@@ -27,6 +30,17 @@ final case class Sources(
   files: DirectiveValueParser.WithScopePath[List[Positioned[String]]] =
     DirectiveValueParser.WithScopePath.empty(Nil)
 ) extends HasBuildOptions {
+
+  private def codeFile(codeFile: String, root: os.Path): Sources.CodeFile =
+    scala.util.Try {
+      val uri = java.net.URI.create(codeFile)
+      uri.getScheme match {
+        case "file" | "http" | "https" => uri
+      }
+    }.getOrElse {
+      os.Path(codeFile, root)
+    }
+
   def buildOptions: Either[BuildException, BuildOptions] = either {
 
     val paths = files
@@ -35,7 +49,7 @@ final case class Sources(
         for {
           root <- Directive.osRoot(files.scopePath, positioned.positions.headOption)
           path <- {
-            try Right(positioned.map(os.Path(_, root)))
+            try Right(positioned.map(codeFile(_, root)))
             catch {
               case e: IllegalArgumentException =>
                 Left(new WrongSourcePathError(positioned.value, e, positioned.positions))
@@ -55,5 +69,8 @@ final case class Sources(
 }
 
 object Sources {
+
+  type CodeFile = os.Path | java.net.URI
+
   val handler: DirectiveHandler[Sources] = DirectiveHandler.derive
 }

modules/options/src/main/scala/scala/build/options/BuildOptions.scala

@@ -583,6 +583,8 @@ final case class BuildOptions(
     }
   }
 
+  lazy val downloader: BuildOptions.Download = BuildOptions.Download(finalCache)
+
   lazy val interactive: Either[BuildException, Interactive] =
     internal.interactive.map(_()).getOrElse(Right(InteractiveNop))
 }
@@ -632,6 +634,23 @@ object BuildOptions {
     }
   }
 
+  type Download = String => Either[String, Array[Byte]]
+  object Download {
+    def apply(
+      cache: FileCache[Task],
+      toArtifact: String => Artifact = Artifact.fromUrl
+    ): Download = {
+      import scala.build.options.ScalaVersionUtil.fileWithTtl0
+      url =>
+        cache.fileWithTtl0(toArtifact(url))
+          .left
+          .map(_.describe)
+          .map(f => os.read.bytes(os.Path(f, Os.pwd)))
+    }
+    def changing(cache: FileCache[Task]): Download = apply(cache, Artifact(_).withChanging(true))
+    val notSupported: Download                     = _ => Left("URL not supported")
+  }
+
   implicit val hasHashData: HasHashData[BuildOptions] = HasHashData.derive
   implicit val monoid: ConfigMonoid[BuildOptions]     = ConfigMonoid.derive
 }

modules/options/src/main/scala/scala/build/options/InternalOptions.scala

@@ -8,6 +8,8 @@ import scala.build.errors.BuildException
 import scala.build.interactive.Interactive
 import scala.build.interactive.Interactive.InteractiveNop
 
+type CodeFile = os.Path | java.net.URI
+
 final case class InternalOptions(
   keepDiagnostics: Boolean = false,
   cache: Option[FileCache[Task]] = None,
@@ -24,7 +26,7 @@ final case class InternalOptions(
     * really needed.
     */
   keepResolution: Boolean = false,
-  extraSourceFiles: Seq[Positioned[os.Path]] = Nil,
+  extraSourceFiles: Seq[Positioned[CodeFile]] = Nil,
   exclude: Seq[Positioned[String]] = Nil,
   offline: Option[Boolean] = None
 ) {

website/docs/reference/directives.md

@@ -126,6 +126,8 @@ Manually add sources to the project. Does not support chaining, sources are adde
 #### Examples
 `//> using file utils.scala`
 
+`//> using file https://raw.githubusercontent.com/softwaremill/sttp/refs/heads/master/examples/src/main/scala/sttp/client4/examples/json/GetAndParseJsonCatsEffectCirce.scala`
+
 ### Dependency
 
 Add dependencies

website/docs/reference/scala-command/directives.md

@@ -197,6 +197,8 @@ Manually add sources to the project. Does not support chaining, sources are adde
 #### Examples
 `//> using file utils.scala`
 
+`//> using file https://raw.githubusercontent.com/softwaremill/sttp/refs/heads/master/examples/src/main/scala/sttp/client4/examples/json/GetAndParseJsonCatsEffectCirce.scala`
+
 ### Exclude sources
 
 Exclude sources from the project