Unify generation of Scala 3 API and Reference documentations (#23606)

This PR does unify how snapshot and stable scaladoc artifacts are
published:

- `docs/sidebar.yml` now contains only entries visible for stable
releases
- `docs/sidebar.nightly.template.yml` contains entries specific for
Snapshot builds (contributing guide, blog posted at
https://dotty.epfl.ch/) - it does not contain `reference` subsection
entries - this part is now generated when executing
`generateScalaDocumentation` task. It emits a patch to
`docs/sidebar.yml` containg combined inputs from both files
-
`project/resources/referenceReplacements/_layouts/static-site-main.html`
was removed, instead we now inject the warning only when building
snapshot docs using `generateScalaDocumentation`
- `project/resources/referenceReplacements/sidebar.yml` is removed, we
would no longer need to keep 2 files in sync
- `generateStableScala3Documentation` no longer uses calculated paths to
obtain compilation outputs - it was only needed in the past to allow for
generation of new scaladoc for older compiler/library

Author: WojciechMazur

docs/_layouts/static-site-main.html

@@ -4,17 +4,7 @@
 
 <div class="site-container">
   <div id="site-header"></div>
-  {% if page.nightlyOf %}
-  <aside class="warning">
-    <div class='icon'></div>
-    <div class='content'>
-    This is a nightly documentation. The content of this page may not be
-    consistent with the current stable version of language. Click
-    <a href="{{ page.nightlyOf }}">here</a> to find the stable version of this
-    page.
-    </div>
-  </aside>
-  {% endif %} {{ content }}
+  {{ content }}
   <div class="divider" />
   <nav class="arrow-navigation" aria-label="Page navigation">
     {% if page.previous %}

docs/sidebar.nightly.template.yml

@@ -0,0 +1,63 @@
+index: index.md
+subsection:
+  - title: Reference
+    directory: docs/reference
+    # Embed sidebar.reference.yml, performed by `generateScalaDocumentation` sbt task
+  - title: Contributing
+    directory: docs/contributing
+    index: contributing/index.md
+    subsection:
+      - page: contributing/getting-started.md
+      - page: contributing/setting-up-your-ide.md
+      - index: contributing/issues/diagnosing-your-issue.md
+        subsection:
+          - page: contributing/issues/reproduce.md
+          - page: contributing/issues/cause.md
+          - page: contributing/issues/areas.md
+      - index: contributing/debugging/debugging.md
+        subsection:
+          - page: contributing/debugging/ide-debugging.md
+          - page: contributing/debugging/inspection.md
+          - page: contributing/debugging/other-debugging.md
+      - page: contributing/testing.md
+      - page: contributing/scaladoc.md
+      - page: contributing/community-build.md
+      - page: contributing/sending-in-a-pr.md
+      - page: contributing/cheatsheet.md
+      - title: Procedures
+        directory: procedures
+        index: contributing/procedures/index.md
+        subsection:
+          - page: contributing/procedures/release.md
+          - page: contributing/procedures/vulpix.md
+      - title: High Level Architecture
+        directory: architecture
+        index: contributing/architecture/index.md
+        subsection:
+          - page: contributing/architecture/lifecycle.md
+          - page: contributing/architecture/context.md
+          - page: contributing/architecture/phases.md
+          - page: contributing/architecture/types.md
+          - page: contributing/architecture/time.md
+          - page: contributing/architecture/symbols.md
+  - title: Internals
+    directory: docs/internals
+    index: internals/index.md
+    subsection:
+      - page: internals/backend.md
+      - page: internals/classpaths.md
+      - page: internals/contexts.md
+      - page: internals/dotc-scalac.md
+      - page: internals/higher-kinded-v2.md
+      - page: internals/overall-structure.md
+      - page: internals/explicit-nulls.md
+      - page: internals/periods.md
+      - page: internals/syntax.md
+      - page: internals/type-system.md
+      - page: internals/dotty-internals-1-notes.md
+      - page: internals/debug-macros.md
+      - page: internals/gadts.md
+      - page: internals/coverage.md
+      - page: internals/best-effort-compilation.md
+  - page: release-notes-0.1.2.md  # Referenced from https://www.scala-lang.org/blog/2017/05/31/first-dotty-milestone-release.html
+    hidden: true

docs/sidebar.yml

@@ -3394,11 +3394,17 @@ object Build {
   val testcasesSourceRoot = taskKey[String]("Root directory where tests sources are generated")
   val testDocumentationRoot = taskKey[String]("Root directory where tests documentation are stored")
   val generateSelfDocumentation = taskKey[Unit]("Generate example documentation")
-  // Note: the two tasks below should be one, but a bug in Tasty prevents that
-  val generateScalaDocumentation = inputKey[Unit]("Generate documentation for dotty lib")
-  val generateStableScala3Documentation  = inputKey[Unit]("Generate documentation for stable dotty lib")
   val generateTestcasesDocumentation  = taskKey[Unit]("Generate documentation for testcases, useful for debugging tests")
 
+  // Published on https://dotty.epfl.ch/ by nightly builds
+  // Contains additional internal/contributing docs
+  val generateScalaDocumentation = inputKey[Unit]("Generate documentation for snapshot release")
+
+  // Published on https://docs.scala-lang.org/api/all.html
+  val generateStableScala3Documentation  = inputKey[Unit]("Generate documentation for stable release")
+
+  // Published on https://docs.scala-lang.org/scala3/reference/
+  // Does not produce API docs, contains additional redirects for improved stablity
   val generateReferenceDocumentation = inputKey[Unit]("Generate language reference documentation for Scala 3")
 
   lazy val `scaladoc-testcases` = project.in(file("scaladoc-testcases")).
@@ -3525,31 +3531,135 @@ object Build {
         val outputDirOverride = extraArgs.headOption.fold(identity[GenerationConfig](_))(newDir => {
           config: GenerationConfig => config.add(OutputDir(newDir))
         })
-        val justAPIArg: Option[String] = extraArgs.drop(1).find(_ == "--justAPI")
-        val justAPI = justAPIArg.fold(identity[GenerationConfig](_))(_ => {
-          config: GenerationConfig => config.remove[SiteRoot]
-        })
-        val overrideFunc = outputDirOverride.andThen(justAPI)
+        val justAPI = extraArgs.contains("--justAPI")
+        def justAPIOverride(config: GenerationConfig): GenerationConfig = {
+          if (!justAPI) config
+          else {
+            val siteRoot = IO.createTemporaryDirectory.getAbsolutePath()
+            config.add(SiteRoot(siteRoot))
+          }
+        }
+
+        // It would be the easiest to create a temp directory and apply patches there, but this task would be used frequently during development
+        // If we'd build using copy the emitted warnings would point developers to copies instead of original sources. Any fixes made in there would be lost.
+        // Instead let's apply revertable patches to the files as part snapshot doc generation process
+        abstract class SourcePatch(val file: File) {
+          def apply(): Unit
+          def revert(): Unit
+        }
+        val docs = file("docs")
+        val sourcePatches = if (justAPI) Nil else Seq(
+          // Generate full sidebar.yml based on template and reference content
+          new SourcePatch(docs / "sidebar.yml") {
+            val referenceSideBarCopy = IO.temporaryDirectory / "sidebar.yml.copy"
+            IO.copyFile(file, referenceSideBarCopy)
+
+            override def apply(): Unit = {
+              val yaml = new org.yaml.snakeyaml.Yaml()
+              type YamlObject = java.util.Map[String, AnyRef]
+              type YamlList[T] = java.util.List[T]
+              def loadYaml(file: File): YamlObject = {
+                val reader = Files.newBufferedReader(file.toPath)
+                try yaml.load(reader).asInstanceOf[YamlObject]
+                finally reader.close()
+              }
+              // Ensure to always operate on original (Map, List) instances
+              val template = loadYaml(docs / "sidebar.nightly.template.yml")
+              template.get("subsection")
+                .asInstanceOf[YamlList[YamlObject]]
+                .stream()
+                .filter(_.get("title") == "Reference")
+                .findFirst()
+                .orElseThrow(() => new IllegalStateException("Reference subsection not found in sidebar.nightly.template.yml"))
+                .putAll(loadYaml(referenceSideBarCopy))
+
+              val sidebarWriter = Files.newBufferedWriter(this.file.toPath)
+              try yaml.dump(template, sidebarWriter)
+              finally sidebarWriter.close()
+            }
+            override def revert(): Unit = IO.move(referenceSideBarCopy, file)
+          },
+          // Add patch about nightly version usage
+          new SourcePatch(docs / "_layouts" / "static-site-main.html") {
+            lazy val originalContent = IO.read(file)
+
+            val warningMessage = """{% if page.nightlyOf %}
+              |  <aside class="warning">
+              |    <div class='icon'></div>
+              |    <div class='content'>
+              |      This is a nightly documentation. The content of this page may not be consistent with the current stable version of language. 
+              |      Click <a href="{{ page.nightlyOf }}">here</a> to find the stable version of this page.
+              |    </div>
+              |  </aside>
+              |{% endif %}""".stripMargin
+
+            override def apply(): Unit = {
+              IO.write(file,
+                originalContent
+                .replace("{{ content }}", s"$warningMessage {{ content }}")
+                .ensuring(_.contains(warningMessage), "patch to static-site-main layout not applied!")
+              )
+            }
+            override def revert(): Unit = IO.write(file, originalContent)
+          }
+        )
 
         val config = Def.task {
-          overrideFunc(Scala3.value)
+          outputDirOverride
+          .andThen(justAPIOverride)
+          .apply(Scala3.value)
         }
 
         val writeAdditionalFiles = Def.task {
           val dest = file(config.value.get[OutputDir].get.value)
-          if (justAPIArg.isEmpty) {
+          if (!justAPI) {
             IO.write(dest / "versions" / "latest-nightly-base", majorVersion)
             // This file is used by GitHub Pages when the page is available in a custom domain
             IO.write(dest / "CNAME", "dotty.epfl.ch")
           }
         }
+        val applyPatches = Def.task {
+          streams.value.log.info(s"Generating snapshot scaladoc, would apply patches to ${sourcePatches.map(_.file)}")
+          sourcePatches.foreach(_.apply())
+        }
+        val revertPatches = Def.task {
+          streams.value.log.info(s"Generated snapshot scaladoc, reverting changes made to ${sourcePatches.map(_.file)}")
+          sourcePatches.foreach(_.revert())
+        }
 
-        writeAdditionalFiles.dependsOn(generateDocumentation(config))
+        writeAdditionalFiles.dependsOn(
+          revertPatches.dependsOn(
+            generateDocumentation(config)
+              .dependsOn(applyPatches)
+          )
+        )
       }.evaluated,
 
       generateStableScala3Documentation := Def.inputTaskDyn {
         val extraArgs = spaceDelimited("<version>").parsed
-        val config = stableScala3(extraArgs.head)
+        val version = baseVersion
+        // In the early days of scaladoc there was a practice to precompile artifacts of Scala 3 and generate docs using different version of scaladoc
+        // It's no longer needed after its stablisation.
+        // Allow to use explcit version check to detect using incorrect revision during release process
+        extraArgs.headOption.foreach { explicitVersion =>
+          assert(
+            explicitVersion == version,
+            s"Version of the build ($version) does not match the explicit verion ($explicitVersion)"
+          )
+        }
+
+        val docs = IO.createTemporaryDirectory
+        IO.copyDirectory(file("docs"), docs)
+        IO.delete(docs / "_blog")
+
+        val config = Def.task {
+          Scala3.value
+          .add(ProjectVersion(version))
+          .add(Revision(version))
+          .add(OutputDir(s"scaladoc/output/${version}"))
+          .add(SiteRoot(docs.getAbsolutePath))
+          .remove[ApiSubdirectory]
+        }
         generateDocumentation(config)
       }.evaluated,
 
@@ -3564,20 +3674,13 @@ object Build {
         generateStaticAssetsTask.value
 
         // Move all the source files to a temporary directory and apply some changes specific to the reference documentation
-        val temp = IO.createTemporaryDirectory
-        IO.copyDirectory(file("docs"), temp / "docs")
-        IO.delete(temp / "docs" / "_blog")
-
-        // Overwrite the main layout and the sidebar
-        IO.copyDirectory(
-          file("project") / "resources" / "referenceReplacements",
-          temp / "docs",
-          overwrite = true
-        )
+        val docs = IO.createTemporaryDirectory
+        IO.copyDirectory(file("docs"), docs)
+        IO.delete(docs / "_blog")
 
         // Add redirections from previously supported URLs, for some pages
         for (name <- Seq("changed-features", "contextual", "dropped-features", "metaprogramming", "other-new-features")) {
-          val path = temp / "docs" / "_docs" / "reference" / name / s"${name}.md"
+          val path = docs / "_docs" / "reference" / name / s"${name}.md"
           val contentLines = IO.read(path).linesIterator.to[collection.mutable.ArrayBuffer]
           contentLines.insert(1, s"redirectFrom: /${name}.html") // Add redirection
           val newContent = contentLines.mkString("\n")
@@ -3587,12 +3690,12 @@ object Build {
         val languageReferenceConfig = Def.task {
           Scala3.value
             .add(OutputDir("scaladoc/output/reference"))
-            .add(SiteRoot(s"${temp.getAbsolutePath}/docs"))
+            .add(SiteRoot(docs.getAbsolutePath))
             .add(ProjectName("Scala 3 Reference"))
             .add(ProjectVersion(baseVersion))
             .remove[VersionsDictionaryUrl]
             .add(SourceLinks(List(
-              s"${temp.getAbsolutePath}=github://scala/scala3/language-reference-stable"
+              s"${docs.getParentFile().getAbsolutePath}=github://scala/scala3/language-reference-stable"
             )))
             .withTargets(List("___fake___.scala"))
         }
@@ -4063,57 +4166,40 @@ object ScaladocConfigs {
   }
 
   lazy val Scala3 = Def.task {
+    val stdlib = { // relative path to the stdlib directory ('library/')
+      val projectRoot = (ThisBuild/baseDirectory).value.toPath
+      val stdlibRoot = (`scala-library-bootstrapped` / baseDirectory).value
+      projectRoot.relativize(stdlibRoot.toPath.normalize())
+    }
+
     DefaultGenerationSettings.value
       .add(ProjectName("Scala 3"))
       .add(OutputDir(file("scaladoc/output/scala3").getAbsoluteFile.getAbsolutePath))
       .add(Revision("main"))
       .add(ExternalMappings(List(javaExternalMapping)))
-      .add(DocRootContent(((`scala-library-bootstrapped` / baseDirectory).value / "src" / "rootdoc.txt").toString))
+      .add(DocRootContent((stdlib / "src" / "rootdoc.txt").toString))
       .add(CommentSyntax(List(
-        //s"${dottyLibRoot}=markdown",
-        //s"${stdLibRoot}=wiki",
+        // Only the files below use markdown syntax (Scala 3 specific sources)
+        s"$stdlib/src/scala/NamedTuple.scala=markdown",
+        s"$stdlib/src/scala/Tuple.scala=markdown",
+        s"$stdlib/src/scala/compiletime=markdown",
+        s"$stdlib/src/scala/quoted=markdown",
+        s"$stdlib/src/scala/util/boundary.scala=markdown",
+        // Scala 2 sources use wiki syntax, we keep it as the default
         "wiki"
       )))
       .add(VersionsDictionaryUrl("https://scala-lang.org/api/versions.json"))
       .add(DocumentSyntheticTypes(true))
-      //.add(SnippetCompiler(List(
-        //s"$dottyLibRoot/src/scala=compile",
-        //s"$dottyLibRoot/src/scala/compiletime=compile",
-        //s"$dottyLibRoot/src/scala/util=compile",
-        //s"$dottyLibRoot/src/scala/util/control=compile"
-      //)))
+      .add(SnippetCompiler(List(
+        s"$stdlib/src/scala/compiletime=compile",
+        s"$stdlib/src/scala/quoted=compile",
+        s"$stdlib/src/scala/util/control=compile",
+        s"$stdlib/src/scala/util=compile",
+        s"$stdlib/src/scala=compile",
+      )))
       .add(SiteRoot("docs"))
       .add(ApiSubdirectory(true))
       .withTargets((`scala-library-bootstrapped` / Compile / products).value.map(_.getAbsolutePath))
   }
 
-  def stableScala3(version: String) = Def.task {
-    val scalaLibrarySrc = s"out/bootstrap/scala2-library-bootstrapped/scala-$version-bin-SNAPSHOT-nonbootstrapped/src_managed"
-    val dottyLibrarySrc = "library/src"
-    Scala3.value
-      .add(defaultSourceLinks(version = version))
-      .add(ProjectVersion(version))
-      .add(SnippetCompiler(
-        List(
-          s"$dottyLibrarySrc/scala/quoted=compile",
-          s"$dottyLibrarySrc/scala/compiletime=compile",
-          s"$dottyLibrarySrc/scala/util=compile",
-          s"$dottyLibrarySrc/scala/util/control=compile"
-        )
-      ))
-      .add(CommentSyntax(List(
-        s"$dottyLibrarySrc=markdown",
-        s"$scalaLibrarySrc=wiki",
-        "wiki"
-      )))
-      .add(DocRootContent(s"$scalaLibrarySrc/rootdoc.txt"))
-      .withTargets(
-        Seq(
-          s"tmp/interfaces/target/classes",
-          s"out/bootstrap/tasty-core-bootstrapped/scala-$version-bin-SNAPSHOT-nonbootstrapped/classes"
-        )
-      )
-      .remove[SiteRoot]
-      .remove[ApiSubdirectory]
-  }
 }

project/Build.scala

@@ -2,3 +2,6 @@
 libraryDependencies += "org.eclipse.jgit" % "org.eclipse.jgit" % "4.11.0.201803080745-r"
 
 libraryDependencies += Dependencies.`jackson-databind`
+
+// Used for manipulating YAML files in sidebar generation script
+libraryDependencies += "org.yaml" % "snakeyaml" % "2.4"