Implement LSIF emitter in Java.

Previously, we shelled out to `lsif-semanticdb` to convert SemanticDB
files into LSIF. This commit ports the original Go implemementation to
Java so that we can call a normal method instead of shelling out to a
separate process.

The biggest problem with the previous solution is that it complicated
installation for users. As we evolve lsif-java, the conversion logic
will also evolve and we want to avoid the situation where users have
an outdated version of lsif-semanticdb installed. With this commit,
users only need to install `lsif-java` and everything should work
correctly.

This commit also adds a new `snapshot-lsif` command to pretty-print an
LSIF index, similar to how we pretty-print SemanticDB payloads. This
command is mostly useful for testing purposes, but could be helpful
to troubleshoot user issues.

Author: olafurpg

.github/workflows/native.yml

@@ -1,11 +1,13 @@
 import scala.xml.{Node => XmlNode, NodeSeq => XmlNodeSeq, _}
 import scala.xml.transform.{RewriteRule, RuleTransformer}
 import java.io.File
+import java.nio.file.Files
 import java.util.Properties
 import scala.collection.mutable.ListBuffer
 
 lazy val V =
   new {
+    val protobuf = "3.15.6"
     val coursier = "2.0.8"
     val bloop = "1.4.7"
     val bsp = "2.0.0-M13"
@@ -64,22 +66,27 @@ commands +=
       "javafmtCheckAll" :: "publishLocal" :: "docs/docusaurusCreateSite" :: s
   }
 
+lazy val semanticdb = project
+  .in(file("semanticdb-java"))
+  .settings(
+    moduleName := "semanticdb-java",
+    javaOnlySettings,
+    PB.targets.in(Compile) :=
+      Seq(PB.gens.java(V.protobuf) -> (Compile / sourceManaged).value)
+  )
+
 lazy val agent = project
   .in(file("semanticdb-agent"))
   .settings(
     fatjarPackageSettings,
-    autoScalaLibrary := false,
+    javaOnlySettings,
     moduleName := "semanticdb-agent",
     libraryDependencies ++=
       List(
         "org.javassist" % "javassist" % "3.27.0-GA",
         "net.bytebuddy" % "byte-buddy" % "1.10.20",
         "net.bytebuddy" % "byte-buddy-agent" % "1.10.20"
       ),
-    incOptions ~= { old =>
-      old.withEnabled(false)
-    },
-    crossPaths := false,
     Compile / packageBin / packageOptions +=
       Package.ManifestAttributes(
         "Agent-Class" -> "com.sourcegraph.semanticdb_javac.SemanticdbAgent",
@@ -92,13 +99,10 @@ lazy val agent = project
 lazy val plugin = project
   .in(file("semanticdb-javac"))
   .settings(
+    fatjarPackageSettings,
+    javaOnlySettings,
     moduleName := "semanticdb-javac",
     javaToolchainVersion := "1.8",
-    autoScalaLibrary := false,
-    incOptions ~= { old =>
-      old.withEnabled(false)
-    },
-    fatjarPackageSettings,
     assemblyShadeRules.in(assembly) :=
       Seq(
         ShadeRule
@@ -108,14 +112,26 @@ lazy val plugin = project
             "org.relaxng.**" -> "com.sourcegraph.shaded.relaxng.@1"
           )
           .inAll
-      ),
-    crossPaths := false,
+      )
+  )
+  .dependsOn(semanticdb)
+
+lazy val lsif = project
+  .in(file("lsif-semanticdb"))
+  .settings(
+    moduleName := "lsif-semanticdb",
+    javaToolchainVersion := "1.8",
+    javaOnlySettings,
+    libraryDependencies +=
+      "com.google.protobuf" % "protobuf-java-util" % V.protobuf,
     PB.targets.in(Compile) :=
-      Seq(PB.gens.java -> (Compile / sourceManaged).value)
+      Seq(PB.gens.java(V.protobuf) -> (Compile / sourceManaged).value),
+    Compile / PB.protocOptions := Seq("--experimental_allow_proto3_optional")
   )
+  .dependsOn(semanticdb)
 
 lazy val cli = project
-  .in(file("cli"))
+  .in(file("lsif-java"))
   .settings(
     moduleName := "lsif-java",
     mainClass.in(Compile) := Some("com.sourcegraph.lsif_java.LsifJava"),
@@ -128,7 +144,11 @@ lazy val cli = project
         "bspVersion" -> V.bsp
       ),
     buildInfoPackage := "com.sourcegraph.lsif_java",
-    libraryDependencies ++= List("org.scalameta" %% "moped" % V.moped),
+    libraryDependencies ++=
+      List(
+        "org.scalameta" %% "moped" % V.moped,
+        "org.scalameta" %% "ascii-graphs" % "0.1.2"
+      ),
     resourceGenerators.in(Compile) +=
       Def
         .task[Seq[File]] {
@@ -161,11 +181,25 @@ lazy val cli = project
           propsFile :: copiedJars.toList
         }
         .taskValue,
-    nativeImageOptions ++= List("-H:IncludeResources=^semanticdb-.*jar$"),
+    nativeImageOptions ++=
+      List(
+        "-H:IncludeResources=^semanticdb-.*jar$",
+        s"-H:ReflectionConfigurationFiles=${target.value / "native-image-configs" / "reflect-config.json"}"
+      ),
     nativeImageOutput := target.in(NativeImage).value / "lsif-java"
   )
   .enablePlugins(NativeImagePlugin, BuildInfoPlugin)
-  .dependsOn(plugin)
+  .dependsOn(lsif)
+
+commands +=
+  Command.command("nativeImageProfiled") { s =>
+    val targetroot =
+      file("tests/minimized/.j11/target/scala-2.13/meta").absolutePath
+    val output = Files.createTempFile("lsif-java", "dump.lsif")
+    "minimized/compile" ::
+      s"""nativeImageRunAgent " index-semanticdb --output=$output $targetroot"""" ::
+      "nativeImage" :: s
+  }
 
 def minimizedSourceDirectory =
   file("tests/minimized/src/main/java").getAbsoluteFile
@@ -264,12 +298,37 @@ lazy val bench = project
   .in(file("tests/benchmarks"))
   .settings(
     moduleName := "lsif-java-bench",
+    javaToolchainVersion := "1.8",
     fork.in(run) := true,
     skip.in(publish) := true
   )
   .dependsOn(unit)
   .enablePlugins(JmhPlugin)
 
+lazy val docs = project
+  .in(file("lsif-java-docs"))
+  .settings(
+    mdocOut :=
+      baseDirectory.in(ThisBuild).value / "website" / "target" / "docs",
+    fork := false,
+    mdocVariables :=
+      Map[String, String](
+        "VERSION" -> version.value,
+        "SCALA_VERSION" -> scalaVersion.value,
+        "STABLE_VERSION" -> version.value.replaceFirst("\\-.*", "")
+      )
+  )
+  .dependsOn(unit)
+  .enablePlugins(DocusaurusPlugin)
+
+lazy val javaOnlySettings = List[Def.Setting[_]](
+  autoScalaLibrary := false,
+  incOptions ~= { old =>
+    old.withEnabled(false)
+  },
+  crossPaths := false
+)
+
 lazy val testSettings = List(
   skip.in(publish) := true,
   autoScalaLibrary := true,
@@ -279,7 +338,6 @@ lazy val testSettings = List(
       "org.scalameta" %% "munit" % "0.7.10",
       "org.scalameta" %% "moped-testkit" % V.moped,
       "org.scalameta" %% "scalameta" % V.scalameta,
-      "org.scala-lang.modules" %% "scala-parallel-collections" % "1.0.0",
       "io.get-coursier" %% "coursier" % V.coursier,
       "com.lihaoyi" %% "pprint" % "0.6.1"
     )
@@ -334,19 +392,3 @@ lazy val fatjarPackageSettings = List[Def.Setting[_]](
     ).transform(node).head
   }
 )
-
-lazy val docs = project
-  .in(file("lsif-java-docs"))
-  .settings(
-    mdocOut :=
-      baseDirectory.in(ThisBuild).value / "website" / "target" / "docs",
-    fork := false,
-    mdocVariables :=
-      Map[String, String](
-        "VERSION" -> version.value,
-        "SCALA_VERSION" -> scalaVersion.value,
-        "STABLE_VERSION" -> version.value.replaceFirst("\\-.*", "")
-      )
-  )
-  .dependsOn(unit)
-  .enablePlugins(DocusaurusPlugin)

build.sbt

@@ -63,7 +63,14 @@ First, install the
 community edition is
 [open source](https://github.com/JetBrains/intellij-community) and free to use.
 
-Next, install the IntelliJ Scala plugin.
+Next, install the following IntelliJ plugins:
+
+- IntelliJ Scala plugin.
+- Google Java Format
+
+Next, follow
+[these instructions](https://github.com/HPI-Information-Systems/Metanome/wiki/Installing-the-google-styleguide-settings-in-intellij-and-eclipse)
+here to configure the Google Java formatter.
 
 Finally, run "File > Project From Existing Sources" to import the sbt build into
 IntelliJ. Select the "sbt" option if it asks you to choose between

cli/src/main/scala/com/sourcegraph/lsif_java/IndexSemanticdbCommand.scala

@@ -1,4 +1,4 @@
-package com.sourcegraph.lsif_java
+package com.sourcegraph.io
 
 import java.nio.file.Path
 import java.nio.file.Paths

docs/contributing.md

@@ -1,4 +1,4 @@
-package com.sourcegraph.lsif_java
+package com.sourcegraph.io
 
 import java.io.IOException
 import java.nio.file.FileVisitResult

…sourcegraph/lsif_java/AbsolutePath.scala …la/com/sourcegraph/io/AbsolutePath.scalacli/src/main/scala/com/sourcegraph/lsif_java/AbsolutePath.scala renamed to lsif-java/src/main/scala/com/sourcegraph/io/AbsolutePath.scala cli/src/main/scala/com/sourcegraph/lsif_java/AbsolutePath.scala renamed to lsif-java/src/main/scala/com/sourcegraph/io/AbsolutePath.scala

@@ -2,6 +2,10 @@ package com.sourcegraph.lsif_java
 
 import java.io.PrintStream
 
+import com.sourcegraph.lsif_java.commands.IndexCommand
+import com.sourcegraph.lsif_java.commands.IndexSemanticdbCommand
+import com.sourcegraph.lsif_java.commands.SnapshotCommand
+import com.sourcegraph.lsif_java.commands.SnapshotLsifCommand
 import moped.cli.Application
 import moped.cli.CommandParser
 import moped.commands.HelpCommand
@@ -17,7 +21,8 @@ object LsifJava {
       CommandParser[VersionCommand],
       CommandParser[IndexCommand],
       CommandParser[IndexSemanticdbCommand],
-      CommandParser[SnapshotCommand]
+      CommandParser[SnapshotCommand],
+      CommandParser[SnapshotLsifCommand]
     )
   )
   def main(args: Array[String]): Unit = {

…ourcegraph/lsif_java/DeleteVisitor.scala …a/com/sourcegraph/io/DeleteVisitor.scalacli/src/main/scala/com/sourcegraph/lsif_java/DeleteVisitor.scala renamed to lsif-java/src/main/scala/com/sourcegraph/io/DeleteVisitor.scala cli/src/main/scala/com/sourcegraph/lsif_java/DeleteVisitor.scala renamed to lsif-java/src/main/scala/com/sourcegraph/io/DeleteVisitor.scala

@@ -17,7 +17,7 @@ object SemanticdbPrinters {
       .linesWithSeparators
       .zipWithIndex
       .foreach { case (line, i) =>
-        out.append(line)
+        out.append(line.replace("\t", "→"))
         val occurences = occurrencesByLine
           .getOrElse(i, Nil)
           .sortBy(o =>

…com/sourcegraph/lsif_java/Embedded.scala …com/sourcegraph/lsif_java/Embedded.scalacli/src/main/scala/com/sourcegraph/lsif_java/Embedded.scala renamed to lsif-java/src/main/scala/com/sourcegraph/lsif_java/Embedded.scala cli/src/main/scala/com/sourcegraph/lsif_java/Embedded.scala renamed to lsif-java/src/main/scala/com/sourcegraph/lsif_java/Embedded.scala

@@ -2,8 +2,8 @@ package com.sourcegraph.lsif_java.buildtools
 
 import java.nio.file.Path
 
-import com.sourcegraph.lsif_java.AbsolutePath
-import com.sourcegraph.lsif_java.IndexCommand
+import com.sourcegraph.io.AbsolutePath
+import com.sourcegraph.lsif_java.commands.IndexCommand
 
 /**
  * A build tool such as Gradle, Maven or Bazel.