diff --git a/Dockerfile b/Dockerfile index 2bc748d73..149a2e316 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,7 +18,7 @@ RUN git config --global --add safe.directory * COPY . . RUN sbt publishLocal dumpScipJavaVersion -RUN mkdir -p /app && coursier bootstrap "com.sourcegraph:scip-java_2.13:$(cat VERSION)" -f -o /app/scip-java -M com.sourcegraph.scip_java.ScipJava +RUN mkdir -p /app && coursier bootstrap "com.sourcegraph:scip-java:$(cat VERSION)" -f -o /app/scip-java -M com.sourcegraph.scip_java.ScipJava COPY ./bin/scip-java-docker-script.sh /usr/bin/scip-java diff --git a/build.sbt b/build.sbt index 638d7ae9c..e62603569 100644 --- a/build.sbt +++ b/build.sbt @@ -11,13 +11,14 @@ lazy val V = val protobuf = "4.34.2" val scipBindings = "0.8.0" val scalaXml = "2.1.0" - val moped = "0.2.0" val gradle = "8.10" val scala213 = "2.13.13" val scalameta = "4.9.3" val kotlinVersion = "2.2.0" val kotest = "4.6.3" val kctfork = "0.7.1" + val clikt = "5.0.3" + val kotlinxSerialization = "1.9.0" } // sbt-git's bundled JGit can't read linked worktrees; shell out to @@ -196,28 +197,29 @@ lazy val mavenPlugin = project lazy val cli = project .in(file("scip-java")) + .enablePlugins(KotlinPlugin, PackPlugin, DockerPlugin) .settings( moduleName := "scip-java", + crossPaths := false, + autoScalaLibrary := false, + kotlinVersion := V.kotlinVersion, + kotlincJvmTarget := "11", + Compile / javacOptions ++= Seq("--release", "11"), (Compile / mainClass) := Some("com.sourcegraph.scip_java.ScipJava"), (run / baseDirectory) := (ThisBuild / baseDirectory).value, // ScipJava.main can call System.exit, so we always fork the JVM when // sbt invokes it directly (e.g. from the scip-kotlinc snapshots // task) so it cannot kill the surrounding sbt process. Compile / run / fork := true, - buildInfoKeys := - Seq[BuildInfoKey]( - version, - sbtVersion, - scalaVersion, - "javacModuleOptions" -> javacModuleOptions, - "scalametaVersion" -> V.scalameta, - "scala213" -> V.scala213 - ), - buildInfoPackage := "com.sourcegraph.scip_java", + // Generate a tiny Java `BuildInfo` class replacing the previous + // sbt-buildinfo-generated Scala object. Same shape as the Gradle plugin's + // `GradlePluginBuildInfo` (introduced in the Gradle plugin Kotlin port). + Compile / sourceGenerators += scipJavaCliBuildInfoGenerator.taskValue, libraryDependencies ++= List( - "org.scala-lang.modules" %% "scala-xml" % V.scalaXml, - "org.scalameta" %% "moped" % V.moped, + "com.github.ajalt.clikt" % "clikt-jvm" % V.clikt, + "org.jetbrains.kotlinx" % "kotlinx-serialization-json-jvm" % + V.kotlinxSerialization, "org.jetbrains.kotlin" % "kotlin-compiler-embeddable" % V.kotlinVersion, "org.jetbrains.kotlin" % "kotlin-scripting-common" % V.kotlinVersion, "org.jetbrains.kotlin" % "kotlin-scripting-jvm" % V.kotlinVersion, @@ -289,10 +291,59 @@ lazy val cli = project docker / dockerfile := NativeDockerfile((ThisBuild / baseDirectory).value / "Dockerfile") ) - .enablePlugins(PackPlugin, DockerPlugin, BuildInfoPlugin) .dependsOn(scip) -// Task key for regenerating the SCIP/SCIP golden snapshots emitted by +// Source-generator for the CLI's build-info Java class. Replaces the +// sbt-buildinfo-generated Scala BuildInfo object so the CLI module stays +// Kotlin/Java-only (and the generated class is straightforward to consume +// from Kotlin). +lazy val scipJavaCliBuildInfoGenerator = Def.task { + val out = (Compile / sourceManaged).value / "com" / "sourcegraph" / + "scip_java" / "BuildInfo.java" + IO.createDirectory(out.getParentFile) + val optionsLiteral = javacModuleOptions + .map(javaStringLiteral) + .mkString("Arrays.asList(", ", ", ")") + val versionLiteral = javaStringLiteral(version.value) + val contents = + s"""package com.sourcegraph.scip_java; + | + |import java.util.Arrays; + |import java.util.Collections; + |import java.util.List; + | + |public final class BuildInfo { + | private BuildInfo() {} + | public static final String version = $versionLiteral; + | public static final List javacModuleOptions = + | Collections.unmodifiableList($optionsLiteral); + |} + |""".stripMargin + IO.write(out, contents) + Seq(out) +} + +def javaStringLiteral(value: String): String = { + val escaped = value.flatMap { + case '\\' => + "\\\\" + case '"' => + "\\\"" + case '\n' => + "\\n" + case '\r' => + "\\r" + case '\t' => + "\\t" + case c if c.isControl => + f"\\u${c.toInt}%04x" + case c => + c.toString + } + "\"" + escaped + "\"" +} + +// Task key for regenerating the SCIP golden snapshots emitted by // the scip-kotlinc compiler plugin over the Kotlin minimized fixtures. // We deliberately do NOT call this `snapshots` to avoid colliding with the // existing top-level `snapshots` test project (`lazy val snapshots = project`). @@ -615,8 +666,8 @@ val testSettings = List( libraryDependencies ++= List( "org.scalameta" %% "munit" % "0.7.29", - "org.scalameta" %% "moped-testkit" % V.moped, "org.scalameta" %% "scalameta" % V.scalameta, + "com.lihaoyi" %% "os-lib" % "0.9.3", "com.lihaoyi" %% "pprint" % "0.6.6" ) ) diff --git a/docs/getting-started.md b/docs/getting-started.md index c295ced8d..b1533f921 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -194,7 +194,7 @@ Run `scip-java index --help` to learn more about the available command-line options. ```scala mdoc:passthrough -com.sourcegraph.scip_java.ScipJava.printHelp(Console.out) +com.sourcegraph.scip_java.ScipJava.INSTANCE.printHelp(Console.out) ``` ## Supported programming languages diff --git a/scip-java/src/main/kotlin/com/sourcegraph/scip_aggregator/ConsoleScipAggregatorReporter.kt b/scip-java/src/main/kotlin/com/sourcegraph/scip_aggregator/ConsoleScipAggregatorReporter.kt new file mode 100644 index 000000000..fb9a7364a --- /dev/null +++ b/scip-java/src/main/kotlin/com/sourcegraph/scip_aggregator/ConsoleScipAggregatorReporter.kt @@ -0,0 +1,31 @@ +package com.sourcegraph.scip_aggregator + +import com.sourcegraph.scip_java.CliReporter +import java.nio.file.NoSuchFileException + +/** + * Console reporter for the `aggregate` command. + * + * The old reporter rendered an `InteractiveProgressBar` via moped's + * `paiges`-based renderer. Progress reporting is dropped here to keep the + * CLI runtime free of Scala libraries; the renderer was always silent + * when fewer than 100 files were processed anyway. + */ +class ConsoleScipAggregatorReporter(private val reporter: CliReporter) : ScipAggregatorReporter() { + + override fun error(e: Throwable) { + if (e is NoSuchFileException) { + reporter.error("no such file: ${e.message}") + } else { + reporter.error(e) + } + } + + override fun hasErrors(): Boolean = reporter.hasErrors() + + override fun startProcessing(taskSize: Int) {} + + override fun processedOneItem() {} + + override fun endProcessing() {} +} diff --git a/scip-java/src/main/kotlin/com/sourcegraph/scip_java/CliEnvironment.kt b/scip-java/src/main/kotlin/com/sourcegraph/scip_java/CliEnvironment.kt new file mode 100644 index 000000000..be5584717 --- /dev/null +++ b/scip-java/src/main/kotlin/com/sourcegraph/scip_java/CliEnvironment.kt @@ -0,0 +1,26 @@ +package com.sourcegraph.scip_java + +import java.io.PrintStream +import java.nio.file.Path +import java.nio.file.Paths + +/** + * Captures the per-invocation environment of a scip-java CLI run. + * + * Tests inject a custom environment to redirect stdout/stderr into a + * byte buffer, point the working directory at a temporary fixture + * directory, and so on. + */ +data class CliEnvironment( + val workingDirectory: Path = Paths.get("").toAbsolutePath(), + val environmentVariables: Map = System.getenv(), + val standardOutput: PrintStream = System.out, + val standardError: PrintStream = System.err, + val isProgressBarEnabled: Boolean = System.console() != null, +) { + fun withWorkingDirectory(cwd: Path): CliEnvironment = copy(workingDirectory = cwd) + + fun withStandardOutput(out: PrintStream): CliEnvironment = copy(standardOutput = out) + + fun withStandardError(err: PrintStream): CliEnvironment = copy(standardError = err) +} diff --git a/scip-java/src/main/kotlin/com/sourcegraph/scip_java/CliReporter.kt b/scip-java/src/main/kotlin/com/sourcegraph/scip_java/CliReporter.kt new file mode 100644 index 000000000..10999e3e7 --- /dev/null +++ b/scip-java/src/main/kotlin/com/sourcegraph/scip_java/CliReporter.kt @@ -0,0 +1,48 @@ +package com.sourcegraph.scip_java + +import java.util.concurrent.atomic.AtomicInteger + +/** + * Minimal reporter that mirrors the moped `ConsoleReporter` API surface that + * scip-java actually uses (info/warning/error/debug/hasErrors/exitCode). + * + * `info` is written to stdout to match the previous behaviour of the + * default moped reporter; `warning` and `error` go to stderr. + */ +class CliReporter(private val env: CliEnvironment) { + private val errorCount = AtomicInteger() + + fun info(message: String) { + env.standardOutput.println(message) + } + + fun warning(message: String) { + env.standardError.println("warning: $message") + } + + fun error(message: String) { + errorCount.incrementAndGet() + env.standardError.println("error: $message") + } + + /** + * Debug messages are dropped to avoid leaking noise into snapshot tests. + */ + @Suppress("UNUSED_PARAMETER") + fun debug(message: String) { + // intentional no-op + } + + fun error(e: Throwable) { + errorCount.incrementAndGet() + e.printStackTrace(env.standardError) + } + + fun hasErrors(): Boolean = errorCount.get() > 0 + + fun exitCode(): Int = if (hasErrors()) 1 else 0 + + fun reset() { + errorCount.set(0) + } +} diff --git a/scip-java/src/main/kotlin/com/sourcegraph/scip_java/Embedded.kt b/scip-java/src/main/kotlin/com/sourcegraph/scip_java/Embedded.kt new file mode 100644 index 000000000..059d75f21 --- /dev/null +++ b/scip-java/src/main/kotlin/com/sourcegraph/scip_java/Embedded.kt @@ -0,0 +1,105 @@ +package com.sourcegraph.scip_java + +import com.sourcegraph.scip_java.buildtools.ProcessResult +import java.nio.charset.StandardCharsets +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.StandardCopyOption + +object Embedded { + + fun scipJar(tmpDir: Path): Path = copyFile(tmpDir, "scip-plugin.jar") + + fun gradlePluginJar(tmpDir: Path): Path = copyFile(tmpDir, "gradle-plugin.jar") + + fun scipKotlincJar(tmpDir: Path): Path = copyFile(tmpDir, "scip-kotlinc.jar") + + private fun javacErrorpath(tmp: Path): Path = tmp.resolve("errorpath.txt") + + fun customJavac(sourceroot: Path, targetroot: Path, tmp: Path): Path { + val bin = tmp.resolve("bin") + val javac = bin.resolve("javac") + val java = bin.resolve("java") + val pluginpath = scipJar(tmp) + val errorpath = javacErrorpath(tmp) + val javacopts = targetroot.resolve("javacopts.txt") + Files.createDirectories(targetroot) + Files.createDirectories(bin) + Files.write( + java, + ("#!/usr/bin/env bash\n" + + "java \"\$@\"\n").toByteArray(StandardCharsets.UTF_8), + ) + val newJavacopts = tmp.resolve("javac_newarguments") + // --add-exports flags required to access internal javac APIs from our + // SCIP plugin. Always set; Java 11+ is the supported baseline. + val javacModuleOptions = BuildInfo.javacModuleOptions.joinToString(" ") + val injectScipArguments = + listOf( + "java", + "-Dscip.errorpath=$errorpath", + "-Dscip.pluginpath=$pluginpath", + "-Dscip.sourceroot=$sourceroot", + "-Dscip.targetroot=$targetroot", + "-Dscip.output=\$NEW_JAVAC_OPTS", + "-Dscip.old-output=$javacopts", + "-classpath $pluginpath", + "com.sourcegraph.scip_javac.InjectScipOptions", + "\"\$@\"", + ).joinToString(" ") + val script = buildString { + append("#!/usr/bin/env bash\n") + append("set -eu\n") + append("LAUNCHER_ARGS=()\n") + append("NEW_JAVAC_OPTS=\"$newJavacopts-\$RANDOM\"\n") + append("for arg in \"\$@\"; do\n") + append(" if [[ \$arg == -J* ]]; then\n") + append(" LAUNCHER_ARGS+=(\"\$arg\")\n") + append(" fi\n") + append("done\n") + append(injectScipArguments).append('\n') + append("if [ \${#LAUNCHER_ARGS[@]} -eq 0 ]; then\n") + append(" javac $javacModuleOptions \"@\$NEW_JAVAC_OPTS\"\n") + append("else\n") + append(" javac $javacModuleOptions \"@\$NEW_JAVAC_OPTS\" \"\${LAUNCHER_ARGS[@]}\"\n") + append("fi\n") + } + Files.write(javac, script.toByteArray(StandardCharsets.UTF_8)) + javac.toFile().setExecutable(true) + java.toFile().setExecutable(true) + return javac + } + + /** + * The custom javac wrapper reports errors to a specific file if unexpected + * errors happen. The javac wrapper gets invoked by builds tools like + * Gradle/Maven, which hide the actual errors from the script because they + * assume the standard output is from javac. This file is used a side-channel + * to avoid relying on the error reporting from Gradle/Maven. + */ + fun reportUnexpectedJavacErrors(reporter: CliReporter, tmp: Path): ProcessResult? { + val errorpath = javacErrorpath(tmp) + if (!Files.isRegularFile(errorpath)) return null + reporter.error("unexpected javac compile errors") + Files.readAllLines(errorpath).forEach { reporter.error(it) } + return ProcessResult(1) + } + + /** Returns the string contents of the scip_java.bzl file on disk. */ + fun bazelAspectFile(tmpDir: Path): String { + val tmpFile = copyFile(tmpDir, "scip-java/scip_java.bzl") + val contents = String(Files.readAllBytes(tmpFile), StandardCharsets.UTF_8) + Files.deleteIfExists(tmpFile) + return contents + } + + private fun copyFile(tmpDir: Path, filename: String): Path { + val input = + Embedded::class.java.getResourceAsStream("/$filename") + ?: error("missing embedded resource: /$filename") + val out = tmpDir.resolve(filename) + Files.createDirectories(out.parent) + input.use { Files.copy(it, out, StandardCopyOption.REPLACE_EXISTING) } + return out + } +} diff --git a/scip-java/src/main/kotlin/com/sourcegraph/scip_java/ScipJava.kt b/scip-java/src/main/kotlin/com/sourcegraph/scip_java/ScipJava.kt new file mode 100644 index 000000000..99219e828 --- /dev/null +++ b/scip-java/src/main/kotlin/com/sourcegraph/scip_java/ScipJava.kt @@ -0,0 +1,28 @@ +package com.sourcegraph.scip_java + +import java.io.PrintStream + +/** + * Public entry point for the scip-java CLI. The single [app] instance is + * shared across test suites and reset between invocations. + */ +object ScipJava { + + @JvmField + val app: ScipJavaApp = ScipJavaApp() + + @JvmStatic + fun main(args: Array) { + app.runAndExitIfNonZero(args.toList()) + } + + fun printHelp(out: PrintStream) { + out.println("```text") + out.println("$ scip-java index --help") + val replacement = ScipJavaApp().apply { + env = env.withStandardOutput(out).withStandardError(out) + } + replacement.run(listOf("index", "--help")) + out.println("```") + } +} diff --git a/scip-java/src/main/kotlin/com/sourcegraph/scip_java/ScipJavaApp.kt b/scip-java/src/main/kotlin/com/sourcegraph/scip_java/ScipJavaApp.kt new file mode 100644 index 000000000..d11c12f15 --- /dev/null +++ b/scip-java/src/main/kotlin/com/sourcegraph/scip_java/ScipJavaApp.kt @@ -0,0 +1,184 @@ +package com.sourcegraph.scip_java + +import com.github.ajalt.clikt.core.CliktCommand +import com.github.ajalt.clikt.core.CliktError +import com.github.ajalt.clikt.core.Context +import com.github.ajalt.clikt.core.PrintHelpMessage +import com.github.ajalt.clikt.core.PrintMessage +import com.github.ajalt.clikt.core.ProgramResult +import com.github.ajalt.clikt.core.UsageError +import com.github.ajalt.clikt.core.findOrSetObject +import com.github.ajalt.clikt.core.parse +import com.github.ajalt.clikt.core.subcommands +import com.sourcegraph.scip_java.buildtools.ProcessResult +import com.sourcegraph.scip_java.buildtools.ProcessRunner +import com.sourcegraph.scip_java.commands.IndexCommand +import com.sourcegraph.scip_java.commands.AggregateCommand +import com.sourcegraph.scip_java.commands.SnapshotCommand +import java.nio.file.Paths + +/** + * Stateful, mutable container for the scip-java CLI runtime. Tests inject + * a fresh environment (with redirected stdout/stderr, a temp working + * directory, etc.) before invoking [run]. + * + * Each invocation of [run] builds a fresh root clikt command tree so option + * state from a previous run never leaks into the next one. + */ +class ScipJavaApp { + + var env: CliEnvironment = CliEnvironment() + set(value) { + field = value + // Rebind the reporter so subsequent writes hit the new + // PrintStreams. Test harnesses swap [env] between invocations. + currentReporter = CliReporter(value) + } + + private var currentReporter: CliReporter = CliReporter(env) + + val reporter: CliReporter + get() = currentReporter + + fun info(message: String) = reporter.info(message) + + fun warning(message: String) = reporter.warning(message) + + fun error(message: String) = reporter.error(message) + + /** + * Spawn an external process using the current working directory. + * Stdout and stderr are streamed to the env's PrintStreams line-by-line. + */ + fun runProcess( + command: List, + env: Map = emptyMap(), + ): ProcessResult { + val syntax = command.joinToString(" ") { if (' ' in it) "'$it'" else it } + this.env.standardOutput.println("$ $syntax") + return ProcessRunner.run( + command, + cwd = this.env.workingDirectory, + env = env, + onStdout = { this.env.standardOutput.println(it) }, + onStderr = { this.env.standardError.println(it) }, + ) + } + + fun run(args: List): Int { + reporter.reset() + val processedArgs = applyGlobalCwd(rewriteNestedOptions(args)) + val root = RootCommand(this) + root.subcommands(IndexCommand(), AggregateCommand(), SnapshotCommand()) + return try { + root.parse(processedArgs) + reporter.exitCode() + } catch (e: PrintHelpMessage) { + env.standardOutput.println(root.getFormattedHelp(e)) + e.statusCode + } catch (e: PrintMessage) { + env.standardOutput.println(e.message) + e.statusCode + } catch (e: ProgramResult) { + e.statusCode + } catch (e: UsageError) { + env.standardError.println(root.getFormattedHelp(e)) + e.statusCode + } catch (e: CliktError) { + e.message?.let { env.standardError.println(it) } + e.statusCode + } + } + + /** Terminate the JVM with the exit code returned by [run]. */ + fun runAndExitIfNonZero(args: List) { + val exit = run(args) + if (exit != 0) kotlin.system.exitProcess(exit) + } + + /** + * The previous moped CLI accepted nested subcommand options written as + * `--aggregate.` on `scip-java index` (e.g. the Bazel aspect + * passes `--aggregate.allow-empty-index`). clikt forbids `.` in + * option names, so we rewrite the dotted prefix to a `-` separator to match + * the options declared on [IndexCommand]. Tokens after a `--` separator are + * left untouched. + */ + private fun rewriteNestedOptions(args: List): List { + var sawDoubleDash = false + return args.map { arg -> + when { + sawDoubleDash -> arg + arg == "--" -> { + sawDoubleDash = true + arg + } + arg.startsWith("--aggregate.") -> + "--aggregate-" + arg.removePrefix("--aggregate.") + else -> arg + } + } + } + + /** + * `--cwd` is a global flag that, unlike a regular clikt parent option, may + * appear in any position (including after the subcommand name), mirroring + * the previous moped-based CLI where `--cwd` was an application-level + * parameter. We extract it here before handing the remaining arguments to + * clikt, and apply it to the working directory. Tokens after a `--` + * separator are passed through untouched so a trailing build command can + * legitimately contain `--cwd`. + */ + private fun applyGlobalCwd(args: List): List { + val result = ArrayList(args.size) + var cwd: String? = null + var sawDoubleDash = false + var i = 0 + while (i < args.size) { + val arg = args[i] + when { + sawDoubleDash -> { + result.add(arg) + i += 1 + } + arg == "--" -> { + sawDoubleDash = true + result.add(arg) + i += 1 + } + arg == "--cwd" && i + 1 < args.size -> { + cwd = args[i + 1] + i += 2 + } + arg.startsWith("--cwd=") -> { + cwd = arg.removePrefix("--cwd=") + i += 1 + } + else -> { + result.add(arg) + i += 1 + } + } + } + cwd?.let { env = env.withWorkingDirectory(Paths.get(it).toAbsolutePath()) } + return result + } + + /** + * Root clikt command that plumbs the parent [ScipJavaApp] into the + * clikt context so subcommands can pick it up via + * `currentContext.findObject`. + */ + private class RootCommand(val app: ScipJavaApp) : CliktCommand(name = "scip-java") { + + override fun help(context: Context) = + "scip-java: index Java/Kotlin codebases into SCIP." + + private val sharedApp by findOrSetObject { app } + + override fun run() { + // Touch sharedApp so the context object is set even when no subcommand uses it. + @Suppress("UNUSED_VARIABLE") val ignored = sharedApp + } + } +} diff --git a/scip-java/src/main/kotlin/com/sourcegraph/scip_java/ScipPrinters.kt b/scip-java/src/main/kotlin/com/sourcegraph/scip_java/ScipPrinters.kt new file mode 100644 index 000000000..025c58729 --- /dev/null +++ b/scip-java/src/main/kotlin/com/sourcegraph/scip_java/ScipPrinters.kt @@ -0,0 +1,332 @@ +package com.sourcegraph.scip_java + +import kotlin.math.max +import org.scip_code.scip.Document +import org.scip_code.scip.Occurrence +import org.scip_code.scip.SymbolInformation +import org.scip_code.scip.SymbolRole + +/** + * Renders a SCIP `Document` (parsed from an `index.scip` file) into the + * human-readable golden-snapshot format used by the test suite. + * + * The output is annotated source code: every source line is followed by + * comment lines that point at the SCIP occurrences/symbols defined on + * that line. + */ +object ScipPrinters { + + /** + * Indent prefix prepended to each source line so that caret-based + * indicators in snapshot comments can point at arbitrary columns. + */ + const val sourceIndent: String = " " + + // scip-java only indexes Java and Kotlin sources, both of which use `//`. + private const val commentSyntax: String = "//" + + @JvmStatic + fun printTextDocument(doc: Document, text: String): String { + val out = StringBuilder() + val occurrencesByLine: Map> = + doc.occurrencesList.groupBy { it.getRange(0) } + val symtab: Map = + doc.symbolsList.associateBy { it.symbol } + val input = SourceInput(text) + + val syntheticDefinitions: Map> = + doc.symbolsList + .flatMap { info -> + info.relationshipsList + .filter { it.isDefinition } + .map { it.symbol to info } + } + .groupBy({ it.first }, { it.second }) + + val allEnclosingRanges: List = + doc.occurrencesList.mapNotNull { occ -> + val r = occ.enclosingRangeList.map { it.toInt() } + when (r.size) { + 3 -> EnclosingRange(r[0], r[1], r[0], r[2], occ.symbol) + 4 -> EnclosingRange(r[0], r[1], r[2], r[3], occ.symbol) + else -> null + } + } + val enclosingByStartLine = + allEnclosingRanges.groupBy { it.startLine }.mapValues { it.value.sortedBy { r -> r.startChar } } + val enclosingByEndLine = + allEnclosingRanges.groupBy { it.endLine }.mapValues { it.value.sortedBy { r -> r.endChar } } + + linesWithSeparators(text).forEachIndexed { i, line -> + enclosingByStartLine[i]?.forEach { er -> + out.append(commentSyntax) + .append(" ".repeat(er.startChar)) + .append("⌄ enclosing_range_start ") + .append(er.symbol) + .append("\n") + } + out.append(sourceIndent).append(line.replace("\t", " ")) + val occurrences = + (occurrencesByLine[i] ?: emptyList()).sortedWith(occurrenceOrdering) + for (occ in occurrences) { + formatOccurrence(input, out, occ, line, symtab, null) + if ((occ.symbolRoles and SymbolRole.Definition_VALUE) > 0) { + syntheticDefinitions[occ.symbol]?.forEach { syntheticDefinition -> + formatOccurrence(input, out, occ, line, symtab, syntheticDefinition) + } + } + } + enclosingByEndLine[i]?.forEach { er -> + val indent = max(0, er.endChar - 1) + out.append(commentSyntax) + .append(" ".repeat(indent)) + .append("⌃ enclosing_range_end ") + .append(er.symbol) + .append("\n") + } + } + return out.toString() + } + + private data class EnclosingRange( + val startLine: Int, + val startChar: Int, + val endLine: Int, + val endChar: Int, + val symbol: String, + ) + + /** + * Element-wise integer-list comparator for `Occurrence.rangeList`, + * matching Scala's `seqOrdering` semantics. Falls back to `symbol` + * for a tie-breaker. + */ + private val occurrenceOrdering: Comparator = + Comparator { a, b -> + val ra = a.rangeList + val rb = b.rangeList + val n = minOf(ra.size, rb.size) + for (i in 0 until n) { + val cmp = ra[i].toInt().compareTo(rb[i].toInt()) + if (cmp != 0) return@Comparator cmp + } + val sizeCmp = ra.size.compareTo(rb.size) + if (sizeCmp != 0) sizeCmp else a.symbol.compareTo(b.symbol) + } + + private data class OccurrencePos( + val startLine: Int, + val startColumn: Int, + val endLine: Int, + val endColumn: Int, + ) + + /** + * Faithful port of `moped.reporters.Position.range` + `RangePosition`: + * the SCIP `(line, column)` pair is converted to a flat character offset + * and back. The round-trip matters for occurrences whose end column + * overflows its start line (e.g. Kotlin `companion object` definitions), + * where the overflow "carries" the end onto a later line, turning a raw + * single-line range into a rendered multi-line range. + */ + private fun positionOf(input: SourceInput, occ: Occurrence): OccurrencePos { + val rawStartLine: Int + val rawStartColumn: Int + val rawEndLine: Int + val rawEndColumn: Int + when (occ.rangeCount) { + 3 -> { + rawStartLine = occ.getRange(0) + rawStartColumn = occ.getRange(1) + rawEndLine = occ.getRange(0) + rawEndColumn = occ.getRange(2) + } + 4 -> { + rawStartLine = occ.getRange(0) + rawStartColumn = occ.getRange(1) + rawEndLine = occ.getRange(2) + rawEndColumn = occ.getRange(3) + } + else -> throw IllegalArgumentException("Invalid range: $occ") + } + // moped's Position.range returns NoPosition (all -1) for empty input. + if (input.isEmpty) return OccurrencePos(-1, -1, -1, -1) + val start = input.lineToOffset(rawStartLine) + rawStartColumn + val end = input.lineToOffset(rawEndLine) + rawEndColumn + val startLine = input.offsetToLine(start) + val endLine = input.offsetToLine(end) + return OccurrencePos( + startLine = startLine, + startColumn = start - input.lineToOffset(startLine), + endLine = endLine, + endColumn = end - input.lineToOffset(endLine), + ) + } + + private fun formatOccurrence( + input: SourceInput, + out: StringBuilder, + occ: Occurrence, + line: String, + symtab: Map, + syntheticDefinition: SymbolInformation?, + ) { + val pos = positionOf(input, occ) + val isMultiline = pos.startLine != pos.endLine + val width = + if (isMultiline) line.length - pos.startColumn - 1 + else max(1, pos.endColumn - pos.startColumn) + + val isDefinition = (occ.symbolRoles and SymbolRole.Definition_VALUE) > 0 + val role = + when { + syntheticDefinition != null -> "synthetic_definition" + isDefinition -> "definition" + else -> "reference" + } + val indent = + if (pos.startColumn + sourceIndent.length > commentSyntax.length) + " ".repeat(pos.startColumn + sourceIndent.length - commentSyntax.length) + else "" + val caretCharacter = if (syntheticDefinition != null) "_" else "^" + // Scala's `"x" * n` returns "" for n <= 0; Kotlin's `String.repeat` + // throws on a negative argument. Clamp to 0 to preserve behaviour. + val carets = + if (pos.startColumn == 1) caretCharacter.repeat(max(0, width - 1)) + else caretCharacter.repeat(max(0, width)) + + val symbol = syntheticDefinition?.symbol ?: occ.symbol + + // Fail the tests if the index contains symbols that don't parse as valid SCIP symbols. + ScipSymbol.parseOrThrowExceptionIfInvalid(symbol) + + out.append(commentSyntax) + .append(indent) + .append(carets) + .append(' ') + .append(role) + .append(' ') + .append(symbol) + if (isMultiline) { + out.append(" ${pos.endLine - pos.startLine}:${pos.endColumn}") + } + out.append('\n') + + val info = syntheticDefinition ?: symtab[occ.symbol] + if (info != null && isDefinition) { + val prefix = commentSyntax + " ".repeat(indent.length) + " ".repeat(carets.length) + " " + if (info.displayName.isNotEmpty()) { + out.append(prefix).append("display_name ").append(info.displayName).append('\n') + } + if (info.hasSignatureDocumentation()) { + out.append(prefix) + .append("signature_documentation ") + .append(info.signatureDocumentation.language) + .append(' ') + .append( + info.signatureDocumentation.text + .replace("\n", "\\n") + .replace("\t", "\\t") + ) + .append('\n') + } + if (info.enclosingSymbol.isNotEmpty()) { + out.append(prefix).append("enclosing_symbol ").append(info.enclosingSymbol).append('\n') + } + if (info.kind != SymbolInformation.Kind.UnspecifiedKind) { + out.append(prefix).append("kind ").append(info.kind).append('\n') + } + for (n in 0 until info.documentationCount) { + val documentation = info.getDocumentation(n) + out.append(prefix) + .append("documentation ") + .append(documentation.replace("\n", "\\n").replace("\t", "\\t")) + .append('\n') + } + info.relationshipsList + .sortedBy { it.symbol } + .forEach { relationship -> + out.append(prefix).append("relationship") + if (relationship.isReference) out.append(" is_reference") + if (relationship.isDefinition) out.append(" is_definition") + if (relationship.isImplementation) out.append(" is_implementation") + if (relationship.isTypeDefinition) out.append(" is_type_definition") + out.append(' ').append(relationship.symbol).append('\n') + } + } + } + + /** + * Mirrors Scala's `linesWithSeparators`: each yielded element keeps its + * trailing line separator (if any) so that `\r\n`/`\n` line endings round- + * trip exactly when reassembled. + */ + private fun linesWithSeparators(text: String): List { + val result = mutableListOf() + var start = 0 + var i = 0 + while (i < text.length) { + val c = text[i] + if (c == '\n') { + result += text.substring(start, i + 1) + start = i + 1 + } + i++ + } + if (start < text.length) result += text.substring(start) + return result + } + + /** + * Port of `moped.reporters.Input`'s offset/line bookkeeping. Only the + * pieces needed by [positionOf] are reproduced ([lineToOffset], + * [offsetToLine], [isEmpty]). + */ + private class SourceInput(text: String) { + private val chars: CharArray = text.toCharArray() + + val isEmpty: Boolean = text.isEmpty() + + // Offset of the first character of each line; a trailing sentinel + // (== chars.size) is appended when the text does not end in '\n'. + private val lineIndices: IntArray = run { + val buf = ArrayList() + buf.add(0) + var i = 0 + while (i < chars.size) { + if (chars[i] == '\n') buf.add(i + 1) + i++ + } + if (buf[buf.size - 1] != chars.size) buf.add(chars.size) + buf.toIntArray() + } + + fun lineToOffset(line: Int): Int { + require(line in 0..(lineIndices.size - 1)) { + "$line is not a valid line number, allowed [0..${lineIndices.size - 1}]" + } + return lineIndices[line] + } + + fun offsetToLine(offset: Int): Int { + require(offset in 0..chars.size) { + "$offset is not a valid offset, allowed [0..${chars.size}]" + } + // File ending in '\n': an offset at EOF is last_line+1:0. + if (offset == chars.size && chars.isNotEmpty() && chars[offset - 1] == '\n') { + return lineIndices.size - 1 + } + var lo = 0 + var hi = lineIndices.size - 1 + while (hi - lo > 1) { + val mid = (hi + lo) / 2 + when { + offset < lineIndices[mid] -> hi = mid + lineIndices[mid] == offset -> return mid + else -> lo = mid + } + } + return lo + } + } +} diff --git a/scip-java/src/main/kotlin/com/sourcegraph/scip_java/ScipSymbol.kt b/scip-java/src/main/kotlin/com/sourcegraph/scip_java/ScipSymbol.kt new file mode 100644 index 000000000..ff816d964 --- /dev/null +++ b/scip-java/src/main/kotlin/com/sourcegraph/scip_java/ScipSymbol.kt @@ -0,0 +1,25 @@ +package com.sourcegraph.scip_java + +import com.sourcegraph.scip.ScipSymbols +import com.sourcegraph.scip_aggregator.SymbolDescriptor + +/** + * Validates that a string is a syntactically valid SCIP symbol. + * + * The snapshot printer uses this to fail fast when an index contains a symbol + * that doesn't parse. The parsed structure isn't needed by any caller, so only + * the throw-on-invalid behaviour is kept. + */ +object ScipSymbol { + fun parseOrThrowExceptionIfInvalid(scipSymbol: String) { + if (scipSymbol.startsWith("local ")) return + val parts = scipSymbol.split(" ", limit = 5) + require(parts.size == 5) { "Invalid scip symbol: $scipSymbol" } + var current = parts[4] + while (true) { + val descriptor = SymbolDescriptor.parseFromSymbol(current) + if (descriptor.owner == ScipSymbols.ROOT_PACKAGE) return + current = descriptor.owner + } + } +} diff --git a/scip-java/src/main/kotlin/com/sourcegraph/scip_java/buildtools/BazelBuildTool.kt b/scip-java/src/main/kotlin/com/sourcegraph/scip_java/buildtools/BazelBuildTool.kt new file mode 100644 index 000000000..1df5f3911 --- /dev/null +++ b/scip-java/src/main/kotlin/com/sourcegraph/scip_java/buildtools/BazelBuildTool.kt @@ -0,0 +1,229 @@ +package com.sourcegraph.scip_java.buildtools + +import com.sourcegraph.scip_java.Embedded +import com.sourcegraph.scip_java.commands.IndexCommand +import java.io.File +import java.nio.charset.StandardCharsets +import java.nio.file.FileSystems +import java.nio.file.FileVisitResult +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.Paths +import java.nio.file.SimpleFileVisitor +import java.nio.file.StandardOpenOption +import java.nio.file.attribute.BasicFileAttributes + +class BazelBuildTool(index: IndexCommand) : BuildTool("Bazel", index) { + + override val isHidden: Boolean = true + + override fun usedInCurrentDirectory(): Boolean { + val cwd = index.workingDirectory + return listOf("MODULE.bazel", "WORKSPACE", "WORKSPACE.bazel").any { + Files.isRegularFile(cwd.resolve(it)) + } + } + + private val targetSpecs: List + get() = if (index.buildCommand.isEmpty()) listOf("//...") else index.buildCommand + + /** + * Prefer `bazelisk` over `bazel` when both are available: `bazelisk` + * respects the project's `.bazelversion`, while `bazel` may be a pinned + * system version. + */ + private val bazelExecutable: String + get() { + val pathDirs = (System.getenv("PATH") ?: "").split(File.pathSeparator) + return listOf("bazelisk", "bazel").firstOrNull { name -> + pathDirs.any { dir -> Files.isExecutable(Paths.get(dir, name)) } + } ?: "bazel" + } + + override fun generateScip(): Int { + val aspectLabel = generateAspectFile() ?: return 1 + + val scipJavaBinary = index.bazelScipJavaBinary.orEmpty() + if (scipJavaBinary.isEmpty()) { + index.app.error( + "the flag --bazel-scip-java-binary is required to index Bazel codebases. " + + "To fix this problem, run scip-java index again with the flag --scip-java-binary=/path/to/scip-java", + ) + return 1 + } + + val javaHome = index.app.env.environmentVariables.getOrDefault("JAVA_HOME", "") + if (javaHome.isEmpty()) { + index.app.error( + "environment variable JAVA_HOME is not set. " + + "To fix this problem run `export JAVA_HOME=/path/to/java` and run scip-java index again.", + ) + return 1 + } + + val buildCommand = + listOf( + bazelExecutable, + "build", + "--noshow_progress", + // The local strategy is required for now because we write + // SCIP and SCIP files to the provided targetroot directory. + "--spawn_strategy=local", + "--aspects", + "$aspectLabel%scip_java_aspect", + "--output_groups=scip", + "--define=sourceroot=${index.workingDirectory}", + "--define=java_home=$javaHome", + "--define=scip_java_binary=$scipJavaBinary", + "--verbose_failures", + ) + targetSpecs + + val buildExitCode = runBazelBuild(buildCommand) + return if (buildExitCode != 0) buildExitCode + else { + aggregateScipFiles() + 0 + } + } + + private fun runBazelBuild(buildCommand: List): Int { + val sandbox = SandboxCommandExtractor() + index.app.info(buildCommand.joinToString(" ")) + val result = + ProcessRunner.run( + buildCommand, + cwd = index.workingDirectory, + onStdout = { index.app.env.standardOutput.println(it) }, + onStderr = { sandbox.accept(it) }, + ) + if (result.exitCode == 0) return 0 + if (index.bazelAutorunSandboxCommand && sandbox.commandLines().isNotEmpty()) { + index.app.info("Automatically re-running sandbox command to help debug the problem.") + ProcessBuilder("bash", "-c", sandbox.commandLines().joinToString("\n")) + .directory(index.workingDirectory.toFile()) + .inheritIO() + .start() + .waitFor() + } + index.app.error( + """To reproduce the failed Bazel command without scip-java, run the following command: + + bazel build ${targetSpecs.joinToString(" ")} + +To narrow the set of targets to index or pass additional flags to Bazel, include extra arguments index after -- like below: + + scip-java index --bazel-scip-java-binary=... -- //custom/target --sandbox_debug +""", + ) + return result.exitCode + } + + private fun aggregateScipFiles() { + // Final step after running the aspect: aggregate all the generated + // `*.scip` files into a single index.scip file. We do this step + // outside of Bazel because Bazel does not allow actions to generate + // outputs outside of the bazel-out directory. Ideally we would + // implement the aggregation step inside Bazel and only copy the + // resulting index.scip file into the root of the workspace. + Files.deleteIfExists(index.finalOutput) + Files.createDirectories(index.finalOutput.parent) + val scipPattern = FileSystems.getDefault().getPathMatcher("glob:**.scip") + val bazelOut = index.workingDirectory.resolve("bazel-out") + if (!Files.exists(bazelOut)) { + index.app.error( + "doing nothing, the directory $bazelOut does not exist. " + + "The most likely cause for this problem is that there are no Java targets in this Bazel workspace. " + + "Please report an issue to the scip-java issue tracker if the command " + + "`bazel query 'kind(java_*, //...)'` returns non-empty output.", + ) + return + } + val bazelOutLink = Files.readSymbolicLink(bazelOut) + Files.walkFileTree( + bazelOutLink, + object : SimpleFileVisitor() { + override fun visitFile(file: Path, attrs: BasicFileAttributes): FileVisitResult { + if (scipPattern.matches(file)) { + val bytes = Files.readAllBytes(file) + Files.write( + index.finalOutput, + bytes, + StandardOpenOption.APPEND, + StandardOpenOption.CREATE, + ) + } + return super.visitFile(file, attrs) + } + }, + ) + } + + /** + * Processes Bazel's stderr line-by-line to extract the sandbox command. + * When `--sandbox_debug` is enabled Bazel prints the sandbox command that + * failed but it doesn't show the stdout/stderr of that command. This + * extractor captures the command so it can be re-run after the fact. + */ + private inner class SandboxCommandExtractor { + private var isSandboxCommandPrinting = false + private val lines = mutableListOf() + + fun commandLines(): List = lines.toList() + + fun accept(line: String) { + if (!isSandboxCommandPrinting && line.startsWith("ERROR:") && line.contains("error executing command")) { + isSandboxCommandPrinting = true + } else if (isSandboxCommandPrinting && !line.startsWith(" ")) { + isSandboxCommandPrinting = false + } else if (isSandboxCommandPrinting) { + lines += line + } + index.app.env.standardError.println(line) + } + } + + /** + * Reads the scip_java.bzl file from resources and writes it to the + * aspect/scip_java.bzl file inside the Bazel workspace. + */ + private fun generateAspectFile(): String? { + val aspectPath = index.workingDirectory.resolve(index.bazelAspect) + val aspectContents = TemporaryFiles.withDirectory(index) { tmp -> Embedded.bazelAspectFile(tmp) } + if (index.bazelOverwriteAspectFile || !Files.exists(aspectPath)) { + Files.deleteIfExists(aspectPath) + Files.createDirectories(aspectPath.parent) + Files.write(aspectPath, aspectContents.toByteArray(StandardCharsets.UTF_8)) + } else if (Files.isRegularFile(aspectPath)) { + val existingContents = String(Files.readAllBytes(aspectPath)) + if (existingContents != aspectContents) { + index.app.reporter.error( + "Outdated Bazel aspect file found at $aspectPath. To fix this problem, " + + "either run again with the flag --bazel-overwrite-aspect-file or update " + + "the contents of the file to the following:\n\n$aspectContents\n\n", + ) + return null + } + } else if (Files.exists(aspectPath)) { + index.app.reporter.error( + "path $aspectPath already exists and is not a file. To fix this problem, remove this path and try again.", + ) + return null + } + return aspectLabel(aspectPath) + } + + /** Returns the target name (aka. "label") to reference the given path. */ + private fun aspectLabel(aspectPath: Path): String { + var parent: Path? = aspectPath.parent + while (parent != null && parent.startsWith(index.workingDirectory)) { + if (Files.isRegularFile(parent.resolve("BUILD"))) { + val path = index.workingDirectory.relativize(parent) + val name = parent.relativize(aspectPath) + return "//$path:$name" + } + parent = parent.parent + } + Files.createFile(aspectPath.resolveSibling("BUILD")) + return index.workingDirectory.relativize(aspectPath).toString() + } +} diff --git a/scip-java/src/main/kotlin/com/sourcegraph/scip_java/buildtools/BuildTool.kt b/scip-java/src/main/kotlin/com/sourcegraph/scip_java/buildtools/BuildTool.kt new file mode 100644 index 000000000..5e627154f --- /dev/null +++ b/scip-java/src/main/kotlin/com/sourcegraph/scip_java/buildtools/BuildTool.kt @@ -0,0 +1,62 @@ +package com.sourcegraph.scip_java.buildtools + +import com.sourcegraph.scip_java.commands.AggregateRunner +import com.sourcegraph.scip_java.commands.IndexCommand +import java.nio.file.Files +import java.nio.file.Path + +/** A build tool such as Gradle, Maven or Bazel. */ +abstract class BuildTool(val name: String, protected val index: IndexCommand) { + open val isHidden: Boolean = false + + val sourceroot: Path + get() = index.workingDirectory + + abstract fun usedInCurrentDirectory(): Boolean + + abstract fun generateScip(): Int + + companion object { + fun all(index: IndexCommand): List = + // We don't support Bazel for auto-indexing, but if it's + // detected, we should at least give a meaningful error message. + autoOrdered(index) + BazelBuildTool(index) + + fun autoOrdered(index: IndexCommand): List = + // The order in this list is important - first detected build + // tool will be used in `auto` mode. Bazel is missing because + // it isn't supported by auto-indexing. + listOf( + ScipBuildTool(index), + MavenBuildTool(index), + GradleBuildTool(index), + ) + + fun allNames(): String = + all(IndexCommand()).filterNot { it.isHidden }.joinToString(", ") { it.name } + + /** + * After the wrapped build tool finished invoking `javac`/`kotlinc`, + * convert the resulting SCIP targetroot into a SCIP index. + */ + fun generateScipFromTargetroot( + generateScipResult: ProcessResult, + targetroot: Path, + index: IndexCommand, + ): Int { + if (!Files.isDirectory(targetroot)) return generateScipResult.exitCode + if (index.app.reporter.hasErrors()) return index.app.reporter.exitCode() + if (generateScipResult.exitCode != 0) return generateScipResult.exitCode + return AggregateRunner.run( + output = index.finalOutput, + targetroots = listOf(targetroot), + app = index.app, + parallel = index.aggregateParallel, + emitInverseRelationships = index.aggregateEmitInverseRelationships, + allowEmptyIndex = index.aggregateAllowEmptyIndex, + allowExportingGlobalSymbolsFromDirectoryEntries = + index.aggregateAllowExportingGlobalSymbolsFromDirectoryEntries, + ) + } + } +} diff --git a/scip-java/src/main/kotlin/com/sourcegraph/scip_java/buildtools/ClasspathEntry.kt b/scip-java/src/main/kotlin/com/sourcegraph/scip_java/buildtools/ClasspathEntry.kt new file mode 100644 index 000000000..53414d423 --- /dev/null +++ b/scip-java/src/main/kotlin/com/sourcegraph/scip_java/buildtools/ClasspathEntry.kt @@ -0,0 +1,164 @@ +package com.sourcegraph.scip_java.buildtools + +import com.sourcegraph.scip_aggregator.MavenPackage +import java.io.File +import java.nio.charset.StandardCharsets +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.Paths +import javax.xml.parsers.DocumentBuilderFactory +import org.w3c.dom.Element +import org.w3c.dom.Node + +/** + * Represents a single classpath entry on the classpath of a project, used to + * emit SCIP "packageInformation" nodes. A classpath entry can either be a jar + * file or a directory path. + */ +data class ClasspathEntry( + val entry: Path, + val groupId: String, + val artifactId: String, + val version: String, +) { + fun mavenCoordinate(): String = "maven:$groupId:$artifactId:$version" + + fun toPackageInformation(): MavenPackage = MavenPackage(entry, groupId, artifactId, version) + + companion object { + /** + * Parses ClasspathEntry from the SCIP targetroot directory. + * + * Two separate formats are supported: + * - javacopts.txt: line-separated list of Java compiler options. + * - dependencies.txt: line-separated list of dependency information. + * + * Note that the targetroot can contain several files with names + * ending in "dependencies.txt" - for example if they come from a + * multi-module build. + */ + @JvmStatic + fun fromTargetroot(targetroot: Path, sourceroot: Path): List { + val javacopts = targetroot.resolve("javacopts.txt") + return if (Files.isRegularFile(javacopts)) { + fromJavacopts(javacopts, sourceroot) + } else { + discoverDependenciesFromFiles(targetroot) + } + } + + private fun discoverDependenciesFromFiles(targetroot: Path): List { + if (!Files.isDirectory(targetroot)) return emptyList() + return Files.list(targetroot).use { stream -> + stream + .filter { Files.isRegularFile(it) && it.fileName.toString().endsWith("dependencies.txt") } + .toArray() + .map { it as Path } + .flatMap { fromDependencies(it) } + .distinct() + } + } + + private fun fromDependencies(dependencies: Path): List = + Files.readAllLines(dependencies, StandardCharsets.UTF_8).mapNotNull { line -> + val parts = line.split('\t') + if (parts.size == 4) { + val (groupId, artifactId, version, entry) = parts + ClasspathEntry( + entry = Paths.get(entry), + groupId = groupId, + artifactId = artifactId, + version = version, + ) + } else null + } + + private fun fromJavacopts(javacopts: Path, sourceroot: Path): List { + val lines = Files.readAllLines(javacopts, StandardCharsets.UTF_8).map { + it.removePrefix("\"").removeSuffix("\"") + } + val result = mutableListOf() + for (i in 0 until lines.size - 1) { + val key = lines[i] + val value = lines[i + 1] + when (key) { + "-d" -> { + val entry = fromClassesDirectory(Paths.get(value), sourceroot) + if (entry != null) result += entry + } + "-cp", "-classpath" -> { + value.split(File.pathSeparator).forEach { jarPath -> + val entry = fromClasspathJarFile(Paths.get(jarPath)) + if (entry != null) result += entry + } + } + } + } + return result + } + + private tailrec fun fromClassesDirectory( + classesDirectory: Path, + sourceroot: Path, + currentDir: Path? = classesDirectory.parent, + ): ClasspathEntry? { + if (currentDir == null || !currentDir.startsWith(sourceroot)) return null + val pomEntry = fromPomXml(currentDir.resolve("pom.xml"), classesDirectory) + if (pomEntry != null) return pomEntry + return fromClassesDirectory(classesDirectory, sourceroot, currentDir.parent) + } + + /** + * Tries to parse a ClasspathEntry from the POM file that lies next + * to the given jar file. + */ + private fun fromClasspathJarFile(jar: Path): ClasspathEntry? { + val fileName = jar.fileName?.toString() ?: return null + val base = fileName.removeSuffix(".jar") + val pom = jar.resolveSibling("$base.pom") + return fromPomXml(pom, jar) + } + + private fun fromPomXml(pom: Path, classpathEntry: Path): ClasspathEntry? { + if (!Files.isRegularFile(pom)) return null + val factory = DocumentBuilderFactory.newInstance().apply { + isNamespaceAware = false + isValidating = false + // Defensive: disable external entity resolution to avoid + // accidental XXE against attacker-controlled pom.xml files. + setFeature("http://apache.org/xml/features/disallow-doctype-decl", true) + setFeature("http://xml.org/sax/features/external-general-entities", false) + setFeature("http://xml.org/sax/features/external-parameter-entities", false) + isXIncludeAware = false + isExpandEntityReferences = false + } + val document = factory.newDocumentBuilder().parse(Files.newInputStream(pom)) + val root = document.documentElement ?: return null + + fun textOf(parent: Element, key: String): String { + val direct = childElementByName(parent, key) + if (direct != null) return direct.textContent.orEmpty().trim() + val parentTag = childElementByName(parent, "parent") ?: return "" + return childElementByName(parentTag, key)?.textContent.orEmpty().trim() + } + + return ClasspathEntry( + entry = classpathEntry, + groupId = textOf(root, "groupId"), + artifactId = textOf(root, "artifactId"), + version = textOf(root, "version"), + ) + } + + private fun childElementByName(parent: Element, name: String): Element? { + val nodes = parent.childNodes + for (i in 0 until nodes.length) { + val node = nodes.item(i) + if (node.nodeType == Node.ELEMENT_NODE && (node as Element).tagName == name) { + return node + } + } + return null + } + } +} diff --git a/scip-java/src/main/kotlin/com/sourcegraph/scip_java/buildtools/GradleBuildTool.kt b/scip-java/src/main/kotlin/com/sourcegraph/scip_java/buildtools/GradleBuildTool.kt new file mode 100644 index 000000000..d979994ac --- /dev/null +++ b/scip-java/src/main/kotlin/com/sourcegraph/scip_java/buildtools/GradleBuildTool.kt @@ -0,0 +1,132 @@ +package com.sourcegraph.scip_java.buildtools + +import com.sourcegraph.scip_java.Embedded +import com.sourcegraph.scip_java.commands.IndexCommand +import java.nio.charset.StandardCharsets +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.Paths + +class GradleBuildTool(index: IndexCommand) : BuildTool("Gradle", index) { + + override fun usedInCurrentDirectory(): Boolean { + val gradleFiles = listOf("settings.gradle", "gradlew", "build.gradle", "build.gradle.kts") + return gradleFiles.any { name -> Files.isRegularFile(index.workingDirectory.resolve(name)) } + } + + override fun generateScip(): Int { + val gradleResult = runBuild() + if (gradleResult.exitCode == 0) { + reportMissingScipOutput() + } + return generateScipFromTargetroot(gradleResult, targetroot(), index) + } + + /** + * Diagnose the case where Gradle finished successfully but our SCIP + * compiler plugin never produced any `.scip` shards. This used to be + * silently rescued by a `-javaagent` fallback; now it surfaces as a + * clear error pointing at the two known causes. + */ + private fun reportMissingScipOutput() { + if (containsFileWithSuffix(targetroot(), ".scip")) return + if (!containsFileWithSuffix(index.workingDirectory, ".class")) { + // Project produced no compiled JVM output — nothing to index, stay quiet. + return + } + index.app.reporter.error( + """scip-java: Gradle finished successfully but produced no SCIP shards in ${targetroot()}. + +This means our SCIP compiler plugin was not attached to one or more JavaCompile tasks. Two known causes: + + 1. The 'compileOnly' configuration was already resolved before our init script ran. + Check the Gradle output above for warnings of the form: + "scip-java: failed to attach SCIP compiler plugin to project ''" + Workaround: apply the SCIP plugin earlier (e.g. via a settings plugin), + or restructure the build so that 'compileOnly' is not resolved at evaluation time. + + 2. Another Gradle plugin is replacing the compiler arguments we add (rather than appending). + Verify with: ./gradlew compileJava --info | grep -- '-Xplugin:scip' + If '-Xplugin:scip' is missing from the printed javac command, another plugin + is overwriting JavaCompile.options.compilerArgs. +""", + ) + } + + private fun containsFileWithSuffix(root: Path, suffix: String): Boolean { + if (!Files.isDirectory(root)) return false + return try { + Files.find( + root, + Integer.MAX_VALUE, + { p, attrs -> attrs.isRegularFile && p.fileName.toString().endsWith(suffix) }, + ).use { stream -> stream.findFirst().isPresent } + } catch (_: Exception) { + false + } + } + + fun targetroot(): Path = index.finalTargetroot(defaultTargetroot) + + private val defaultTargetroot: Path = Paths.get("build", "scip-targetroot") + + private fun runBuild(): ProcessResult { + val gradleWrapperName = + if (System.getProperty("os.name").lowercase().contains("win")) "gradlew.bat" else "gradlew" + val gradleWrapper = index.workingDirectory.resolve(gradleWrapperName) + val gradleCommand = + if (Files.isRegularFile(gradleWrapper) && Files.isExecutable(gradleWrapper)) + gradleWrapper.toString() + else "gradle" + return TemporaryFiles.withDirectory(index) { tmp -> + runCompileCommand(tmp, gradleCommand) + } + } + + private fun runCompileCommand(tmp: Path, gradleCommand: String): ProcessResult { + val script = initScript(tmp).toString() + val cmd = mutableListOf() + cmd += gradleCommand + cmd += "--no-daemon" + cmd += "--init-script" + cmd += script + cmd += "-Pkotlin.compiler.execution.strategy=in-process" + cmd += "-Dscip.targetroot=${targetroot()}" + cmd += index.finalBuildCommand(listOf("clean", "scipPrintDependencies", "scipCompileAll")) + + targetroot().toFile().deleteRecursively() + val result = index.app.runProcess(cmd, env = mapOf("TERM" to "dumb")) + return Embedded.reportUnexpectedJavacErrors(index.app.reporter, tmp) ?: result + } + + private fun initScript(tmp: Path): Path { + val pluginpath = Embedded.scipJar(tmp) + val gradlePluginPath = Embedded.gradlePluginJar(tmp) + val scipKotlincPath = Embedded.scipKotlincJar(tmp) + val dependenciesPath = targetroot().resolve("dependencies.txt") + Files.deleteIfExists(dependenciesPath) + + val script = + """ + initscript { + dependencies{ + classpath(files("${gradlePluginPath}")) + } + } + + import com.sourcegraph.gradle.scip.ScipGradlePlugin + + allprojects { + project.ext["scipTarget"] = "${targetroot()}" + project.ext["javacPluginJar"] = "$pluginpath" + project.ext["dependenciesOut"] = "$dependenciesPath" + project.ext["scipKotlincJar"] = "$scipKotlincPath" + apply plugin: ScipGradlePlugin + } + """.trimIndent() + + val out = tmp.resolve("init-script.gradle") + Files.write(out, script.toByteArray(StandardCharsets.UTF_8)) + return out + } +} diff --git a/scip-java/src/main/kotlin/com/sourcegraph/scip_java/buildtools/MavenBuildTool.kt b/scip-java/src/main/kotlin/com/sourcegraph/scip_java/buildtools/MavenBuildTool.kt new file mode 100644 index 000000000..51fceef13 --- /dev/null +++ b/scip-java/src/main/kotlin/com/sourcegraph/scip_java/buildtools/MavenBuildTool.kt @@ -0,0 +1,61 @@ +package com.sourcegraph.scip_java.buildtools + +import com.sourcegraph.scip_java.Embedded +import com.sourcegraph.scip_java.commands.IndexCommand +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.Paths + +class MavenBuildTool(index: IndexCommand) : BuildTool("Maven", index) { + + override fun usedInCurrentDirectory(): Boolean = + Files.isRegularFile(index.workingDirectory.resolve("pom.xml")) + + override fun generateScip(): Int { + val result = runBuild() + return generateScipFromTargetroot( + result, + index.finalTargetroot(defaultTargetroot), + index, + ) + } + + private val defaultTargetroot: Path = Paths.get("target", "scip-targetroot") + + private fun runBuild(): ProcessResult = + TemporaryFiles.withDirectory(index) { tmp -> + val mvnw = index.workingDirectory.resolve("mvnw") + val mavenScript = + if (Files.isRegularFile(mvnw) && Files.isExecutable(mvnw)) mvnw.toString() + else "mvn" + val executable = + Embedded.customJavac( + index.workingDirectory, + index.finalTargetroot(defaultTargetroot), + tmp, + ) + val command = mutableListOf() + command += mavenScript + command += "-Dmaven.compiler.useIncrementalCompilation=false" + // NOTE(olafur): the square/javapoet repo sets compilerId to + // 'javac-with-javac', which appears to override the + // '-Dmaven.compiler.executable' setting. Forcing the compilerId to + // 'javac' fixes the issue for this repo. + command += "-Dmaven.compiler.compilerId=javac" + command += "-Dmaven.compiler.executable=$executable" + command += "-Dmaven.compiler.fork=true" + command += index.finalBuildCommand( + listOf( + "--batch-mode", + "clean", + // Default to the "verify" command, as recommended by the official docs + // https://maven.apache.org/guides/introduction/introduction-to-the-lifecycle.html#usual-command-line-calls + "verify", + "-DskipTests", + ), + ) + + val exit = index.app.runProcess(command) + Embedded.reportUnexpectedJavacErrors(index.app.reporter, tmp) ?: exit + } +} diff --git a/scip-java/src/main/kotlin/com/sourcegraph/scip_java/buildtools/ProcessRunner.kt b/scip-java/src/main/kotlin/com/sourcegraph/scip_java/buildtools/ProcessRunner.kt new file mode 100644 index 000000000..5dd7251e6 --- /dev/null +++ b/scip-java/src/main/kotlin/com/sourcegraph/scip_java/buildtools/ProcessRunner.kt @@ -0,0 +1,55 @@ +package com.sourcegraph.scip_java.buildtools + +import java.io.BufferedReader +import java.io.InputStreamReader +import java.nio.charset.StandardCharsets +import java.nio.file.Path +import java.util.concurrent.Executors +import java.util.concurrent.TimeUnit + +/** Outcome of running an external process. */ +data class ProcessResult(val exitCode: Int) + +/** + * Tiny `ProcessBuilder` wrapper that streams stdout/stderr line-by-line to + * caller-provided sinks. + * + * Replaces the os-lib `os.proc(...).call(...)` calls from the previous Scala + * implementation. Stdout and stderr are read on dedicated threads so the + * spawned process cannot deadlock on a full pipe. + */ +object ProcessRunner { + fun run( + command: List, + cwd: Path, + env: Map = emptyMap(), + onStdout: (String) -> Unit = {}, + onStderr: (String) -> Unit = {}, + ): ProcessResult { + val builder = ProcessBuilder(command).directory(cwd.toFile()) + if (env.isNotEmpty()) { + val merged = builder.environment() + for ((k, v) in env) { + merged[k] = v + } + } + val process = builder.start() + val pool = Executors.newFixedThreadPool(2) + try { + val outFuture = pool.submit { drain(process.inputStream, onStdout) } + val errFuture = pool.submit { drain(process.errorStream, onStderr) } + val exit = process.waitFor() + outFuture.get(30, TimeUnit.SECONDS) + errFuture.get(30, TimeUnit.SECONDS) + return ProcessResult(exit) + } finally { + pool.shutdown() + } + } + + private fun drain(input: java.io.InputStream, sink: (String) -> Unit) { + BufferedReader(InputStreamReader(input, StandardCharsets.UTF_8)).useLines { lines -> + for (line in lines) sink(line) + } + } +} diff --git a/scip-java/src/main/kotlin/com/sourcegraph/scip_java/buildtools/ScipBuildTool.kt b/scip-java/src/main/kotlin/com/sourcegraph/scip_java/buildtools/ScipBuildTool.kt new file mode 100644 index 000000000..47e45d088 --- /dev/null +++ b/scip-java/src/main/kotlin/com/sourcegraph/scip_java/buildtools/ScipBuildTool.kt @@ -0,0 +1,487 @@ +package com.sourcegraph.scip_java.buildtools + +import com.sourcegraph.scip_java.BuildInfo +import com.sourcegraph.scip_java.Embedded +import com.sourcegraph.scip_java.commands.IndexCommand +import java.io.File +import java.io.IOException +import java.nio.file.FileSystems +import java.nio.file.FileVisitResult +import java.nio.file.Files +import java.nio.file.NoSuchFileException +import java.nio.file.Path +import java.nio.file.Paths +import java.nio.file.SimpleFileVisitor +import java.nio.file.attribute.BasicFileAttributes +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.JsonElement +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.JsonPrimitive +import kotlinx.serialization.json.boolean +import kotlinx.serialization.json.contentOrNull +import kotlinx.serialization.json.jsonArray +import kotlinx.serialization.json.jsonObject +import kotlinx.serialization.json.jsonPrimitive +import org.jetbrains.kotlin.cli.common.arguments.K2JVMCompilerArguments +import org.jetbrains.kotlin.cli.common.arguments.parseCommandLineArguments +import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity +import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSourceLocation +import org.jetbrains.kotlin.cli.common.messages.MessageCollector +import org.jetbrains.kotlin.cli.common.messages.MessageRenderer +import org.jetbrains.kotlin.cli.jvm.K2JVMCompiler +import org.jetbrains.kotlin.config.Services + +/** + * A custom build tool that is specifically made for scip-java. + * + * The purpose of this build tool is to SCIP-index source code defined by an + * external JSON config (`scip-java.json`) instead of by Maven/Gradle/Bazel. + * Callers are expected to pre-resolve dependencies and pass the resulting + * classpath via the `classpath` field. `scip-java` does not fetch anything + * from the network. + */ +class ScipBuildTool(index: IndexCommand) : BuildTool("SCIP", index) { + + private val javaPattern = FileSystems.getDefault().getPathMatcher("glob:**.java") + private val kotlinPattern = FileSystems.getDefault().getPathMatcher("glob:**.kt") + private val allPatterns = FileSystems.getDefault().getPathMatcher("glob:**.{java,kt}") + private val moduleInfo: Path = Paths.get("module-info.java") + + override fun usedInCurrentDirectory(): Boolean = configFiles().any { Files.isRegularFile(it) } + + override val isHidden: Boolean = true + + override fun generateScip(): Int { + return generateScipFromTargetroot( + runBuild(), + index.finalTargetroot(defaultTargetroot), + index, + ) + } + + private val targetroot: Path + get() = index.finalTargetroot(defaultTargetroot) + + private val defaultTargetroot: Path = Paths.get("target") + + private fun configFiles(): List { + val list = ArrayList() + index.scipConfig?.let { list.add(it) } + ConfigFileNames.forEach { list.add(index.workingDirectory.resolve(it)) } + return list + } + + private fun runBuild(): ProcessResult { + val config = + try { + parseConfig() + } catch (e: Exception) { + index.app.error(e.message ?: e.toString()) + return ProcessResult(1) + } + if (index.cleanup) clean() + return try { + compile(config) + } catch (e: Exception) { + e.printStackTrace(index.app.env.standardOutput) + ProcessResult(1) + } + } + + /** Parses the lsif-java.json file into a Config object. */ + private fun parseConfig(): Config { + val configFile = configFiles().firstOrNull { Files.isRegularFile(it) } + ?: throw IOException( + "no config file found. To fix this problem, create a config file in the path '${configFiles().first()}'", + ) + val raw = Files.readString(configFile) + val element = Json.parseToJsonElement(raw) + if (element !is JsonObject) { + throw IOException("expected JSON object at $configFile, got $element") + } + return Config.fromJson(element) + } + + /** + * Shells out to "javac" (and runs `kotlinc` in-process) to compile the + * sources with the SCIP compiler plugin enabled. + */ + private fun compile(config: Config): ProcessResult { + if (config.dependencies.isNotEmpty()) { + index.app.error( + "scip-java no longer resolves Maven coordinates from the 'dependencies' field " + + "of scip-java.json. Pre-resolve dependencies and populate the 'classpath' " + + "field with absolute JAR paths instead.", + ) + return ProcessResult(1) + } + val tmp = Files.createTempDirectory("scip-java") + Files.createDirectories(targetroot) + val sourceroot = index.workingDirectory + if (!Files.isDirectory(sourceroot)) throw NoSuchFileException(sourceroot.toString()) + val allSourceFiles = collectAllSourceFiles(config, sourceroot) + val javaFiles = allSourceFiles.filter { javaPattern.matches(it) } + val kotlinFiles = allSourceFiles.filter { kotlinPattern.matches(it) } + if (javaFiles.isEmpty() && kotlinFiles.isEmpty()) { + if (config.reportWarningOnEmptyIndex) { + index.app.warning( + "doing nothing, no files matching pattern '$sourceroot/**.{java,kt}'", + ) + } + return ProcessResult(0) + } + + val errors = mutableListOf() + compileJavaFiles(tmp, config, javaFiles)?.let { errors += it } + compileKotlinFiles(config, kotlinFiles, tmp)?.let { errors += it } + + if (index.cleanup) { + tmp.toFile().deleteRecursively() + } + val isScipGenerated = Files.isDirectory(targetroot.resolve("META-INF")) + return if (errors.isNotEmpty() && (index.strictCompilation || !isScipGenerated)) { + for (error in errors) index.app.reporter.error(error) + ProcessResult(1) + } else { + if (errors.isNotEmpty() && isScipGenerated) { + index.app.reporter.info( + "Some SCIP files got generated even if there were compile errors. " + + "In most cases, this means that scip-java managed to index everything except " + + "the locations that had compile errors and you can ignore the compile errors.", + ) + for (error in errors) { + index.app.reporter.info(error.message ?: error.toString()) + } + } + ProcessResult(0) + } + } + + private fun compileKotlinFiles( + config: Config, + allKotlinFiles: List, + tmp: Path, + ): Throwable? { + if (allKotlinFiles.isEmpty()) return null + val sourceroot = index.workingDirectory + val filesPaths = allKotlinFiles.map { it.toString() } + + // The scip-kotlinc compiler plugin is built and shipped together + // with the scip-java CLI as an embedded resource (see Embedded.kt and + // the cli/resourceGenerators task in build.sbt). + val plugin = Embedded.scipKotlincJar(tmp) + + val classpath = + config.classpath + .joinToString(File.pathSeparator) { + index.workingDirectory.resolve(it).toString() + } + + val kargs = K2JVMCompilerArguments() + val args = mutableListOf( + "-nowarn", + "-no-reflect", + "-no-stdlib", + "-Xmulti-platform", + "-Xno-check-actual", + "-verbose:class", + "-opt-in=kotlin.RequiresOptIn", + "-opt-in=kotlin.ExperimentalUnsignedTypes", + "-opt-in=kotlin.ExperimentalStdlibApi", + "-opt-in=kotlin.ExperimentalMultiplatform", + "-opt-in=kotlin.contracts.ExperimentalContracts", + "-Xallow-kotlin-package", + "-Xplugin=$plugin", + "-P", + "plugin:scip-kotlinc:sourceroot=$sourceroot", + "-P", + "plugin:scip-kotlinc:targetroot=$targetroot", + "-classpath", + classpath, + ) + args += filesPaths + + parseCommandLineArguments(args, kargs) + + val exit = + K2JVMCompiler().exec( + object : MessageCollector { + private var sawError = false + + override fun clear() { + sawError = false + } + + override fun hasErrors(): Boolean = sawError + + override fun report( + severity: CompilerMessageSeverity, + message: String, + location: CompilerMessageSourceLocation?, + ) { + if ( + message.endsWith("without a body must be abstract") || + message.endsWith("must have a body") + ) { + // We get these when indexing the stdlib; + // no other solution found yet. + return + } + val rendered = MessageRenderer.PLAIN_FULL_PATHS.render(severity, message, location) + index.app.reporter.debug(rendered) + // Only treat ERROR / EXCEPTION as failures. + // Kotlin 2.2.0's K2JVMCompiler emits LOGGING/INFO/WARNING + // messages during normal compilation; pushing those onto + // `errors` would cause hasErrors to return true. + if (severity.isError) { + sawError = true + } + } + }, + Services.EMPTY, + kargs, + ) + return if (exit.code == 0) null else Exception(exit.toString()) + } + + private fun compileJavaFiles(tmp: Path, config: Config, allJavaFiles: List): Throwable? { + val (moduleInfos, javaFiles) = allJavaFiles.partition { it.endsWith(moduleInfo) } + if (javaFiles.isEmpty()) return null + val sourceroot = index.workingDirectory + val scipJar = Embedded.scipJar(tmp) + val classpath = mutableListOf() + classpath += scipJar.toString() + classpath += config.classpath.map { + index.workingDirectory.resolve(it).toString() + } + val argsfile = targetroot.resolve("javacopts.txt") + val arguments = mutableListOf() + arguments += "-encoding" + arguments += "utf8" + arguments += "-nowarn" + arguments += "-d"; arguments += generatedDir(tmp, "d") + arguments += "-s"; arguments += generatedDir(tmp, "s") + arguments += "-h"; arguments += generatedDir(tmp, "h") + arguments += "-classpath" + arguments += classpath.joinToString(File.pathSeparator) + arguments += "-Xplugin:scip -targetroot:$targetroot -sourceroot:$sourceroot" + if (config.processorpath.isNotEmpty()) { + arguments += "-processorpath" + val processorpath = + listOf(scipJar.toString()) + + config.processorpath + .mapNotNull { guessBazelJar(it, index.workingDirectory)?.toString() } + arguments += processorpath.joinToString(File.pathSeparator) + } + val isIgnoredAnnotationProcessor = + isIgnoredAnnotationProcessor + index.scipIgnoredAnnotationProcessors.toSet() + val processors = config.processors.filterNot { it in isIgnoredAnnotationProcessor } + if (processors.isNotEmpty()) { + arguments += "-processor" + arguments += processors.joinToString(",") + } + arguments += fixJavacOptions(config.javacOptions) + if (config.kind == "jdk" && moduleInfos.isNotEmpty()) { + for (module in moduleInfos) { + arguments += "--module" + arguments += module.parent.fileName.toString() + } + arguments += "--module-source-path" + arguments += sourceroot.toString() + } else { + arguments += javaFiles.map { it.toString() } + } + val quotedArguments = arguments.map { "\"$it\"" } + Files.write(argsfile, quotedArguments) + if (javaFiles.size > 1) { + index.app.reporter.info(String.format("Compiling %,d Java sources", javaFiles.size)) + } + val javac = javacPath(config) + index.app.reporter.info("$ $javac @$argsfile") + val javacModuleOptions = BuildInfo.javacModuleOptions + val jvmOptions = config.jvmOptions.map { "-J$it" } + + val cmd = mutableListOf() + cmd += javac.toString() + cmd += "@$argsfile" + cmd += javacModuleOptions + cmd += jvmOptions + val result = + ProcessRunner.run( + cmd, + cwd = sourceroot, + onStdout = { index.app.reporter.info(it) }, + onStderr = { index.app.reporter.info(it) }, + ) + return if (result.exitCode == 0) null else Exception("javac exited with code ${result.exitCode}") + } + + private fun fixJavacOptions(options: List): List { + val out = mutableListOf() + var i = 0 + while (i < options.size) { + val option = options[i] + when { + option == "--release" -> { + // Skip --release because it's not strictly needed for indexing, + // and it fails the build if -source/-target are also provided. + if (i + 1 < options.size) i++ // drop the value + } + option.startsWith("-Xep") || // ErrorProne flag, which fails the build + option.startsWith("-Xplugin:scip") || // Redundant SCIP + option.startsWith("-XD") || // unsure what this one does + index.scipIgnoredJavacOptionPrefixes.any { option.startsWith(it) } -> { + // skip + } + else -> out += option + } + i++ + } + return out + } + + private fun javacPath(config: Config): Path { + val home = + config.javaHome + ?: index.app.env.environmentVariables["JAVA_HOME"] + ?: throw RuntimeException( + "scip-java requires either the 'javaHome' field in scip-java.json or the " + + "JAVA_HOME environment variable to be set to a JDK installation.", + ) + return Paths.get(home, "bin", "javac") + } + + private fun clean() { + targetroot.toFile().deleteRecursively() + } + + private fun collectAllSourceFiles(dir: Path): List { + val out = ArrayList() + Files.walkFileTree( + dir, + object : SimpleFileVisitor() { + override fun preVisitDirectory(d: Path, attrs: BasicFileAttributes): FileVisitResult = + if (d == targetroot) FileVisitResult.SKIP_SUBTREE else FileVisitResult.CONTINUE + + override fun visitFile(file: Path, attrs: BasicFileAttributes): FileVisitResult { + if (allPatterns.matches(file)) out.add(file) + return FileVisitResult.CONTINUE + } + + override fun visitFileFailed(file: Path, exc: IOException): FileVisitResult = + FileVisitResult.CONTINUE + }, + ) + return out + } + + private fun collectAllSourceFiles(config: Config, dir: Path): List { + return if (config.sourceFiles.isNotEmpty()) { + config.sourceFiles.flatMap { relativePath -> + val path = dir.resolve(relativePath) + when { + Files.isRegularFile(path) && allPatterns.matches(path) -> listOf(path) + Files.isDirectory(path) -> collectAllSourceFiles(path) + else -> emptyList() + } + } + } else { + collectAllSourceFiles(dir) + } + } + + /** + * HACK: We haven't figured out a reliable way to get annotation processor + * jars on the processorpath. The Bazel aspect sometimes says a NAME.jar + * file is on the processorpath when it doesn't exist but processed_NAME.jar + * or header_NAME.jar exists. + */ + private fun guessBazelJar(pathString: String, workingDirectory: Path): Path? { + var path = workingDirectory.resolve(pathString) + if (Files.isRegularFile(path)) return path + + if (!pathString.startsWith("bazel-bin") && !pathString.startsWith("bazel-out")) { + path = workingDirectory.resolve(Paths.get("bazel-bin", pathString)) + if (Files.isRegularFile(path)) return path + } + + val processed = path.resolveSibling("processed_${path.fileName}") + if (Files.isRegularFile(processed)) return processed + + val header = path.resolveSibling("header_${path.fileName}") + if (Files.isRegularFile(header)) return header + + index.app.warning("annotation processor jar does not exist: $path") + return null + } + + private fun generatedDir(tmp: Path, name: String): String = + Files.createDirectory(tmp.resolve(name)).toString() + + /** Configuration parsed from `scip-java.json` / `lsif-java.json`. */ + private data class Config( + val reportWarningOnEmptyIndex: Boolean = true, + val javaHome: String? = null, + val dependencies: List = emptyList(), + val sourceFiles: List = emptyList(), + val classpath: List = emptyList(), + val processorpath: List = emptyList(), + val processors: List = emptyList(), + val javacOptions: List = emptyList(), + val jvmOptions: List = emptyList(), + val kind: String = "", + ) { + companion object { + fun fromJson(obj: JsonObject): Config { + val defaults = Config() + return Config( + reportWarningOnEmptyIndex = obj.boolean("reportWarningOnEmptyIndex", defaults.reportWarningOnEmptyIndex), + javaHome = obj["javaHome"]?.stringOrNull(), + dependencies = obj.stringList("dependencies"), + sourceFiles = obj.stringList("sourceFiles"), + classpath = obj.stringList("classpath"), + processorpath = obj.stringList("processorpath"), + processors = obj.stringList("processors"), + javacOptions = obj.stringList("javacOptions"), + jvmOptions = obj.stringList("jvmOptions"), + kind = obj["kind"]?.stringOrNull() ?: defaults.kind, + ) + } + + private fun JsonObject.boolean(key: String, default: Boolean): Boolean { + val v = this[key] as? JsonPrimitive ?: return default + return v.boolean + } + + private fun JsonObject.stringList(key: String): List { + val v = this[key] ?: return emptyList() + return v.jsonArray.mapNotNull { + when (it) { + is JsonPrimitive -> it.contentOrNull + is JsonObject -> { + // The dependencies field used to accept either a "g:a:v" + // string or an object form. We don't support either now; + // just stringify so the caller can detect it. + it.toString() + } + else -> null + } + } + } + + private fun JsonElement.stringOrNull(): String? = + (this as? JsonPrimitive)?.contentOrNull + } + } + + companion object { + // This file is named "lsif-java.json" instead of "scip-java.json" in + // order to preserve compatibility with "JVM dependencies" repos + // (https://docs.sourcegraph.com/integration/jvm). If we rename to + // "scip-java.json" then the git commit SHAs of these repos change + // and old canonical URLs will become 404 links. + val ConfigFileNames = listOf("scip-java.json", "lsif-java.json") + val isIgnoredAnnotationProcessor: Set = + setOf("org.openjdk.jmh.generators.BenchmarkProcessor") + } +} diff --git a/scip-java/src/main/kotlin/com/sourcegraph/scip_java/buildtools/TemporaryFiles.kt b/scip-java/src/main/kotlin/com/sourcegraph/scip_java/buildtools/TemporaryFiles.kt new file mode 100644 index 000000000..74f30c1dc --- /dev/null +++ b/scip-java/src/main/kotlin/com/sourcegraph/scip_java/buildtools/TemporaryFiles.kt @@ -0,0 +1,22 @@ +package com.sourcegraph.scip_java.buildtools + +import com.sourcegraph.scip_java.commands.IndexCommand +import java.nio.file.Files +import java.nio.file.Path + +object TemporaryFiles { + fun withDirectory(index: IndexCommand, fn: (Path) -> T): T { + val explicit = index.temporaryDirectory + if (explicit != null) { + return fn(explicit) + } + val tmp = Files.createTempDirectory("scip-java") + try { + return fn(tmp) + } finally { + if (index.cleanup) { + tmp.toFile().deleteRecursively() + } + } + } +} diff --git a/scip-java/src/main/kotlin/com/sourcegraph/scip_java/commands/AggregateCommand.kt b/scip-java/src/main/kotlin/com/sourcegraph/scip_java/commands/AggregateCommand.kt new file mode 100644 index 000000000..336c8e459 --- /dev/null +++ b/scip-java/src/main/kotlin/com/sourcegraph/scip_java/commands/AggregateCommand.kt @@ -0,0 +1,87 @@ +package com.sourcegraph.scip_java.commands + +import com.github.ajalt.clikt.core.CliktCommand +import com.github.ajalt.clikt.core.Context +import com.github.ajalt.clikt.core.ProgramResult +import com.github.ajalt.clikt.core.requireObject +import com.github.ajalt.clikt.parameters.arguments.argument +import com.github.ajalt.clikt.parameters.arguments.multiple +import com.github.ajalt.clikt.parameters.options.default +import com.github.ajalt.clikt.parameters.options.flag +import com.github.ajalt.clikt.parameters.options.multiple +import com.github.ajalt.clikt.parameters.options.option +import com.github.ajalt.clikt.parameters.types.path +import com.sourcegraph.scip_java.ScipJavaApp +import java.nio.file.Path +import java.nio.file.Paths + +/** + * `scip-java aggregate`: aggregates per-source SCIP shards into a single + * SCIP index file. + */ +class AggregateCommand : + CliktCommand(name = "aggregate") { + + override fun help(context: Context) = + "Aggregates per-source SCIP shards into a single SCIP index file." + + private val app: ScipJavaApp by requireObject() + + private val output: Path by option( + "--output", + help = "The name of the output file.", + ).path().default(Paths.get("index.scip")) + + private val parallel: Boolean by option( + "--parallel", + "--no-parallel", + help = "Whether to process the SCIP shards in parallel.", + ).flag("--no-parallel", default = true) + + private val emitInverseRelationships: Boolean by option( + "--emit-inverse-relationships", + "--no-emit-inverse-relationships", + help = + "Whether to emit parent->child relationships for 'Find references' and 'Find implementations'. " + + "This flag exists as a workaround for https://github.com/sourcegraph/sourcegraph/issues/50927", + ).flag("--no-emit-inverse-relationships", default = true) + + private val targetrootOption: List by option( + "--targetroot", + help = "Directories that contain SCIP shards.", + ).path().multiple() + + private val targetrootArgs: List by argument().path().multiple() + + private val allowEmptyIndex: Boolean by option( + "--allow-empty-index", + help = + "If true, don't report an error when no documents have been indexed. " + + "The resulting SCIP index will silently be empty instead.", + ).flag() + + private val allowExportingGlobalSymbolsFromDirectoryEntries: Boolean by option( + "--allow-exporting-global-symbols-from-directory-entries", + "--no-allow-exporting-global-symbols-from-directory-entries", + help = + "Determines how to index symbols that are compiled to classfiles inside directories.", + ).flag( + "--no-allow-exporting-global-symbols-from-directory-entries", + default = true, + ) + + override fun run() { + val exit = + AggregateRunner.run( + output = output, + targetroots = targetrootOption + targetrootArgs, + app = app, + parallel = parallel, + emitInverseRelationships = emitInverseRelationships, + allowEmptyIndex = allowEmptyIndex, + allowExportingGlobalSymbolsFromDirectoryEntries = + allowExportingGlobalSymbolsFromDirectoryEntries, + ) + if (exit != 0) throw ProgramResult(exit) + } +} diff --git a/scip-java/src/main/kotlin/com/sourcegraph/scip_java/commands/AggregateRunner.kt b/scip-java/src/main/kotlin/com/sourcegraph/scip_java/commands/AggregateRunner.kt new file mode 100644 index 000000000..2f31ca1da --- /dev/null +++ b/scip-java/src/main/kotlin/com/sourcegraph/scip_java/commands/AggregateRunner.kt @@ -0,0 +1,67 @@ +package com.sourcegraph.scip_java.commands + +import com.sourcegraph.scip_aggregator.ConsoleScipAggregatorReporter +import com.sourcegraph.scip_aggregator.ScipAggregator +import com.sourcegraph.scip_aggregator.ScipAggregatorOptions +import com.sourcegraph.scip_java.BuildInfo +import com.sourcegraph.scip_java.ScipJavaApp +import com.sourcegraph.scip_java.buildtools.ClasspathEntry +import java.nio.file.Path +import org.scip_code.scip.ToolInfo + +/** + * Pure logic backing the `aggregate` command. + * + * `IndexCommand` shells out to a build tool that drops SCIP shards + * under a targetroot directory, then calls into this runner to aggregate + * those shards into a single SCIP index file. The separation makes it + * possible to invoke the runner directly from in-process callers + * (build tools) without going through clikt. + */ +object AggregateRunner { + fun run( + output: Path, + targetroots: List, + app: ScipJavaApp, + parallel: Boolean, + emitInverseRelationships: Boolean, + allowEmptyIndex: Boolean, + allowExportingGlobalSymbolsFromDirectoryEntries: Boolean, + ): Int { + val sourceroot = app.env.workingDirectory.toAbsolutePath() + val absoluteTargetroots = + if (targetroots.isEmpty()) listOf(sourceroot) + else targetroots.map { sourceroot.resolve(it) } + val absoluteOutput = sourceroot.resolve(output) + + val reporter = ConsoleScipAggregatorReporter(app.reporter) + val packages = + absoluteTargetroots + .asSequence() + .flatMap { ClasspathEntry.fromTargetroot(it, sourceroot).asSequence() } + .distinct() + .toList() + + val options = + ScipAggregatorOptions( + absoluteTargetroots, + absoluteOutput, + sourceroot, + reporter, + ToolInfo.newBuilder() + .setName("scip-java") + .setVersion(BuildInfo.version) + .build(), + parallel, + packages.map { it.toPackageInformation() }, + emitInverseRelationships, + allowEmptyIndex, + allowExportingGlobalSymbolsFromDirectoryEntries, + ) + ScipAggregator.run(options) + if (!app.reporter.hasErrors()) { + app.info(options.output.toString()) + } + return app.reporter.exitCode() + } +} diff --git a/scip-java/src/main/kotlin/com/sourcegraph/scip_java/commands/IndexCommand.kt b/scip-java/src/main/kotlin/com/sourcegraph/scip_java/commands/IndexCommand.kt new file mode 100644 index 000000000..9c8e9140c --- /dev/null +++ b/scip-java/src/main/kotlin/com/sourcegraph/scip_java/commands/IndexCommand.kt @@ -0,0 +1,297 @@ +package com.sourcegraph.scip_java.commands + +import com.github.ajalt.clikt.core.CliktCommand +import com.github.ajalt.clikt.core.ProgramResult +import com.github.ajalt.clikt.core.requireObject +import com.github.ajalt.clikt.parameters.arguments.argument +import com.github.ajalt.clikt.parameters.arguments.multiple +import com.github.ajalt.clikt.parameters.options.default +import com.github.ajalt.clikt.parameters.options.flag +import com.github.ajalt.clikt.parameters.options.multiple +import com.github.ajalt.clikt.parameters.options.option +import com.github.ajalt.clikt.parameters.types.path +import com.sourcegraph.scip_java.ScipJavaApp +import com.sourcegraph.scip_java.buildtools.BuildTool +import com.sourcegraph.scip_java.buildtools.ScipBuildTool +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.Paths +import kotlin.math.abs +import kotlin.math.min + +/** + * `scip-java index`: detects a build tool in the current working directory + * and shells out to it (Maven/Gradle/Bazel/scip-java.json) to produce a + * SCIP index in `index.scip`. + * + * The public API surface (in particular all the fields read by the + * per-build-tool implementations) mirrors the old Scala `IndexCommand` + * so the per-build-tool logic ports straight over. + */ +class IndexCommand : CliktCommand(name = "index") { + + override fun help(context: com.github.ajalt.clikt.core.Context): String = + "Automatically generate an SCIP index in the current working directory." + + /** + * Resolved from the clikt context (set by the root command) at run-time. + * When an `IndexCommand` is constructed outside of a clikt parse flow + * (e.g. to enumerate build-tool names), [app] falls back to a fresh app. + */ + private val sharedApp by requireObject() + private var explicitApp: ScipJavaApp? = null + + val app: ScipJavaApp + get() = explicitApp ?: runCatching { sharedApp }.getOrElse { + // No clikt context (e.g. someone constructed `IndexCommand()` to + // enumerate build tool names). Fall back to a fresh app so calls + // that only touch `.name` / `.isHidden` don't crash. + ScipJavaApp().also { explicitApp = it } + } + + val output: Path by option( + "--output", + help = "The path where to generate the SCIP index.", + ).path().default(Paths.get("index.scip")) + + val targetroot: Path? by option( + "--targetroot", + help = + "The directory where to generate SCIP files. Defaults to a build-specific path. " + + "For example, the default value for Gradle is 'build/scip-targetroot' " + + "and for Maven it's 'target/scip-targetroot'.", + ).path() + + val buildTool: String? by option( + "--build-tool", + help = + "Explicitly specify which build tool to use. By default, the build tool is automatically detected. " + + "Use this flag if the automatic build tool detection is not working correctly.", + metavar = "Gradle", + ) + + val cleanup: Boolean by option( + "--cleanup", + "--no-cleanup", + help = "Whether to remove generated temporary files on exit.", + ).flag("--no-cleanup", default = true) + + val temporaryDirectory: Path? by option( + "--temporary-directory", + hidden = true, + ).path() + + val scipIgnoredJavacOptionPrefixes: List by option( + "--scip-ignored-javac-option-prefixes", + help = + "List of Java compiler option prefixes that should be excluded from compilation during indexing. " + + "This flag is only used when indexing via scip-java.json files or Bazel.", + ).multiple() + + val scipIgnoredAnnotationProcessors: List by option( + "--scip-ignored-annotation-processors", + help = + "List of fully qualified annotation processors that should be ignored when indexing a codebase. " + + "This flag is only used when indexing via scip-java.json files or Bazel.", + ).multiple() + + val scipConfig: Path? by option( + "--scip-config", + help = "Path to a scip-java.json file with build configuration. By default, the path scip-java.json is used.", + ).path() + + val bazelScipJavaBinary: String? by option( + "--bazel-scip-java-binary", + help = "Optional path to a `scip-java` binary. Required to index a Bazel codebase.", + ) + + val bazelAspect: Path by option( + "--bazel-aspect", + help = "Relative path to a Bazel aspect file with an aspect named 'scip_java_aspect'.", + ).path().default(Paths.get("aspects/scip_java.bzl")) + + val bazelOverwriteAspectFile: Boolean by option( + "--bazel-overwrite-aspect-file", + help = "If true, overwrites the existing Bazel aspect file (if any).", + ).flag() + + val bazelAutorunSandboxCommand: Boolean by option( + "--bazel-autorun-sandbox-command", + "--no-bazel-autorun-sandbox-command", + help = + "If true, automatically tries to extract the printed out sandbox command " + + "and re-run the command to reveal the underlying problem.", + ).flag("--no-bazel-autorun-sandbox-command", default = true) + + val strictCompilation: Boolean by option( + "--strict-compilation", + hidden = true, + help = "Fail command invocation if compiler produces any errors.", + ).flag() + + val buildCommand: List by argument( + help = "Optional. The build command to use to compile all sources. Defaults to a build-specific command.", + ).multiple() + + // Forwarded options for the embedded `aggregate` step. The previous + // moped CLI exposed the nested `aggregate` command's flags on + // `scip-java index` using the `--aggregate.` naming (e.g. the + // Bazel aspect passes `--aggregate.allow-empty-index`). clikt forbids + // `.` in option names, so these are registered with a `-` separator and the + // dotted form is rewritten to it during argument preprocessing + // (ScipJavaApp.run). Consumed by BuildTool.generateScipFromTargetroot. + val aggregateParallel: Boolean by option( + "--aggregate-parallel", + "--aggregate-no-parallel", + hidden = true, + ).flag("--aggregate-no-parallel", default = true) + + val aggregateEmitInverseRelationships: Boolean by option( + "--aggregate-emit-inverse-relationships", + "--aggregate-no-emit-inverse-relationships", + hidden = true, + ).flag("--aggregate-no-emit-inverse-relationships", default = true) + + val aggregateAllowEmptyIndex: Boolean by option( + "--aggregate-allow-empty-index", + hidden = true, + ).flag() + + val aggregateAllowExportingGlobalSymbolsFromDirectoryEntries: Boolean by option( + "--aggregate-allow-exporting-global-symbols-from-directory-entries", + "--aggregate-no-allow-exporting-global-symbols-from-directory-entries", + hidden = true, + ).flag( + "--aggregate-no-allow-exporting-global-symbols-from-directory-entries", + default = true, + ) + + val workingDirectory: Path + get() = app.env.workingDirectory.toAbsolutePath() + + fun finalTargetroot(default: Path): Path = + workingDirectory.resolve(targetroot ?: default) + + val finalOutput: Path + get() = workingDirectory.resolve(output) + + fun finalBuildCommand(default: List): List = + if (buildCommand.isEmpty()) default else buildCommand + + override fun run() { + val exit = doRun() + if (exit != 0) throw ProgramResult(exit) + } + + fun doRun(): Int { + val allBuildTools = BuildTool.all(this) + val usedBuildTools = allBuildTools.filter { it.usedInCurrentDirectory() } + val matchingBuildTools = + usedBuildTools.filter { tool -> + val name = buildTool + name == null || tool.name.compareTo(name, ignoreCase = true) == 0 + } + + val name = buildTool + if (name != null && name.equals("auto", ignoreCase = true)) { + return runAutoBuildTool() + } + + return when (matchingBuildTools.size) { + 0 -> unknownBuildTool(buildTool, usedBuildTools) + 1 -> matchingBuildTools[0].generateScip() + else -> { + val first = matchingBuildTools[0] + if (first is ScipBuildTool && scipConfig != null) { + first.generateScip() + } else { + val names = matchingBuildTools.joinToString(", ") { it.name } + app.error( + "Multiple build tools detected: $names. " + + "To fix this problem, use the '--build-tool=BUILD_TOOL_NAME' flag to specify which build tool to run.", + ) + 1 + } + } + } + } + + private fun unknownBuildTool(explicit: String?, usedBuildTools: List): Int { + if (explicit != null && usedBuildTools.isNotEmpty()) { + val toFix = + closestCandidate(explicit, usedBuildTools.map { it.name }) + ?.let { "Did you mean --build-tool=$it?" } + ?: "To fix this problem, run again with the --build-tool flag set to one of the detected build tools." + val autoDetected = usedBuildTools.joinToString(", ") { it.name } + app.error( + "Automatically detected the build tool(s) $autoDetected but none of them match the explicitly provided flag '--build-tool=$explicit'. " + + toFix, + ) + } else { + if (Files.isDirectory(workingDirectory)) { + app.error( + "No build tool detected in workspace '$workingDirectory'. " + + "At the moment, the only supported build tools are: ${BuildTool.allNames()}.", + ) + } else { + val cause = + if (Files.exists(workingDirectory)) "Workspace '$workingDirectory' is not a directory" + else "The directory '$workingDirectory' does not exist" + app.error("$cause. To fix this problem, make sure the working directory is an actual directory.") + } + } + return 1 + } + + private fun runAutoBuildTool(): Int { + val usedInOrder = BuildTool.autoOrdered(this).filter { it.usedInCurrentDirectory() } + if (usedInOrder.isEmpty()) { + app.error("Build tool mode set to `auto`, but no supported build tools were detected") + return 1 + } + val first = usedInOrder.first() + val rest = usedInOrder.drop(1) + val restMessage = + if (rest.isEmpty()) "" + else rest.joinToString( + ", ", + prefix = ", other tools that were detected: [", + postfix = "]", + ) { it.name } + app.info("Auto mode: `${first.name}` will be used in this workspace$restMessage") + return first.generateScip() + } + + /** + * Port of `moped.internal.reporters.Levenshtein.closestCandidate`. Returns + * the nearest candidate only when it is "close enough" (normalized edit + * distance ratio below 0.4); otherwise no suggestion is offered. + */ + private fun closestCandidate(query: String, candidates: List): String? { + if (candidates.isEmpty()) return null + val candidate = candidates.minByOrNull { levenshtein(query, it) } ?: return null + val minDifference = abs(query.length - candidate.length) + val difference = levenshtein(candidate, query).toDouble() - minDifference + val ratio = difference / min(query.length, candidate.length) + return if (ratio < 0.4) candidate else null + } + + /** + * Port of `moped.internal.reporters.Levenshtein.distance`: the raw + * Levenshtein edit distance minus the absolute length difference of the + * two strings. + */ + private fun levenshtein(s1: String, s2: String): Int { + val m = s1.length + val n = s2.length + val dist = Array(n + 1) { j -> IntArray(m + 1) { i -> if (j == 0) i else if (i == 0) j else 0 } } + for (j in 1..n) { + for (i in 1..m) { + dist[j][i] = + if (s2[j - 1] == s1[i - 1]) dist[j - 1][i - 1] + else min(dist[j - 1][i], min(dist[j][i - 1], dist[j - 1][i - 1])) + 1 + } + } + return dist[n][m] - abs(m - n) + } +} diff --git a/scip-java/src/main/kotlin/com/sourcegraph/scip_java/commands/SnapshotCommand.kt b/scip-java/src/main/kotlin/com/sourcegraph/scip_java/commands/SnapshotCommand.kt new file mode 100644 index 000000000..93c9cbf46 --- /dev/null +++ b/scip-java/src/main/kotlin/com/sourcegraph/scip_java/commands/SnapshotCommand.kt @@ -0,0 +1,100 @@ +package com.sourcegraph.scip_java.commands + +import com.github.ajalt.clikt.core.CliktCommand +import com.github.ajalt.clikt.core.Context +import com.github.ajalt.clikt.core.ProgramResult +import com.github.ajalt.clikt.core.requireObject +import com.github.ajalt.clikt.parameters.arguments.argument +import com.github.ajalt.clikt.parameters.arguments.multiple +import com.github.ajalt.clikt.parameters.options.default +import com.github.ajalt.clikt.parameters.options.flag +import com.github.ajalt.clikt.parameters.options.option +import com.github.ajalt.clikt.parameters.types.path +import com.sourcegraph.scip_java.ScipJavaApp +import com.sourcegraph.scip_java.ScipPrinters +import java.net.URI +import java.nio.charset.StandardCharsets +import java.nio.file.FileSystems +import java.nio.file.FileVisitResult +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.Paths +import java.nio.file.SimpleFileVisitor +import java.nio.file.attribute.BasicFileAttributes +import org.scip_code.scip.Index + +/** + * `scip-java snapshot`: generates annotated snapshots for each `*.scip` + * file in the given target roots. + */ +class SnapshotCommand : CliktCommand(name = "snapshot") { + + override fun help(context: Context) = + "Generates annotated snapshots for each `*.scip` file in the given target roots." + + private val app: ScipJavaApp by requireObject() + + private val targetroot: List by argument( + help = "List of directories containing SCIP files", + ).path().multiple() + + private val output: Path by option( + "--output", + help = "Output directory for the annotated snapshots", + ).path().default(Paths.get("generated")) + + private val cleanup: Boolean by option( + "--cleanup", + "--no-cleanup", + ).flag("--no-cleanup", default = true) + + override fun run() { + val scipPattern = FileSystems.getDefault().getPathMatcher("glob:**.scip") + if (cleanup) { + output.toFile().deleteRecursively() + } + Files.createDirectories(output) + var foundScipFile = false + for (root in targetroot) { + if (!Files.exists(root)) continue + Files.walkFileTree( + root, + object : SimpleFileVisitor() { + override fun visitFile(file: Path, attrs: BasicFileAttributes): FileVisitResult { + if (scipPattern.matches(file)) { + val index = Index.parseFrom(Files.readAllBytes(file)) + // Per-source SCIP shards under META-INF/scip/ carry no Metadata; + // only the aggregated index does. Skip shards so `scip-java + // snapshot ` doesn't trip over them. + val projectRoot = index.metadata.projectRoot + if (projectRoot.isNotEmpty()) { + foundScipFile = true + val rootUri = URI.create(projectRoot) + for (doc in index.documentsList) { + val sourcepath = Paths.get(rootUri.resolve(doc.relativePath)) + val source = + String(Files.readAllBytes(sourcepath), StandardCharsets.UTF_8) + val document = ScipPrinters.printTextDocument(doc, source) + val snapshotOutput = output.resolve(doc.relativePath) + Files.createDirectories(snapshotOutput.parent) + Files.write( + snapshotOutput, + document.toByteArray(StandardCharsets.UTF_8), + ) + } + } + } + return super.visitFile(file, attrs) + } + }, + ) + } + if (!foundScipFile) { + app.error( + "no SCIP files found. To fix this problem, make sure that one of the directories " + + "in ${targetroot.joinToString(", ")} contains a `*.scip` file.", + ) + throw ProgramResult(1) + } + } +} diff --git a/scip-java/src/main/scala/com/sourcegraph/io/AbsolutePath.scala b/scip-java/src/main/scala/com/sourcegraph/io/AbsolutePath.scala deleted file mode 100644 index 80e771744..000000000 --- a/scip-java/src/main/scala/com/sourcegraph/io/AbsolutePath.scala +++ /dev/null @@ -1,16 +0,0 @@ -package com.sourcegraph.io - -import java.nio.file.Path -import java.nio.file.Paths - -object AbsolutePath { - def systemWorkingDirectory: Path = Paths.get(".").toAbsolutePath.normalize() - def of(path: Path): Path = of(path, systemWorkingDirectory) - def of(path: Path, cwd: Path): Path = - if (path.isAbsolute) - path - else if (cwd.isAbsolute) - cwd.resolve(path) - else - systemWorkingDirectory.resolve(cwd).resolve(path) -} diff --git a/scip-java/src/main/scala/com/sourcegraph/io/AutoDeletedFile.scala b/scip-java/src/main/scala/com/sourcegraph/io/AutoDeletedFile.scala deleted file mode 100644 index 2574e0f77..000000000 --- a/scip-java/src/main/scala/com/sourcegraph/io/AutoDeletedFile.scala +++ /dev/null @@ -1,45 +0,0 @@ -package com.sourcegraph.io - -import java.nio.charset.StandardCharsets -import java.nio.file.Files -import java.nio.file.Path -import java.nio.file.StandardOpenOption - -import scala.util.Using.Releasable - -class AutoDeletedFile private ( - val path: Path, - val oldContent: Option[Array[Byte]] -) - -object AutoDeletedFile { - def fromPath(path: Path, newContent: String): AutoDeletedFile = { - val oldContent = - if (Files.isRegularFile(path)) - Some(Files.readAllBytes(path)) - else - None - Files.createDirectories(path.getParent) - Files.write( - path, - newContent.getBytes(StandardCharsets.UTF_8), - StandardOpenOption.CREATE, - StandardOpenOption.TRUNCATE_EXISTING - ) - new AutoDeletedFile(path, oldContent) - } - implicit val releasableAutoDeletedFile: Releasable[AutoDeletedFile] = { - file => - file.oldContent match { - case Some(oldBytes) => - Files.write( - file.path, - oldBytes, - StandardOpenOption.CREATE, - StandardOpenOption.TRUNCATE_EXISTING - ) - case None => - Files.deleteIfExists(file.path) - } - } -} diff --git a/scip-java/src/main/scala/com/sourcegraph/io/DeleteVisitor.scala b/scip-java/src/main/scala/com/sourcegraph/io/DeleteVisitor.scala deleted file mode 100644 index 24344652c..000000000 --- a/scip-java/src/main/scala/com/sourcegraph/io/DeleteVisitor.scala +++ /dev/null @@ -1,53 +0,0 @@ -package com.sourcegraph.io - -import java.io.IOException -import java.nio.file.FileVisitResult -import java.nio.file.Files -import java.nio.file.Path -import java.nio.file.SimpleFileVisitor -import java.nio.file.attribute.BasicFileAttributes - -class DeleteVisitor(deleteFile: Path => Boolean = _ => true) - extends SimpleFileVisitor[Path] { - override def preVisitDirectory( - dir: Path, - attrs: BasicFileAttributes - ): FileVisitResult = { - if (!deleteFile(dir)) - FileVisitResult.SKIP_SUBTREE - else - super.preVisitDirectory(dir, attrs) - } - override def visitFile( - file: Path, - attrs: BasicFileAttributes - ): FileVisitResult = { - if (deleteFile(file)) { - Files.deleteIfExists(file) - } - super.visitFile(file, attrs) - } - - override def postVisitDirectory( - dir: Path, - exc: IOException - ): FileVisitResult = { - val ls = Files.list(dir) - try { - if (!ls.iterator().hasNext) { - Files.deleteIfExists(dir) - } - } finally { - ls.close() - } - super.postVisitDirectory(dir, exc) - } - - override def visitFileFailed( - file: Path, - exc: IOException - ): FileVisitResult = { - FileVisitResult.CONTINUE - } - -} diff --git a/scip-java/src/main/scala/com/sourcegraph/scip_aggregator/ConsoleScipAggregatorReporter.scala b/scip-java/src/main/scala/com/sourcegraph/scip_aggregator/ConsoleScipAggregatorReporter.scala deleted file mode 100644 index fe0db55db..000000000 --- a/scip-java/src/main/scala/com/sourcegraph/scip_aggregator/ConsoleScipAggregatorReporter.scala +++ /dev/null @@ -1,43 +0,0 @@ -package com.sourcegraph.scip_aggregator - -import java.io.PrintWriter -import java.nio.file.NoSuchFileException - -import moped.cli.Application -import moped.progressbars.InteractiveProgressBar - -/** - * Console reporter for the aggregate command. - */ -class ConsoleScipAggregatorReporter(app: Application) - extends ScipAggregatorReporter { - - val renderer = new ScipAggregatorProgressRenderer - val progressbar = - new InteractiveProgressBar( - new PrintWriter(app.env.standardError), - renderer, - isDynamicPartEnabled = app.env.isProgressBarEnabled - ) - override def error(e: Throwable): Unit = { - e match { - case _: NoSuchFileException => - app.reporter.error(s"no such file: ${e.getMessage}") - case _ => - e.printStackTrace(app.err) - } - } - - override def hasErrors: Boolean = app.reporter.hasErrors() - override def startProcessing(taskSize: Int): Unit = { - renderer.totalSize = taskSize - progressbar.start() - } - override def processedOneItem(): Unit = { - renderer.currentSize.incrementAndGet() - } - override def endProcessing(): Unit = { - progressbar.stop() - } - -} diff --git a/scip-java/src/main/scala/com/sourcegraph/scip_aggregator/ScipAggregatorProgressRenderer.scala b/scip-java/src/main/scala/com/sourcegraph/scip_aggregator/ScipAggregatorProgressRenderer.scala deleted file mode 100644 index d93f0a074..000000000 --- a/scip-java/src/main/scala/com/sourcegraph/scip_aggregator/ScipAggregatorProgressRenderer.scala +++ /dev/null @@ -1,28 +0,0 @@ -package com.sourcegraph.scip_aggregator - -import java.util.concurrent.atomic.AtomicInteger - -import moped.progressbars.ProgressRenderer -import moped.progressbars.ProgressStep -import org.typelevel.paiges.Doc - -/** - * Progress bar for the scip-aggregator command. - */ -class ScipAggregatorProgressRenderer() extends ProgressRenderer { - var totalSize = 0 - val currentSize = new AtomicInteger() - override def renderStep(): ProgressStep = { - if (totalSize < 100) - return ProgressStep.empty - - val current = currentSize.get() - val ratio = current.toDouble / totalSize - val progress: Int = (ratio * 10).toInt - val percentage: String = s"${(ratio * 100).toInt}%".padTo(4, ' ') - val bars = ("#" * progress).padTo(10, ' ') - val render = - f"Generating SCIP... [$bars] $percentage $current%,.0f files processed" - ProgressStep(static = Doc.empty, dynamic = Doc.text(render)) - } -} diff --git a/scip-java/src/main/scala/com/sourcegraph/scip_java/Embedded.scala b/scip-java/src/main/scala/com/sourcegraph/scip_java/Embedded.scala deleted file mode 100644 index a58e500f8..000000000 --- a/scip-java/src/main/scala/com/sourcegraph/scip_java/Embedded.scala +++ /dev/null @@ -1,125 +0,0 @@ -package com.sourcegraph.scip_java - -import java.nio.charset.StandardCharsets -import java.nio.file.Files -import java.nio.file.Path -import java.nio.file.StandardCopyOption - -import com.sourcegraph.scip_java.BuildInfo -import moped.reporters.Reporter -import os.CommandResult - -object Embedded { - - def scipJar(tmpDir: Path): Path = copyFile(tmpDir, "scip-plugin.jar") - - def gradlePluginJar(tmpDir: Path): Path = copyFile( - tmpDir, - "gradle-plugin.jar" - ) - - def scipKotlincJar(tmpDir: Path): Path = copyFile(tmpDir, "scip-kotlinc.jar") - - private def javacErrorpath(tmp: Path) = tmp.resolve("errorpath.txt") - - def customJavac(sourceroot: Path, targetroot: Path, tmp: Path): Path = { - val bin = tmp.resolve("bin") - val javac = bin.resolve("javac") - val java = bin.resolve("java") - val pluginpath = Embedded.scipJar(tmp) - val errorpath = javacErrorpath(tmp) - val javacopts = targetroot.resolve("javacopts.txt") - Files.createDirectories(targetroot) - Files.createDirectories(bin) - Files.write( - java, - """#!/usr/bin/env bash - |java "$@" - |""".stripMargin.getBytes(StandardCharsets.UTF_8) - ) - val newJavacopts = tmp.resolve("javac_newarguments") - // --add-exports flags required to access internal javac APIs from our - // SCIP plugin. Always set; Java 11+ is the supported baseline. - val javacModuleOptions = BuildInfo.javacModuleOptions.mkString(" ") - val injectScipArguments = List[String]( - "java", - s"-Dscip.errorpath=$errorpath", - s"-Dscip.pluginpath=$pluginpath", - s"-Dscip.sourceroot=$sourceroot", - s"-Dscip.targetroot=$targetroot", - s"-Dscip.output=$$NEW_JAVAC_OPTS", - s"-Dscip.old-output=$javacopts", - s"-classpath $pluginpath", - "com.sourcegraph.scip_javac.InjectScipOptions", - """"$@"""" - ).mkString(" ") - val script = - s"""#!/usr/bin/env bash - |set -eu - |LAUNCHER_ARGS=() - |NEW_JAVAC_OPTS="$newJavacopts-$$RANDOM" - |for arg in "$$@"; do - | if [[ $$arg == -J* ]]; then - | LAUNCHER_ARGS+=("$$arg") - | fi - |done - |$injectScipArguments - |if [ $${#LAUNCHER_ARGS[@]} -eq 0 ]; then - | javac $javacModuleOptions "@$$NEW_JAVAC_OPTS" - |else - | javac $javacModuleOptions "@$$NEW_JAVAC_OPTS" "$${LAUNCHER_ARGS[@]}" - |fi - |""".stripMargin - Files.write(javac, script.getBytes(StandardCharsets.UTF_8)) - javac.toFile.setExecutable(true) - java.toFile.setExecutable(true) - javac - } - - /** - * The custom javac wrapper reports errors to a specific file if unexpected - * errors happen. The javac wrapper gets invoked by builds tools like - * Gradle/Maven, which hide the actual errors from the script because they - * assume the standard output is from javac. This file is used a side-channel - * to avoid relying on the error reporting from Gradle/Maven. - */ - def reportUnexpectedJavacErrors( - reporter: Reporter, - tmp: Path - ): Option[CommandResult] = { - val errorpath = javacErrorpath(tmp) - if (Files.isRegularFile(errorpath)) { - reporter.error("unexpected javac compile errors") - Files - .readAllLines(errorpath) - .forEach { line => - reporter.error(line) - } - Some(CommandResult(Nil, 1, Nil)) - } else { - None - } - } - - /** - * Returns the string contents of the scip_java.bzl file on disk. - */ - def bazelAspectFile(tmpDir: Path): String = { - // We could in theory load the resource straight into a string but it was - // easier to copy it to a file and read it from there. - val tmpFile = copyFile(tmpDir, "scip-java/scip_java.bzl") - val contents = - new String(Files.readAllBytes(tmpFile), StandardCharsets.UTF_8) - Files.deleteIfExists(tmpFile) - contents - } - - private def copyFile(tmpDir: Path, filename: String): Path = { - val in = this.getClass.getResourceAsStream(s"/$filename") - val out = tmpDir.resolve(filename) - Files.createDirectories(out.getParent()) - try Files.copy(in, out, StandardCopyOption.REPLACE_EXISTING) - finally in.close() - out - } -} diff --git a/scip-java/src/main/scala/com/sourcegraph/scip_java/ScipJava.scala b/scip-java/src/main/scala/com/sourcegraph/scip_java/ScipJava.scala deleted file mode 100644 index 5c550ef85..000000000 --- a/scip-java/src/main/scala/com/sourcegraph/scip_java/ScipJava.scala +++ /dev/null @@ -1,39 +0,0 @@ -package com.sourcegraph.scip_java - -import java.io.PrintStream - -import com.sourcegraph.scip_java.commands.AggregateCommand -import com.sourcegraph.scip_java.commands.IndexCommand -import com.sourcegraph.scip_java.commands.SnapshotCommand -import moped.cli.Application -import moped.cli.CommandParser -import moped.commands.HelpCommand -import moped.commands.VersionCommand -import moped.reporters.Tput - -object ScipJava { - val app: Application = Application.fromName( - binaryName = "scip-java", - BuildInfo.version, - List( - CommandParser[HelpCommand], - CommandParser[VersionCommand], - CommandParser[IndexCommand], - CommandParser[AggregateCommand], - CommandParser[SnapshotCommand] - ) - ) - def main(args: Array[String]): Unit = { - app.runAndExitIfNonZero(args.toList) - } - - def printHelp(out: PrintStream): Unit = { - out.println("```text") - out.println("$ scip-java index --help") - val newApp = app - .withTput(Tput.constant(100)) - .withEnv(app.env.withStandardOutput(out)) - newApp.run(List("index", "--help")) - out.println("```") - } -} diff --git a/scip-java/src/main/scala/com/sourcegraph/scip_java/ScipPrinters.scala b/scip-java/src/main/scala/com/sourcegraph/scip_java/ScipPrinters.scala deleted file mode 100644 index 7bd8b3112..000000000 --- a/scip-java/src/main/scala/com/sourcegraph/scip_java/ScipPrinters.scala +++ /dev/null @@ -1,298 +0,0 @@ -package com.sourcegraph.scip_java - -import scala.collection.mutable -import scala.jdk.CollectionConverters.CollectionHasAsScala -import scala.math.Ordering.Implicits.seqOrdering - -import moped.reporters.Input -import moped.reporters.Position -import org.scip_code.scip.Document -import org.scip_code.scip.Occurrence -import org.scip_code.scip.SymbolInformation -import org.scip_code.scip.SymbolRole - -object ScipPrinters { - - /** - * Indent prefix prepended to each source line so that caret-based indicators - * in snapshot comments can point at arbitrary columns. - */ - val sourceIndent = " " - - // scip-java only indexes Java and Kotlin sources, both of which use `//`. - private val commentSyntax = "//" - - def printTextDocument(doc: Document, text: String): String = { - val out = new mutable.StringBuilder() - val occurrencesByLine = doc - .getOccurrencesList - .asScala - .groupBy(_.getRange(0)) - val symtab = - doc.getSymbolsList.asScala.map(info => info.getSymbol -> info).toMap - - val syntheticDefinitions = - doc - .getSymbolsList - .asScala - .flatMap { info => - info - .getRelationshipsList - .asScala - .collect { - case relationship if relationship.getIsDefinition => - info -> relationship - } - } - .groupBy { case (_, relationship) => - relationship.getSymbol - } - .view - .mapValues( - _.map { case (info, _) => - info - } - ) - .toMap - val input = Input.filename(doc.getRelativePath, text) - - // Collect enclosing ranges from all occurrences, grouped by start/end line. - case class EnclosingRange( - startLine: Int, - startChar: Int, - endLine: Int, - endChar: Int, - symbol: String - ) - val allEnclosingRanges = - doc - .getOccurrencesList - .asScala - .flatMap { occ => - occ.getEnclosingRangeList.asScala.map(_.toInt).toList match { - case List(sl, sc, ec) => - Some(EnclosingRange(sl, sc, sl, ec, occ.getSymbol)) - case List(sl, sc, el, ec) => - Some(EnclosingRange(sl, sc, el, ec, occ.getSymbol)) - case _ => - None - } - } - .toList - val enclosingByStartLine = - allEnclosingRanges - .groupBy(_.startLine) - .view - .mapValues(_.sortBy(_.startChar)) - .toMap - val enclosingByEndLine = - allEnclosingRanges - .groupBy(_.endLine) - .view - .mapValues(_.sortBy(_.endChar)) - .toMap - - text - .linesWithSeparators - .zipWithIndex - .foreach { case (line, i) => - enclosingByStartLine - .getOrElse(i, Nil) - .foreach { er => - out - .append(commentSyntax) - .append(" " * er.startChar) - .append("⌄ enclosing_range_start ") - .append(er.symbol) - .append("\n") - } - out.append(sourceIndent).append(line.replace("\t", " ")) - val occurrences = occurrencesByLine - .getOrElse(i, Nil) - .toSeq - .sortBy(o => - (o.getRangeList.asScala.toList.map(_.toInt), o.getSymbol) - ) - occurrences.foreach { occ => - formatOccurrence(input, out, occ, line, symtab) - if ((occ.getSymbolRoles & SymbolRole.Definition_VALUE) > 0) { - syntheticDefinitions - .getOrElse(occ.getSymbol, Nil) - .foreach { syntheticDefinition => - formatOccurrence( - input, - out, - occ, - line, - symtab, - syntheticDefinition = Some(syntheticDefinition) - ) - } - } - } - enclosingByEndLine - .getOrElse(i, Nil) - .foreach { er => - val indent = math.max(0, er.endChar - 1) - out - .append(commentSyntax) - .append(" " * indent) - .append("⌃ enclosing_range_end ") - .append(er.symbol) - .append("\n") - } - } - out.toString() - } - - private def mopedPosition(input: Input, occ: Occurrence): Position = { - if (occ.getRangeCount == 3) - Position.range( - input, - occ.getRange(0), - occ.getRange(1), - occ.getRange(0), - occ.getRange(2) - ) - else if (occ.getRangeCount == 4) - Position.range( - input, - occ.getRange(0), - occ.getRange(1), - occ.getRange(2), - occ.getRange(3) - ) - else - throw new IllegalArgumentException(s"Invalid range: $occ") - } - - private def formatOccurrence( - input: Input, - out: mutable.StringBuilder, - occ: Occurrence, - line: String, - symtab: Map[String, SymbolInformation], - syntheticDefinition: Option[SymbolInformation] = None - ): Unit = { - val pos = mopedPosition(input, occ) - val isMultiline = pos.startLine != pos.endLine - val width = - if (isMultiline) { - line.length - pos.startColumn - 1 - } else { - math.max(1, pos.endColumn - pos.startColumn) - } - - val isDefinition = - (occ.getSymbolRoles & SymbolRole.Definition.getNumber) > 0 - val role = - if (syntheticDefinition.isDefined) - "synthetic_definition" - else if (isDefinition) - "definition" - else - "reference" - val indent = - if (pos.startColumn + sourceIndent.length > commentSyntax.length) - " " * (pos.startColumn + sourceIndent.length - commentSyntax.length) - else - "" - val caretCharacter = - if (syntheticDefinition.isDefined) - "_" - else - "^" - val carets = - if (pos.startColumn == 1) - caretCharacter * (width - 1) - else - caretCharacter * width - - val symbol = syntheticDefinition.fold(occ.getSymbol)(_.getSymbol) - - // Fail the tests if the index contains symbols that don't parse as valid SCIP symbols. - val _ = ScipSymbol.parseOrThrowExceptionIfInvalid(symbol) - - out - .append(commentSyntax) - .append(indent) - .append(carets) - .append(" ") - .append(role) - .append(" ") - .append(symbol) - if (isMultiline) { - out.append(s" ${pos.endLine - pos.startLine}:${pos.endColumn}") - } - out.append("\n") - - syntheticDefinition.orElse(symtab.get(occ.getSymbol)) match { - case Some(info) if isDefinition => - val prefix = - commentSyntax + (" " * indent.length) + (" " * carets.length) + " " - if (!info.getDisplayName.isEmpty) { - out - .append(prefix) - .append("display_name ") - .append(info.getDisplayName) - .append("\n") - } - if (info.hasSignatureDocumentation) { - out - .append(prefix) - .append("signature_documentation ") - .append(info.getSignatureDocumentation.getLanguage) - .append(" ") - .append( - info - .getSignatureDocumentation - .getText - .replace("\n", "\\n") - .replace("\t", "\\t") - ) - .append("\n") - } - if (!info.getEnclosingSymbol.isEmpty) { - out - .append(prefix) - .append("enclosing_symbol ") - .append(info.getEnclosingSymbol) - .append("\n") - } - if (info.getKind != SymbolInformation.Kind.UnspecifiedKind) { - out.append(prefix).append("kind ").append(info.getKind).append("\n") - } - 0.until(info.getDocumentationCount) - .foreach { n => - val documentation = info.getDocumentation(n) - out - .append(prefix) - .append("documentation ") - .append(documentation.replace("\n", "\\n").replace("\t", "\\t")) - .append("\n") - } - info - .getRelationshipsList - .asScala - .toList - .sortBy(_.getSymbol) // sort for deterministic order - .foreach { relationship => - out.append(prefix).append("relationship") - if (relationship.getIsReference) { - out.append(" is_reference") - } - if (relationship.getIsDefinition) { - out.append(" is_definition") - } - if (relationship.getIsImplementation) { - out.append(" is_implementation") - } - if (relationship.getIsTypeDefinition) { - out.append(" is_type_definition") - } - out.append(" ").append(relationship.getSymbol).append("\n") - } - case _ => - } - } -} diff --git a/scip-java/src/main/scala/com/sourcegraph/scip_java/ScipSymbol.scala b/scip-java/src/main/scala/com/sourcegraph/scip_java/ScipSymbol.scala deleted file mode 100644 index d31bc4527..000000000 --- a/scip-java/src/main/scala/com/sourcegraph/scip_java/ScipSymbol.scala +++ /dev/null @@ -1,53 +0,0 @@ -package com.sourcegraph.scip_java - -import com.sourcegraph.scip.ScipSymbols -import com.sourcegraph.scip_aggregator.SymbolDescriptor - -sealed abstract class ScipSymbol {} -final case class LocalScipSymbol(identifier: String) extends ScipSymbol -final case class GlobalScipSymbol( - scheme: String, - packageManager: String, - packageName: String, - packageVersion: String, - descriptors: List[SymbolDescriptor] -) extends ScipSymbol - -object ScipSymbol { - - def parseOrThrowExceptionIfInvalid(scipSymbol: String): ScipSymbol = { - if (scipSymbol.startsWith("local ")) { - LocalScipSymbol(scipSymbol.stripPrefix("local ")) - } else { - scipSymbol.split(" ", 5) match { - case Array( - scheme, - packageManager, - packageName, - packageVersion, - descriptor - ) => - GlobalScipSymbol( - scheme, - packageManager, - packageName, - packageVersion, - parseDescriptors(descriptor) - ) - case _ => - throw new IllegalArgumentException( - s"Invalid scip symbol: $scipSymbol" - ) - } - } - } - - private def parseDescriptors(scipSymbol: String): List[SymbolDescriptor] = { - val descriptor = SymbolDescriptor.parseFromSymbol(scipSymbol) - if (descriptor.owner == ScipSymbols.ROOT_PACKAGE) - Nil - else - descriptor :: parseDescriptors(descriptor.owner) - } - -} diff --git a/scip-java/src/main/scala/com/sourcegraph/scip_java/buildtools/BazelBuildTool.scala b/scip-java/src/main/scala/com/sourcegraph/scip_java/buildtools/BazelBuildTool.scala deleted file mode 100644 index 4c7661a31..000000000 --- a/scip-java/src/main/scala/com/sourcegraph/scip_java/buildtools/BazelBuildTool.scala +++ /dev/null @@ -1,266 +0,0 @@ -package com.sourcegraph.scip_java.buildtools - -import java.nio.charset.StandardCharsets -import java.nio.file.FileSystems -import java.nio.file.FileVisitResult -import java.nio.file.Files -import java.nio.file.Path -import java.nio.file.Paths -import java.nio.file.SimpleFileVisitor -import java.nio.file.StandardOpenOption -import java.nio.file.attribute.BasicFileAttributes - -import scala.collection.mutable.ListBuffer - -import com.sourcegraph.io.AbsolutePath -import com.sourcegraph.scip_java.Embedded -import com.sourcegraph.scip_java.commands.IndexCommand -import moped.cli.Application -import os.ProcessOutput.Readlines - -class BazelBuildTool(index: IndexCommand) extends BuildTool("Bazel", index) { - - override def isHidden: Boolean = true - - override def usedInCurrentDirectory(): Boolean = { - val cwd = index.workingDirectory - List("MODULE.bazel", "WORKSPACE", "WORKSPACE.bazel").exists(f => - Files.isRegularFile(cwd.resolve(f)) - ) - } - - private def targetSpecs = - if (index.buildCommand.isEmpty) - List("//...") - else - index.buildCommand - - // Prefer `bazelisk` over `bazel` when both are available: bazelisk respects - // the project's `.bazelversion`, while `bazel` may be a pinned system version. - private def bazelExecutable: String = { - val pathDirs = - sys.env.getOrElse("PATH", "").split(java.io.File.pathSeparator).toList - List("bazelisk", "bazel") - .find { name => - pathDirs.exists(dir => Files.isExecutable(Paths.get(dir, name))) - } - .getOrElse("bazel") - } - - override def generateScip(): Int = { - val aspectLabel = this.generateAspectFile().getOrElse("") - if (aspectLabel.isEmpty) { - return 1 - } - - val scipJavaBinary = index.bazelScipJavaBinary.getOrElse("") - if (scipJavaBinary.isEmpty) { - index - .app - .error( - "the flag --bazel-scip-java-binary is required to index Bazel codebases. To fix this problem, run scip-java index again with the flag --scip-java-binary=/path/to/scip-java" - ) - return 1 - } - - val javaHome = index.app.env.environmentVariables.getOrElse("JAVA_HOME", "") - if (javaHome.isEmpty) { - index - .app - .error( - "environment variable JAVA_HOME is not set. " + - "To fix this problem run `export JAVA_HOME=/path/to/java` and run scip-java index again." - ) - return 1 - } - - val buildCommand = - List( - bazelExecutable, - "build", - "--noshow_progress", - // The local strategy is required for now because we write SCIP and SCIP files - // to the provided targetroot directory. - "--spawn_strategy=local", - "--aspects", - s"$aspectLabel%scip_java_aspect", - "--output_groups=scip", - s"--define=sourceroot=${index.workingDirectory}", - s"--define=java_home=$javaHome", - s"--define=scip_java_binary=$scipJavaBinary", - "--verbose_failures" - ) ++ targetSpecs - - val buildExitCode = runBazelBuild(buildCommand) - if (buildExitCode != 0) { - buildExitCode - } else { - aggregateScipFiles() - 0 - } - } - - private def runBazelBuild(buildCommand: List[String]): Int = { - val sandbox = new SandboxCommandExtractor(index.app) - index.app.info(buildCommand.mkString(" ")) - val commandResult = index - .app - .process(buildCommand) - .call(check = false, stderr = Readlines(sandbox)) - if (commandResult.exitCode != 0) { - if (index.bazelAutorunSandboxCommand && sandbox.commandLines().nonEmpty) { - index - .app - .info( - "Automatically re-running sandbox command to help debug the problem." - ) - index - .app - .process("bash", "-c", sandbox.commandLines().mkString("\n")) - .call(check = false, stdout = os.Inherit, stderr = os.Inherit) - } - index - .app - .error( - s"""To reproduce the failed Bazel command without scip-java, run the following command: - | - | bazel build ${targetSpecs.mkString(" ")} - | - |To narrow the set of targets to index or pass additional flags to Bazel, include extra arguments index after -- like below: - | - | scip-java index --bazel-scip-java-binary=... -- //custom/target --sandbox_debug - |""".stripMargin - ) - commandResult.exitCode - } else { - 0 - } - } - - private def aggregateScipFiles(): Unit = { - // Final step after running the aspect: aggregate all the generated `*.scip` files into a single index.scip file. - // We have to do this step outside of Bazel because Bazel does not allow actions to generate outputs outside - // of the bazel-out directory. Ideally, we should be able to implement the aggregation step inside Bazel - // and only copy the resulting index.scip file into the root of the workspace. However, to begin with, we - // walk the bazel-bin/ directory to concatenate the `*.scip` files even if this is not the idiomatic way to - // do it (in general, scanning the bazel-bin/ directory is a big no no). - Files.deleteIfExists(index.finalOutput) - Files.createDirectories(index.finalOutput.getParent) - val scipPattern = FileSystems.getDefault.getPathMatcher("glob:**.scip") - val bazelOut = index.workingDirectory.resolve("bazel-out") - if (!Files.exists(bazelOut)) { - index - .app - .error( - s"doing nothing, the directory $bazelOut does not exist. " + - "The most likely cause for this problem is that there are no Java targets in this Bazel workspace. " + - "Please report an issue to the scip-java issue tracker if the command `bazel query 'kind(java_*, //...)'` returns non-empty output." - ) - } else { - val bazelOutLink = Files.readSymbolicLink(bazelOut) - Files.walkFileTree( - bazelOutLink, - new SimpleFileVisitor[Path] { - override def visitFile( - file: Path, - attrs: BasicFileAttributes - ): FileVisitResult = { - if (scipPattern.matches(file)) { - val bytes = Files.readAllBytes(file) - Files.write( - index.finalOutput, - bytes, - StandardOpenOption.APPEND, - StandardOpenOption.CREATE - ) - } - super.visitFile(file, attrs) - } - } - ) - } - } - - /** - * Callback to `os.ProcessOutput.Readlines` that processes the stderr output - * of a running Bazel process and extract the sandbox command. When the - * --sandbox_debug flag is enabled, Bazel prints out the sandbox command that - * failed but it doesn't show the stdout/stderr of that command. This - * extractor captures the command so that we can automatically re-run the - * command to print out the stdout/stderr. - */ - private class SandboxCommandExtractor(app: Application) - extends (String => Unit) { - private var isSandboxCommandPrinting = false - private val lines = ListBuffer.empty[String] - def commandLines(): List[String] = lines.toList - override def apply(line: String): Unit = { - if ( - !isSandboxCommandPrinting && line.startsWith("ERROR:") && - line.contains("error executing command") - ) { - isSandboxCommandPrinting = true - } else if (isSandboxCommandPrinting && !line.startsWith(" ")) { - isSandboxCommandPrinting = false - } else if (isSandboxCommandPrinting) { - lines += line - } - app.env.standardError.println(line) - } - } - - /** - * Reads the scip_java.bzl file from resources and writes it to the - * aspect/scip_java.bzl file inside the Bazel workspace. - */ - private def generateAspectFile(): Option[String] = { - val aspectPath = AbsolutePath.of(index.bazelAspect, index.workingDirectory) - val aspectContents = - TemporaryFiles.withDirectory(index) { tmp => - Embedded.bazelAspectFile(tmp) - } - if (index.bazelOverwriteAspectFile || !Files.exists(aspectPath)) { - Files.deleteIfExists(aspectPath) - Files.createDirectories(aspectPath.getParent) - Files.write(aspectPath, aspectContents.getBytes(StandardCharsets.UTF_8)) - } else if (Files.isRegularFile(aspectPath)) { - val existingContents = new String(Files.readAllBytes(aspectPath)) - if (existingContents != aspectContents) { - index - .app - .reporter - .error( - s"Outdated Bazel aspect file found at $aspectPath. To fix this problem, either run again with the flag --bazel-overwrite-aspect-file or update the contents of the file to the following:\n\n$aspectContents\n\n" - ) - return None - } - } else if (Files.exists(aspectPath)) { - index - .app - .reporter - .error( - s"path $aspectPath already exists and is not a file. To fix this problem, remove this path and try again." - ) - return None - } - - Some(aspectLabel(aspectPath)) - } - - /** - * Returns the target name (aka. "label") to reference the given path. - */ - private def aspectLabel(aspectPath: Path): String = { - var parent = aspectPath.getParent - while (parent.startsWith(index.workingDirectory)) { - if (Files.isRegularFile(parent.resolve("BUILD"))) { - val path = index.workingDirectory.relativize(parent) - val name = parent.relativize(aspectPath) - return s"//$path:$name" - } - parent = parent.getParent - } - Files.createFile(aspectPath.resolveSibling("BUILD")) - index.workingDirectory.relativize(aspectPath).toString - } -} diff --git a/scip-java/src/main/scala/com/sourcegraph/scip_java/buildtools/BuildTool.scala b/scip-java/src/main/scala/com/sourcegraph/scip_java/buildtools/BuildTool.scala deleted file mode 100644 index e5fbb5017..000000000 --- a/scip-java/src/main/scala/com/sourcegraph/scip_java/buildtools/BuildTool.scala +++ /dev/null @@ -1,65 +0,0 @@ -package com.sourcegraph.scip_java.buildtools - -import java.nio.file.Files -import java.nio.file.Path - -import com.sourcegraph.scip_java.commands.IndexCommand -import os.CommandResult - -/** - * A build tool such as Gradle, Maven or Bazel. - */ -abstract class BuildTool(val name: String, index: IndexCommand) { - def isHidden: Boolean = false - final def sourceroot: Path = index.workingDirectory - def usedInCurrentDirectory(): Boolean - def generateScip(): Int -} - -object BuildTool { - def all(index: IndexCommand): List[BuildTool] = - // We don't support Bazel for auto-indexing, but if it's - // detected, we should at least give a meaningful error message - autoOrdered(index) :+ new BazelBuildTool(index) - - def autoOrdered(index: IndexCommand): List[BuildTool] = List( - // The order in this list is important - - // first detected build tool will be used in `auto` mode - // Bazel is missing because it isn't supported by auto-indexing - - // first as it indicates user's intent to use SCIP auto-indexing - new ScipBuildTool(index), - // Maven first, then Gradle - // To match the order indicated in IntelliJ Java developer survey 2022: - // https://www.jetbrains.com/lp/devecosystem-2022/java/#which-build-systems-do-you-regularly-use-if-any- - new MavenBuildTool(index), - new GradleBuildTool(index) - ) - def allNames: String = all(IndexCommand()) - .filterNot(_.isHidden) - .map(_.name) - .mkString(", ") - - def generateScipFromTargetroot( - generateScipResult: CommandResult, - targetroot: Path, - index: IndexCommand - ): Int = { - if (!Files.isDirectory(targetroot)) { - generateScipResult.exitCode - } else if (index.app.reporter.hasErrors()) { - index.app.reporter.exitCode() - } else if (generateScipResult.exitCode != 0) { - generateScipResult.exitCode - } else { - index - .aggregate - .copy( - output = index.finalOutput, - targetroot = List(targetroot), - app = index.app - ) - .run() - } - } -} diff --git a/scip-java/src/main/scala/com/sourcegraph/scip_java/buildtools/ClasspathEntry.scala b/scip-java/src/main/scala/com/sourcegraph/scip_java/buildtools/ClasspathEntry.scala deleted file mode 100644 index 87d1711fe..000000000 --- a/scip-java/src/main/scala/com/sourcegraph/scip_java/buildtools/ClasspathEntry.scala +++ /dev/null @@ -1,194 +0,0 @@ -package com.sourcegraph.scip_java.buildtools - -import java.io.File -import java.nio.charset.StandardCharsets -import java.nio.file.Files -import java.nio.file.Path -import java.nio.file.Paths - -import scala.annotation.tailrec -import scala.jdk.CollectionConverters._ - -import com.sourcegraph.scip_aggregator.MavenPackage - -/** - * Represents a single classpath entry on the classpath of a project, used to - * emit SCIP "packageInformation" nodes. A classpath entry can either be a jar - * file or a directory path. - */ -case class ClasspathEntry( - entry: Path, - sources: Option[Path], - groupId: String, - artifactId: String, - version: String -) { - def mavenCoordinate: String = s"maven:$groupId:$artifactId:$version" - def toPackageInformation: MavenPackage = - new MavenPackage(entry, groupId, artifactId, version) -} - -object ClasspathEntry { - - /** - * Parses ClasspathEntry from the SCIP targetroot directory. - * - * Two separate formats are supported: - * - * - javacopts.txt: line-separated list of Java compiler options. - * - dependencies.txt: line-separated list of dependency information. - * - * Note that the targetroot can contain several files with names ending in - * "dependencies.txt" - for example if they come from a multi-module build. - * - * @param targetroot - * @return - */ - def fromTargetroot( - targetroot: Path, - sourceroot: Path - ): List[ClasspathEntry] = { - val javacopts = targetroot.resolve("javacopts.txt") - if (Files.isRegularFile(javacopts)) - fromJavacopts(javacopts, sourceroot) - else - discoverDependenciesFromFiles(targetroot) - } - - /** - * Discover all files that end in "dependencies.txt" directly under - * targetroot. There can be many files because we will be writing dependencies - * for multiple projects. - * - * @param targetroot - * @return - * classpath entries read from the discovered files - */ - private def discoverDependenciesFromFiles( - targetroot: Path - ): List[ClasspathEntry] = { - os.list - .stream(os.Path(targetroot)) - .filter(p => os.isFile(p) && p.last.endsWith("dependencies.txt")) - .map(path => fromDependencies(path.toNIO)) - .toList - .flatten - .distinct - } - - /** - * Parses ClasspathEntry from a "dependencies.txt" file - * - * Every line of the file is a tab separated value with the following columns: - * groupId, artifactId, version, path to the jar file OR classes directory - * path. - */ - private def fromDependencies(dependencies: Path): List[ClasspathEntry] = { - Files - .readAllLines(dependencies, StandardCharsets.UTF_8) - .asScala - .iterator - .map(_.split("\t")) - .collect { case Array(groupId, artifactId, version, entry) => - ClasspathEntry( - entry = Paths.get(entry), - sources = None, - groupId = groupId, - artifactId = artifactId, - version = version - ) - } - .toList - } - - /** - * Parses ClasspathEntry from a "javacopts.txt" file in the targetroot. - * - * Every line of the file represents a Java compiler options, such as - * "-classpath" or "-encoding". - */ - private def fromJavacopts( - javacopts: Path, - sourceroot: Path - ): List[ClasspathEntry] = { - Files - .readAllLines(javacopts, StandardCharsets.UTF_8) - .asScala - .iterator - .map(_.stripPrefix("\"").stripSuffix("\"")) - .sliding(2) - .collect { - case Seq("-d", classesDirectory) => - fromClassesDirectory(Paths.get(classesDirectory), sourceroot).toList - case Seq("-cp" | "-classpath", classpath) => - classpath - .split(File.pathSeparator) - .iterator - .map(Paths.get(_)) - .flatMap(ClasspathEntry.fromClasspathJarFile) - .toList - } - .flatten - .toList - } - - private def fromClassesDirectory( - classesDirectory: Path, - sourceroot: Path - ): Option[ClasspathEntry] = { - @tailrec - def loop(dir: Path): Option[ClasspathEntry] = { - if (dir == null || !dir.startsWith(sourceroot)) - None - else - fromPomXml(dir.resolve("pom.xml"), classesDirectory, None) match { - case None => - loop(dir.getParent()) - case Some(value) => - Some(value) - } - } - loop(classesDirectory.getParent()) - } - - /** - * Tries to parse a ClasspathEntry from the POM file that lies next to the - * given jar file. - */ - private def fromClasspathJarFile(jar: Path): Option[ClasspathEntry] = { - val pom = jar.resolveSibling( - jar.getFileName.toString.stripSuffix(".jar") + ".pom" - ) - val sources = Option( - jar.resolveSibling( - jar.getFileName.toString.stripSuffix(".jar") + ".sources" - ) - ).filter(Files.isRegularFile(_)) - fromPomXml(pom, jar, sources) - } - - private def fromPomXml( - pom: Path, - classpathEntry: Path, - sources: Option[Path] - ): Option[ClasspathEntry] = { - if (Files.isRegularFile(pom)) { - val xml = scala.xml.XML.loadFile(pom.toFile) - def xmlValue(key: String): String = { - val node = xml \ key - if (node.isEmpty) - (xml \ "parent" \ key).text - else - node.text - }.trim - val groupId = xmlValue("groupId") - val artifactId = xmlValue("artifactId") - val version = xmlValue("version") - Some( - ClasspathEntry(classpathEntry, sources, groupId, artifactId, version) - ) - } else { - None - } - } -} diff --git a/scip-java/src/main/scala/com/sourcegraph/scip_java/buildtools/GradleBuildTool.scala b/scip-java/src/main/scala/com/sourcegraph/scip_java/buildtools/GradleBuildTool.scala deleted file mode 100644 index 36e5346ad..000000000 --- a/scip-java/src/main/scala/com/sourcegraph/scip_java/buildtools/GradleBuildTool.scala +++ /dev/null @@ -1,167 +0,0 @@ -package com.sourcegraph.scip_java.buildtools - -import java.nio.charset.StandardCharsets -import java.nio.file._ - -import scala.collection.mutable.ListBuffer -import scala.util.Properties -import scala.util.Try - -import com.sourcegraph.io.DeleteVisitor -import com.sourcegraph.scip_java.Embedded -import com.sourcegraph.scip_java.commands.IndexCommand -import os.CommandResult - -class GradleBuildTool(index: IndexCommand) extends BuildTool("Gradle", index) { - - override def usedInCurrentDirectory(): Boolean = { - val gradleFiles = List( - "settings.gradle", - "gradlew", - "build.gradle", - "build.gradle.kts" - ) - gradleFiles.exists(name => - Files.isRegularFile(index.workingDirectory.resolve(name)) - ) - } - - override def generateScip(): Int = { - val gradleResult = runBuild() - if (gradleResult.exitCode == 0) { - reportMissingScipOutput() - } - BuildTool.generateScipFromTargetroot(gradleResult, targetroot, index) - } - - /** - * Diagnose the case where Gradle finished successfully but our SCIP compiler - * plugin never produced any `.scip` shards. This used to be silently rescued - * by a `-javaagent` fallback; now it surfaces as a clear error pointing at - * the two known causes. - */ - private def reportMissingScipOutput(): Unit = { - if (containsFileWithSuffix(targetroot, ".scip")) - return - if (!containsFileWithSuffix(index.workingDirectory, ".class")) - // Project produced no compiled JVM output — nothing to index, stay quiet. - return - index - .app - .reporter - .error( - s"""scip-java: Gradle finished successfully but produced no SCIP shards in $targetroot. - | - |This means our SCIP compiler plugin was not attached to one or more JavaCompile tasks. Two known causes: - | - | 1. The 'compileOnly' configuration was already resolved before our init script ran. - | Check the Gradle output above for warnings of the form: - | "scip-java: failed to attach SCIP compiler plugin to project ''" - | Workaround: apply the SCIP plugin earlier (e.g. via a settings plugin), - | or restructure the build so that 'compileOnly' is not resolved at evaluation time. - | - | 2. Another Gradle plugin is replacing the compiler arguments we add (rather than appending). - | Verify with: ./gradlew compileJava --info | grep -- '-Xplugin:scip' - | If '-Xplugin:scip' is missing from the printed javac command, another plugin - | is overwriting JavaCompile.options.compilerArgs. - |""".stripMargin - ) - } - - private def containsFileWithSuffix(root: Path, suffix: String): Boolean = - Files.isDirectory(root) && - Try { - val stream = Files.find( - root, - Integer.MAX_VALUE, - (p, attrs) => - attrs.isRegularFile && p.getFileName.toString.endsWith(suffix) - ) - try stream.findFirst().isPresent - finally stream.close() - }.getOrElse(false) - - def targetroot: Path = index.finalTargetroot(defaultTargetroot) - - private def defaultTargetroot: Path = Paths.get("build", "scip-targetroot") - private def runBuild(): CommandResult = { - val gradleWrapper: Path = index - .workingDirectory - .resolve( - if (Properties.isWin) - "gradlew.bat" - else - "gradlew" - ) - val gradleCommand: String = - if ( - Files.isRegularFile(gradleWrapper) && Files.isExecutable(gradleWrapper) - ) - gradleWrapper.toString - else - "gradle" - - TemporaryFiles.withDirectory(index) { tmp => - runCompileCommand(tmp, gradleCommand) - } - } - - private def runCompileCommand( - tmp: Path, - gradleCommand: String - ): CommandResult = { - val script = initScript(tmp).toString - val buildCommand = ListBuffer.empty[String] - buildCommand += gradleCommand - buildCommand += "--no-daemon" - buildCommand += "--init-script" - buildCommand += script - buildCommand += "-Pkotlin.compiler.execution.strategy=in-process" - buildCommand += s"-Dscip.targetroot=$targetroot" - buildCommand ++= - index.finalBuildCommand( - List("clean", "scipPrintDependencies", "scipCompileAll") - ) - - Files.walkFileTree(targetroot, new DeleteVisitor()) - val result = index.process(buildCommand, env = Map("TERM" -> "dumb")) - Embedded - .reportUnexpectedJavacErrors(index.app.reporter, tmp) - .getOrElse(result) - } - - private def scipJavaDependencies = "scipJavaDependencies" - - private def initScript(tmp: Path): Path = { - val pluginpath = Embedded.scipJar(tmp) - val gradlePluginPath = Embedded.gradlePluginJar(tmp) - val scipKotlincPath = Embedded.scipKotlincJar(tmp) - val dependenciesPath = targetroot.resolve("dependencies.txt") - Files.deleteIfExists(dependenciesPath) - - val script = - s""" - | initscript { - | dependencies{ - | classpath(files("${gradlePluginPath}")) - | } - | } - | - | import com.sourcegraph.gradle.scip.ScipGradlePlugin - | - | allprojects { - | project.ext["scipTarget"] = "$targetroot" - | project.ext["javacPluginJar"] = "$pluginpath" - | project.ext["dependenciesOut"] = "$dependenciesPath" - | project.ext["scipKotlincJar"] = "$scipKotlincPath" - | apply plugin: ScipGradlePlugin - | } - """.stripMargin.trim - - Files.write( - tmp.resolve("init-script.gradle"), - script.getBytes(StandardCharsets.UTF_8) - ) - } - -} diff --git a/scip-java/src/main/scala/com/sourcegraph/scip_java/buildtools/MavenBuildTool.scala b/scip-java/src/main/scala/com/sourcegraph/scip_java/buildtools/MavenBuildTool.scala deleted file mode 100644 index 2579d8c70..000000000 --- a/scip-java/src/main/scala/com/sourcegraph/scip_java/buildtools/MavenBuildTool.scala +++ /dev/null @@ -1,73 +0,0 @@ -package com.sourcegraph.scip_java.buildtools - -import java.nio.file.Files -import java.nio.file.Path -import java.nio.file.Paths - -import scala.collection.mutable.ListBuffer - -import com.sourcegraph.scip_java.Embedded -import com.sourcegraph.scip_java.commands.IndexCommand -import os.CommandResult - -class MavenBuildTool(index: IndexCommand) extends BuildTool("Maven", index) { - - override def usedInCurrentDirectory(): Boolean = Files.isRegularFile( - index.workingDirectory.resolve("pom.xml") - ) - - override def generateScip(): Int = { - BuildTool.generateScipFromTargetroot( - runBuild(), - index.finalTargetroot(defaultTargetroot), - index - ) - } - - private def defaultTargetroot: Path = Paths.get("target", "scip-targetroot") - - private def runBuild(): CommandResult = { - TemporaryFiles.withDirectory(index) { tmp => - val mvnw = index.workingDirectory.resolve("mvnw") - val mavenScript = - if (Files.isRegularFile(mvnw) && Files.isExecutable(mvnw)) - mvnw.toString - else { - "mvn" - } - val buildCommand = ListBuffer.empty[String] - val executable = Embedded.customJavac( - index.workingDirectory, - index.finalTargetroot(defaultTargetroot), - tmp - ) - buildCommand ++= - List( - mavenScript, - s"-Dmaven.compiler.useIncrementalCompilation=false", - // NOTE(olafur) the square/javapoet repo sets compilerId to 'javac-with-javac', which appears to - // override the '-Dmaven.compiler.executable' setting.. Forcing the compilerId to 'javac' fixes the - // issue for this repo. - s"-Dmaven.compiler.compilerId=javac", - s"-Dmaven.compiler.executable=$executable", - s"-Dmaven.compiler.fork=true" - ) - buildCommand ++= - index.finalBuildCommand( - List( - s"--batch-mode", - s"clean", - // Default to the "verify" command, as recommended by the official docs - // https://maven.apache.org/guides/introduction/introduction-to-the-lifecycle.html#usual-command-line-calls - "verify", - "-DskipTests" - ) - ) - - val exit = index.process(buildCommand) - Embedded - .reportUnexpectedJavacErrors(index.app.reporter, tmp) - .getOrElse(exit) - } - } -} diff --git a/scip-java/src/main/scala/com/sourcegraph/scip_java/buildtools/ScipBuildTool.scala b/scip-java/src/main/scala/com/sourcegraph/scip_java/buildtools/ScipBuildTool.scala deleted file mode 100644 index 764f07e4c..000000000 --- a/scip-java/src/main/scala/com/sourcegraph/scip_java/buildtools/ScipBuildTool.scala +++ /dev/null @@ -1,619 +0,0 @@ -package com.sourcegraph.scip_java.buildtools - -import java.io.File -import java.io.IOException -import java.nio.file.FileSystems -import java.nio.file.FileVisitResult -import java.nio.file.Files -import java.nio.file.NoSuchFileException -import java.nio.file.Path -import java.nio.file.Paths -import java.nio.file.SimpleFileVisitor -import java.nio.file.attribute.BasicFileAttributes -import java.util - -import scala.collection.mutable.ArrayBuffer -import scala.collection.mutable.ListBuffer -import scala.jdk.CollectionConverters._ -import scala.util.Failure -import scala.util.Success -import scala.util.Try -import scala.util.control.NonFatal - -import com.sourcegraph.io.AbsolutePath -import com.sourcegraph.io.DeleteVisitor -import com.sourcegraph.scip_java.BuildInfo -import com.sourcegraph.scip_java.Embedded -import com.sourcegraph.scip_java.commands.IndexCommand -import moped.json.DecodingContext -import moped.json.ErrorResult -import moped.json.JsonCodec -import moped.json.JsonElement -import moped.json.JsonString -import moped.json.Result -import moped.json.ValueResult -import moped.macros.ClassShape -import moped.parsers.JsonParser -import moped.reporters.Diagnostic -import moped.reporters.Input -import org.jetbrains.kotlin.cli.common.arguments.K2JVMCompilerArguments -import org.jetbrains.kotlin.cli.common.arguments.ParseCommandLineArgumentsKt.parseCommandLineArguments -import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity -import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSourceLocation -import org.jetbrains.kotlin.cli.common.messages.MessageCollector -import org.jetbrains.kotlin.cli.common.messages.MessageRenderer -import org.jetbrains.kotlin.cli.jvm.K2JVMCompiler -import org.jetbrains.kotlin.config.Services -import os.CommandResult -import os.ProcessOutput.Readlines -import os.SubprocessException - -/** - * A custom build tool that is specifically made for scip-java. - * - * The purpose of this build tool is to SCIP index the source code inside - * `*-sources.jar` files of Maven dependencies. Builds are written in a JSON - * file with the following format: - * - * {{{ - * { - * "classpath": ["/abs/path/to/junit-4.13.1.jar"], - * "javaHome": "/path/to/jdk" - * } - * }}} - * - * Callers are expected to pre-resolve dependencies and pass the resulting - * classpath via the `classpath` field. The `javaHome` field (or the `JAVA_HOME` - * environment variable) must point at a JDK installation that provides - * `bin/javac`. scip-java does not fetch anything from the network. - */ -class ScipBuildTool(index: IndexCommand) extends BuildTool("SCIP", index) { - - private val javaPattern = FileSystems - .getDefault - .getPathMatcher("glob:**.java") - private val kotlinPattern = FileSystems - .getDefault - .getPathMatcher("glob:**.kt") - private val allPatterns = FileSystems - .getDefault - .getPathMatcher("glob:**.{java,kt}") - private val moduleInfo = Paths.get("module-info.java") - - override def usedInCurrentDirectory(): Boolean = configFiles.exists(path => - Files.isRegularFile(path) - ) - override def isHidden: Boolean = true - override def generateScip(): Int = { - BuildTool.generateScipFromTargetroot( - runBuild(), - index.finalTargetroot(defaultTargetroot), - index - ) - } - - private def targetroot: Path = index.finalTargetroot(defaultTargetroot) - private def defaultTargetroot: Path = Paths.get("target") - private def configFiles = - index.scipConfig.toList ++ - ScipBuildTool - .ConfigFileNames - .map(name => index.workingDirectory.resolve(name)) - private def runBuild(): CommandResult = { - parsedConfig match { - case ValueResult(value) => - if (index.cleanup) { - clean() - } - try { - compile(value) - } catch { - case NonFatal(e) => - e.printStackTrace(index.app.out) - CommandResult(Nil, 1, Nil) - } - case ErrorResult(error) => - error - .all - .foreach { d => - index.app.error(d.message) - } - CommandResult(Nil, 1, Nil) - } - } - - /** Parses the lsif-java.json file into a Config object. */ - private def parsedConfig: Result[Config] = { - configFiles.find(path => Files.isRegularFile(path)) match { - case None => - ErrorResult( - Diagnostic.error( - s"no config file found. To fix this problem, create a config file in the path '${configFiles - .head}'" - ) - ) - case Some(configFile) => - val input = Input.path(configFile) - JsonParser - .parse(input) - .flatMap(json => - Config - .codec - .decode(DecodingContext(json).withFatalUnknownFields(true)) - ) - } - } - - /** - * Shells out to "javac" to compile the sources with the SCIP compiler plugin - * enabled. - */ - private def compile(config: Config): CommandResult = { - if (config.dependencies.nonEmpty) { - index - .app - .error( - "scip-java no longer resolves Maven coordinates from the 'dependencies' " + - "field of scip-java.json. Pre-resolve dependencies and populate the " + - "'classpath' field with absolute JAR paths instead." - ) - return CommandResult(Nil, 1, Nil) - } - val tmp = Files.createTempDirectory("scip-java") - Files.createDirectories(tmp) - Files.createDirectories(targetroot) - val sourceroot = index.workingDirectory - if (!Files.isDirectory(sourceroot)) { - throw new NoSuchFileException(sourceroot.toString) - } - val allSourceFiles = collectAllSourceFiles(config, sourceroot) - val javaFiles = allSourceFiles.filter(javaPattern.matches) - val kotlinFiles = allSourceFiles.filter(kotlinPattern.matches) - if (javaFiles.isEmpty && kotlinFiles.isEmpty) { - if (config.reportWarningOnEmptyIndex) { - index - .app - .warning( - s"doing nothing, no files matching pattern '$sourceroot/**.{java,kt}'" - ) - } - return CommandResult(Nil, 0, Nil) - } - - val compileAttempts = ListBuffer.empty[Try[Unit]] - compileAttempts += compileJavaFiles(tmp, config, javaFiles) - compileAttempts += compileKotlinFiles(config, kotlinFiles, tmp) - val errors = compileAttempts.collect { case Failure(exception) => - exception - } - - if (index.cleanup) { - Files.walkFileTree(tmp, new DeleteVisitor) - } - val isScipGenerated = Files.isDirectory(targetroot.resolve("META-INF")) - if (errors.nonEmpty && (index.strictCompilation || !isScipGenerated)) { - errors.foreach { error => - index.app.reporter.log(Diagnostic.exception(error)) - } - CommandResult(Nil, 1, Nil) - } else { - if (errors.nonEmpty && isScipGenerated) { - index - .app - .reporter - .info( - "Some SCIP files got generated even if there were compile errors. " + - "In most cases, this means that scip-java managed to index everything " + - "except the locations that had compile errors and you can ignore the compile errors." - ) - errors.foreach { error => - index.app.reporter.info(error.getMessage()) - } - } - CommandResult(Nil, 0, Nil) - } - } - - private def compileKotlinFiles( - config: Config, - allKotlinFiles: List[Path], - tmp: Path - ): Try[Unit] = { - if (allKotlinFiles.isEmpty) - return Success() - val filesPaths = allKotlinFiles.map(_.toString) - - // The scip-kotlinc compiler plugin is now built and shipped together - // with the scip-java CLI as an embedded resource (see Embedded.scala and - // the cli/resourceGenerators task in build.sbt), so we no longer need to - // resolve a separately-published artifact from Maven Central. - val plugin = Embedded.scipKotlincJar(tmp) - - val classpath = config - .classpath - .map(path => - AbsolutePath.of(Paths.get(path), index.workingDirectory).toString - ) - .mkString(File.pathSeparator) - - val kargs: K2JVMCompilerArguments = new K2JVMCompilerArguments() - val args = ListBuffer[String]( - "-nowarn", - "-no-reflect", - "-no-stdlib", - "-Xmulti-platform", - "-Xno-check-actual", - "-verbose:class", - "-opt-in=kotlin.RequiresOptIn", - "-opt-in=kotlin.ExperimentalUnsignedTypes", - "-opt-in=kotlin.ExperimentalStdlibApi", - "-opt-in=kotlin.ExperimentalMultiplatform", - "-opt-in=kotlin.contracts.ExperimentalContracts", - "-Xallow-kotlin-package", - s"-Xplugin=$plugin", - "-P", - s"plugin:scip-kotlinc:sourceroot=$sourceroot", - "-P", - s"plugin:scip-kotlinc:targetroot=$targetroot", - "-classpath", - classpath - ) - - args ++= filesPaths - - parseCommandLineArguments(args.asJava, kargs, false) - - val exit = new K2JVMCompiler().exec( - new MessageCollector { - private val errors = new util.LinkedList[String] - override def clear(): Unit = errors.clear() - - override def hasErrors: Boolean = !errors.isEmpty - - override def report( - compilerMessageSeverity: CompilerMessageSeverity, - s: String, - compilerMessageSourceLocation: CompilerMessageSourceLocation - ): Unit = { - if ( - s.endsWith("without a body must be abstract") || - s.endsWith("must have a body") - ) - return // we get these when indexing the stdlib, no other solution found yet - val msg = MessageRenderer - .PLAIN_FULL_PATHS - .render(compilerMessageSeverity, s, compilerMessageSourceLocation) - index.app.reporter.debug(msg) - // Only treat ERROR / EXCEPTION as failures. Kotlin 2.2.0's - // K2JVMCompiler emits LOGGING messages at startup (e.g. about the - // missing scripting plugin) and INFO/WARNING messages during - // normal compilation; pushing those onto `errors` would cause - // hasErrors to return true, which makes the compiler return - // COMPILATION_ERROR even when nothing is actually wrong. - if (compilerMessageSeverity.isError) { - errors.push(msg) - } - } - }, - Services.EMPTY, - kargs - ) - if (exit.getCode == 0) - Success(()) - else - Failure(new Exception(exit.toString)) - } - - private def compileJavaFiles( - tmp: Path, - config: Config, - allJavaFiles: List[Path] - ): Try[Unit] = { - val (moduleInfos, javaFiles) = allJavaFiles.partition( - _.endsWith(moduleInfo) - ) - if (javaFiles.isEmpty) - return Success(()) - val scipJar = Embedded.scipJar(tmp) - val actualClasspath = ArrayBuffer.empty[String] - actualClasspath += scipJar.toString - actualClasspath ++= - config - .classpath - .map(path => - AbsolutePath.of(Paths.get(path), index.workingDirectory).toString - ) - val argsfile = targetroot.resolve("javacopts.txt") - val arguments = ListBuffer.empty[String] - arguments += "-encoding" - arguments += "utf8" - arguments += "-nowarn" - arguments += "-d" - arguments += generatedDir(tmp, "d") - arguments += "-s" - arguments += generatedDir(tmp, "s") - arguments += "-h" - arguments += generatedDir(tmp, "h") - if (actualClasspath.nonEmpty) { - arguments += "-classpath" - arguments += actualClasspath.mkString(File.pathSeparator) - } - arguments += - s"-Xplugin:scip -targetroot:$targetroot -sourceroot:$sourceroot" - if (config.processorpath.nonEmpty) { - arguments += "-processorpath" - val processorpath = - scipJar.toString :: - config - .processorpath - .flatMap(path => guessBazelJar(path, index.workingDirectory)) - .map(_.toString) - arguments += processorpath.mkString(File.pathSeparator) - } - val isIgnoredAnnotationProcessor = - ScipBuildTool.isIgnoredAnnotationProcessor ++ - index.scipIgnoredAnnotationProcessors - val processors = config.processors.filterNot(isIgnoredAnnotationProcessor) - if (processors.nonEmpty) { - arguments += "-processor" - arguments += processors.mkString(",") - } - arguments ++= fixJavacOptions(config.javacOptions) - if (config.kind == "jdk" && moduleInfos.nonEmpty) { - moduleInfos.foreach { module => - arguments += "--module" - arguments += module.getParent.getFileName.toString - } - arguments += "--module-source-path" - arguments += sourceroot.toString - } else { - arguments ++= javaFiles.map(_.toString) - } - val quotedArguments = arguments.map(a => "\"" + a + "\"") - Files.write(argsfile, quotedArguments.asJava) - if (javaFiles.size > 1) { - index.app.reporter.info(f"Compiling ${javaFiles.size}%,.0f Java sources") - } - val pipe = Readlines(line => { - index.app.reporter.info(line) - }) - val javac = javacPath(config) - index.app.reporter.info(s"$$ $javac @$argsfile") - val javacModuleOptions: Seq[String] = BuildInfo.javacModuleOptions - - val jvmOptions = config.jvmOptions.map("-J" + _) - - val result = os - .proc(javac.toString, s"@$argsfile", javacModuleOptions, jvmOptions) - .call( - stdout = pipe, - stderr = pipe, - cwd = os.Path(sourceroot), - check = false - ) - if (result.exitCode == 0) - Success(()) - else - Failure(SubprocessException(result)) - } - - private def fixJavacOptions(options: List[String]): List[String] = - options match { - case "--release" :: _ :: rest => - // Skip --release because it's not strictly needed for indexing, - // and it fails the build if -source/-target are also provided. - // In real-world testing, there were some builds that defined - // both -source/-target and --release even if javac rejects - // this combination of flags (because --release implies -source/-target). - // It could be that the Java rules have built-in support to automatically - // exclude duplicate -source/-target/--release flags. - fixJavacOptions(rest) - case option :: rest => - val isIgnored = - option.startsWith("-Xep") || // ErrorProne flag, which fails the build - option.startsWith("-Xplugin:scip") || // Redundant SCIP - option.startsWith("-XD") || // unsure what this one does - index // User-provided flag - .scipIgnoredJavacOptionPrefixes - .exists(prefix => option.startsWith(prefix)) - - if (isIgnored) - fixJavacOptions(rest) - else - option :: fixJavacOptions(rest) - case Nil => - Nil - } - - private def javacPath(config: Config): Path = { - val home = config - .javaHome - .orElse(index.app.env.environmentVariables.get("JAVA_HOME")) - .getOrElse { - throw new RuntimeException( - "scip-java requires either the 'javaHome' field in scip-java.json or " + - "the JAVA_HOME environment variable to be set to a JDK installation." - ) - } - Paths.get(home, "bin", "javac") - } - - private def clean(): Unit = { - Files.walkFileTree(targetroot, new DeleteVisitor) - } - - private def collectAllSourceFiles(dir: Path) = { - val buf = List.newBuilder[Path] - Files.walkFileTree( - dir, - new SimpleFileVisitor[Path] { - override def preVisitDirectory( - dir: Path, - attrs: BasicFileAttributes - ): FileVisitResult = { - if (dir == targetroot) - FileVisitResult.SKIP_SUBTREE - else - FileVisitResult.CONTINUE - } - override def visitFile( - file: Path, - attrs: BasicFileAttributes - ): FileVisitResult = { - if (allPatterns.matches(file)) { - buf += file - } - FileVisitResult.CONTINUE - } - override def visitFileFailed( - file: Path, - exc: IOException - ): FileVisitResult = FileVisitResult.CONTINUE - } - ) - buf.result() - } - - /** Recursively collects all Java files in the working directory */ - private def collectAllSourceFiles(config: Config, dir: Path): List[Path] = { - if (config.sourceFiles.nonEmpty) { - config - .sourceFiles - .flatMap { relativePath => - val path = AbsolutePath.of(Paths.get(relativePath), dir) - - if (Files.isRegularFile(path) && allPatterns.matches(path)) - List(path) - else if (Files.isDirectory(path)) - collectAllSourceFiles(path) - else - Nil - } - } else - collectAllSourceFiles(dir) - } - - // HACK(olafurpg): I haven't figured out a reliable way to get annotation processor jars on the processorpath. - // The Bazel aspect sometimes says a NAME.jar file is on the processorpath when it doesn't exist but processed_NAME.jar or header_NAME.jar exists. - // The long-term solution is figuring out how to get the exact same processorpath as Bazel uses for compilation. - private def guessBazelJar( - pathString: String, - workingDirectory: Path - ): Option[Path] = { - var path = AbsolutePath.of(Paths.get(pathString), workingDirectory) - if (Files.isRegularFile(path)) - return Some(path) - - // In some cases, the bazel-out/ prefix is missing from the path that Bazel gives us. - if ( - !pathString.startsWith("bazel-bin") && !pathString.startsWith("bazel-out") - ) { - path = AbsolutePath.of( - Paths.get("bazel-bin", pathString), - workingDirectory - ) - - if (Files.isRegularFile(path)) - return Some(path) - } - - val processed = path.resolveSibling( - "processed_" + path.getFileName.toString - ) - if (Files.isRegularFile(processed)) - return Some(processed) - - val header = path.resolveSibling("header_" + path.getFileName.toString) - if (Files.isRegularFile(header)) - return Some(header) - - index.app.warning("annotation processor jar does not exist: " + path) - None - } - - private def generatedDir(tmp: Path, name: String): String = { - Files.createDirectory(tmp.resolve(name)).toString() - } - - /** - * Gets parsed from "junit:junit:4.13.1" strings inside lsif-java.json files. - */ - private case class Dependency( - groupId: String = "", - artifactId: String = "", - version: String = "" - ) { - def repr: String = s"$groupId:$artifactId:$version" - } - private object Dependency { - def unapply(syntax: String): Option[Dependency] = - syntax match { - case s"$groupId:$artifactId:$version" => - Some( - Dependency( - groupId = groupId, - artifactId = artifactId, - version = version - ) - ) - case _ => - None - } - val automatic = moped.macros.deriveCodec(Dependency()) - implicit lazy val codec = - new JsonCodec[Dependency] { - def decode(context: DecodingContext): Result[Dependency] = - context.json match { - case JsonString(value) => - value match { - case Dependency(x) => - ValueResult(x) - case other => - ErrorResult( - Diagnostic.error( - s"expected format 'GROUP_ID:ARTIFACT_ID:VERSION', obtained $other" - ) - ) - } - case _ => - automatic.decode(context) - } - def encode(value: Dependency): JsonElement = automatic.encode(value) - def shape: ClassShape = automatic.shape - } - } - - /** Gets parsed from lsif-java.json files. */ - private case class Config( - reportWarningOnEmptyIndex: Boolean = true, - javaHome: Option[String] = None, - dependencies: List[Dependency] = Nil, - sourceFiles: List[String] = Nil, - classpath: List[String] = Nil, - bootclasspath: List[String] = Nil, - processorpath: List[String] = Nil, - processors: List[String] = Nil, - javacOptions: List[String] = Nil, - jvmOptions: List[String] = Nil, - jvm: String = "17", - kind: String = "" - ) - private object Config { - implicit lazy val codec = moped.macros.deriveCodec(Config()) - } - -} - -object ScipBuildTool { - // This file is named "lsif-java.json" instead of "scip-java.json" in order to - // preserve compatibility with "JVM dependencies" repos - // (https://docs.sourcegraph.com/integration/jvm). If we rename to - // "scip-java.json" then the git commit SHAs of these repos changes and old - // canonical URLs will become 404 links. The lsif-java.json file is not - // supposed to be written by end-users anyways. It's mostly an implementation - // default for how we support cross-repo navigation with scip-java. - val ConfigFileNames = List("scip-java.json", "lsif-java.json") - val isIgnoredAnnotationProcessor = Set( - "org.openjdk.jmh.generators.BenchmarkProcessor" - ) -} diff --git a/scip-java/src/main/scala/com/sourcegraph/scip_java/buildtools/TemporaryFiles.scala b/scip-java/src/main/scala/com/sourcegraph/scip_java/buildtools/TemporaryFiles.scala deleted file mode 100644 index 57e5dc08a..000000000 --- a/scip-java/src/main/scala/com/sourcegraph/scip_java/buildtools/TemporaryFiles.scala +++ /dev/null @@ -1,24 +0,0 @@ -package com.sourcegraph.scip_java.buildtools - -import java.nio.file.Files -import java.nio.file.Path - -import com.sourcegraph.io.DeleteVisitor -import com.sourcegraph.scip_java.commands.IndexCommand - -object TemporaryFiles { - def withDirectory[T](index: IndexCommand)(fn: Path => T): T = { - index.temporaryDirectory match { - case Some(tmp) => - fn(tmp) - case None => - val tmp = Files.createTempDirectory("scip-java") - try fn(tmp) - finally { - if (index.cleanup) { - Files.walkFileTree(tmp, new DeleteVisitor()) - } - } - } - } -} diff --git a/scip-java/src/main/scala/com/sourcegraph/scip_java/commands/AggregateCommand.scala b/scip-java/src/main/scala/com/sourcegraph/scip_java/commands/AggregateCommand.scala deleted file mode 100644 index a5d694915..000000000 --- a/scip-java/src/main/scala/com/sourcegraph/scip_java/commands/AggregateCommand.scala +++ /dev/null @@ -1,100 +0,0 @@ -package com.sourcegraph.scip_java.commands - -import java.nio.file.Path -import java.nio.file.Paths - -import scala.jdk.CollectionConverters._ - -import com.sourcegraph.io.AbsolutePath -import com.sourcegraph.scip_aggregator.ConsoleScipAggregatorReporter -import com.sourcegraph.scip_aggregator.ScipAggregator -import com.sourcegraph.scip_aggregator.ScipAggregatorOptions -import com.sourcegraph.scip_java.BuildInfo -import com.sourcegraph.scip_java.buildtools.ClasspathEntry -import moped.annotations._ -import moped.cli.Application -import moped.cli.Command -import moped.cli.CommandParser -import org.scip_code.scip.ToolInfo - -@Description("Aggregates per-source SCIP shards into a single SCIP index file.") -@Usage("scip-java aggregate [OPTIONS ...] [POSITIONAL ARGUMENTS ...]") -@ExampleUsage( - "scip-java aggregate --out=myindex.scip my/targetroot1 my/targetroot2" -) -@CommandName("aggregate") -final case class AggregateCommand( - @Description("The name of the output file.") - output: Path = Paths.get("index.scip"), - @Description("Whether to process the SCIP shards in parallel") - parallel: Boolean = true, - @Description( - "Whether to emit parent->child relationships for 'Find references' and 'Find implementations'. " + - "This flag exists as a workaround for the issue https://github.com/sourcegraph/sourcegraph/issues/50927" - ) - emitInverseRelationships: Boolean = true, - @Description("Directories that contain SCIP shards.") - @PositionalArguments() - targetroot: List[Path] = Nil, - @Description( - "If true, don't report an error when no documents have been indexed. " + - "The resulting SCIP index will silently be empty instead." - ) - allowEmptyIndex: Boolean = false, - @Description( - "Determines how to index symbols that are compiled to classfiles inside directories. " + - "If true, symbols inside directory entries are allowed to be publicly visible outside of the generated SCIP index. " + - "If false, symbols inside directory entries are only visible inside the generated SCIP index. " + - "The practical consequences of making this flag false is that cross-index (or cross-repository) navigation does not work between " + - "Maven->Maven or Gradle->Gradle projects because those build tools compile sources to classfiles inside directories." - ) - allowExportingGlobalSymbolsFromDirectoryEntries: Boolean = true, - @Inline() - app: Application = Application.default -) extends Command { - def sourceroot: Path = AbsolutePath.of(app.env.workingDirectory) - def absoluteTargetroots: List[Path] = - if (targetroot.isEmpty) - List(sourceroot) - else - targetroot.map(AbsolutePath.of(_, sourceroot)) - - def run(): Int = { - val reporter = new ConsoleScipAggregatorReporter(app) - val packages = - absoluteTargetroots - .iterator - .flatMap(targetroot => - ClasspathEntry.fromTargetroot(targetroot, sourceroot) - ) - .distinct - .toList - val options = - new ScipAggregatorOptions( - absoluteTargetroots.asJava, - AbsolutePath.of(output, sourceroot), - sourceroot, - reporter, - ToolInfo - .newBuilder() - .setName("scip-java") - .setVersion(BuildInfo.version) - .build(), - parallel, - packages.map(_.toPackageInformation).asJava, - emitInverseRelationships, - allowEmptyIndex, - allowExportingGlobalSymbolsFromDirectoryEntries - ) - ScipAggregator.run(options) - if (!app.reporter.hasErrors()) { - app.info(options.output.toString) - } - app.reporter.exitCode() - } -} - -object AggregateCommand { - val default = AggregateCommand() - implicit val parser = CommandParser.derive(default) -} diff --git a/scip-java/src/main/scala/com/sourcegraph/scip_java/commands/IndexCommand.scala b/scip-java/src/main/scala/com/sourcegraph/scip_java/commands/IndexCommand.scala deleted file mode 100644 index bf3c3df37..000000000 --- a/scip-java/src/main/scala/com/sourcegraph/scip_java/commands/IndexCommand.scala +++ /dev/null @@ -1,245 +0,0 @@ -package com.sourcegraph.scip_java.commands - -import java.nio.file.Files -import java.nio.file.Path -import java.nio.file.Paths - -import com.sourcegraph.io.AbsolutePath -import com.sourcegraph.scip_java.buildtools.BuildTool -import com.sourcegraph.scip_java.buildtools.ScipBuildTool -import fansi.Color -import moped.annotations._ -import moped.cli.Application -import moped.cli.Command -import moped.cli.CommandParser -import moped.internal.reporters.Levenshtein -import os.CommandResult -import os.ProcessOutput -import os.Shellable - -@Description( - "Automatically generate an SCIP index in the current working directory." -) -@Usage("scip-java index [OPTIONS ...] -- [TRAILING_ARGUMENTS ...]") -@ExampleUsage( - """|# Running the `index` command with no flags should work most of the time. - |$ scip-java index""".stripMargin -) -case class IndexCommand( - @Description("The path where to generate the SCIP index.") - output: Path = Paths.get("index.scip"), - @Description( - "The directory where to generate SCIP files. " + - "Defaults to a build-specific path. " + - "For example, the default value for Gradle is 'build/scip-targetroot' and for Maven it's 'target/scip-targetroot'" - ) - targetroot: Option[Path] = None, - @Description( - "Explicitly specify which build tool to use. " + - "By default, the build tool is automatically detected. " + - "Use this flag if the automatic build tool detection is not working correctly." - ) - @ExampleValue("Gradle") - buildTool: Option[String] = None, - @Description("Whether to remove generated temporary files on exit.") - cleanup: Boolean = true, - @Hidden // Hidden because it's only used for testing purposes - temporaryDirectory: Option[Path] = None, - @Section("SCIP Build Tool") - @Description( - "List of Java compiler option prefixes that should be excluded from compilation during indexing. " + - "This flag is only used when indexing via scip-java.json files or Bazel." - ) - scipIgnoredJavacOptionPrefixes: List[String] = Nil, - @Description( - "List of fully qualified annotation processors that should be ignored when indexing a codebase. " + - "This flag is only used when indexing via scip-java.json files or Bazel." - ) - scipIgnoredAnnotationProcessors: List[String] = Nil, - @Description( - "Path to a scip-java.json file with build configuration. By default, the path scip-java.json is used." - ) - scipConfig: Option[Path] = None, - @Section("Bazel") - @Description( - "Optional path to a `scip-java` binary. Required to index a Bazel codebase." - ) - bazelScipJavaBinary: Option[String] = None, - @Description( - "Relative path to a Bazel aspect file with an aspect named 'scip_java_aspect'." - ) - bazelAspect: Path = Paths.get("aspects/scip_java.bzl"), - @Description("If true, overwrites the existing Bazel aspect file (if any)") - bazelOverwriteAspectFile: Boolean = false, - @Description( - "If true, automatically tries to extract the printed out sandbox command and re-run the command to reveal the underlying problem." - ) - bazelAutorunSandboxCommand: Boolean = true, - @Description( - "Optional. The build command to use to compile all sources. " + - "Defaults to a build-specific command. For example, the default command for Maven command is 'clean verify -DskipTests'." + - "To override the default, pass in the build command after a double dash: 'scip-java index -- compile test:compile'" - ) - - @Hidden - @Description("Fail command invocation if compiler produces any errors") - strictCompilation: Boolean = false, - @TrailingArguments() - buildCommand: List[String] = Nil, - @Hidden - aggregate: AggregateCommand = AggregateCommand(), - @Inline - app: Application = Application.default -) extends Command { - - def process( - shellable: Shellable, - env: Map[String, String] = Map.empty - ): CommandResult = { - val commandSyntax = shellable - .value - .map { line => - if (line.contains(" ")) - s"""'$line'""" - else - line - } - .mkString("$ ", " ", "") - app.out.println(Color.DarkGray(commandSyntax)) - app - .process(shellable) - .call( - check = false, - stdout = ProcessOutput.Readlines(line => app.out.println(line)), - stderr = ProcessOutput.Readlines(line => app.err.println(line)), - cwd = workingDirectory, - env = env - ) - } - - def workingDirectory: Path = AbsolutePath.of(app.env.workingDirectory) - def finalTargetroot(default: Path): Path = AbsolutePath.of( - targetroot.getOrElse(default), - workingDirectory - ) - def finalOutput: Path = AbsolutePath.of(output, workingDirectory) - def finalBuildCommand(default: List[String]): List[String] = - if (buildCommand.isEmpty) - default - else - buildCommand - - override def run(): Int = { - val allBuildTools = BuildTool.all(this) - val usedBuildTools = allBuildTools.filter(_.usedInCurrentDirectory()) - val matchingBuildTools = usedBuildTools.filter(tool => - buildTool match { - case Some(explicitName) => - tool.name.compareToIgnoreCase(explicitName) == 0 - case None => - true - } - ) - - buildTool match { - case Some(auto) if auto.equalsIgnoreCase("auto") => - runAutoBuildTool() - case _ => - matchingBuildTools match { - case Nil => - unknownBuildTool(buildTool, usedBuildTools) - case tool :: Nil => - tool.generateScip() - case many @ (first :: rest) => - if (first.isInstanceOf[ScipBuildTool] && scipConfig.isDefined) { - first.generateScip() - } else { - val names = many.map(_.name).mkString(", ") - app.error( - s"Multiple build tools detected: $names. " + - s"To fix this problem, use the '--build-tool=BUILD_TOOL_NAME' flag to specify which build tool to run." - ) - 1 - } - } - } - } - - private def unknownBuildTool( - buildTool: Option[String], - usedBuildTools: List[BuildTool] - ): Int = { - buildTool match { - case Some(explicit) if usedBuildTools.nonEmpty => - val toFix = - Levenshtein.closestCandidate( - explicit, - usedBuildTools.map(_.name) - ) match { - case Some(closest) => - s"Did you mean --build-tool=$closest?" - case None => - "To fix this problem, run again with the --build-tool flag set to one of the detected build tools." - } - val autoDetected = usedBuildTools.map(_.name).mkString(", ") - app.error( - s"Automatically detected the build tool(s) $autoDetected but none of them match the explicitly provided flag '--build-tool=$explicit'. " + - toFix - ) - case _ => - if (Files.isDirectory(workingDirectory)) { - app.error( - s"No build tool detected in workspace '$workingDirectory'. " + - s"At the moment, the only supported build tools are: ${BuildTool - .allNames}." - ) - } else { - val cause = - if (Files.exists(workingDirectory)) { - s"Workspace '$workingDirectory' is not a directory" - } else { - s"The directory '$workingDirectory' does not exist" - } - app.error( - cause + - s". To fix this problem, make sure the working directory is an actual directory." - ) - } - } - - 1 - } - - private def runAutoBuildTool(): Int = { - val usedInOrder = BuildTool - .autoOrdered(this) - .filter(_.usedInCurrentDirectory()) - - usedInOrder match { - case Nil => - app.error( - "Build tool mode set to `auto`, but no supported build tools were detected" - ) - 1 - case first :: rest => - val restMessage = rest - .map(_.name) - .mkString(", other tools that were detected: [", ", ", "]") - - app.info( - s"Auto mode: `${first - .name}` will be used in this workspace${restMessage}" - ) - - first.generateScip() - } - } - -} - -object IndexCommand { - val default: IndexCommand = IndexCommand() - implicit val parser: CommandParser[IndexCommand] = CommandParser.derive( - default - ) -} diff --git a/scip-java/src/main/scala/com/sourcegraph/scip_java/commands/SnapshotCommand.scala b/scip-java/src/main/scala/com/sourcegraph/scip_java/commands/SnapshotCommand.scala deleted file mode 100644 index 4908754e8..000000000 --- a/scip-java/src/main/scala/com/sourcegraph/scip_java/commands/SnapshotCommand.scala +++ /dev/null @@ -1,103 +0,0 @@ -package com.sourcegraph.scip_java.commands - -import java.net.URI -import java.nio.charset.StandardCharsets -import java.nio.file._ -import java.nio.file.attribute.BasicFileAttributes - -import scala.jdk.CollectionConverters._ - -import com.sourcegraph.io.DeleteVisitor -import com.sourcegraph.scip_java.ScipPrinters -import moped.annotations._ -import moped.cli.Application -import moped.cli.Command -import moped.cli.CommandParser -import org.scip_code.scip.Index - -@Description( - "Generates annotated snapshots for each `*.scip` file in the given target roots." -) -@Usage("scip-java snapshot [OPTIONS ...] [POSITIONAL ARGUMENTS ...]") -@ExampleUsage( - "scip-java snapshot --output=generated/ my/targetroo1 my/targetroot2" -) -@CommandName("snapshot") -case class SnapshotCommand( - @PositionalArguments - @Description("List of directories containing SCIP files") - targetroot: List[Path] = Nil, - @Description("Output directory for the annotated snapshots") - output: Path = Paths.get("generated"), - cleanup: Boolean = true, - @Inline() - app: Application = Application.default -) extends Command { - def sourceroot: Path = app.env.workingDirectory - - override def run(): Int = { - val scipPattern = FileSystems.getDefault.getPathMatcher("glob:**.scip") - if (cleanup) { - Files.walkFileTree(output, new DeleteVisitor()) - } - Files.createDirectories(output) - var foundScipFile = false - targetroot.foreach { root => - Files.walkFileTree( - root, - new SimpleFileVisitor[Path] { - override def visitFile( - file: Path, - attrs: BasicFileAttributes - ): FileVisitResult = { - if (scipPattern.matches(file)) { - val index = Index.parseFrom(Files.readAllBytes(file)) - // Per-source SCIP shards under META-INF/scip/ carry no Metadata; - // only the aggregated index does. Skip shards so `scip-java - // snapshot ` doesn't trip over them. - val projectRoot = index.getMetadata.getProjectRoot - if (!projectRoot.isEmpty) { - foundScipFile = true - val root = URI.create(projectRoot) - index - .getDocumentsList - .asScala - .foreach { doc => - val sourcepath = Paths.get( - root.resolve(doc.getRelativePath) - ) - val source = - new String( - Files.readAllBytes(sourcepath), - StandardCharsets.UTF_8 - ) - val document = ScipPrinters.printTextDocument(doc, source) - val snapshotOutput = output.resolve(doc.getRelativePath) - Files.createDirectories(snapshotOutput.getParent) - Files.write( - snapshotOutput, - document.getBytes(StandardCharsets.UTF_8) - ) - } - } - } - super.visitFile(file, attrs) - } - } - ) - } - if (foundScipFile) { - 0 - } else { - app.error( - s"no SCIP files found. To fix this problem, make sure that one of the directories " + - s"in ${targetroot.mkString(", ")} contains a `*.scip` file." - ) - 1 - } - } -} - -object SnapshotCommand { - implicit val parser = CommandParser.derive(SnapshotCommand()) -} diff --git a/tests/buildTools/src/test/scala/tests/BaseBuildToolSuite.scala b/tests/buildTools/src/test/scala/tests/BaseBuildToolSuite.scala index 2bae48d07..60faf285f 100644 --- a/tests/buildTools/src/test/scala/tests/BaseBuildToolSuite.scala +++ b/tests/buildTools/src/test/scala/tests/BaseBuildToolSuite.scala @@ -4,6 +4,7 @@ import java.nio.file.FileSystems import java.nio.file.Files import java.nio.file.Path +import scala.jdk.CollectionConverters._ import scala.util.Properties import scala.meta.internal.io.FileIO @@ -11,14 +12,11 @@ import scala.meta.io.AbsolutePath import com.sourcegraph.scip_java.ScipJava import com.sourcegraph.scip_java.buildtools.ClasspathEntry -import moped.testkit.DeleteVisitor -import moped.testkit.FileLayout -import moped.testkit.MopedSuite import munit.Tag import munit.TestOptions import os.Shellable -abstract class BaseBuildToolSuite extends MopedSuite(ScipJava.app) { +abstract class BaseBuildToolSuite extends ScipJavaSuite(ScipJava.app) { self => override def environmentVariables: Map[String, String] = sys.env @@ -50,7 +48,7 @@ abstract class BaseBuildToolSuite extends MopedSuite(ScipJava.app) { path } override def beforeEach(context: BeforeEach): Unit = { - DeleteVisitor.deleteRecursively(path) + os.remove.all(os.Path(path)) } } @@ -112,7 +110,7 @@ abstract class BaseBuildToolSuite extends MopedSuite(ScipJava.app) { "--targetroot", targetroot.toString ) ++ extraArguments - val exit = app().run(arguments) + val exit = app.run(arguments) expectedError match { case Some(fn) => assert(clue(exit) != 0, clues(app.capturedOutput)) @@ -136,7 +134,8 @@ abstract class BaseBuildToolSuite extends MopedSuite(ScipJava.app) { if (expectedPackages.nonEmpty) { val obtainedPackages = ClasspathEntry .fromTargetroot(targetroot, workingDirectory) - .map(_.mavenCoordinate) + .asScala + .map(_.mavenCoordinate()) .sorted .distinct .mkString("\n") diff --git a/tests/snapshots/src/main/scala/tests/MinimizedSnapshotScipGenerator.scala b/tests/snapshots/src/main/scala/tests/MinimizedSnapshotScipGenerator.scala index 6fd12c074..92a319913 100644 --- a/tests/snapshots/src/main/scala/tests/MinimizedSnapshotScipGenerator.scala +++ b/tests/snapshots/src/main/scala/tests/MinimizedSnapshotScipGenerator.scala @@ -6,18 +6,18 @@ import java.nio.file.Files import java.nio.file.Paths import scala.jdk.CollectionConverters.CollectionHasAsScala +import scala.jdk.CollectionConverters.SeqHasAsJava import scala.meta.internal.io.FileIO import scala.meta.io.AbsolutePath -import com.sourcegraph.io.DeleteVisitor import com.sourcegraph.scip_java.ScipJava import com.sourcegraph.scip_java.ScipPrinters import org.scip_code.scip.Index class MinimizedSnapshotScipGenerator { def run(args: List[String]): Unit = { - val exit = ScipJava.app.run(args) + val exit = ScipJava.app.run(args.asJava) require(exit == 0) } def run(context: SnapshotContext, handler: SnapshotHandler): Unit = { @@ -75,8 +75,8 @@ class MinimizedSnapshotScipGenerator { ) } } finally { - Files.walkFileTree(scipOutput, new DeleteVisitor()) - Files.walkFileTree(snapshotOutput.toNIO, new DeleteVisitor()) + os.remove.all(os.Path(scipOutput)) + os.remove.all(os.Path(snapshotOutput.toNIO)) } } } diff --git a/tests/snapshots/src/main/scala/tests/SaveSnapshotHandler.scala b/tests/snapshots/src/main/scala/tests/SaveSnapshotHandler.scala index 5283d622b..b93bae385 100644 --- a/tests/snapshots/src/main/scala/tests/SaveSnapshotHandler.scala +++ b/tests/snapshots/src/main/scala/tests/SaveSnapshotHandler.scala @@ -1,14 +1,16 @@ package tests +import java.io.IOException import java.nio.charset.StandardCharsets +import java.nio.file.FileVisitResult import java.nio.file.Files import java.nio.file.Path +import java.nio.file.SimpleFileVisitor +import java.nio.file.attribute.BasicFileAttributes import java.util.concurrent.ConcurrentLinkedDeque import scala.jdk.CollectionConverters._ -import com.sourcegraph.io.DeleteVisitor - class SaveSnapshotHandler extends SnapshotHandler { private val writtenTests = new ConcurrentLinkedDeque[Path]() override def onSnapshotTest( @@ -28,7 +30,28 @@ class SaveSnapshotHandler extends SnapshotHandler { val isWritten = writtenTests.asScala.toSet Files.walkFileTree( context.expectDirectory, - new DeleteVisitor(deleteFile = file => !isWritten.contains(file)) + new SimpleFileVisitor[Path] { + override def visitFile( + file: Path, + attrs: BasicFileAttributes + ): FileVisitResult = { + if (!isWritten.contains(file)) + Files.deleteIfExists(file) + FileVisitResult.CONTINUE + } + override def postVisitDirectory( + dir: Path, + exc: IOException + ): FileVisitResult = { + val entries = Files.list(dir) + val isEmpty = + try !entries.iterator().hasNext() + finally entries.close() + if (isEmpty) + Files.deleteIfExists(dir) + FileVisitResult.CONTINUE + } + } ) } diff --git a/tests/unit/src/main/scala/tests/FileLayout.scala b/tests/unit/src/main/scala/tests/FileLayout.scala new file mode 100644 index 000000000..fdd05c43c --- /dev/null +++ b/tests/unit/src/main/scala/tests/FileLayout.scala @@ -0,0 +1,115 @@ +package tests + +import java.nio.charset.Charset +import java.nio.charset.StandardCharsets +import java.nio.file.FileVisitResult +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.SimpleFileVisitor +import java.nio.file.StandardOpenOption +import java.nio.file.attribute.BasicFileAttributes + +import scala.collection.mutable + +/** + * Tiny port of `moped.testkit.FileLayout` so the tests no longer have a + * test-time dependency on the moped runtime. + */ +object FileLayout { + + def asString( + root: Path, + includePath: Path => Boolean = _ => true, + charset: Charset = StandardCharsets.UTF_8 + ): String = { + if (!Files.isDirectory(root)) + return "" + import scala.jdk.CollectionConverters._ + val out = new StringBuilder() + val buf = mutable.ArrayBuffer.empty[Path] + Files.walkFileTree( + root, + new SimpleFileVisitor[Path] { + override def preVisitDirectory( + dir: Path, + attrs: BasicFileAttributes + ): FileVisitResult = + if (!includePath(dir)) + FileVisitResult.SKIP_SUBTREE + else + FileVisitResult.CONTINUE + + override def visitFile( + file: Path, + attrs: BasicFileAttributes + ): FileVisitResult = { + if (includePath(file)) + buf += file + FileVisitResult.CONTINUE + } + } + ) + buf + .sorted + .foreach { file => + val relpath = root.relativize(file).iterator().asScala.mkString("/") + out.append("/").append(relpath) + if (Files.isSymbolicLink(file)) { + out.append(" -> ").append(Files.readSymbolicLink(file)) + } else { + val bytes = Files.readAllBytes(file) + out.append("\n").append(new String(bytes, charset)) + } + out.append("\n") + } + out.toString() + } + + def mapFromString(layout: String): Map[String, String] = { + if (layout.trim.isEmpty) + return Map.empty + val normalized = layout.replace("\r\n", "\n") + normalized + .split("(?=\n/)") + .map { row => + row.stripPrefix("\n").split("\n", 2).toList match { + case path :: contents :: Nil => + val withEndOfFileLine = + if (contents.endsWith("\n")) + contents + else + contents + "\n" + path.stripPrefix("/") -> withEndOfFileLine + case other => + throw new IllegalArgumentException( + s"Unable to split argument into path/contents! \n$other" + ) + } + } + .toMap + } + + def fromString( + layout: String, + root: Path = Files.createTempDirectory("scip-java"), + charset: Charset = StandardCharsets.UTF_8 + ): Path = { + if (layout.trim.isEmpty) + return root + mapFromString(layout).foreach { case (path, contents) => + val file = path.split("/").foldLeft(root)(_ resolve _) + val parent = file.getParent + // Don't try to mkdir under a symlinked parent. + if (!Files.exists(parent)) + Files.createDirectories(parent) + Files.deleteIfExists(file) + Files.write( + file, + contents.getBytes(charset), + StandardOpenOption.WRITE, + StandardOpenOption.CREATE + ) + } + root + } +} diff --git a/tests/unit/src/main/scala/tests/ScipJavaSuite.scala b/tests/unit/src/main/scala/tests/ScipJavaSuite.scala new file mode 100644 index 000000000..47031185b --- /dev/null +++ b/tests/unit/src/main/scala/tests/ScipJavaSuite.scala @@ -0,0 +1,143 @@ +package tests + +import java.io.ByteArrayOutputStream +import java.io.PrintStream +import java.nio.charset.StandardCharsets +import java.nio.file.Files +import java.nio.file.Path + +import scala.jdk.CollectionConverters._ + +import com.sourcegraph.scip_java.CliEnvironment +import com.sourcegraph.scip_java.ScipJavaApp +import munit.FunSuite +import munit.Location +import munit.TestOptions +import munit.internal.console.AnsiColors + +/** + * Test base class that boots a [[ScipJavaApp]] with stdout/stderr redirected + * into an in-memory buffer (exposed via [[ApplicationFixture#capturedOutput]]) + * and a fresh per-test working/cache directory layout. + * + * Mirrors the surface area of `moped.testkit.MopedSuite` that the test suites + * in this repository actually use, without requiring the Scala-only moped + * runtime. + */ +abstract class ScipJavaSuite(applicationToTest: ScipJavaApp) extends FunSuite { + + val temporaryDirectory: DirectoryFixture = new DirectoryFixture + + def createTempDirectory(): Path = Files.createTempDirectory("scip-java") + + def workingDirectory: Path = temporaryDirectory().resolve("workingDirectory") + + def cacheDirectory: Path = temporaryDirectory().resolve("cache") + + def environmentVariables: Map[String, String] = Map.empty[String, String] + + val app = new ApplicationFixture(applicationToTest) + + class DirectoryFixture extends Fixture[Path]("Directory") { + private var path: Path = _ + def apply(): Path = path + override def beforeAll(): Unit = { + path = createTempDirectory() + } + override def afterEach(context: AfterEach): Unit = { + os.remove.all(os.Path(path)) + } + } + + class ApplicationFixture(applicationToTest: ScipJavaApp) + extends Fixture[ScipJavaApp]("Application") { + private val out = new ByteArrayOutputStream + private val ps = + new PrintStream( + out, /* autoFlush = */ true, + StandardCharsets.UTF_8.name() + ) + + // NOTE: every invocation gets its own freshly-instrumented [[ScipJavaApp]] + // (all of them writing into the shared `out` buffer of this fixture) so + // that suites running in parallel never clobber each other's redirected + // stdout/stderr. This mirrors `moped.testkit`'s per-invocation app model. + def apply(): ScipJavaApp = { + val app = new ScipJavaApp() + app.setEnv( + new CliEnvironment( + workingDirectory, + environmentVariables.asJava, + ps, + ps, + /* isProgressBarEnabled = */ false + ) + ) + app + } + + /** Run the CLI with the given Scala-list arguments. */ + def run(args: List[String]): Int = apply().run(args.asJava) + + def reset(): Unit = out.reset() + + def capturedRawOutput: String = out.toString(StandardCharsets.UTF_8.name()) + + def capturedOutput: String = AnsiColors.filterAnsi(capturedRawOutput) + + override def beforeEach(context: BeforeEach): Unit = { + Files.createDirectories(workingDirectory) + reset() + } + } + + override def munitFixtures: Seq[Fixture[_]] = + super.munitFixtures ++ List(temporaryDirectory, app) + + def checkErrorOutput( + name: TestOptions, + arguments: List[String], + expectedOutput: String, + workingDirectoryLayout: String = "" + )(implicit loc: Location): Unit = { + checkOutput( + name, + arguments, + expectedOutput, + expectedExit = 1, + workingDirectoryLayout = workingDirectoryLayout + ) + } + + def checkOutput( + name: TestOptions, + arguments: => List[String], + expectedOutput: String, + expectedExit: Int = 0, + workingDirectoryLayout: String = "" + )(implicit loc: Location): Unit = { + test(name) { + if (workingDirectoryLayout.nonEmpty) { + FileLayout.fromString(workingDirectoryLayout, workingDirectory) + } + val exit = app.run(arguments) + assertEquals(exit, expectedExit, clues(app.capturedOutput)) + assertNoDiff(app.capturedOutput, expectedOutput) + } + } + + override def assertNoDiff(obtained: String, expected: String, clue: => Any)( + implicit loc: Location + ): Unit = { + val sanitized = obtained.replace(temporaryDirectory().toString(), "") + // Workaround for https://github.com/scalameta/munit/issues/179 + super.assertNoDiff( + if (sanitized == "\n") + "" + else + sanitized, + expected, + clue + ) + } +} diff --git a/tests/unit/src/main/scala/tests/TempDirectories.scala b/tests/unit/src/main/scala/tests/TempDirectories.scala index 50f32789c..dd6ebcebd 100644 --- a/tests/unit/src/main/scala/tests/TempDirectories.scala +++ b/tests/unit/src/main/scala/tests/TempDirectories.scala @@ -3,7 +3,6 @@ package tests import java.nio.file.Files import java.nio.file.Path -import com.sourcegraph.io.DeleteVisitor import munit.FunSuite trait TempDirectories { @@ -18,7 +17,7 @@ trait TempDirectories { } override def afterEach(context: AfterEach): Unit = { - Files.walkFileTree(path, new DeleteVisitor()) + os.remove.all(os.Path(path)) } } diff --git a/tests/unit/src/test/scala/tests/SnapshotCommandSuite.scala b/tests/unit/src/test/scala/tests/SnapshotCommandSuite.scala index 659fda041..ccca20925 100644 --- a/tests/unit/src/test/scala/tests/SnapshotCommandSuite.scala +++ b/tests/unit/src/test/scala/tests/SnapshotCommandSuite.scala @@ -5,10 +5,8 @@ import java.nio.file.Files import scala.meta.inputs.Input import com.sourcegraph.scip_java.ScipJava -import moped.testkit.FileLayout -import moped.testkit.MopedSuite -class SnapshotCommandSuite extends MopedSuite(ScipJava.app) { +class SnapshotCommandSuite extends ScipJavaSuite(ScipJava.app) { test("snapshot") { FileLayout.fromString( """/main/Sample.java @@ -35,7 +33,7 @@ class SnapshotCommandSuite extends MopedSuite(ScipJava.app) { val generatedpath = workingDirectory.resolve("generated") - val indexExit = app().run( + val indexExit = app.run( List( "aggregate", "--output", @@ -46,7 +44,7 @@ class SnapshotCommandSuite extends MopedSuite(ScipJava.app) { assertEquals(indexExit, 0, clues(app.capturedOutput)) - val snapshotExit = app().run( + val snapshotExit = app.run( List("snapshot", "--output", generatedpath.toString, targetroot.toString) ) assertEquals(snapshotExit, 0, clues(app.capturedOutput))