Merge pull request #40 from gradle/dd/job-correlator

Add configuration options for github-actions integration

Author: bigdaz

.github/workflows/gradle.yml

@@ -69,14 +69,16 @@ jobs:
             uses: gradle/gradle-build-action@v2
 
           - name: Self Test :plugin
-            run: ./plugin-self-test
+            run: ./plugin-self-test GitHubDependencyGraphPlugin_generateDependencyGraph
             env:
                 GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+                GITHUB_DEPENDENCY_GRAPH_JOB_ID: ${{ github.run_id }}
+                GITHUB_DEPENDENCY_GRAPH_JOB_CORRELATOR: "plugin-self-test"
 
           - name: Save plugin JSON report
             uses: actions/upload-artifact@v3
             with:
                 name: plugin-json
-                path: build/reports/github-dependency-graph-plugin/github-dependency-snapshot.json
+                path: build/reports/github-dependency-graph-snapshots/plugin-self-test.json
                 if-no-files-found: error
 

README.md

@@ -24,9 +24,19 @@ apply plugin: GitHubDependencyGraphPlugin
 ```
 
 This causes 2 separate plugins to be applied, that can be used independently:
-- `GitHubDependencyExtractorPlugin` collects all dependencies that are resolved during a build execution and writes these to a file. The output file can be found at `<root>/build/reports/github-depenency-graph-gradle-plugin/github-dependency-snapshot.json`.
+- `GitHubDependencyExtractorPlugin` collects all dependencies that are resolved during a build execution and writes these to a file. The output file can be found at `<root>/build/reports/github-depenency-graph-snapshots/<job-correlator>.json`.
 - `ForceDependencyResolutionPlugin` creates a `GitHubDependencyGraphPlugin_generateDependencyGraph` task that will attempt to resolve all dependencies for a Gradle build, by simply invoking `dependencies` on all projects.
 
+### Required environment variables
+
+The following environment variables configure the snapshot generated by the `GitHubDependencyExtractorPlugin`. See the [GitHub Dependency Submission API docs](https://docs.github.com/en/rest/dependency-graph/dependency-submission?apiVersion=2022-11-28) for details:
+- `GITHUB_DEPENDENCY_GRAPH_JOB_CORRELATOR`: Sets the `job.correlator` value for the dependency submission
+- `GITHUB_DEPENDENCY_GRAPH_JOB_ID`: Sets the `job.id` value for the dependency submission
+- `GITHUB_REF`: Sets the `ref` value for the dependency submission
+- `GITHUB_SHA`: Sets the `sha` value for the dependency submission
+- `GITHUB_WORKSPACE`: Sets the root directory of the github repository
+- `GITHUB_DEPENDENCY_GRAPH_REPORT_DIR` (optional): Specifies where the dependency graph report will be generated
+
 ## Building/Testing
 
 To build and test this plugin, run the following task:

plugin-self-test

@@ -16,5 +16,5 @@ else
     -H "Authorization: Bearer $GITHUB_TOKEN"\
     -H "X-GitHub-Api-Version: 2022-11-28" \
     https://api.github.com/repos/gradle/github-dependency-graph-gradle-plugin/dependency-graph/snapshots \
-    -d @build/reports/github-dependency-graph-plugin/github-dependency-snapshot.json
+    -d @build/reports/github-dependency-graph-snapshots/plugin-self-test.json
 fi

plugin-self-test-local

@@ -1,7 +1,7 @@
 #!/bin/sh
 
-export GITHUB_JOB="Build"
-export GITHUB_RUN_NUMBER="42"
+export GITHUB_DEPENDENCY_GRAPH_JOB_ID="42"
+export GITHUB_DEPENDENCY_GRAPH_JOB_CORRELATOR="plugin-self-test"
 export GITHUB_REF="refs/heads/main"
 export GITHUB_SHA=$( git rev-parse HEAD )
 export GITHUB_WORKSPACE=$( pwd )

plugin-test/src/test/groovy/org/gradle/github/dependencygraph/BaseExtractorTest.groovy

@@ -49,8 +49,7 @@ abstract class BaseExtractorTest extends Specification {
 
     SimpleGradleExecuter createExecuter() {
         // Create a new JsonManifestLoader for each invocation of the executer
-        File manifestFile =
-            testDirectory.file("build/reports/github-dependency-graph-plugin/github-dependency-snapshot.json")
+        File manifestFile = reportDir.file("dummy-job-correlator.json")
         loader = new JsonRepositorySnapshotLoader(manifestFile)
         return createExecuter(testGradleVersion)
     }
@@ -73,6 +72,10 @@ abstract class BaseExtractorTest extends Specification {
         return new TestFile(testDir)
     }
 
+    TestFile getReportDir() {
+        return getTestDirectory().file("reports")
+    }
+
     TestFile file(Object... path) {
         if (path.length == 1 && path[0] instanceof TestFile) {
             return path[0] as TestFile
@@ -108,7 +111,7 @@ abstract class BaseExtractorTest extends Specification {
     }
 
     protected void establishEnvironmentVariables() {
-        environmentVars = new TestEnvironmentVars(testDirectory)
+        environmentVars = new TestEnvironmentVars(testDirectory, reportDir)
         getExecuter().withEnvironmentVars(environmentVars.asEnvironmentMap())
     }
 
@@ -134,8 +137,8 @@ abstract class BaseExtractorTest extends Specification {
         assert json.sha == environmentVars.sha
         assert json.ref == environmentVars.ref
         def job = json.job as Map
-        assert job.correlator == environmentVars.job
-        assert job.id == environmentVars.runNumber
+        assert job.correlator == environmentVars.jobCorrelator
+        assert job.id == environmentVars.jobId
         def detector = json.detector as Map
         assert detector.name.contains("Gradle")
         assert detector.version != null
@@ -255,25 +258,28 @@ abstract class BaseExtractorTest extends Specification {
     }
 
     static class TestEnvironmentVars {
-        final String job = "Build_" + System.currentTimeMillis()
-        final String runNumber = UUID.randomUUID().toString()
+        final String jobId = UUID.randomUUID().toString()
+        final String jobCorrelator = "dummy-job-correlator"
         final String ref = "refs/head/feature/test" + UUID.randomUUID().toString()
         final String sha = fakeSha()
         final String workspace
+        final String reportDir
         final String gitHubToken = UUID.randomUUID().toString()
 
-        TestEnvironmentVars(TestFile testDirectory) {
+        TestEnvironmentVars(TestFile testDirectory, TestFile reportDir) {
             workspace = testDirectory.absolutePath
+            this.reportDir = reportDir.absolutePath
         }
 
         Map<String, String> asEnvironmentMap() {
             return [
-                "GITHUB_JOB"       : job,
-                "GITHUB_RUN_NUMBER": runNumber,
-                "GITHUB_REF"       : ref,
-                "GITHUB_SHA"       : sha,
-                "GITHUB_WORKSPACE" : workspace,
-                "GITHUB_TOKEN"     : gitHubToken
+                "GITHUB_DEPENDENCY_GRAPH_JOB_ID"        : jobId,
+                "GITHUB_DEPENDENCY_GRAPH_JOB_CORRELATOR": jobCorrelator,
+                "GITHUB_DEPENDENCY_GRAPH_REPORT_DIR"    : reportDir,
+                "GITHUB_REF"                            : ref,
+                "GITHUB_SHA"                            : sha,
+                "GITHUB_WORKSPACE"                      : workspace,
+                "GITHUB_TOKEN"                          : gitHubToken
             ]
         }
 

plugin-test/src/test/groovy/org/gradle/github/dependencygraph/SingleProjectDependencyExtractorTest.groovy

@@ -32,17 +32,6 @@ class SingleProjectDependencyExtractorTest extends BaseExtractorTest {
         """
     }
 
-    def "includes build path in job correlator"() {
-        when:
-        executer.withArgument("-Dorg.gradle.github.env.GRADLE_BUILD_PATH=foo/bar")
-        run()
-
-        then:
-        def json = jsonRepositorySnapshot()
-        def job = json.job as Map
-        job.correlator == environmentVars.job + ":foo/bar"
-    }
-
     def "extracts implementation and test dependencies for a java project"() {
         given:
         buildFile << """

plugin/src/main/kotlin/org/gradle/github/dependencygraph/GitHubDependencyExtractorPlugin.kt

@@ -22,14 +22,15 @@ import java.nio.file.Paths
  */
 class GitHubDependencyExtractorPlugin : Plugin<Gradle> {
     private companion object : PluginCompanionUtils() {
-        const val ENV_GITHUB_JOB = "GITHUB_JOB"
-        const val ENV_GITHUB_RUN_NUMBER = "GITHUB_RUN_NUMBER"
+        const val ENV_DEPENDENCY_GRAPH_JOB_ID = "GITHUB_DEPENDENCY_GRAPH_JOB_ID"
+        const val ENV_DEPENDENCY_GRAPH_JOB_CORRELATOR = "GITHUB_DEPENDENCY_GRAPH_JOB_CORRELATOR"
+        const val ENV_DEPENDENCY_GRAPH_REPORT_DIR = "GITHUB_DEPENDENCY_GRAPH_REPORT_DIR"
         const val ENV_GITHUB_REF = "GITHUB_REF"
         const val ENV_GITHUB_SHA = "GITHUB_SHA"
-        const val ENV_GRADLE_BUILD_PATH = "GRADLE_BUILD_PATH"
 
         /**
          * Environment variable should be set to the workspace directory that the Git repository is checked out in.
+         * This is used to determine relative path to build files referenced in the dependency graph.
          */
         const val ENV_GITHUB_WORKSPACE = "GITHUB_WORKSPACE"
     }
@@ -50,7 +51,7 @@ class GitHubDependencyExtractorPlugin : Plugin<Gradle> {
         gradle.rootProject { project ->
             dependencyExtractorProvider
                 .get()
-                .setRootProjectBuildDirectory(project.buildDir)
+                .rootProjectBuildDirectory = project.buildDir
         }
 
         // Register the service to listen for Build Events
@@ -92,18 +93,18 @@ class GitHubDependencyExtractorPlugin : Plugin<Gradle> {
                 // Create a constant value that the provider will always return.
                 // IE. Memoize the value
                 val constantDependencyExtractor = object : DependencyExtractor() {
-                    override val gitHubJobName: String
-                        get() = gradle.loadEnvironmentVariable(ENV_GITHUB_JOB)
-                    override val gitHubRunNumber: String
-                        get() = gradle.loadEnvironmentVariable(ENV_GITHUB_RUN_NUMBER)
+                    override val dependencyGraphJobCorrelator: String
+                        get() = gradle.loadEnvironmentVariable(ENV_DEPENDENCY_GRAPH_JOB_CORRELATOR)
+                    override val dependencyGraphJobId: String
+                        get() = gradle.loadEnvironmentVariable(ENV_DEPENDENCY_GRAPH_JOB_ID)
+                    override val dependencyGraphReportDir: String
+                        get() = gradle.loadEnvironmentVariable(ENV_DEPENDENCY_GRAPH_REPORT_DIR, "")
                     override val gitSha: String
                         get() = gradle.loadEnvironmentVariable(ENV_GITHUB_SHA)
                     override val gitRef: String
                         get() = gradle.loadEnvironmentVariable(ENV_GITHUB_REF)
                     override val gitWorkspaceDirectory: Path
                         get() = gitWorkspaceDirectory
-                    override val gradleBuildPath: String
-                        get() = gradle.loadEnvironmentVariable(ENV_GRADLE_BUILD_PATH, "")
                 }
                 return providerFactory.provider { constantDependencyExtractor }
             }
@@ -154,12 +155,12 @@ class GitHubDependencyExtractorPlugin : Plugin<Gradle> {
                     DependencyExtractorBuildService::class.java
                 ) { spec ->
                     spec.parameters {
-                        it.gitHubJobName.convention(gradle.loadEnvironmentVariable(ENV_GITHUB_JOB))
-                        it.gitHubRunNumber.convention(gradle.loadEnvironmentVariable(ENV_GITHUB_RUN_NUMBER))
+                        it.dependencyGraphJobCorrelator.convention(gradle.loadEnvironmentVariable(ENV_DEPENDENCY_GRAPH_JOB_CORRELATOR))
+                        it.dependencyGraphJobId.convention(gradle.loadEnvironmentVariable(ENV_DEPENDENCY_GRAPH_JOB_ID))
+                        it.dependencyGraphReportDir.convention(gradle.loadEnvironmentVariable(ENV_DEPENDENCY_GRAPH_REPORT_DIR, ""))
                         it.gitSha.convention(gradle.loadEnvironmentVariable(ENV_GITHUB_SHA))
                         it.gitRef.convention(gradle.loadEnvironmentVariable(ENV_GITHUB_REF))
                         it.gitWorkspaceDirectory.convention(gitWorkspaceDirectory)
-                        it.gradleBuildPath.convention(gradle.loadEnvironmentVariable(ENV_GRADLE_BUILD_PATH, ""))
                     }
                 }
             }

plugin/src/main/kotlin/org/gradle/github/dependencygraph/internal/DependencyExtractor.kt

@@ -5,8 +5,8 @@ import org.gradle.api.artifacts.result.ResolvedComponentResult
 import org.gradle.api.artifacts.result.ResolvedDependencyResult
 import org.gradle.api.internal.artifacts.DefaultProjectComponentIdentifier
 import org.gradle.api.internal.artifacts.configurations.ResolveConfigurationDependenciesBuildOperationType
+import org.gradle.api.logging.Logging
 import org.gradle.github.GitHubDependencyGraphPlugin
-import org.gradle.github.dependencygraph.GitHubDependencyExtractorPlugin
 import org.gradle.github.dependencygraph.internal.model.ComponentCoordinates
 import org.gradle.github.dependencygraph.internal.model.ResolvedComponent
 import org.gradle.github.dependencygraph.internal.model.ResolvedConfiguration
@@ -23,27 +23,22 @@ abstract class DependencyExtractor :
     BuildOperationListener,
     AutoCloseable {
 
-    protected abstract val gitHubJobName: String
-    protected abstract val gitHubRunNumber: String
+    protected abstract val dependencyGraphJobCorrelator: String
+    protected abstract val dependencyGraphJobId: String
+    protected abstract val dependencyGraphReportDir: String
     protected abstract val gitSha: String
     protected abstract val gitRef: String
     protected abstract val gitWorkspaceDirectory: Path
-    protected abstract val gradleBuildPath: String
 
-    /**
-     * Can't use this as a proper input:
-     * https://github.com/gradle/gradle/issues/19562
-     */
-    private var fileWriter: DependencyFileWriter = DependencyFileWriter.create()
+    var rootProjectBuildDirectory: File? = null
 
     private val gitHubRepositorySnapshotBuilder by lazy {
         GitHubRepositorySnapshotBuilder(
-            gitHubJobName = gitHubJobName,
-            gitHubRunNumber = gitHubRunNumber,
+            dependencyGraphJobCorrelator = dependencyGraphJobCorrelator,
+            dependencyGraphJobId = dependencyGraphJobId,
             gitSha = gitSha,
             gitRef = gitRef,
-            gitWorkspaceDirectory = gitWorkspaceDirectory,
-            gradleBuildPath = gradleBuildPath
+            gitWorkspaceDirectory = gitWorkspaceDirectory
         )
     }
 
@@ -53,10 +48,6 @@ abstract class DependencyExtractor :
         println("Creating: DependencyExtractorService")
     }
 
-    internal fun setRootProjectBuildDirectory(rootProjectBuildDirectory: File) {
-        fileWriter = DependencyFileWriter.create(rootProjectBuildDirectory)
-    }
-
     override fun started(buildOperation: BuildOperationDescriptor, startEvent: OperationStartEvent) {
         // This method will never be called when registered in a `BuildServiceRegistry` (ie. Gradle 6.1 & higher)
         // No-op
@@ -185,8 +176,25 @@ abstract class DependencyExtractor :
         }
     }
 
-    private fun writeAndGetSnapshotFile(): File =
+    private fun writeAndGetSnapshotFile() {
+        val outputFile = File(getOutputDir(), "${dependencyGraphJobCorrelator}.json")
+        val fileWriter = DependencyFileWriter(outputFile)
         fileWriter.writeDependencyManifest(gitHubRepositorySnapshotBuilder.build())
+    }
+
+    private fun getOutputDir(): File {
+        if (dependencyGraphReportDir.isNotEmpty()) {
+            return File(dependencyGraphReportDir)
+        }
+
+        if (rootProjectBuildDirectory == null) {
+            throw RuntimeException("Cannot determine report file location")
+        }
+        return File(
+            rootProjectBuildDirectory,
+            "reports/github-dependency-graph-snapshots"
+        )
+    }
 
     override fun close() {
         if (thrownExceptions.isNotEmpty()) {
@@ -206,6 +214,9 @@ abstract class DependencyExtractor :
             )
         }
     }
+    companion object {
+        private val LOGGER = Logging.getLogger(DependencyExtractor::class.java)
+    }
 }
 
 private inline fun <reified D, reified R> handleBuildOperationTypeRaw(

plugin/src/main/kotlin/org/gradle/github/dependencygraph/internal/DependencyExtractorBuildService.kt

@@ -11,24 +11,24 @@ abstract class DependencyExtractorBuildService :
     BuildService<DependencyExtractorBuildService.Parameters> {
 
     interface Parameters : BuildServiceParameters {
-        val gitHubJobName: Property<String>
-        val gitHubRunNumber: Property<String>
+        val dependencyGraphJobId: Property<String>
+        val dependencyGraphJobCorrelator: Property<String>
+        val dependencyGraphReportDir: Property<String>
         val gitSha: Property<String>
         val gitRef: Property<String>
         val gitWorkspaceDirectory: DirectoryProperty
-        val gradleBuildPath: Property<String>
     }
 
-    override val gitHubJobName: String
-        get() = parameters.gitHubJobName.get()
-    override val gitHubRunNumber: String
-        get() = parameters.gitHubRunNumber.get()
+    override val dependencyGraphJobId: String
+        get() = parameters.dependencyGraphJobId.get()
+    override val dependencyGraphJobCorrelator: String
+        get() = parameters.dependencyGraphJobCorrelator.get()
+    override val dependencyGraphReportDir: String
+        get() = parameters.dependencyGraphReportDir.get()
     override val gitSha: String
         get() = parameters.gitSha.get()
     override val gitRef: String
         get() = parameters.gitRef.get()
     override val gitWorkspaceDirectory: Path
         get() = parameters.gitWorkspaceDirectory.get().asFile.toPath()
-    override val gradleBuildPath: String
-        get() = parameters.gradleBuildPath.get()
 }

plugin/src/main/kotlin/org/gradle/github/dependencygraph/internal/DependencyFileWriter.kt

@@ -4,19 +4,13 @@ import org.gradle.api.logging.Logging
 import org.gradle.github.dependencygraph.internal.json.GitHubRepositorySnapshot
 import java.io.File
 
-class DependencyFileWriter
-private constructor(
-    private val manifestFile: File,
-    private val loggerWarning: () -> Unit
-) {
-
+class DependencyFileWriter(val manifestFile: File) {
     private var writtenFile: Boolean = false
 
     fun writeDependencyManifest(graph: GitHubRepositorySnapshot): File {
         if (writtenFile) {
             return manifestFile
         }
-        loggerWarning()
         manifestFile.parentFile.mkdirs()
         manifestFile.writeText(JacksonJsonSerializer.serializeToJson(graph))
         writtenFile = true
@@ -26,28 +20,5 @@ private constructor(
 
     companion object {
         private val LOGGER = Logging.getLogger(DependencyFileWriter::class.java)
-
-        fun create(buildDirectory: File): DependencyFileWriter =
-            create(buildDirectory) {
-                // No-op
-            }
-
-        fun create(): DependencyFileWriter =
-            create(File(".")) {
-                LOGGER.warn(
-                    "[WARNING] Something went wrong configuring the GithubDependencyExtractorPlugin. " +
-                        "Using JVM working directory as root of build"
-                )
-            }
-
-        private fun create(buildDirectory: File, loggerWarning: () -> Unit): DependencyFileWriter {
-            return DependencyFileWriter(
-                File(
-                    buildDirectory,
-                    "reports/github-dependency-graph-plugin/github-dependency-snapshot.json"
-                ),
-                loggerWarning
-            )
-        }
     }
 }