Avoid overlapping outputs for indexing tasks (#9725)

* chore: Refactor instrumentation index tasks to avoid overlapping inputs

* Emit index in proper generated source location and use the location as source dir.
* Deduplicate code.

* chore: Refactor dd-java-agent index tasks and included agent to avoid overlapping output

* Expand the included agent jars in another location buildDir/included, this location
  contributes to the resources.
* Emit index in separate location, modified index to receive input folder and output folder,
  this location contributes to the resources.
* Fix indexer logging.

* style: Access properties via `it` in Groovy DSL

* chore: Fix indexer warning logs

For logs like

```
[main] WARN datadog.trace.bootstrap.AgentJarIndex - Detected duplicate content 'datadog.trace.civisibility.writer.ddintake.*' under 'trace', already seen in <root>. Ensure your content is under a distinct directory.
```

* chore: Fix indexer warning log, due to duplicate license files

For logs like

```
Detected duplicate content under 'META-INF.AL2.0'. Ensure your content is under a distinct directory.
Detected duplicate content under 'META-INF.LGPL2.1'. Ensure your content is under a distinct directory.
```

* chore: Fix indexer warning log, due to duplicate dd-javac-plugin-client files

For logs like

```
[main] WARN datadog.trace.bootstrap.AgentJarIndex - Detected duplicate content under 'datadog.compiler.utils.*'. Ensure your content is under a distinct directory.
[main] WARN datadog.trace.bootstrap.AgentJarIndex - Detected duplicate content under 'datadog.compiler.annotations.*'. Ensure your content is under a distinct
```

* fix: Use Sync task rather than Copy, which may leave deleted files in place

* fix: Replace weak configuration using tasks.matching {} by proper filetree.builtBy call

* style: Fix star import

Author: bric3

dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/AgentJarIndex.java

@@ -15,8 +15,10 @@
 import java.nio.file.attribute.BasicFileAttributes;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 import java.util.jar.JarFile;
 import java.util.zip.ZipEntry;
@@ -101,6 +103,7 @@ static class IndexGenerator extends SimpleFileVisitor<Path> {
 
     private Path prefixRoot;
     private int prefixId;
+    private Map<Integer, String> prefixMappings = new HashMap<>();
 
     IndexGenerator(Path resourcesDir) {
       this.resourcesDir = resourcesDir;
@@ -125,6 +128,7 @@ public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) {
         prefixRoot = dir;
         prefixes.add(dir.getFileName() + "/");
         prefixId = prefixes.size();
+        prefixMappings.put(prefixId, dir.getFileName().toString());
       }
       return FileVisitResult.CONTINUE;
     }
@@ -143,10 +147,16 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) {
         String entryKey = computeEntryKey(prefixRoot.relativize(file));
         if (null != entryKey) {
           int existingPrefixId = prefixTrie.apply(entryKey);
-          if (-1 != existingPrefixId && prefixId != existingPrefixId) {
+          // warn if two subsections contain content under the same package prefix
+          // because we're then unable to redirect requests to the right submodule
+          // (ignore the two 'datadog.compiler' packages which allow duplication)
+          if (existingPrefixId > 0 && prefixId != existingPrefixId) {
             log.warn(
-                "Detected duplicate content under '{}'. Ensure your content is under a distinct directory.",
-                entryKey);
+                "Detected duplicate content '{}' under '{}', already seen in {}. Ensure your content is under a distinct directory.",
+                entryKey,
+                resourcesDir.relativize(file).getName(0), // prefix
+                prefixMappings.get(existingPrefixId) // previous prefix
+                );
           }
           prefixTrie.put(entryKey, prefixId);
           if (entryKey.endsWith("*")) {
@@ -185,10 +195,18 @@ private static String computeEntryKey(Path path) {
     }
 
     public static void main(String[] args) throws IOException {
+      if (args.length < 1) {
+        throw new IllegalArgumentException("Expected: resources-dir");
+      }
+
       Path resourcesDir = Paths.get(args[0]).toAbsolutePath();
+      Path indexDir = resourcesDir;
+      if (args.length == 2) {
+        indexDir = Paths.get(args[1]).toAbsolutePath();
+      }
       IndexGenerator indexGenerator = new IndexGenerator(resourcesDir);
       Files.walkFileTree(resourcesDir, indexGenerator);
-      indexGenerator.writeIndex(resourcesDir.resolve(AGENT_INDEX_FILE_NAME));
+      indexGenerator.writeIndex(indexDir.resolve(AGENT_INDEX_FILE_NAME));
     }
   }
 }

dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/InstrumenterIndex.java

@@ -334,18 +334,18 @@ public void writeIndex(Path indexFile) throws IOException {
      */
     public static void main(String[] args) throws IOException {
       if (args.length < 1) {
-        throw new IllegalArgumentException("Expected: resources-dir");
+        throw new IllegalArgumentException("Expected: index-dir");
       }
 
-      Path resourcesDir = Paths.get(args[0]).toAbsolutePath();
+      Path indexDir = Paths.get(args[0]).toAbsolutePath();
 
       // satisfy some instrumenters that cache matchers in initializers
       HierarchyMatchers.registerIfAbsent(HierarchyMatchers.simpleChecks());
       SharedTypePools.registerIfAbsent(SharedTypePools.simpleCache());
 
       IndexGenerator indexGenerator = new IndexGenerator();
       indexGenerator.buildIndex();
-      indexGenerator.writeIndex(resourcesDir.resolve(INSTRUMENTER_INDEX_NAME));
+      indexGenerator.writeIndex(indexDir.resolve(INSTRUMENTER_INDEX_NAME));
     }
   }
 }

dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/KnownTypesIndex.java

@@ -154,18 +154,18 @@ public void writeIndex(Path indexFile) throws IOException {
      */
     public static void main(String[] args) throws IOException {
       if (args.length < 1) {
-        throw new IllegalArgumentException("Expected: resources-dir");
+        throw new IllegalArgumentException("Expected: index-dir");
       }
 
-      Path resourcesDir = Paths.get(args[0]).toAbsolutePath();
+      Path indexDir = Paths.get(args[0]).toAbsolutePath();
 
       // satisfy some instrumenters that cache matchers in initializers
       HierarchyMatchers.registerIfAbsent(HierarchyMatchers.simpleChecks());
       SharedTypePools.registerIfAbsent(SharedTypePools.simpleCache());
 
       IndexGenerator indexGenerator = new IndexGenerator();
       indexGenerator.buildIndex();
-      indexGenerator.writeIndex(resourcesDir.resolve(KNOWN_TYPES_INDEX_NAME));
+      indexGenerator.writeIndex(indexDir.resolve(KNOWN_TYPES_INDEX_NAME));
     }
   }
 }

dd-java-agent/build.gradle

@@ -17,12 +17,21 @@ configurations {
   traceShadowInclude
 }
 
+def includedAgentDir = project.layout.buildDirectory.dir("generated/included")
+def includedJarFileTree = fileTree(includedAgentDir)
+
+tasks.named("processResources") {
+  dependsOn(includedJarFileTree)
+}
+
 // The special pre-check should be compiled with Java 6 to detect unsupported Java versions
 // and prevent issues for users that still using them.
 sourceSets {
   "main_java6" {
     java.srcDirs "${project.projectDir}/src/main/java6"
+
   }
+  main.resources.srcDir(includedAgentDir)
 }
 
 def java6CompileTask = tasks.named("compileMain_java6Java") {
@@ -52,9 +61,6 @@ def generalShadowJarConfig(ShadowJar shadowJarTask) {
 
     duplicatesStrategy = DuplicatesStrategy.FAIL
 
-    // Include AgentPreCheck compiled with Java 6.
-    from sourceSets.main_java6.output
-
     // Remove some cruft from the final jar.
     // These patterns should NOT include **/META-INF/maven/**/pom.properties, which is
     // used to report our own dependencies, but we should remove the top-level metadata
@@ -114,29 +120,32 @@ def generalShadowJarConfig(ShadowJar shadowJarTask) {
   }
 }
 
-def includeShadowJar(TaskProvider<ShadowJar> includedShadowJarTask, String destinationDir) {
-  def opentracingFound = new AtomicBoolean()
-  project.tasks.named("processResources", ProcessResources) {
-    doFirst {
+def includeShadowJar(TaskProvider<ShadowJar> includedShadowJarTask, String agentDir, FileTree includedJarFileTree) {
+  def expandTask = project.tasks.register("expandAgentShadowJar${agentDir.capitalize()}", Sync) {
+    it.group = LifecycleBasePlugin.BUILD_GROUP
+    it.description = "Expand the included shadow jar into the agent jar under ${agentDir}"
+
+    def opentracingFound = new AtomicBoolean()
+    it.doFirst("detect-open-tracing") {
       eachFile {
         // We seem unlikely to use this name somewhere else.
         if (it.path.contains("opentracing") && it.name.contains("Format\$Builtin")) {
           opentracingFound.set(true)
         }
       }
     }
-    doLast {
+    it.doLast("fail-on-detected-opentracing") {
       if (opentracingFound.get()) {
         throw new GradleException("OpenTracing direct dependency found!")
       }
     }
 
-    from(zipTree(includedShadowJarTask.map { it.archiveFile })) {
-      into destinationDir
+    it.into providers.provider { new File(includedJarFileTree.dir, agentDir) }
+    it.from(zipTree(includedShadowJarTask.map { it.archiveFile })) {
       rename '(^.*)\\.class$', '$1.classdata'
       // Rename LICENSE file since it clashes with license dir on non-case sensitive FSs (i.e. Mac)
       rename '^LICENSE$', 'LICENSE.renamed'
-      if (destinationDir == 'inst') {
+      if (agentDir == 'inst') {
         // byte-buddy now ships classes optimized for Java8+ under META-INF/versions/9
         // since we target Java8+ we can promote these classes over the pre-Java8 ones
         duplicatesStrategy = DuplicatesStrategy.EXCLUDE
@@ -148,39 +157,41 @@ def includeShadowJar(TaskProvider<ShadowJar> includedShadowJarTask, String desti
       }
     }
 
-    dependsOn includedShadowJarTask
+    it.dependsOn includedShadowJarTask
   }
 
+  includedJarFileTree.builtBy(expandTask)
+
   includedShadowJarTask.configure {
     generalShadowJarConfig(it as ShadowJar)
   }
 }
 
-def includeSubprojShadowJar(Project includedProjectJar, String destinationDir) {
+def includeSubprojShadowJar(Project includedProjectJar, String destinationDir, FileTree includedJarFileTree) {
   evaluationDependsOn(includedProjectJar.path)
-  includeShadowJar(includedProjectJar.tasks.named("shadowJar", ShadowJar), destinationDir)
+  includeShadowJar(includedProjectJar.tasks.named("shadowJar", ShadowJar), destinationDir, includedJarFileTree)
 }
 
-includeSubprojShadowJar(project(':dd-java-agent:instrumentation'), 'inst')
-includeSubprojShadowJar(project(':dd-java-agent:agent-jmxfetch'), 'metrics')
-includeSubprojShadowJar(project(':dd-java-agent:agent-profiling'), 'profiling')
-includeSubprojShadowJar(project(':dd-java-agent:appsec'), 'appsec')
-includeSubprojShadowJar(project(':dd-java-agent:agent-aiguard'), 'aiguard')
-includeSubprojShadowJar(project(':dd-java-agent:agent-iast'), 'iast')
-includeSubprojShadowJar(project(':dd-java-agent:agent-debugger'), 'debugger')
-includeSubprojShadowJar(project(':dd-java-agent:agent-ci-visibility'), 'ci-visibility')
-includeSubprojShadowJar(project(':dd-java-agent:agent-llmobs'), 'llm-obs')
-includeSubprojShadowJar(project(':dd-java-agent:agent-logs-intake'), 'logs-intake')
-includeSubprojShadowJar(project(':dd-java-agent:cws-tls'), 'cws-tls')
+includeSubprojShadowJar(project(':dd-java-agent:instrumentation'), 'inst', includedJarFileTree)
+includeSubprojShadowJar(project(':dd-java-agent:agent-jmxfetch'), 'metrics', includedJarFileTree)
+includeSubprojShadowJar(project(':dd-java-agent:agent-profiling'), 'profiling', includedJarFileTree)
+includeSubprojShadowJar(project(':dd-java-agent:appsec'), 'appsec', includedJarFileTree)
+includeSubprojShadowJar(project(':dd-java-agent:agent-aiguard'), 'aiguard', includedJarFileTree)
+includeSubprojShadowJar(project(':dd-java-agent:agent-iast'), 'iast', includedJarFileTree)
+includeSubprojShadowJar(project(':dd-java-agent:agent-debugger'), 'debugger', includedJarFileTree)
+includeSubprojShadowJar(project(':dd-java-agent:agent-ci-visibility'), 'ci-visibility', includedJarFileTree)
+includeSubprojShadowJar(project(':dd-java-agent:agent-llmobs'), 'llm-obs', includedJarFileTree)
+includeSubprojShadowJar(project(':dd-java-agent:agent-logs-intake'), 'logs-intake', includedJarFileTree)
+includeSubprojShadowJar(project(':dd-java-agent:cws-tls'), 'cws-tls', includedJarFileTree)
 
 def sharedShadowJar = tasks.register('sharedShadowJar', ShadowJar) {
-  configurations = [project.configurations.sharedShadowInclude]
+  it.configurations = [project.configurations.sharedShadowInclude]
   // Put the jar in a different directory so we don't overwrite the normal shadow jar and
   // break caching, and also to not interfere with CI scripts that copy everything in the
   // libs directory
   it.destinationDirectory.set(project.layout.buildDirectory.dir("shared-lib"))
   // Add a classifier so we don't confuse the jar file with the normal shadow jar
-  archiveClassifier = 'shared'
+  it.archiveClassifier = 'shared'
   it.dependencies {
     exclude(project(':dd-java-agent:agent-bootstrap'))
     exclude(project(':dd-java-agent:agent-logging'))
@@ -192,18 +203,21 @@ def sharedShadowJar = tasks.register('sharedShadowJar', ShadowJar) {
     exclude(dependency('org.slf4j::'))
   }
 }
-includeShadowJar(sharedShadowJar, 'shared')
+includeShadowJar(sharedShadowJar, 'shared', includedJarFileTree)
 
 // place the tracer in its own shadow jar separate to instrumentation
 def traceShadowJar = tasks.register('traceShadowJar', ShadowJar) {
-  configurations = [project.configurations.traceShadowInclude]
+  it.configurations = [project.configurations.traceShadowInclude]
   it.destinationDirectory.set(project.layout.buildDirectory.dir("trace-lib"))
-  archiveClassifier = 'trace'
+  it.archiveClassifier = 'trace'
   it.dependencies deps.excludeShared
 }
-includeShadowJar(traceShadowJar, 'trace')
+includeShadowJar(traceShadowJar, 'trace', includedJarFileTree)
 
 tasks.named("shadowJar", ShadowJar) {
+  // Include AgentPreCheck compiled with Java 6.
+  from sourceSets.main_java6.output
+
   generalShadowJarConfig(it)
 
   configurations = [project.configurations.shadowInclude]
@@ -221,24 +235,35 @@ tasks.named("shadowJar", ShadowJar) {
   }
 }
 
-tasks.register('generateAgentJarIndex', JavaExec) {
-  def indexName = 'dd-java-agent.index'
-  def contentDir = "${sourceSets.main.output.resourcesDir}"
-  def indexFile = "${contentDir}/${indexName}"
+// temporary config to add slf4j-simple so we get logging while indexing
+project.configurations.register('slf4j-simple') {
+  it.dependencies.add(project.dependencyFactory.create("org.slf4j:slf4j-simple:${libs.versions.slf4j.get()}"))
+}
+
+def generateAgentJarIndex = tasks.register('generateAgentJarIndex', JavaExec) {
+  def destinationDir = project.layout.buildDirectory.dir("generated/${it.name}")
 
-  it.group = 'Build'
+  it.group = LifecycleBasePlugin.BUILD_GROUP
   it.description = "Generate dd-java-agent.index"
-  it.inputs.files(fileTree(contentDir).exclude(indexName))
-  it.outputs.files(indexFile)
   it.mainClass = 'datadog.trace.bootstrap.AgentJarIndex$IndexGenerator'
-  it.classpath = project.configurations.shadowInclude
-  it.args = [contentDir]
 
-  dependsOn 'processResources'
-  dependsOn 'writeVersionNumberFile'
+  it.inputs.files(includedJarFileTree)
+  it.inputs.files(it.classpath)
+  it.outputs.dir(destinationDir)
+  it.classpath = objects.fileCollection().tap {
+    it.from(project.configurations.named("shadowInclude"))
+    it.from(project.configurations.named('slf4j-simple'))
+  }
+  // debuggable within gradle using:
+  // it.jvmArgs("-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5005")
+  it.argumentProviders.add(new CommandLineArgumentProvider() {
+      @Override
+      Iterable<String> asArguments() {
+        return [includedAgentDir.get().asFile.path, destinationDir.get().asFile.path,]
+      }
+    })
 }
-
-compileJava.dependsOn 'generateAgentJarIndex'
+sourceSets.main.resources.srcDir(generateAgentJarIndex)
 
 subprojects { Project subProj ->
   // Don't need javadoc task run for internal projects.

dd-java-agent/cws-tls/build.gradle

@@ -1,4 +1,5 @@
 import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar
+import org.apache.maven.model.License
 
 plugins {
   id 'com.gradleup.shadow'
@@ -22,6 +23,10 @@ dependencies {
 
 tasks.named("shadowJar", ShadowJar) {
   dependencies deps.excludeShared
-  // exclude this since it's available in the instrumentation jar
-  exclude 'com/sun/jna/**/*'
+
+  // exclude 'jna' this since it's available in the instrumentation jar
+  dependencies {
+    exclude(dependency("net.java.dev.jna:jna"))
+    exclude(dependency("net.java.dev.jna:jna-platform"))
+  }
 }