Use a custom task to deploy the files from the class loader. (apache-spark-on-k8s#440)

It's impossible to use zipTree or fileTree to deploy files out of the resources folder.

To make matters cleaner, build the resources into a zip file in the jar, then use a simple stream copy to write the zip file to disk. Then use a sync task to actually unpack the zip file. The result is avoiding any need to do any zip-specific magic to extract the files.

Author: mccheah

gradle/publish.gradle

@@ -36,7 +36,7 @@ bintray {
     name = rootProject.name
     userOrg = 'palantir'
     licenses = ['Apache-2.0']
-    publications = ['nebula']
+    publications = ['nebula', 'dockerBundle']
   }
 }
 

settings.gradle

@@ -19,3 +19,4 @@
 rootProject.name = 'spark'
 
 include ':spark-docker-image-generator'
+enableFeaturePreview('STABLE_PUBLISHING')

spark-docker-image-generator/build.gradle

@@ -41,7 +41,7 @@ test {
   maxHeapSize = "512m"
 }
 
-task prepareDockerBundleDir(type: Sync) {
+task prepareDockerBundleDir(type: Zip) {
   from("${rootDir}/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/Dockerfile") {
     into 'kubernetes/dockerfiles/spark'
     rename 'Dockerfile', 'Dockerfile.original'
@@ -57,9 +57,33 @@ task prepareDockerBundleDir(type: Sync) {
   from(fileTree("${rootDir}/sbin")) {
     into 'sbin'
   }
-  into file("src/main/resources/docker-resources")
+  destinationDir file("${buildDir}/docker-resources")
+  archiveName "docker-resources.zip"
   includeEmptyDirs = false
 }
 
 tasks.compileJava.dependsOn tasks.prepareDockerBundleDir
 tasks.idea.dependsOn tasks.prepareDockerBundleDir
+
+publishing {
+  publications {
+    dockerBundle(MavenPublication) {
+      artifactId 'spark-docker-resources'
+      artifact tasks.prepareDockerBundleDir
+    }
+  }
+
+  repositories {
+    maven {
+      name = "mavenTest"
+      url = "${buildDir}/mavenTest"
+    }
+  }
+}
+
+test {
+    systemProperty 'docker-resources-zip-path', "${buildDir}/docker-resources/docker-resources.zip"
+}
+
+tasks.test.dependsOn tasks.publishDockerBundlePublicationToMavenTestRepository
+tasks.test.environment 'MAVEN_REPO', "${buildDir.getAbsolutePath()}/mavenTest"

spark-docker-image-generator/plugin-test-project/build.gradle

@@ -29,6 +29,7 @@ plugins {
 repositories {
   jcenter()
   maven { url "http://palantir.bintray.com/releases" }
+  maven { url System.env.MAVEN_REPO }
 }
 
 apply plugin: 'java'

spark-docker-image-generator/src/main/java/org/apache/spark/deploy/kubernetes/docker/gradle/SparkDockerPlugin.java

@@ -17,11 +17,11 @@
 package org.apache.spark.deploy.kubernetes.docker.gradle;
 
 import java.io.File;
-import java.net.URL;
 import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
+import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;
 
@@ -77,10 +77,13 @@ public void apply(Project project) {
               sparkAppJar)
               .into(jarsDirProvider));
       copySparkAppLibTask.dependsOn(jarTask);
-      URL dockerResourcesUrl = getClass().getResource("/docker-resources");
+      String version = Optional.ofNullable(getClass().getPackage().getImplementationVersion())
+              .orElse("latest.release");
+      Configuration dockerResourcesConf = project.getConfigurations().detachedConfiguration(
+            project.getDependencies().create("org.apache.spark:spark-docker-resources:" + version));
       Sync deployScriptsTask = project.getTasks().create(
           "sparkDockerDeployScripts", Sync.class, task -> {
-            task.from(project.fileTree(dockerResourcesUrl));
+            task.from(project.zipTree(dockerResourcesConf.getSingleFile()));
             task.setIncludeEmptyDirs(false);
             task.into(dockerBuildDirectory);
           });

spark-docker-image-generator/src/test/java/org/apache/spark/deploy/kubernetes/docker/gradle/GenerateDockerFileTaskSuite.java

@@ -18,6 +18,7 @@
 
 import java.io.BufferedReader;
 import java.io.File;
+import java.io.FileInputStream;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
@@ -26,6 +27,8 @@
 import java.nio.file.Files;
 import java.util.List;
 import java.util.stream.Collectors;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipInputStream;
 
 import org.apache.commons.io.IOUtils;
 import org.assertj.core.api.Assertions;
@@ -37,7 +40,6 @@
 import org.mockito.Mockito;
 
 public final class GenerateDockerFileTaskSuite {
-
   @Rule
   public TemporaryFolder tempFolder = new TemporaryFolder();
 
@@ -49,10 +51,24 @@ public void before() throws IOException {
     File dockerFileDir = tempFolder.newFolder("docker");
     destDockerFile = new File(dockerFileDir, "Dockerfile");
     srcDockerFile = tempFolder.newFile("Dockerfile.original");
-    try (InputStream originalDockerFileResource = getClass().getResourceAsStream(
-        "/docker-resources/kubernetes/dockerfiles/spark/Dockerfile.original");
-         FileOutputStream srcDockerFileStream = new FileOutputStream(srcDockerFile)) {
-      IOUtils.copy(originalDockerFileResource, srcDockerFileStream);
+
+    try (InputStream originalDockerBundleZipped = new FileInputStream(
+            new File(System.getProperty("docker-resources-zip-path")));
+        ZipInputStream unzipped = new ZipInputStream(originalDockerBundleZipped);
+        FileOutputStream srcDockerFileStream = new FileOutputStream(srcDockerFile)) {
+      ZipEntry currentEntry = unzipped.getNextEntry();
+      boolean foundDockerFile = false;
+      while (currentEntry != null && !foundDockerFile) {
+        if (currentEntry.getName().equals("kubernetes/dockerfiles/spark/Dockerfile.original")) {
+          IOUtils.copy(unzipped, srcDockerFileStream);
+          foundDockerFile = true;
+        } else {
+          currentEntry = unzipped.getNextEntry();
+        }
+      }
+      if (!foundDockerFile) {
+        throw new IllegalStateException("Dockerfile not found.");
+      }
     }
   }
 
@@ -73,7 +89,7 @@ public void testGenerateDockerFile() throws IOException {
         getClass().getResourceAsStream("/ExpectedDockerfile");
         InputStreamReader expectedDockerFileReader =
             new InputStreamReader(expectedDockerFileInput, StandardCharsets.UTF_8);
-         BufferedReader expectedDockerFileBuffered =
+        BufferedReader expectedDockerFileBuffered =
             new BufferedReader(expectedDockerFileReader)) {
       List<String> expectedFileLines = expectedDockerFileBuffered
           .lines()