Update to Embulk v0.10.18, and not to depend on "embulk-core"

Along with Embulk v0.10.18,
* Change the Maven artifact group name to "org.embulk"
* Update the version to "0.5.0-SNAPSHOT"
* Update org.embulk:embulk-util-config to "0.1.3" to use its newConfigDiff and newTaskReport
* Use org.embulk:embulk-util-retryhelper instead of org.embulk.spi.util.RetryExecutor
* Use org.embulk:embulk-util-file instead of org.embulk.spi.util.*
* Remove use of commons-lang3

Author: Dai MIKURUBE

README.md

@@ -2,7 +2,7 @@
 
 ## Overview
 
-embulk-input-s3 v0.3.0+ requires Embulk v0.9.12+
+embulk-input-s3 v0.5.0+ requires Embulk v0.9.23+
 
 * Plugin type: **file input**
 * Resume supported: **yes**

build.gradle

@@ -7,8 +7,8 @@ plugins {
     id "org.embulk.embulk-plugins" version "0.4.2" apply false
 }
 
-group = "org.embulk.input.s3"
-version = "0.4.1-SNAPSHOT"
+group = "org.embulk"
+version = "0.5.0-SNAPSHOT"
 
 allprojects {
     apply plugin: "java"

embulk-input-riak_cs/build.gradle

@@ -10,12 +10,13 @@ repositories {
 }
 
 dependencies {
-    compileOnly "org.embulk:embulk-api:0.10.5"
-    compileOnly "org.embulk:embulk-core:0.10.5"
+    compileOnly "org.embulk:embulk-api:0.10.18"
+    compileOnly "org.embulk:embulk-spi:0.10.18"
+    // No longer need to depend on "embulk-core"!
 
     compile(project(":embulk-input-s3")) {
         // They conflict with embulk-core. They are once excluded here,
-        // and added explicitly with versions exactly the same with embulk-core:0.10.5.
+        // and added explicitly with versions exactly the same with embulk-core:0.10.18.
         exclude group: "com.fasterxml.jackson.core", module: "jackson-annotations"
         exclude group: "com.fasterxml.jackson.core", module: "jackson-core"
         exclude group: "com.fasterxml.jackson.core", module: "jackson-databind"
@@ -25,7 +26,7 @@ dependencies {
     }
 
     // They are once excluded from transitive dependencies of other dependencies,
-    // and added explicitly with versions exactly the same with embulk-core:0.10.5.
+    // and added explicitly with versions exactly the same with embulk-core:0.10.18.
     compile "com.fasterxml.jackson.core:jackson-annotations:2.6.7"
     compile "com.fasterxml.jackson.core:jackson-core:2.6.7"
     compile "com.fasterxml.jackson.core:jackson-databind:2.6.7"
@@ -35,31 +36,20 @@ dependencies {
 
     testCompile "junit:junit:4.13"
     testCompile "org.mockito:mockito-core:1.10.19"
-    testCompile "org.embulk:embulk-api:0.10.5"
-    testCompile "org.embulk:embulk-core:0.10.5"
-    testCompile "org.embulk:embulk-core:0.10.5:tests"
-    testCompile "org.embulk:embulk-deps-buffer:0.10.5"
-    testCompile "org.embulk:embulk-deps-config:0.10.5"
-    testCompile "org.embulk:embulk-deps-timestamp:0.10.5"
-    testCompile "org.embulk:embulk-standards:0.10.5"
+    testCompile "org.embulk:embulk-api:0.10.18"
+    testCompile "org.embulk:embulk-spi:0.10.18"
+    testCompile "org.embulk:embulk-core:0.10.18"
+    testCompile "org.embulk:embulk-core:0.10.18:tests"
+    testCompile "org.embulk:embulk-deps:0.10.18"
+    testCompile "org.embulk:embulk-standards:0.10.18"
     testCompile project(":embulk-input-s3").sourceSets.test.output
 }
 
 embulkPlugin {
     mainClass = "org.embulk.input.riak_cs.RiakCsFileInputPlugin"
     category = "input"
     type = "riak_cs"
-    ignoreConflicts = [
-        // They conflict with embulk-core:0.10.5, but embulk-core will remove them from its dependencies.
-        // They are intentionally included in the plugin's dependencies to keep it working before and after the removal.
-        // Warning messages against them are marked to ignore explicitly here.
-        [ group: "com.fasterxml.jackson.core", module: "jackson-annotations" ],
-        [ group: "com.fasterxml.jackson.core", module: "jackson-core" ],
-        [ group: "com.fasterxml.jackson.core", module: "jackson-databind" ],
-        [ group: "com.fasterxml.jackson.datatype", module: "jackson-datatype-jdk8" ],
-        [ group: "javax.validation", module: "validation-api" ],
-        [ group: "joda-time", module: "joda-time" ],
-    ]
+    // "ignoreConflicts" is no longer needed because this plugin depends only on "embulk-api" and "embulk-spi".
 }
 
 gem {

embulk-input-riak_cs/gradle/dependency-locks/embulkPluginRuntime.lockfile

@@ -17,6 +17,8 @@ joda-time:joda-time:2.9.2
 org.apache.httpcomponents:httpclient:4.5.5
 org.apache.httpcomponents:httpcore:4.4.9
 org.embulk:embulk-util-aws-credentials:0.4.0
-org.embulk:embulk-util-config:0.1.1
+org.embulk:embulk-util-config:0.1.3
+org.embulk:embulk-util-file:0.1.1
+org.embulk:embulk-util-retryhelper:0.8.0
 org.slf4j:jcl-over-slf4j:1.7.12
 software.amazon.ion:ion-java:1.0.2

embulk-input-s3/build.gradle

@@ -10,12 +10,12 @@ repositories {
 }
 
 dependencies {
-    compileOnly "org.embulk:embulk-api:0.10.5"
-    compileOnly "org.embulk:embulk-core:0.10.5"
+    compileOnly "org.embulk:embulk-api:0.10.18"
+    compileOnly "org.embulk:embulk-spi:0.10.18"
 
     compile("com.amazonaws:aws-java-sdk-s3:1.11.466") {
         // They conflict with embulk-core. They are once excluded here,
-        // and added explicitly with versions exactly the same with embulk-core:0.10.5.
+        // and added explicitly with versions exactly the same with embulk-core:0.10.18.
         exclude group: "com.fasterxml.jackson.core", module: "jackson-annotations"
         exclude group: "com.fasterxml.jackson.core", module: "jackson-core"
         exclude group: "com.fasterxml.jackson.core", module: "jackson-databind"
@@ -27,7 +27,7 @@ dependencies {
 
     compile("com.amazonaws:aws-java-sdk-sts:1.11.466") {
         // They conflict with embulk-core. They are once excluded here,
-        // and added explicitly with versions exactly the same with embulk-core:0.10.5.
+        // and added explicitly with versions exactly the same with embulk-core:0.10.18.
         exclude group: "com.fasterxml.jackson.core", module: "jackson-annotations"
         exclude group: "com.fasterxml.jackson.core", module: "jackson-core"
         exclude group: "com.fasterxml.jackson.core", module: "jackson-databind"
@@ -45,9 +45,13 @@ dependencies {
         exclude group: "org.slf4j", module: "slf4j-api"
     }
 
+    compile "org.embulk:embulk-util-retryhelper:0.8.0"
+
+    compile "org.embulk:embulk-util-file:0.1.1"
+
     compile("org.embulk:embulk-util-aws-credentials:0.4.0") {
         // They conflict with embulk-core. They are once excluded here,
-        // and added explicitly with versions exactly the same with embulk-core:0.10.5.
+        // and added explicitly with versions exactly the same with embulk-core:0.10.18.
         exclude group: "com.fasterxml.jackson.core", module: "jackson-annotations"
         exclude group: "com.fasterxml.jackson.core", module: "jackson-core"
         exclude group: "com.fasterxml.jackson.core", module: "jackson-databind"
@@ -56,8 +60,18 @@ dependencies {
         exclude group: "joda-time", module: "joda-time"
     }
 
+    compile("org.embulk:embulk-util-config:0.1.3") {
+        // They conflict with embulk-core. They are once excluded here,
+        // and added explicitly with versions exactly the same with embulk-core:0.10.18.
+        exclude group: "com.fasterxml.jackson.core", module: "jackson-annotations"
+        exclude group: "com.fasterxml.jackson.core", module: "jackson-core"
+        exclude group: "com.fasterxml.jackson.core", module: "jackson-databind"
+        exclude group: "com.fasterxml.jackson.datatype", module: "jackson-datatype-jdk8"
+        exclude group: "javax.validation", module: "validation-api"
+    }
+
     // They are once excluded from transitive dependencies of other dependencies,
-    // and added explicitly with versions exactly the same with embulk-core:0.10.5.
+    // and added explicitly with versions exactly the same with embulk-core:0.10.18.
     compile "com.fasterxml.jackson.core:jackson-annotations:2.6.7"
     compile "com.fasterxml.jackson.core:jackson-core:2.6.7"
     compile "com.fasterxml.jackson.core:jackson-databind:2.6.7"
@@ -67,30 +81,19 @@ dependencies {
 
     testCompile "junit:junit:4.13"
     testCompile "org.mockito:mockito-core:1.10.19"
-    testCompile "org.embulk:embulk-api:0.10.5"
-    testCompile "org.embulk:embulk-core:0.10.5"
-    testCompile "org.embulk:embulk-core:0.10.5:tests"
-    testCompile "org.embulk:embulk-deps-buffer:0.10.5"
-    testCompile "org.embulk:embulk-deps-config:0.10.5"
-    testCompile "org.embulk:embulk-deps-timestamp:0.10.5"
-    testCompile "org.embulk:embulk-standards:0.10.5"
+    testCompile "org.embulk:embulk-api:0.10.18"
+    testCompile "org.embulk:embulk-spi:0.10.18"
+    testCompile "org.embulk:embulk-core:0.10.18"
+    testCompile "org.embulk:embulk-core:0.10.18:tests"
+    testCompile "org.embulk:embulk-deps:0.10.18"
+    testCompile "org.embulk:embulk-standards:0.10.18"
 }
 
 embulkPlugin {
     mainClass = "org.embulk.input.s3.S3FileInputPlugin"
     category = "input"
     type = "s3"
-    ignoreConflicts = [
-        // They conflict with embulk-core:0.10.5, but embulk-core will remove them from its dependencies.
-        // They are intentionally included in the plugin's dependencies to keep it working before and after the removal.
-        // Warning messages against them are marked to ignore explicitly here.
-        [ group: "com.fasterxml.jackson.core", module: "jackson-annotations" ],
-        [ group: "com.fasterxml.jackson.core", module: "jackson-core" ],
-        [ group: "com.fasterxml.jackson.core", module: "jackson-databind" ],
-        [ group: "com.fasterxml.jackson.datatype", module: "jackson-datatype-jdk8" ],
-        [ group: "javax.validation", module: "validation-api" ],
-        [ group: "joda-time", module: "joda-time" ],
-    ]
+    // "ignoreConflicts" is no longer needed because this plugin depends only on "embulk-api" and "embulk-spi".
 }
 
 gem {

embulk-input-s3/gradle/dependency-locks/embulkPluginRuntime.lockfile

@@ -17,6 +17,8 @@ joda-time:joda-time:2.9.2
 org.apache.httpcomponents:httpclient:4.5.5
 org.apache.httpcomponents:httpcore:4.4.9
 org.embulk:embulk-util-aws-credentials:0.4.0
-org.embulk:embulk-util-config:0.1.1
+org.embulk:embulk-util-config:0.1.3
+org.embulk:embulk-util-file:0.1.1
+org.embulk:embulk-util-retryhelper:0.8.0
 org.slf4j:jcl-over-slf4j:1.7.12
 software.amazon.ion:ion-java:1.0.2

embulk-input-s3/src/main/java/org/embulk/input/s3/AbstractS3FileInputPlugin.java

@@ -38,10 +38,6 @@
 import org.embulk.spi.Exec;
 import org.embulk.spi.FileInputPlugin;
 import org.embulk.spi.TransactionalFileInput;
-import org.embulk.spi.util.InputStreamFileInput;
-import org.embulk.spi.util.InputStreamFileInput.InputStreamWithHints;
-import org.embulk.spi.util.ResumableInputStream;
-import org.embulk.spi.util.RetryExecutor;
 import org.embulk.util.aws.credentials.AwsCredentials;
 import org.embulk.util.aws.credentials.AwsCredentialsTask;
 import org.embulk.util.config.Config;
@@ -50,6 +46,9 @@
 import org.embulk.util.config.ConfigMapperFactory;
 import org.embulk.util.config.Task;
 import org.embulk.util.config.TaskMapper;
+import org.embulk.util.file.InputStreamFileInput;
+import org.embulk.util.file.ResumableInputStream;
+import org.embulk.util.retryhelper.RetryExecutor;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -63,7 +62,6 @@
 import java.util.Optional;
 
 import static java.lang.String.format;
-import static org.embulk.spi.util.RetryExecutor.retryExecutor;
 
 public abstract class AbstractS3FileInputPlugin
         implements FileInputPlugin
@@ -169,7 +167,7 @@ public ConfigDiff resume(TaskSource taskSource,
         control.run(taskSource, taskCount);
 
         // build next config
-        ConfigDiff configDiff = Exec.newConfigDiff();
+        ConfigDiff configDiff = CONFIG_MAPPER_FACTORY.newConfigDiff();
 
         // last_path
         if (task.getIncremental()) {
@@ -273,10 +271,11 @@ private void setHttpProxyInAwsClient(ClientConfiguration clientConfig, HttpProxy
      */
     private static RetryExecutor retryExecutorFrom(RetrySupportPluginTask task)
     {
-        return retryExecutor()
+        return RetryExecutor.builder()
             .withRetryLimit(task.getMaximumRetries())
-            .withInitialRetryWait(task.getInitialRetryIntervalMillis())
-            .withMaxRetryWait(task.getMaximumRetryIntervalMillis());
+            .withInitialRetryWaitMillis(task.getInitialRetryIntervalMillis())
+            .withMaxRetryWaitMillis(task.getMaximumRetryIntervalMillis())
+            .build();
     }
 
     private FileList listFiles(final PluginTask task)
@@ -408,7 +407,7 @@ public void abort()
 
         public TaskReport commit()
         {
-            return Exec.newTaskReport();
+            return CONFIG_MAPPER_FACTORY.newTaskReport();
         }
 
         @Override
@@ -442,7 +441,7 @@ public SingleFileProvider(PluginTask task, int taskIndex)
         }
 
         @Override
-        public InputStreamWithHints openNextWithHints() throws IOException
+        public InputStreamFileInput.InputStreamWithHints openNextWithHints() throws IOException
         {
             if (!iterator.hasNext()) {
                 return null;
@@ -463,7 +462,7 @@ public S3Object call()
             // Keep it for now but might be removed in the future.
             LOGGER.info("Open S3Object with bucket [{}], key [{}], with size [{}]", bucket, key, objectSize);
             InputStream inputStream = new ResumableInputStream(object.getObjectContent(), new S3InputStreamReopener(client, request, objectSize, retryExec));
-            return new InputStreamWithHints(inputStream, String.format("s3://%s/%s", bucket, key));
+            return new InputStreamFileInput.InputStreamWithHints(inputStream, String.format("s3://%s/%s", bucket, key));
         }
 
         @Override

embulk-input-s3/src/main/java/org/embulk/input/s3/DefaultRetryable.java

@@ -18,7 +18,9 @@
 
 import com.amazonaws.AmazonServiceException;
 import org.apache.http.HttpStatus;
-import org.embulk.spi.util.RetryExecutor;
+import org.embulk.util.retryhelper.RetryExecutor;
+import org.embulk.util.retryhelper.RetryGiveupException;
+import org.embulk.util.retryhelper.Retryable;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -27,8 +29,6 @@
 import java.util.concurrent.Callable;
 
 import static java.lang.String.format;
-import static org.embulk.spi.util.RetryExecutor.RetryGiveupException;
-import static org.embulk.spi.util.RetryExecutor.Retryable;
 
 /**
  * Retryable utility, regardless the occurred exceptions,

embulk-input-s3/src/main/java/org/embulk/input/s3/explorer/S3FileExplorer.java

@@ -18,7 +18,7 @@
 
 import com.amazonaws.services.s3.AmazonS3;
 import org.embulk.input.s3.FileList;
-import org.embulk.spi.util.RetryExecutor;
+import org.embulk.util.retryhelper.RetryExecutor;
 
 public abstract class S3FileExplorer
 {

embulk-input-s3/src/main/java/org/embulk/input/s3/explorer/S3NameOrderPrefixFileExplorer.java

@@ -21,7 +21,7 @@
 import com.amazonaws.services.s3.model.ObjectListing;
 import com.amazonaws.services.s3.model.S3ObjectSummary;
 import org.embulk.input.s3.DefaultRetryable;
-import org.embulk.spi.util.RetryExecutor;
+import org.embulk.util.retryhelper.RetryExecutor;
 
 import java.util.List;