Merge pull request #43 from aliyun/feature/release-v1.1.7-4-tookits-structure

add client-toolkits module, re-align client project modules structure

Author: Garfier

client/client-common/pom.xml

@@ -0,0 +1,82 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>com.aliyun.dataworks</groupId>
+        <artifactId>client</artifactId>
+        <version>1.1.7-4</version>
+        <relativePath>../pom.xml</relativePath>
+    </parent>
+
+    <artifactId>client-common</artifactId>
+    <packaging>jar</packaging>
+
+    <name>client-common</name>
+
+    <properties>
+        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+    </properties>
+
+    <dependencies>
+        <dependency>
+            <groupId>ch.qos.logback</groupId>
+            <artifactId>logback-classic</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-api</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>com.google.guava</groupId>
+            <artifactId>guava</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-collections4</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>commons-io</groupId>
+            <artifactId>commons-io</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-lang3</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>commons-cli</groupId>
+            <artifactId>commons-cli</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>com.alibaba.fastjson2</groupId>
+            <artifactId>fastjson2</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.projectlombok</groupId>
+            <artifactId>lombok</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+            <version>3.8.1</version>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-source-plugin</artifactId>
+                <version>3.0.1</version>
+                <executions>
+                    <execution>
+                        <id>attach-sources</id>
+                        <goals>
+                            <goal>jar</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+</project>

client/src/main/bin/common.py renamed to client/client-common/src/main/bin/common.py

@@ -1,7 +1,7 @@
 import os
-import sys
 import re
 import subprocess
+import sys
 
 
 def get_cur_path():
@@ -61,14 +61,15 @@ def run_command(app_type, main_class=None):
         "java", "-cp", get_classpath(app_type),
         "-DcurrentDir=" + os.path.abspath(os.path.join(get_cur_path(), "..")),
         "-DappType=" + app_type,
-        "com.aliyun.migrationx.common.command.appbase.CommandAppEntrance" if not main_class else main_class
+        "com.aliyun.dataworks.client.command.CommandAppEntrance" if not main_class else main_class
     ]
 
-    verbose = True if 'MIGRATIONX_VERBOSE' not in os.environ else bool(os.environ['MIGRATIONX_VERBOSE'])
+    verbose = True if 'CLIENT_VERBOSE' not in os.environ else bool(os.environ['CLIENT_VERBOSE'])
     if verbose:
         logging_file = os.path.abspath(os.path.join(get_log_path(), app_type + ".log"))
         logback_xml = os.path.abspath(os.path.join(get_conf_path(), "logback.xml"))
         print("Logging file refer to: " + logging_file)
+        print("Logback file refer to: " + logback_xml)
         cmd.insert(3, "-Dlogging.file=" + logging_file)
         cmd.insert(4, "-Dlogging.appenderRef=CONSOLE")
         cmd.insert(5, "-Dlogback.configurationFile=" + logback_xml)

client/src/main/conf/logback.xml renamed to client/client-common/src/main/conf/logback.xml

@@ -13,15 +13,17 @@
  * limitations under the License.
  */
 
-package com.aliyun.migrationx.common.command.appbase;
+package com.aliyun.dataworks.client.command;
+
+import com.alibaba.fastjson2.JSONObject;
 
-import com.google.gson.JsonObject;
 import lombok.Data;
 import lombok.ToString;
 import lombok.experimental.Accessors;
 
 /**
  * App Meta Info
+ *
  * @author 聿剑
  * @date 2023/03/08
  */
@@ -32,5 +34,5 @@ public class AppMeta {
     private String name;
     private AppType type;
     private String appClass;
-    private JsonObject config;
+    private JSONObject config;
 }

client/migrationx-common/src/main/java/com/aliyun/migrationx/common/command/appbase/AppMeta.java renamed to client/client-common/src/main/java/com/aliyun/dataworks/client/command/AppMeta.java

@@ -13,15 +13,15 @@
  * limitations under the License.
  */
 
-package com.aliyun.migrationx.common.command.appbase;
+package com.aliyun.dataworks.client.command;
 
 /**
- *
  * @author 聿剑
  * @date 2023/9/14
  */
 public enum AppType {
     reader,
     transformer,
-    writer
+    writer,
+    spec
 }

client/migrationx-common/src/main/java/com/aliyun/migrationx/common/command/appbase/AppType.java renamed to client/client-common/src/main/java/com/aliyun/dataworks/client/command/AppType.java

@@ -13,12 +13,13 @@
  * limitations under the License.
  */
 
-package com.aliyun.migrationx.common.command.appbase;
+package com.aliyun.dataworks.client.command;
 
 import java.util.Arrays;
 import java.util.Locale;
 import java.util.Optional;
 
+import lombok.Setter;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.cli.CommandLine;
 import org.apache.commons.cli.CommandLineParser;
@@ -32,26 +33,28 @@
  * @author 聿剑
  * @date 2022/10/20
  */
+@Setter
 @Slf4j
 public abstract class CommandApp {
+    /**
+     * -- SETTER --
+     * set app meta
+     */
     protected AppMeta appMeta;
 
     /**
      * run app
      *
      * @param args String[]
      */
-    public abstract void run(String[] args) throws Exception;
-
-    /**
-     * set app meta
-     *
-     * @param appMeta AppMeta
-     */
-    public void setAppMeta(AppMeta appMeta) {
-        this.appMeta = appMeta;
+    public void run(String[] args) throws Exception {
+        Options options = getOptions();
+        CommandLine cli = getCommandLine(options, args);
+        doCommandRun(options, cli, args);
     }
 
+    protected void doCommandRun(Options options, CommandLine cli, String[] args) {}
+
     protected CommandLine getCommandLine(Options options, String[] args) throws Exception {
         HelpFormatter helpFormatter = new HelpFormatter();
         try {

client/migrationx-common/src/main/java/com/aliyun/migrationx/common/command/appbase/CommandApp.java renamed to client/client-common/src/main/java/com/aliyun/dataworks/client/command/CommandApp.java

@@ -13,19 +13,19 @@
  * limitations under the License.
  */
 
-package com.aliyun.migrationx.common.command.appbase;
+package com.aliyun.dataworks.client.command;
 
 import java.io.File;
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
 import java.util.Map;
+import java.util.Objects;
+import java.util.stream.Stream;
 
-import com.aliyun.migrationx.common.exception.BizException;
-import com.aliyun.migrationx.common.exception.ErrorCode;
-import com.aliyun.migrationx.common.utils.GsonUtils;
+import com.alibaba.fastjson2.JSON;
 
 import com.google.common.base.Joiner;
-import com.google.gson.reflect.TypeToken;
+import com.google.common.reflect.TypeToken;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.cli.CommandLine;
 import org.apache.commons.cli.CommandLineParser;
@@ -46,7 +46,7 @@ public static void main(String[] args) throws IOException {
         String currentDir = System.getProperty("currentDir");
         AppType appType = AppType.valueOf(System.getProperty("appType"));
 
-        Map<AppType, Map<String, AppMeta>> apps = loadApps(Joiner.on(File.separator).join(currentDir, "conf"));
+        Map<AppType, Map<String, AppMeta>> apps = loadApps(Joiner.on(File.separator).join(currentDir, "conf"), appType);
 
         Options options = new Options();
         options.addRequiredOption("a", "app", true, "app name");
@@ -58,7 +58,7 @@ public static void main(String[] args) throws IOException {
             String appName = commandLine.getOptionValue("a");
 
             if (!apps.get(appType).containsKey(appName)) {
-                throw new BizException(ErrorCode.UNKNOWN_COMMAND_APP).with(appName);
+                throw new RuntimeException("unknown app: " + appName);
             }
 
             CommandApp app = CommandAppFactory.create(appType, appName);
@@ -68,7 +68,7 @@ public static void main(String[] args) throws IOException {
             log.info("app command success");
         } catch (ParseException e) {
             log.error("app command parse error: {}", e.getMessage());
-            String footer = Joiner.on(" ").join("\nAvailable apps: \n", GsonUtils.toJsonString(CommandAppFactory.getApps()));
+            String footer = Joiner.on(" ").join("\nAvailable apps: \n", JSON.toJSONString(CommandAppFactory.getApps()));
             helpFormatter.printHelp("Options", "migrationx", options, footer);
         } catch (Exception e) {
             log.error("app command failed: ", e);
@@ -77,28 +77,32 @@ public static void main(String[] args) throws IOException {
     }
 
     @SuppressWarnings("unchecked")
-    private static Map<AppType, Map<String, AppMeta>> loadApps(String conf) throws IOException {
+    private static Map<AppType, Map<String, AppMeta>> loadApps(String conf, AppType... appTypes) throws IOException {
         File config = new File(conf);
         File appsJson = config;
         if (config.isDirectory()) {
             appsJson = new File(config, "apps.json");
         }
 
         String json = FileUtils.readFileToString(appsJson, StandardCharsets.UTF_8);
-        Map<AppType, Map<String, AppMeta>> apps = GsonUtils.fromJsonString(json, new TypeToken<Map<AppType, Map<String, AppMeta>>>() {}.getType());
+        Map<AppType, Map<String, AppMeta>> apps = JSON.parseObject(json, new TypeToken<Map<AppType, Map<String, AppMeta>>>() {}.getType());
         MapUtils.emptyIfNull(apps).forEach((appType, map) -> MapUtils.emptyIfNull(map).forEach((appName, appMeta) -> {
+            if (appTypes != null && Stream.of(appTypes).noneMatch(t -> Objects.equals(t, appType))) {
+                log.info("skip load app type: {}, appType needs load: {}", appType, appTypes);
+                return;
+            }
+
             try {
                 appMeta.setName(appName);
                 appMeta.setType(appType);
                 log.info("register command app type: {}, name: {}, class: {}", appType, appName, appMeta.getAppClass());
-                CommandAppFactory.register(appType, appName, (Class<? extends CommandApp>) Class.forName(appMeta.getAppClass()));
+                CommandAppFactory.register(appType, appName, (Class<? extends CommandApp>)Class.forName(appMeta.getAppClass()));
             } catch (ClassNotFoundException e) {
-                log.info("register command app failed, appType: {}, appName: {}, class: {}, error: {}",
-                        appType, appName, appMeta.getAppClass(), e);
+                log.info("register command app failed, appType: {}, appName: {}, class: {}, error: ", appType, appName, appMeta.getAppClass(), e);
             }
         }));
 
-        log.info("apps map: {}", GsonUtils.toJsonString(apps));
+        log.info("apps map: {}", JSON.toJSONString(apps));
         return apps;
     }
 }

client/migrationx-common/src/main/java/com/aliyun/migrationx/common/command/appbase/CommandAppEntrance.java renamed to client/client-common/src/main/java/com/aliyun/dataworks/client/command/CommandAppEntrance.java

@@ -13,13 +13,12 @@
  * limitations under the License.
  */
 
-package com.aliyun.migrationx.common.command.appbase;
+package com.aliyun.dataworks.client.command;
 
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.function.Function;
 import java.util.stream.Collectors;
 
 import lombok.extern.slf4j.Slf4j;

client/migrationx-common/src/main/java/com/aliyun/migrationx/common/command/appbase/CommandAppFactory.java renamed to client/client-common/src/main/java/com/aliyun/dataworks/client/command/CommandAppFactory.java

@@ -0,0 +1,76 @@
+<!--
+  ~ Copyright (c) 2024, Alibaba Cloud;
+  ~ Licensed under the Apache License, Version 2.0 (the "License");
+  ~ you may not use this file except in compliance with the License.
+  ~ You may obtain a copy of the License at
+  ~
+  ~    http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+
+<assembly
+        xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"
+        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+        xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
+    <formats>
+        <format>tar.gz</format>
+    </formats>
+    <id>all</id>
+    <includeBaseDirectory>true</includeBaseDirectory>
+    <fileSets>
+        <!-- script bin conf -->
+        <fileSet>
+            <directory>${project.basedir}/../client-common/src/main/bin</directory>
+            <includes>
+                <include>**/*</include>
+            </includes>
+            <outputDirectory>bin</outputDirectory>
+            <fileMode>755</fileMode>
+        </fileSet>
+        <fileSet>
+            <directory>${project.basedir}/../client-common/src/main/conf</directory>
+            <includes>
+                <include>**/*</include>
+            </includes>
+            <outputDirectory>conf</outputDirectory>
+        </fileSet>
+
+        <fileSet>
+            <directory>${project.basedir}/src/main/bin</directory>
+            <includes>
+                <include>**/*</include>
+            </includes>
+            <outputDirectory>bin</outputDirectory>
+            <fileMode>755</fileMode>
+        </fileSet>
+        <fileSet>
+            <directory>${project.basedir}/src/main/conf</directory>
+            <includes>
+                <include>**/*</include>
+            </includes>
+            <outputDirectory>conf</outputDirectory>
+        </fileSet>
+
+        <fileSet>
+            <directory>${project.basedir}/target/lib/common</directory>
+            <includes>
+                <include>*.jar</include>
+            </includes>
+            <outputDirectory>lib/common</outputDirectory>
+        </fileSet>
+
+        <fileSet>
+            <directory>${project.basedir}/target</directory>
+            <includes>
+                <include>*jar</include>
+            </includes>
+            <outputDirectory>lib/common</outputDirectory>
+        </fileSet>
+
+    </fileSets>
+</assembly>