diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 58425fdcbed..0a475fa0e09 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -56,17 +56,6 @@ updates: patterns: - "*" - - package-ecosystem: "maven" - directory: "/alluxio" - schedule: - interval: "weekly" - open-pull-requests-limit: 0 - groups: - alluxio-security-updates: - applies-to: security-updates - patterns: - - "*" - - package-ecosystem: "maven" directory: "/bigquery" schedule: diff --git a/.github/workflows/core.yml b/.github/workflows/core.yml index db7344d6ad3..e4fa6d60ab8 100644 --- a/.github/workflows/core.yml +++ b/.github/workflows/core.yml @@ -93,7 +93,7 @@ jobs: matrix: java: [ 11 ] env: - INTERPRETERS: 'hbase,jdbc,file,flink-cmd,cassandra,elasticsearch,bigquery,alluxio,livy,groovy,java,neo4j,sparql,mongodb,influxdb,shell' + INTERPRETERS: 'hbase,jdbc,file,flink-cmd,cassandra,elasticsearch,bigquery,livy,groovy,java,neo4j,sparql,mongodb,influxdb,shell' steps: - name: Checkout uses: actions/checkout@v4 diff --git a/.github/workflows/frontend.yml b/.github/workflows/frontend.yml index f019d213c68..8f6bc4880ed 100644 --- a/.github/workflows/frontend.yml +++ b/.github/workflows/frontend.yml @@ -23,7 +23,7 @@ env: SPARK_PRINT_LAUNCH_COMMAND: "true" SPARK_LOCAL_IP: 127.0.0.1 ZEPPELIN_LOCAL_IP: 127.0.0.1 - INTERPRETERS: '!hbase,!jdbc,!file,!flink,!cassandra,!elasticsearch,!bigquery,!alluxio,!livy,!groovy,!java,!neo4j,!sparql,!mongodb' + INTERPRETERS: '!hbase,!jdbc,!file,!flink,!cassandra,!elasticsearch,!bigquery,!livy,!groovy,!java,!neo4j,!sparql,!mongodb' ZEPPELIN_E2E_TEST_NOTEBOOK_DIR: '/tmp/zeppelin-e2e-notebooks' permissions: diff --git a/AGENTS.md b/AGENTS.md index 085f8213db5..4c78d608d0a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -168,7 +168,6 @@ Each interpreter is an independent Maven module inheriting from `zeppelin-interp | `sparql/` | SPARQL queries | | `influxdb/` | InfluxDB | | `file/` | HDFS/local file browser | -| `alluxio/` | Alluxio file system | ### Plugin Modules (`zeppelin-plugins/`) diff --git a/alluxio/pom.xml b/alluxio/pom.xml deleted file mode 100644 index ff51bb4e1c2..00000000000 --- a/alluxio/pom.xml +++ /dev/null @@ -1,110 +0,0 @@ - - - - - 4.0.0 - - - zeppelin-interpreter-parent - org.apache.zeppelin - 0.13.0-SNAPSHOT - ../zeppelin-interpreter-parent/pom.xml - - - zeppelin-alluxio - jar - Zeppelin: Alluxio interpreter - - - 2.9.0 - alluxio - - - - - org.alluxio - alluxio-shell - ${alluxio.version} - - - - - com.google.guava - guava - 32.0.0-jre - test - - - com.google.protobuf - protobuf-java - 3.25.5 - test - - - - org.mockito - mockito-core - test - - - - org.alluxio - alluxio-minicluster - ${alluxio.version} - test - - - org.apache.hadoop - hadoop-client - - - - - - org.apache.hadoop - hadoop-client-api - test - - - - org.apache.hadoop - hadoop-client-runtime - test - - - - - - - maven-enforcer-plugin - - - maven-resources-plugin - - - maven-shade-plugin - - - org.apache.maven.plugins - maven-checkstyle-plugin - - - - - diff --git a/alluxio/src/main/java/org/apache/zeppelin/alluxio/AlluxioInterpreter.java b/alluxio/src/main/java/org/apache/zeppelin/alluxio/AlluxioInterpreter.java deleted file mode 100644 index 5478926ec9d..00000000000 --- a/alluxio/src/main/java/org/apache/zeppelin/alluxio/AlluxioInterpreter.java +++ /dev/null @@ -1,265 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.zeppelin.alluxio; - -import alluxio.cli.fs.FileSystemShell; -import alluxio.conf.Configuration; -import alluxio.conf.AlluxioConfiguration; -import alluxio.conf.PropertyKey; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.PrintStream; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.LinkedList; -import java.util.List; -import java.util.Properties; -import java.util.stream.Stream; - - -import org.apache.zeppelin.completer.CompletionType; -import org.apache.zeppelin.interpreter.Interpreter; -import org.apache.zeppelin.interpreter.InterpreterContext; -import org.apache.zeppelin.interpreter.InterpreterResult; -import org.apache.zeppelin.interpreter.InterpreterResult.Code; -import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion; - -/** - * Alluxio interpreter for Zeppelin. - */ -public class AlluxioInterpreter extends Interpreter { - - private static final Logger LOGGER = LoggerFactory.getLogger(AlluxioInterpreter.class); - - protected static final String ALLUXIO_MASTER_HOSTNAME = "alluxio.master.hostname"; - protected static final String ALLUXIO_MASTER_PORT = "alluxio.master.port"; - - private FileSystemShell fs; - - private int totalCommands = 0; - private int completedCommands = 0; - - private final String alluxioMasterHostname; - private final String alluxioMasterPort; - - protected final List keywords = Arrays.asList("cat", "chgrp", - "chmod", "chown", "copyFromLocal", "copyToLocal", "count", - "createLineage", "deleteLineage", "du", "fileInfo", "free", - "getCapacityBytes", "getUsedBytes", "listLineages", "load", - "loadMetadata", "location", "ls", "mkdir", "mount", "mv", - "persist", "pin", "report", "rm", "setTtl", "tail", "touch", - "unmount", "unpin", "unsetTtl"); - - public AlluxioInterpreter(Properties property) { - super(property); - - alluxioMasterHostname = property.getProperty(ALLUXIO_MASTER_HOSTNAME); - alluxioMasterPort = property.getProperty(ALLUXIO_MASTER_PORT); - } - - private Stream filteredProperties(String prefix) { - return properties.stringPropertyNames().stream().filter( - propertyKey -> propertyKey.startsWith(prefix) - ); - } - - @Override - public void open() { - LOGGER.info("Starting Alluxio shell to connect to " + alluxioMasterHostname + - " on port " + alluxioMasterPort); - // Setting the extra parameters being set in the interpreter config starting with alluxio - filteredProperties("alluxio.").forEach(x -> System.setProperty(x, properties.getProperty(x))); - - System.setProperty(PropertyKey.USER_RPC_RETRY_MAX_DURATION.getName(), "5s"); - - AlluxioConfiguration conf = Configuration.global(); - // Reduce the RPC retry max duration to fall earlier for CLIs - fs = new FileSystemShell(conf); - } - - @Override - public void close() { - LOGGER.info("Closing Alluxio shell"); - try { - fs.close(); - } catch (IOException e) { - LOGGER.error("Cannot close connection", e); - } - } - - @Override - public InterpreterResult interpret(String st, InterpreterContext context) { - String[] lines = splitAndRemoveEmpty(st, "\n"); - return interpret(lines, context); - } - - private InterpreterResult interpret(String[] commands, InterpreterContext context) { - boolean isSuccess = true; - totalCommands = commands.length; - completedCommands = 0; - - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - PrintStream ps = new PrintStream(baos); - PrintStream old = System.out; - - System.setOut(ps); - - for (String command : commands) { - int commandResult = 1; - String[] args = splitAndRemoveEmpty(command, " "); - if (args.length > 0 && args[0].equals("help")) { - System.out.println(getCommandList()); - } else { - commandResult = fs.run(args); - } - if (commandResult != 0) { - isSuccess = false; - break; - } else { - completedCommands += 1; - } - System.out.println(); - } - - System.out.flush(); - System.setOut(old); - - if (isSuccess) { - return new InterpreterResult(Code.SUCCESS, baos.toString()); - } else { - return new InterpreterResult(Code.ERROR, baos.toString()); - } - } - - private String[] splitAndRemoveEmpty(String st, String splitSeparator) { - String[] voices = st.split(splitSeparator); - ArrayList result = new ArrayList<>(); - for (String voice : voices) { - if (!voice.trim().isEmpty()) { - result.add(voice); - } - } - return result.toArray(new String[result.size()]); - } - - private String[] splitAndRemoveEmpty(String[] sts, String splitSeparator) { - ArrayList result = new ArrayList<>(); - for (String st : sts) { - result.addAll(Arrays.asList(splitAndRemoveEmpty(st, splitSeparator))); - } - return result.toArray(new String[result.size()]); - } - - @Override - public void cancel(InterpreterContext context) { } - - @Override - public FormType getFormType() { - return FormType.NATIVE; - } - - @Override - public int getProgress(InterpreterContext context) { - return completedCommands * 100 / totalCommands; - } - - @Override - public List completion(String buf, int cursor, - InterpreterContext interpreterContext) { - String[] words = splitAndRemoveEmpty(splitAndRemoveEmpty(buf, "\n"), " "); - String lastWord = ""; - if (words.length > 0) { - lastWord = words[ words.length - 1 ]; - } - - List voices = new LinkedList<>(); - for (String command : keywords) { - if (command.startsWith(lastWord)) { - voices.add(new InterpreterCompletion(command, command, CompletionType.command.name())); - } - } - return voices; - } - - private String getCommandList() { - StringBuilder sb = new StringBuilder(); - sb.append("Commands list:"); - sb.append("\n\t[help] - List all available commands."); - sb.append("\n\t[cat ] - Prints the file's contents to the console."); - sb.append("\n\t[chgrp [-R] ] - Changes the group of a file or directory " + - "specified by args. Specify -R to change the group recursively."); - sb.append("\n\t[chmod -R ] - Changes the permission of a file or directory " + - "specified by args. Specify -R to change the permission recursively."); - sb.append("\n\t[chown -R ] - Changes the owner of a file or directory " + - "specified by args. Specify -R to change the owner recursively."); - sb.append("\n\t[copyFromLocal ] - Copies a file or a directory from " + - "local filesystem to Alluxio filesystem."); - sb.append("\n\t[copyToLocal ] - Copies a file or a directory from the " + - "Alluxio filesystem to the local filesystem."); - sb.append("\n\t[count ] - Displays the number of files and directories matching " + - "the specified prefix."); - sb.append("\n\t[createLineage " + - "[ ...]] - Creates a lineage."); - sb.append("\n\t[deleteLineage ] - Deletes a lineage. If " + - "cascade is specified as true, dependent lineages will also be deleted."); - sb.append("\n\t[du ] - Displays the size of the specified file or directory."); - sb.append("\n\t[fileInfo ] - Displays all block info for the specified file."); - sb.append("\n\t[free ] - Removes the file or directory(recursively) " + - "from Alluxio memory space."); - sb.append("\n\t[getCapacityBytes] - Gets the capacity of the Alluxio file system."); - sb.append("\n\t[getUsedBytes] - Gets number of bytes used in the Alluxio file system."); - sb.append("\n\t[listLineages] - Lists all lineages."); - sb.append("\n\t[load ] - Loads a file or directory in Alluxio space, makes it " + - "resident in memory."); - sb.append("\n\t[loadMetadata ] - Loads metadata for the given Alluxio path from the " + - "under file system."); - sb.append("\n\t[location ] - Displays the list of hosts storing the specified file."); - sb.append("\n\t[ls [-R] ] - Displays information for all files and directories " + - "directly under the specified path. Specify -R to display files and " + - "directories recursively."); - sb.append("\n\t[mkdir [path2] ... [pathn]] - Creates the specified directories, " + - "including any parent directories that are required."); - sb.append("\n\t[mount ] - Mounts a UFS path onto an Alluxio path."); - sb.append("\n\t[mv ] - Renames a file or directory."); - sb.append("\n\t[persist ] - Persists a file or directory currently stored " + - "only in Alluxio to the UnderFileSystem."); - sb.append("\n\t[pin ] - Pins the given file or directory in memory (works " + - "recursively for directories). Pinned files are never evicted from memory, unless " + - "TTL is set."); - sb.append("\n\t[report ] - Reports to the master that a file is lost."); - sb.append("\n\t[rm [-R] ] - Removes the specified file. Specify -R to remove file or " + - "directory recursively."); - sb.append("\n\t[setTtl