From 3964c1531fd7a139abccac6a998bee0e5626b883 Mon Sep 17 00:00:00 2001 From: Xinyao Zhang <43081360+zhangxinyao88@users.noreply.github.com> Date: Tue, 6 Oct 2026 23:42:10 -0400 Subject: [PATCH] GH-3289: Test schema and metadata through CLI arguments --- .../org/apache/parquet/cli/CliTestBase.java | 58 +++++++++ .../java/org/apache/parquet/cli/MainTest.java | 8 +- .../apache/parquet/cli/MetadataCliTest.java | 46 +++++++ .../org/apache/parquet/cli/SchemaCliTest.java | 123 ++++++++++++++++++ 4 files changed, 233 insertions(+), 2 deletions(-) create mode 100644 parquet-cli/src/test/java/org/apache/parquet/cli/CliTestBase.java create mode 100644 parquet-cli/src/test/java/org/apache/parquet/cli/MetadataCliTest.java create mode 100644 parquet-cli/src/test/java/org/apache/parquet/cli/SchemaCliTest.java diff --git a/parquet-cli/src/test/java/org/apache/parquet/cli/CliTestBase.java b/parquet-cli/src/test/java/org/apache/parquet/cli/CliTestBase.java new file mode 100644 index 0000000000..f48f8f24c6 --- /dev/null +++ b/parquet-cli/src/test/java/org/apache/parquet/cli/CliTestBase.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.parquet.cli; + +import java.io.PrintWriter; +import java.io.StringWriter; +import java.util.stream.Collectors; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.util.ToolRunner; +import org.apache.parquet.cli.commands.ParquetFileTest; +import org.slf4j.event.LoggingEvent; +import org.slf4j.helpers.MessageFormatter; +import org.slf4j.helpers.SubstituteLoggerFactory; + +abstract class CliTestBase extends ParquetFileTest { + + protected static CliResult cli(String... args) throws Exception { + SubstituteLoggerFactory loggerFactory = new SubstituteLoggerFactory(); + try { + int exitCode = ToolRunner.run(new Configuration(), new Main(loggerFactory.getLogger("cli-test")), args); + String output = loggerFactory.getEventQueue().stream() + .map(CliTestBase::formatEvent) + .collect(Collectors.joining("\n")); + return new CliResult(exitCode, output); + } finally { + loggerFactory.clear(); + } + } + + private static String formatEvent(LoggingEvent event) { + String message = MessageFormatter.arrayFormat(event.getMessage(), event.getArgumentArray()) + .getMessage(); + if (event.getThrowable() == null) { + return message; + } + StringWriter stackTrace = new StringWriter(); + event.getThrowable().printStackTrace(new PrintWriter(stackTrace)); + return message + "\n" + stackTrace; + } + + protected record CliResult(int exitCode, String output) {} +} diff --git a/parquet-cli/src/test/java/org/apache/parquet/cli/MainTest.java b/parquet-cli/src/test/java/org/apache/parquet/cli/MainTest.java index 55d8b8aa4b..73daad87b6 100644 --- a/parquet-cli/src/test/java/org/apache/parquet/cli/MainTest.java +++ b/parquet-cli/src/test/java/org/apache/parquet/cli/MainTest.java @@ -22,6 +22,7 @@ import java.io.File; import java.io.FileWriter; +import java.nio.file.Path; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.util.ToolRunner; import org.junit.jupiter.api.Test; @@ -52,7 +53,9 @@ public void testConfigFileLoading() throws Exception { @Test public void testLocalPropertiesFile() throws Exception { - String configFile = getClass().getResource("/test-config.properties").getPath(); + String configFile = Path.of( + getClass().getResource("/test-config.properties").toURI()) + .toString(); assertThatCode(() -> ToolRunner.run( new Configuration(), new Main(LoggerFactory.getLogger(MainTest.class)), @@ -62,7 +65,8 @@ public void testLocalPropertiesFile() throws Exception { @Test public void testLocalXmlFile() throws Exception { - String configFile = getClass().getResource("/test-config.xml").getPath(); + String configFile = + Path.of(getClass().getResource("/test-config.xml").toURI()).toString(); assertThatCode(() -> ToolRunner.run( new Configuration(), new Main(LoggerFactory.getLogger(MainTest.class)), diff --git a/parquet-cli/src/test/java/org/apache/parquet/cli/MetadataCliTest.java b/parquet-cli/src/test/java/org/apache/parquet/cli/MetadataCliTest.java new file mode 100644 index 0000000000..51e30474a1 --- /dev/null +++ b/parquet-cli/src/test/java/org/apache/parquet/cli/MetadataCliTest.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.parquet.cli; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.nio.file.Files; +import java.nio.file.Path; +import org.junit.jupiter.api.Test; + +class MetadataCliTest extends CliTestBase { + + @Test + void printsMetadataForPathContainingSpaces() throws Exception { + Path input = getTempFolder().toPath().resolve("metadata input.parquet"); + Files.copy(parquetFile().toPath(), input); + + CliResult result = cli("meta", input.toString()); + + assertThat(result.exitCode()).isZero(); + assertThat(result.output()) + .contains( + "File path: " + input, + "Created by: parquet-mr", + "Schema:\nmessage schema {", + "required int32 int32_field;", + "required fixed_len_byte_array(12) flba_field;", + "Row group 0: count: 10"); + } +} diff --git a/parquet-cli/src/test/java/org/apache/parquet/cli/SchemaCliTest.java b/parquet-cli/src/test/java/org/apache/parquet/cli/SchemaCliTest.java new file mode 100644 index 0000000000..9188d40fd6 --- /dev/null +++ b/parquet-cli/src/test/java/org/apache/parquet/cli/SchemaCliTest.java @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.parquet.cli; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import org.apache.avro.Schema; +import org.junit.jupiter.api.Test; + +class SchemaCliTest extends CliTestBase { + + private static final String PARQUET_SCHEMA = String.join( + "\n", + "message schema {", + " required int32 int32_field;", + " required int64 int64_field;", + " required float float_field;", + " required double double_field;", + " required binary binary_field;", + " required fixed_len_byte_array(12) flba_field;", + " required int32 date_field (DATE);", + "}", + ""); + + @Test + void printsAvroSchemaByDefault() throws Exception { + CliResult result = cli("schema", parquetFile().getAbsolutePath()); + + assertThat(result.exitCode()).isZero(); + Schema schema = new Schema.Parser().parse(result.output()); + assertThat(schema.getName()).isEqualTo("schema"); + assertThat(schema.getFields()) + .extracting(Schema.Field::name) + .containsExactly( + "int32_field", + "int64_field", + "float_field", + "double_field", + "binary_field", + "flba_field", + "date_field"); + assertThat(schema.getField("int32_field").schema().getType()).isEqualTo(Schema.Type.INT); + assertThat(schema.getField("flba_field").schema().getFixedSize()).isEqualTo(12); + assertThat(schema.getField("date_field").schema().getLogicalType().getName()) + .isEqualTo("date"); + } + + @Test + void printsParquetSchemaForPathContainingSpaces() throws Exception { + Path input = getTempFolder().toPath().resolve("input with spaces.parquet"); + Files.copy(parquetFile().toPath(), input); + + CliResult result = cli("schema", "--parquet", input.toString()); + + assertThat(result.exitCode()).isZero(); + assertThat(result.output()).isEqualTo(PARQUET_SCHEMA); + } + + @Test + void writesSchemaToOutputFile() throws Exception { + Path output = getTempFolder().toPath().resolve("schema output.txt"); + + CliResult result = cli( + "schema", + "--parquet", + "--output", + output.toString(), + parquetFile().getAbsolutePath()); + + assertThat(result.exitCode()).isZero(); + assertThat(result.output()).isEmpty(); + assertThat(Files.readString(output, StandardCharsets.UTF_8)).isEqualTo(PARQUET_SCHEMA); + } + + @Test + void refusesToOverwriteOutputWithoutFlag() throws Exception { + Path output = getTempFolder().toPath().resolve("schema.txt"); + Files.writeString(output, "existing contents", StandardCharsets.UTF_8); + + CliResult result = + cli("schema", "--output", output.toString(), parquetFile().getAbsolutePath()); + + assertThat(result.exitCode()).isEqualTo(1); + assertThat(result.output()).contains("File already exists", output.toString()); + assertThat(Files.readString(output, StandardCharsets.UTF_8)).isEqualTo("existing contents"); + } + + @Test + void overwritesOutputWithFlag() throws Exception { + Path output = getTempFolder().toPath().resolve("schema.txt"); + Files.writeString(output, "existing contents", StandardCharsets.UTF_8); + + CliResult result = cli( + "schema", + "--parquet", + "-o", + output.toString(), + "--overwrite", + parquetFile().getAbsolutePath()); + + assertThat(result.exitCode()).isZero(); + assertThat(Files.readString(output, StandardCharsets.UTF_8)).isEqualTo(PARQUET_SCHEMA); + } +}