From 4cd9c785617cc1573e90a0af02cb2907bb7e8170 Mon Sep 17 00:00:00 2001 From: Tim Meehan Date: Tue, 22 Sep 2026 14:47:33 +0000 Subject: [PATCH] [SPARK-59417][SQL] Stream multiline top-level JSON arrays Add the internal configuration `spark.sql.json.enableStreamingTopLevelArray` and the per-read JSON option `enableStreamingTopLevelArray` that overrides it. When enabled, multiline JSON reading consumes a top-level array of structs lazily from one Jackson parser instead of materializing the whole array before returning rows, bounding parser-side memory. Both default to false. The two paths agree on valid input and differ on malformed input: streaming makes a parse mode apply to one array element rather than to the whole document, because rows already handed to the consumer cannot be withdrawn. That is why the path is opt-in rather than the default. `FailureSafeParser` gains a lazy failure path that keeps the rows emitted before a later failure and resumes the source iterator for a recoverable one. An element is resumed from only where the parser provably cannot have moved past it, so any other container failure still ends the document. The streaming iterator holds its parser in a cell that every close path clears first, and registers `close()` with the task completion listener so an iterator abandoned before the closing bracket is released at task completion rather than at GC. Co-authored-by: Isaac --- docs/sql-data-sources-json.md | 6 + .../spark/sql/catalyst/json/JSONOptions.scala | 5 + .../sql/catalyst/json/JacksonParser.scala | 197 ++++- .../catalyst/util/BadRecordException.scala | 4 +- .../sql/catalyst/util/FailureSafeParser.scala | 77 +- .../apache/spark/sql/internal/SQLConf.scala | 19 + .../JsonBenchmark-jdk21-results.txt | 146 ++-- .../JsonBenchmark-jdk25-results.txt | 146 ++-- sql/core/benchmarks/JsonBenchmark-results.txt | 146 ++-- .../datasources/json/JsonDataSource.scala | 26 +- .../datasources/JSONArchiveReadBase.scala | 63 +- .../datasources/json/JsonBenchmark.scala | 51 ++ .../datasources/json/JsonSuite.scala | 680 +++++++++++++++--- 13 files changed, 1217 insertions(+), 349 deletions(-) diff --git a/docs/sql-data-sources-json.md b/docs/sql-data-sources-json.md index 6ac5d690868c8..ccb2ea0f08115 100644 --- a/docs/sql-data-sources-json.md +++ b/docs/sql-data-sources-json.md @@ -237,6 +237,12 @@ Data source options of JSON can be set via: Parse one record, which may span multiple lines, per file. JSON built-in functions ignore this option. read + + enableStreamingTopLevelArray + (value of spark.sql.json.enableStreamingTopLevelArray configuration) + When multiLine is enabled and a file holds a top-level JSON array, read the array's elements one at a time instead of materializing the whole array before returning rows. It applies to reads into a struct schema, and has no effect on reads using singleVariantColumn or explodeEmbeddedArray. While streaming, mode applies to an individual element rather than the whole document: PERMISSIVE fills columnNameOfCorruptRecord for the malformed element alone, leaving it null on the valid rows of the same document, and DROPMALFORMED drops that element rather than the document. An element whose failure leaves the parser at an unknown position, such as a nested value of the wrong shape, still ends the document. + read + allowUnquotedControlChars false diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JSONOptions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JSONOptions.scala index c7e8a8e13116f..bad241687fb0e 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JSONOptions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JSONOptions.scala @@ -239,6 +239,10 @@ class JSONOptions( val useUnsafeRow: Boolean = parameters.get(USE_UNSAFE_ROW).map(_.toBoolean).getOrElse( SQLConf.get.getConf(SQLConf.JSON_USE_UNSAFE_ROW)) + val streamMultilineTopLevelArray: Boolean = + parameters.get(ENABLE_STREAMING_TOP_LEVEL_ARRAY).map(_.toBoolean).getOrElse( + SQLConf.get.getConf(SQLConf.JSON_STREAM_MULTILINE_TOP_LEVEL_ARRAY)) + /** Build a Jackson [[JsonFactory]] using JSON options. */ def buildJsonFactory(): JsonFactory = { val streamReadConstraints = StreamReadConstraints @@ -338,6 +342,7 @@ object JSONOptions extends DataSourceOptions { val SINGLE_VARIANT_COLUMN = newOption(DataSourceOptions.SINGLE_VARIANT_COLUMN) val EXPLODE_EMBEDDED_ARRAY = newOption(DataSourceOptions.EXPLODE_EMBEDDED_ARRAY) val USE_UNSAFE_ROW = newOption("useUnsafeRow") + val ENABLE_STREAMING_TOP_LEVEL_ARRAY = newOption("enableStreamingTopLevelArray") // Options with alternative val ENCODING = "encoding" val CHARSET = "charset" diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonParser.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonParser.scala index ea8061b774c43..d15312f7452a3 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonParser.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonParser.scala @@ -19,6 +19,7 @@ package org.apache.spark.sql.catalyst.json import java.io.{ByteArrayOutputStream, CharConversionException} import java.nio.charset.MalformedInputException +import java.util.concurrent.atomic.AtomicReference import scala.collection.mutable import scala.collection.mutable.ArrayBuffer @@ -28,6 +29,7 @@ import com.fasterxml.jackson.core._ import org.apache.hadoop.fs.PositionedReadable import org.apache.spark.SparkUpgradeException +import org.apache.spark.TaskContext import org.apache.spark.internal.Logging import org.apache.spark.sql.catalyst.{InternalRow, NoopFilters, StructFilters} import org.apache.spark.sql.catalyst.expressions._ @@ -61,6 +63,9 @@ class JacksonParser( // `ValueConverter`s for the root schema for all fields in the schema private val rootConverter = makeRootConverter(schema) + // Lazy: a schema that is not a struct never reaches either top-level-array path. + private lazy val arrayElementConverter = makeConverter(schema) + private val factory = options.buildJsonFactory() private lazy val timestampFormatter = TimestampFormatter( @@ -148,7 +153,6 @@ class JacksonParser( } private def makeStructRootConverter(st: StructType): JsonParser => Iterable[InternalRow] = { - val elementConverter = makeConverter(st) val fieldConverters = st.map(_.dataType).map(makeConverter).toArray val jsonFilters = if (SQLConf.get.jsonFilterPushDown) { new JsonFilters(filters, st) @@ -171,7 +175,8 @@ class JacksonParser( // List([str_a_2,null], [null,str_b_3]) // case START_ARRAY if allowArrayAsStructs => - val array = convertArray(parser, elementConverter, isRoot = true, arrayAsStructs = true) + val array = + convertArray(parser, arrayElementConverter, isRoot = true, arrayAsStructs = true) // Here, as we support reading top level JSON arrays and take every element // in such an array as a row, this case is possible. if (array.numElements() == 0) { @@ -693,6 +698,34 @@ class JacksonParser( case _ => err } + private def badRecord(error: Throwable, recordLiteral: () => UTF8String): BadRecordException = + error match { + case e: SparkUpgradeException => throw e + case e: CharConversionException if options.encoding.isEmpty => + val msg = + """JSON parser cannot handle a character in its input. + |Specifying encoding as an input option explicitly might help to resolve the issue. + |""".stripMargin + e.getMessage + val wrappedCharException = new CharConversionException(msg) + wrappedCharException.initCause(e) + BadRecordException(recordLiteral, () => Array.empty, wrappedCharException) + case PartialResultException(row, cause) => + BadRecordException(recordLiteral, () => Array(row), convertCauseForPartialResult(cause)) + case PartialResultArrayException(rows, cause) => + BadRecordException(recordLiteral, () => rows, cause) + case PartialArrayDataResultException(arrayData, cause) => + BadRecordException( + recordLiteral, + () => Array(InternalRow(arrayData)), + convertCauseForPartialResult(cause)) + case PartialMapDataResultException(mapData, cause) => + BadRecordException( + recordLiteral, + () => Array(InternalRow(mapData)), + convertCauseForPartialResult(cause)) + case e => BadRecordException(recordLiteral, () => Array.empty, e) + } + /** * Parse the JSON input to the set of [[InternalRow]]s. * @@ -717,43 +750,131 @@ class JacksonParser( } } catch { case e: SparkUpgradeException => throw e - case e @ (_: RuntimeException | _: JsonProcessingException | _: MalformedInputException) => - // JSON parser currently doesn't support partial results for corrupted records. - // For such records, all fields other than the field configured by - // `columnNameOfCorruptRecord` are set to `null`. - throw BadRecordException(() => recordLiteral(record), () => Array.empty, e) case e: CharConversionException if options.encoding.isEmpty => - val msg = - """JSON parser cannot handle a character in its input. - |Specifying encoding as an input option explicitly might help to resolve the issue. - |""".stripMargin + e.getMessage - val wrappedCharException = new CharConversionException(msg) - wrappedCharException.initCause(e) - throw BadRecordException(() => recordLiteral(record), () => Array.empty, - wrappedCharException) - case PartialResultException(row, cause) => - throw BadRecordException( - record = () => recordLiteral(record), - partialResults = () => Array(row), - convertCauseForPartialResult(cause)) - case PartialResultArrayException(rows, cause) => - throw BadRecordException( - record = () => recordLiteral(record), - partialResults = () => rows, - cause) - // These exceptions should never be thrown outside of JacksonParser. - // They are used for the control flow in the parser. We add them here for completeness - // since they also indicate a bad record. - case PartialArrayDataResultException(arrayData, cause) => - throw BadRecordException( - record = () => recordLiteral(record), - partialResults = () => Array(InternalRow(arrayData)), - convertCauseForPartialResult(cause)) - case PartialMapDataResultException(mapData, cause) => - throw BadRecordException( - record = () => recordLiteral(record), - partialResults = () => Array(InternalRow(mapData)), - convertCauseForPartialResult(cause)) + throw badRecord(e, () => recordLiteral(record)) + case e @ (_: RuntimeException | _: JsonProcessingException | _: MalformedInputException | + _: PartialResultException | _: PartialResultArrayException | + _: PartialArrayDataResultException | _: PartialMapDataResultException) => + throw badRecord(e, () => recordLiteral(record)) + } + } + + private[sql] def parseIterator[T]( + record: T, + createParser: (JsonFactory, T) => JsonParser, + recordLiteral: T => UTF8String): Iterator[InternalRow] = { + val streamArray = allowArrayAsStructs && schema.isInstanceOf[StructType] && + options.singleVariantColumn.isEmpty && options.explodeEmbeddedArray.isEmpty + val jsonParser = try { + createParser(factory, record) + } catch { + case e: SparkUpgradeException => throw e + case e: CharConversionException if options.encoding.isEmpty => + throw badRecord(e, () => recordLiteral(record)) + case e @ (_: RuntimeException | _: JsonProcessingException | _: MalformedInputException | + _: PartialResultException | _: PartialResultArrayException | + _: PartialArrayDataResultException | _: PartialMapDataResultException) => + throw badRecord(e, () => recordLiteral(record)) + } + // An archive read builds one parser per entry inside a single task, so a closed parser left + // reachable from the completion listener keeps its entry's buffer alive for the rest of the + // task. Every close path takes the parser out of this cell first, so exactly one of them closes + // it and the listener stops retaining it as soon as one does. + val openParser = new AtomicReference(jsonParser) + def takeParser(): Option[JsonParser] = Option(openParser.getAndSet(null)) + // Cleanup must not fail a read that already emitted rows, nor a task whose iterator was + // abandoned, so the paths that can still succeed close the way `tryWithResource` does: quietly. + def closeParser(): Unit = takeParser().foreach(parser => Utils.closeQuietly(parser)) + // Abandoning the iterator before END_ARRAY (e.g. a LIMIT) reaches no eager close. + Option(TaskContext.get()).foreach(_.addTaskCompletionListener[Unit](_ => closeParser())) + def fail(error: Throwable): Nothing = { + try takeParser().foreach(_.close()) catch { + case NonFatal(closeError) => error.addSuppressed(closeError) + } + throw badRecord(error, () => recordLiteral(record)) + } + // Resuming is only safe where the parser cannot have moved past the element that failed. A + // scalar element leaves it on that scalar's own token, and `convertObject` consumes through + // END_OBJECT before raising `PartialResultException`; any other container failure can have + // stopped anywhere inside the element. + def resumableAfter(error: Throwable, elementStart: Option[JsonToken]): Boolean = + options.parseMode != FailFastMode && (elementStart match { + case Some(START_OBJECT) | Some(START_ARRAY) => error.isInstanceOf[PartialResultException] + case Some(_) => error.isInstanceOf[RuntimeException] + case None => false + }) + def handleFailure[R](elementStart: Option[JsonToken])(operation: => R): R = { + try operation catch { + case e: SparkUpgradeException => fail(e) + case e: CharConversionException if options.encoding.isEmpty => fail(e) + case e @ (_: RuntimeException | _: JsonProcessingException | _: MalformedInputException | + _: PartialResultException | _: PartialResultArrayException | + _: PartialArrayDataResultException | _: PartialMapDataResultException) => + if (resumableAfter(e, elementStart)) { + throw badRecord(e, () => recordLiteral(record)).copy(recoverable = true) + } else { + fail(e) + } + // Anything else, such as a raw `IOException` off the underlying stream, is not a malformed + // record and so propagates unwrapped -- but it still has to close what the eager path's + // `tryWithResource` would have, rather than leaving it to task completion. + case other: Throwable => + closeParser() + throw other + } + } + + handleFailure(elementStart = None)(jsonParser.nextToken()) match { + case null => + closeParser() + Iterator.empty + case START_ARRAY if streamArray => + new Iterator[InternalRow] { + private var nextRow: InternalRow = _ + private var prepared = false + private var finished = false + + override def hasNext: Boolean = { + prepare() + !finished + } + + override def next(): InternalRow = { + prepare() + if (finished) throw new NoSuchElementException("next on empty iterator") + prepared = false + nextRow + } + + private def prepare(): Unit = { + if (prepared || finished) return + handleFailure(elementStart = None)(jsonParser.nextToken()) match { + case END_ARRAY => finish() + // Unreachable today: jackson-core signals end of input inside an open array by + // throwing JsonEOFException, which handleFailure turns into a malformed record. + // Kept so a backend that returns null cannot fall through with no current token. + case null => + fail(new JsonParseException(jsonParser, "Unexpected end of top-level array")) + case elementStart => + nextRow = handleFailure(Some(elementStart)) { + val row = arrayElementConverter(jsonParser).asInstanceOf[InternalRow] + if (row == null) throw QueryExecutionErrors.rootConverterReturnNullError() + row + } + prepared = true + } + } + + private def finish(): Unit = { + finished = true + closeParser() + } + } + case _ => + val rows = handleFailure(elementStart = None)(rootConverter(jsonParser)) + if (rows == null) fail(QueryExecutionErrors.rootConverterReturnNullError()) + closeParser() + rows.iterator } } } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/BadRecordException.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/BadRecordException.scala index 4fa6a2275e743..54b85a6bde1b3 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/BadRecordException.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/BadRecordException.scala @@ -76,11 +76,13 @@ case class PartialResultArrayException( * @param cause the actual exception about why the record is bad and can't be parsed. It's better * to use `LazyBadRecordCauseWrapper` here to delay heavy cause construction * until it's needed. + * @param recoverable whether the parser can resume at the next record after handling this failure */ case class BadRecordException( @transient record: () => UTF8String, @transient partialResults: () => Array[InternalRow] = () => Array.empty[InternalRow], - cause: Throwable) extends Exception(cause) { + cause: Throwable, + recoverable: Boolean = false) extends Exception(cause) { override def getStackTrace(): Array[StackTraceElement] = new Array[StackTraceElement](0) override def fillInStackTrace(): Throwable = this } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/FailureSafeParser.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/FailureSafeParser.scala index d9946d1b12ec3..3c31e323491c7 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/FailureSafeParser.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/FailureSafeParser.scala @@ -59,30 +59,63 @@ class FailureSafeParser[IN]( try { rawParser.apply(input).iterator.map(row => toResultRow(Some(row), () => null)) } catch { - case e: BadRecordException => mode match { - case PermissiveMode => - val partialResults = e.partialResults() - if (partialResults.nonEmpty) { - partialResults.iterator.map(row => toResultRow(Some(row), e.record)) - } else { - Iterator(toResultRow(None, e.record)) - } - case DropMalformedMode => - Iterator.empty - case FailFastMode => - e.getCause match { - case _: JsonArraysAsStructsException => - // SPARK-42298 we recreate the exception here to make sure the error message - // have the record content. - throw QueryExecutionErrors.cannotParseJsonArraysAsStructsError(e.record().toString) - case StringAsDataTypeException(fieldName, fieldValue, dataType) => - throw QueryExecutionErrors.cannotParseStringAsDataTypeError(e.record().toString, - fieldName, fieldValue, dataType) - case causeWrapper: LazyBadRecordCauseWrapper => - throwMalformedRecordsDetectedInRecordParsingError(e, causeWrapper.cause()) - case cause => throwMalformedRecordsDetectedInRecordParsingError(e, cause) + case e: BadRecordException => parseFailure(e) + } + } + + def parseIterator( + input: IN, + iteratorParser: IN => Iterator[InternalRow]): Iterator[InternalRow] = { + var delegate = try { + iteratorParser.apply(input).map(row => toResultRow(Some(row), () => null)) + } catch { + case e: BadRecordException => parseFailure(e) + } + new Iterator[InternalRow] { + private def handleFailure[T](operation: Iterator[InternalRow] => T): T = { + while (true) { + try { + return operation(delegate) + } catch { + case e: BadRecordException => + val source = delegate + val recovery = parseFailure(e) + delegate = if (e.recoverable) recovery ++ source else recovery } + } + throw new IllegalStateException("unreachable") } + + override def hasNext: Boolean = handleFailure(_.hasNext) + + override def next(): InternalRow = handleFailure(_.next()) + } + } + + private def parseFailure(e: BadRecordException): Iterator[InternalRow] = { + mode match { + case PermissiveMode => + val partialResults = e.partialResults() + if (partialResults.nonEmpty) { + partialResults.iterator.map(row => toResultRow(Some(row), e.record)) + } else { + Iterator(toResultRow(None, e.record)) + } + case DropMalformedMode => + Iterator.empty + case FailFastMode => + e.getCause match { + case _: JsonArraysAsStructsException => + // SPARK-42298 we recreate the exception here to make sure the error message + // has the record content. + throw QueryExecutionErrors.cannotParseJsonArraysAsStructsError(e.record().toString) + case StringAsDataTypeException(fieldName, fieldValue, dataType) => + throw QueryExecutionErrors.cannotParseStringAsDataTypeError(e.record().toString, + fieldName, fieldValue, dataType) + case causeWrapper: LazyBadRecordCauseWrapper => + throwMalformedRecordsDetectedInRecordParsingError(e, causeWrapper.cause()) + case cause => throwMalformedRecordsDetectedInRecordParsingError(e, cause) + } } } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala index b79a5d7a3b537..6a56c5c3f7168 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala @@ -7034,6 +7034,25 @@ object SQLConf { .booleanConf .createWithDefault(true) + val JSON_STREAM_MULTILINE_TOP_LEVEL_ARRAY = + buildConf("spark.sql.json.enableStreamingTopLevelArray") + .internal() + .doc("When true, multiline JSON reads stream the elements of a top-level array one at a " + + "time instead of materializing the entire array before returning rows. This applies only " + + "to reads into a struct schema that take top-level arrays as structs, and has no " + + "effect on reads using the `singleVariantColumn` or `explodeEmbeddedArray` option. " + + "Streaming also makes an array element, rather than the whole document, the record " + + "that a parse mode applies to, since rows already emitted cannot be withdrawn: " + + "PERMISSIVE fills the corrupt record column for the malformed element only, leaving " + + "it null on the valid rows of the same document, and DROPMALFORMED drops that " + + "element rather than the whole document. An element whose failure leaves the parser " + + "at an unknown position, such as a nested value of the wrong shape, still ends the " + + "document. It can be overwritten by the JSON option `enableStreamingTopLevelArray`.") + .version("4.4.0") + .withBindingPolicy(ConfigBindingPolicy.SESSION) + .booleanConf + .createWithDefault(false) + val JSON_USE_UNSAFE_ROW = buildConf("spark.sql.json.useUnsafeRow") .doc("When set to true, use UnsafeRow to represent struct result in the JSON parser. It " + diff --git a/sql/core/benchmarks/JsonBenchmark-jdk21-results.txt b/sql/core/benchmarks/JsonBenchmark-jdk21-results.txt index fa901824b53fd..4f8b1d71af8fb 100644 --- a/sql/core/benchmarks/JsonBenchmark-jdk21-results.txt +++ b/sql/core/benchmarks/JsonBenchmark-jdk21-results.txt @@ -3,128 +3,142 @@ Benchmark for performance of JSON parsing ================================================================================================ Preparing data for benchmarking ... -OpenJDK 64-Bit Server VM 21.0.12+8-LTS on Linux 6.17.0-1020-azure +OpenJDK 64-Bit Server VM 21.0.12+8-LTS on Linux 6.17.0-1022-azure AMD EPYC 9V74 80-Core Processor JSON schema inferring: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -No encoding 1835 1857 22 2.7 366.9 1.0X -UTF-8 is set 3997 4002 8 1.3 799.4 0.5X +No encoding 2081 2116 47 2.4 416.2 1.0X +UTF-8 is set 5253 5423 265 1.0 1050.6 0.4X Preparing data for benchmarking ... -OpenJDK 64-Bit Server VM 21.0.12+8-LTS on Linux 6.17.0-1020-azure +OpenJDK 64-Bit Server VM 21.0.12+8-LTS on Linux 6.17.0-1022-azure AMD EPYC 9V74 80-Core Processor count a short column: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -No encoding 1544 1550 8 3.2 308.9 1.0X -UTF-8 is set 3868 3896 26 1.3 773.6 0.4X +No encoding 1789 1809 31 2.8 357.9 1.0X +UTF-8 is set 4835 4838 2 1.0 967.0 0.4X Preparing data for benchmarking ... -OpenJDK 64-Bit Server VM 21.0.12+8-LTS on Linux 6.17.0-1020-azure +OpenJDK 64-Bit Server VM 21.0.12+8-LTS on Linux 6.17.0-1022-azure AMD EPYC 9V74 80-Core Processor count a wide column: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -No encoding 3723 3749 34 0.3 3723.3 1.0X -UTF-8 is set 3630 3655 24 0.3 3630.4 1.0X +No encoding 4791 4805 14 0.2 4790.5 1.0X +UTF-8 is set 5038 5052 15 0.2 5038.0 1.0X Preparing data for benchmarking ... -OpenJDK 64-Bit Server VM 21.0.12+8-LTS on Linux 6.17.0-1020-azure +OpenJDK 64-Bit Server VM 21.0.12+8-LTS on Linux 6.17.0-1022-azure AMD EPYC 9V74 80-Core Processor select wide row: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -No encoding 7611 7634 20 0.0 152224.9 1.0X -UTF-8 is set 8052 8074 27 0.0 161045.5 0.9X +No encoding 10203 10286 96 0.0 204060.2 1.0X +UTF-8 is set 10939 10969 40 0.0 218787.9 0.9X Preparing data for benchmarking ... -OpenJDK 64-Bit Server VM 21.0.12+8-LTS on Linux 6.17.0-1020-azure +OpenJDK 64-Bit Server VM 21.0.12+8-LTS on Linux 6.17.0-1022-azure AMD EPYC 9V74 80-Core Processor Select a subset of 10 columns: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -Select 10 columns 1348 1362 13 0.7 1347.7 1.0X -Select 1 column 1097 1103 6 0.9 1096.8 1.2X +Select 10 columns 1759 1767 7 0.6 1759.1 1.0X +Select 1 column 1157 1165 8 0.9 1156.7 1.5X Preparing data for benchmarking ... -OpenJDK 64-Bit Server VM 21.0.12+8-LTS on Linux 6.17.0-1020-azure +OpenJDK 64-Bit Server VM 21.0.12+8-LTS on Linux 6.17.0-1022-azure AMD EPYC 9V74 80-Core Processor creation of JSON parser per line: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -Short column without encoding 542 547 9 1.8 541.7 1.0X -Short column with UTF-8 1010 1012 3 1.0 1009.5 0.5X -Wide column without encoding 4096 4120 23 0.2 4095.5 0.1X -Wide column with UTF-8 5713 5736 24 0.2 5712.6 0.1X +Short column without encoding 583 591 8 1.7 583.0 1.0X +Short column with UTF-8 1288 1291 4 0.8 1288.4 0.5X +Wide column without encoding 5623 5632 8 0.2 5622.8 0.1X +Wide column with UTF-8 7468 7483 17 0.1 7467.5 0.1X Preparing data for benchmarking ... -OpenJDK 64-Bit Server VM 21.0.12+8-LTS on Linux 6.17.0-1020-azure +OpenJDK 64-Bit Server VM 21.0.12+8-LTS on Linux 6.17.0-1022-azure AMD EPYC 9V74 80-Core Processor JSON functions: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -Text read 46 51 7 21.6 46.4 1.0X -from_json 919 927 13 1.1 919.1 0.1X -json_tuple 702 704 2 1.4 702.5 0.1X -get_json_object wholestage off 731 736 10 1.4 730.6 0.1X -get_json_object wholestage on 643 649 10 1.6 642.9 0.1X +Text read 66 71 5 15.2 66.0 1.0X +from_json 1029 1034 5 1.0 1029.0 0.1X +json_tuple 971 974 3 1.0 971.0 0.1X +get_json_object wholestage off 986 994 11 1.0 986.0 0.1X +get_json_object wholestage on 886 894 7 1.1 885.5 0.1X Preparing data for benchmarking ... -OpenJDK 64-Bit Server VM 21.0.12+8-LTS on Linux 6.17.0-1020-azure +OpenJDK 64-Bit Server VM 21.0.12+8-LTS on Linux 6.17.0-1022-azure AMD EPYC 9V74 80-Core Processor Dataset of json strings: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -Text read 201 202 1 24.9 40.2 1.0X -schema inferring 1202 1204 2 4.2 240.4 0.2X -parsing 2183 2192 8 2.3 436.7 0.1X +Text read 260 261 2 19.2 52.0 1.0X +schema inferring 1594 1600 9 3.1 318.8 0.2X +parsing 2212 2216 4 2.3 442.4 0.1X Preparing data for benchmarking ... -OpenJDK 64-Bit Server VM 21.0.12+8-LTS on Linux 6.17.0-1020-azure +OpenJDK 64-Bit Server VM 21.0.12+8-LTS on Linux 6.17.0-1022-azure AMD EPYC 9V74 80-Core Processor Json files in the per-line mode: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -Text read 481 483 2 10.4 96.3 1.0X -Schema inferring 1680 1684 4 3.0 335.9 0.3X -Parsing without charset 2418 2425 7 2.1 483.6 0.2X -Parsing with UTF-8 4684 4710 37 1.1 936.8 0.1X +Text read 605 612 7 8.3 121.1 1.0X +Schema inferring 2108 2113 8 2.4 421.5 0.3X +Parsing without charset 2673 2683 13 1.9 534.6 0.2X +Parsing with UTF-8 5951 5955 4 0.8 1190.3 0.1X -OpenJDK 64-Bit Server VM 21.0.12+8-LTS on Linux 6.17.0-1020-azure +OpenJDK 64-Bit Server VM 21.0.12+8-LTS on Linux 6.17.0-1022-azure AMD EPYC 9V74 80-Core Processor Write dates and timestamps: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -Create a dataset of timestamps 91 95 4 11.0 91.3 1.0X -to_json(timestamp) 402 403 1 2.5 402.5 0.2X -write timestamps to files 427 428 1 2.3 427.2 0.2X -Create a dataset of dates 96 98 2 10.4 95.8 1.0X -to_json(date) 276 277 2 3.6 275.8 0.3X -write dates to files 291 293 3 3.4 290.7 0.3X - -OpenJDK 64-Bit Server VM 21.0.12+8-LTS on Linux 6.17.0-1020-azure +Create a dataset of timestamps 111 114 2 9.0 110.8 1.0X +to_json(timestamp) 509 514 7 2.0 509.0 0.2X +write timestamps to files 534 538 4 1.9 533.7 0.2X +Create a dataset of dates 122 129 7 8.2 122.3 0.9X +to_json(date) 366 367 1 2.7 366.4 0.3X +write dates to files 376 383 9 2.7 376.2 0.3X + +OpenJDK 64-Bit Server VM 21.0.12+8-LTS on Linux 6.17.0-1022-azure AMD EPYC 9V74 80-Core Processor Read dates and timestamps: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ----------------------------------------------------------------------------------------------------------------------------------------------------- -read timestamp text from files 132 134 2 7.6 132.4 1.0X -read timestamps from files 901 905 3 1.1 900.9 0.1X -infer timestamps from files 1621 1637 17 0.6 1620.8 0.1X -read date text from files 121 122 1 8.2 121.5 1.1X -read date from files 587 597 8 1.7 587.4 0.2X -timestamp strings 115 118 4 8.7 114.7 1.2X -parse timestamps from Dataset[String] 1055 1060 5 0.9 1055.3 0.1X -infer timestamps from Dataset[String] 1770 1771 2 0.6 1769.6 0.1X -date strings 154 155 1 6.5 153.7 0.9X -parse dates from Dataset[String] 828 840 17 1.2 827.7 0.2X -from_json(timestamp) 1473 1476 5 0.7 1472.7 0.1X -from_json(date) 1235 1255 26 0.8 1235.1 0.1X -infer error timestamps from Dataset[String] with default format 1067 1073 5 0.9 1066.8 0.1X -infer error timestamps from Dataset[String] with user-provided format 1089 1091 2 0.9 1088.7 0.1X -infer error timestamps from Dataset[String] with legacy format 1097 1099 2 0.9 1097.1 0.1X - -OpenJDK 64-Bit Server VM 21.0.12+8-LTS on Linux 6.17.0-1020-azure +read timestamp text from files 160 161 1 6.2 160.1 1.0X +read timestamps from files 1109 1117 8 0.9 1108.8 0.1X +infer timestamps from files 2029 2030 2 0.5 2028.8 0.1X +read date text from files 147 149 3 6.8 146.7 1.1X +read date from files 692 701 14 1.4 691.5 0.2X +timestamp strings 144 147 5 7.0 143.7 1.1X +parse timestamps from Dataset[String] 1309 1312 3 0.8 1309.5 0.1X +infer timestamps from Dataset[String] 2229 2231 3 0.4 2229.3 0.1X +date strings 206 208 2 4.9 205.8 0.8X +parse dates from Dataset[String] 957 960 4 1.0 956.5 0.2X +from_json(timestamp) 1744 1749 5 0.6 1744.0 0.1X +from_json(date) 1433 1437 4 0.7 1433.3 0.1X +infer error timestamps from Dataset[String] with default format 1299 1302 4 0.8 1299.5 0.1X +infer error timestamps from Dataset[String] with user-provided format 1304 1307 4 0.8 1303.9 0.1X +infer error timestamps from Dataset[String] with legacy format 1335 1337 3 0.7 1334.7 0.1X + +OpenJDK 64-Bit Server VM 21.0.12+8-LTS on Linux 6.17.0-1022-azure AMD EPYC 9V74 80-Core Processor Filters pushdown: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -w/o filters 3953 3968 15 0.0 39532.7 1.0X -pushdown disabled 3897 3920 22 0.0 38965.1 1.0X -w/ filters 522 529 7 0.2 5223.6 7.6X +w/o filters 5507 5528 35 0.0 55073.9 1.0X +pushdown disabled 5300 5309 8 0.0 52999.5 1.0X +w/ filters 744 747 3 0.1 7443.5 7.4X -OpenJDK 64-Bit Server VM 21.0.12+8-LTS on Linux 6.17.0-1020-azure +OpenJDK 64-Bit Server VM 21.0.12+8-LTS on Linux 6.17.0-1022-azure AMD EPYC 9V74 80-Core Processor Partial JSON results: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -parse invalid JSON 1801 1808 9 0.0 180098.5 1.0X +parse invalid JSON 2465 2468 3 0.0 246496.7 1.0X + +OpenJDK 64-Bit Server VM 21.0.12+8-LTS on Linux 6.17.0-1022-azure +AMD EPYC 9V74 80-Core Processor +Top-level JSON array with 0-byte payloads: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------- +streaming enabled: false 75 82 8 1.3 752.2 1.0X +streaming enabled: true 74 78 3 1.3 741.4 1.0X + +OpenJDK 64-Bit Server VM 21.0.12+8-LTS on Linux 6.17.0-1022-azure +AMD EPYC 9V74 80-Core Processor +Top-level JSON array with 65536-byte payloads: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +----------------------------------------------------------------------------------------------------------------------------- +streaming enabled: false 136 139 3 0.0 135565.5 1.0X +streaming enabled: true 133 134 1 0.0 133383.2 1.0X diff --git a/sql/core/benchmarks/JsonBenchmark-jdk25-results.txt b/sql/core/benchmarks/JsonBenchmark-jdk25-results.txt index 9da81c8e0212d..e2b256f1b924e 100644 --- a/sql/core/benchmarks/JsonBenchmark-jdk25-results.txt +++ b/sql/core/benchmarks/JsonBenchmark-jdk25-results.txt @@ -3,128 +3,142 @@ Benchmark for performance of JSON parsing ================================================================================================ Preparing data for benchmarking ... -OpenJDK 64-Bit Server VM 25.0.4+7-LTS on Linux 6.17.0-1020-azure +OpenJDK 64-Bit Server VM 25.0.4+7-LTS on Linux 6.17.0-1022-azure AMD EPYC 9V74 80-Core Processor JSON schema inferring: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -No encoding 1905 1945 34 2.6 381.0 1.0X -UTF-8 is set 3738 3770 40 1.3 747.6 0.5X +No encoding 2032 2116 98 2.5 406.4 1.0X +UTF-8 is set 4421 4489 67 1.1 884.2 0.5X Preparing data for benchmarking ... -OpenJDK 64-Bit Server VM 25.0.4+7-LTS on Linux 6.17.0-1020-azure +OpenJDK 64-Bit Server VM 25.0.4+7-LTS on Linux 6.17.0-1022-azure AMD EPYC 9V74 80-Core Processor count a short column: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -No encoding 1468 1479 11 3.4 293.5 1.0X -UTF-8 is set 3291 3292 1 1.5 658.2 0.4X +No encoding 1734 1744 9 2.9 346.8 1.0X +UTF-8 is set 3939 3950 14 1.3 787.9 0.4X Preparing data for benchmarking ... -OpenJDK 64-Bit Server VM 25.0.4+7-LTS on Linux 6.17.0-1020-azure +OpenJDK 64-Bit Server VM 25.0.4+7-LTS on Linux 6.17.0-1022-azure AMD EPYC 9V74 80-Core Processor count a wide column: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -No encoding 3729 3744 13 0.3 3729.2 1.0X -UTF-8 is set 3523 3547 29 0.3 3522.9 1.1X +No encoding 4737 4744 6 0.2 4736.7 1.0X +UTF-8 is set 4481 4502 22 0.2 4481.0 1.1X Preparing data for benchmarking ... -OpenJDK 64-Bit Server VM 25.0.4+7-LTS on Linux 6.17.0-1020-azure +OpenJDK 64-Bit Server VM 25.0.4+7-LTS on Linux 6.17.0-1022-azure AMD EPYC 9V74 80-Core Processor select wide row: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -No encoding 7307 7345 51 0.0 146141.9 1.0X -UTF-8 is set 7645 7658 14 0.0 152906.0 1.0X +No encoding 10142 10236 89 0.0 202833.6 1.0X +UTF-8 is set 10400 10421 28 0.0 208006.9 1.0X Preparing data for benchmarking ... -OpenJDK 64-Bit Server VM 25.0.4+7-LTS on Linux 6.17.0-1020-azure +OpenJDK 64-Bit Server VM 25.0.4+7-LTS on Linux 6.17.0-1022-azure AMD EPYC 9V74 80-Core Processor Select a subset of 10 columns: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -Select 10 columns 1227 1229 2 0.8 1227.2 1.0X -Select 1 column 877 881 4 1.1 876.8 1.4X +Select 10 columns 1553 1557 4 0.6 1553.1 1.0X +Select 1 column 1154 1193 34 0.9 1154.3 1.3X Preparing data for benchmarking ... -OpenJDK 64-Bit Server VM 25.0.4+7-LTS on Linux 6.17.0-1020-azure +OpenJDK 64-Bit Server VM 25.0.4+7-LTS on Linux 6.17.0-1022-azure AMD EPYC 9V74 80-Core Processor creation of JSON parser per line: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -Short column without encoding 445 446 2 2.2 444.6 1.0X -Short column with UTF-8 862 875 22 1.2 861.9 0.5X -Wide column without encoding 3973 3982 13 0.3 3972.5 0.1X -Wide column with UTF-8 5557 5569 15 0.2 5557.1 0.1X +Short column without encoding 470 471 2 2.1 470.2 1.0X +Short column with UTF-8 1008 1010 2 1.0 1007.9 0.5X +Wide column without encoding 5437 5467 34 0.2 5437.4 0.1X +Wide column with UTF-8 7125 7134 10 0.1 7125.4 0.1X Preparing data for benchmarking ... -OpenJDK 64-Bit Server VM 25.0.4+7-LTS on Linux 6.17.0-1020-azure +OpenJDK 64-Bit Server VM 25.0.4+7-LTS on Linux 6.17.0-1022-azure AMD EPYC 9V74 80-Core Processor JSON functions: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -Text read 43 46 4 23.1 43.2 1.0X -from_json 793 796 3 1.3 792.6 0.1X -json_tuple 643 646 4 1.6 643.0 0.1X -get_json_object wholestage off 696 704 9 1.4 696.2 0.1X -get_json_object wholestage on 611 614 2 1.6 611.1 0.1X +Text read 67 71 6 15.0 66.7 1.0X +from_json 938 956 25 1.1 938.2 0.1X +json_tuple 899 901 1 1.1 899.2 0.1X +get_json_object wholestage off 952 962 10 1.0 952.4 0.1X +get_json_object wholestage on 859 862 3 1.2 859.0 0.1X Preparing data for benchmarking ... -OpenJDK 64-Bit Server VM 25.0.4+7-LTS on Linux 6.17.0-1020-azure +OpenJDK 64-Bit Server VM 25.0.4+7-LTS on Linux 6.17.0-1022-azure AMD EPYC 9V74 80-Core Processor Dataset of json strings: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -Text read 180 183 3 27.8 36.0 1.0X -schema inferring 1135 1137 1 4.4 227.1 0.2X -parsing 2012 2016 4 2.5 402.3 0.1X +Text read 224 231 11 22.3 44.8 1.0X +schema inferring 1500 1503 3 3.3 300.0 0.1X +parsing 2146 2154 12 2.3 429.2 0.1X Preparing data for benchmarking ... -OpenJDK 64-Bit Server VM 25.0.4+7-LTS on Linux 6.17.0-1020-azure +OpenJDK 64-Bit Server VM 25.0.4+7-LTS on Linux 6.17.0-1022-azure AMD EPYC 9V74 80-Core Processor Json files in the per-line mode: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -Text read 455 458 3 11.0 91.0 1.0X -Schema inferring 1603 1604 2 3.1 320.5 0.3X -Parsing without charset 2168 2172 6 2.3 433.5 0.2X -Parsing with UTF-8 4135 4141 6 1.2 826.9 0.1X +Text read 574 580 8 8.7 114.7 1.0X +Schema inferring 1848 1854 6 2.7 369.6 0.3X +Parsing without charset 2249 2258 9 2.2 449.9 0.3X +Parsing with UTF-8 4644 4647 3 1.1 928.7 0.1X -OpenJDK 64-Bit Server VM 25.0.4+7-LTS on Linux 6.17.0-1020-azure +OpenJDK 64-Bit Server VM 25.0.4+7-LTS on Linux 6.17.0-1022-azure AMD EPYC 9V74 80-Core Processor Write dates and timestamps: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -Create a dataset of timestamps 86 89 3 11.7 85.5 1.0X -to_json(timestamp) 382 383 1 2.6 381.7 0.2X -write timestamps to files 397 401 4 2.5 396.8 0.2X -Create a dataset of dates 91 94 4 11.0 91.1 0.9X -to_json(date) 262 265 2 3.8 262.4 0.3X -write dates to files 270 273 2 3.7 270.4 0.3X - -OpenJDK 64-Bit Server VM 25.0.4+7-LTS on Linux 6.17.0-1020-azure +Create a dataset of timestamps 106 109 3 9.4 106.3 1.0X +to_json(timestamp) 489 490 1 2.0 488.6 0.2X +write timestamps to files 547 551 4 1.8 546.7 0.2X +Create a dataset of dates 117 119 2 8.6 116.9 0.9X +to_json(date) 344 347 3 2.9 344.4 0.3X +write dates to files 367 369 2 2.7 366.8 0.3X + +OpenJDK 64-Bit Server VM 25.0.4+7-LTS on Linux 6.17.0-1022-azure AMD EPYC 9V74 80-Core Processor Read dates and timestamps: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ----------------------------------------------------------------------------------------------------------------------------------------------------- -read timestamp text from files 117 120 3 8.5 117.3 1.0X -read timestamps from files 786 795 7 1.3 786.4 0.1X -infer timestamps from files 1476 1478 1 0.7 1476.5 0.1X -read date text from files 109 113 4 9.1 109.4 1.1X -read date from files 510 511 1 2.0 510.4 0.2X -timestamp strings 107 110 5 9.3 107.0 1.1X -parse timestamps from Dataset[String] 939 940 1 1.1 938.8 0.1X -infer timestamps from Dataset[String] 1609 1613 7 0.6 1608.7 0.1X -date strings 148 149 1 6.8 147.9 0.8X -parse dates from Dataset[String] 719 721 2 1.4 719.4 0.2X -from_json(timestamp) 1308 1312 5 0.8 1308.1 0.1X -from_json(date) 1105 1110 7 0.9 1105.5 0.1X -infer error timestamps from Dataset[String] with default format 992 995 4 1.0 991.9 0.1X -infer error timestamps from Dataset[String] with user-provided format 968 972 4 1.0 967.5 0.1X -infer error timestamps from Dataset[String] with legacy format 1008 1011 4 1.0 1008.2 0.1X - -OpenJDK 64-Bit Server VM 25.0.4+7-LTS on Linux 6.17.0-1020-azure +read timestamp text from files 150 154 5 6.7 149.6 1.0X +read timestamps from files 878 884 6 1.1 878.4 0.2X +infer timestamps from files 1684 1688 6 0.6 1684.4 0.1X +read date text from files 136 141 6 7.3 136.2 1.1X +read date from files 579 581 2 1.7 579.1 0.3X +timestamp strings 136 142 8 7.4 135.9 1.1X +parse timestamps from Dataset[String] 1046 1052 8 1.0 1046.2 0.1X +infer timestamps from Dataset[String] 1841 1849 7 0.5 1841.0 0.1X +date strings 197 198 1 5.1 197.4 0.8X +parse dates from Dataset[String] 839 848 7 1.2 839.3 0.2X +from_json(timestamp) 1527 1538 16 0.7 1526.8 0.1X +from_json(date) 1328 1345 29 0.8 1327.9 0.1X +infer error timestamps from Dataset[String] with default format 1186 1189 2 0.8 1186.4 0.1X +infer error timestamps from Dataset[String] with user-provided format 1151 1152 2 0.9 1150.7 0.1X +infer error timestamps from Dataset[String] with legacy format 1210 1213 6 0.8 1209.8 0.1X + +OpenJDK 64-Bit Server VM 25.0.4+7-LTS on Linux 6.17.0-1022-azure AMD EPYC 9V74 80-Core Processor Filters pushdown: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -w/o filters 3260 3270 9 0.0 32601.2 1.0X -pushdown disabled 3176 3183 6 0.0 31759.6 1.0X -w/ filters 521 526 4 0.2 5213.7 6.3X +w/o filters 4727 4737 9 0.0 47265.0 1.0X +pushdown disabled 4660 4675 14 0.0 46602.0 1.0X +w/ filters 700 715 14 0.1 7003.7 6.7X -OpenJDK 64-Bit Server VM 25.0.4+7-LTS on Linux 6.17.0-1020-azure +OpenJDK 64-Bit Server VM 25.0.4+7-LTS on Linux 6.17.0-1022-azure AMD EPYC 9V74 80-Core Processor Partial JSON results: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -parse invalid JSON 1830 1836 8 0.0 183015.3 1.0X +parse invalid JSON 2515 2532 16 0.0 251515.4 1.0X + +OpenJDK 64-Bit Server VM 25.0.4+7-LTS on Linux 6.17.0-1022-azure +AMD EPYC 9V74 80-Core Processor +Top-level JSON array with 0-byte payloads: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------- +streaming enabled: false 74 83 10 1.3 742.9 1.0X +streaming enabled: true 76 77 2 1.3 760.3 1.0X + +OpenJDK 64-Bit Server VM 25.0.4+7-LTS on Linux 6.17.0-1022-azure +AMD EPYC 9V74 80-Core Processor +Top-level JSON array with 65536-byte payloads: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +----------------------------------------------------------------------------------------------------------------------------- +streaming enabled: false 108 111 4 0.0 107718.5 1.0X +streaming enabled: true 107 111 5 0.0 106816.1 1.0X diff --git a/sql/core/benchmarks/JsonBenchmark-results.txt b/sql/core/benchmarks/JsonBenchmark-results.txt index 7424f900e7c7c..2241a4c7aed80 100644 --- a/sql/core/benchmarks/JsonBenchmark-results.txt +++ b/sql/core/benchmarks/JsonBenchmark-results.txt @@ -3,128 +3,142 @@ Benchmark for performance of JSON parsing ================================================================================================ Preparing data for benchmarking ... -OpenJDK 64-Bit Server VM 17.0.20+8-LTS on Linux 6.17.0-1020-azure +OpenJDK 64-Bit Server VM 17.0.20+8-LTS on Linux 6.17.0-1022-azure AMD EPYC 7763 64-Core Processor JSON schema inferring: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -No encoding 2651 2664 13 1.9 530.3 1.0X -UTF-8 is set 5057 5062 8 1.0 1011.4 0.5X +No encoding 2438 2486 75 2.1 487.7 1.0X +UTF-8 is set 5905 5916 12 0.8 1180.9 0.4X Preparing data for benchmarking ... -OpenJDK 64-Bit Server VM 17.0.20+8-LTS on Linux 6.17.0-1020-azure +OpenJDK 64-Bit Server VM 17.0.20+8-LTS on Linux 6.17.0-1022-azure AMD EPYC 7763 64-Core Processor count a short column: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -No encoding 2168 2186 16 2.3 433.7 1.0X -UTF-8 is set 4944 4952 9 1.0 988.8 0.4X +No encoding 2243 2289 51 2.2 448.6 1.0X +UTF-8 is set 4872 4892 18 1.0 974.4 0.5X Preparing data for benchmarking ... -OpenJDK 64-Bit Server VM 17.0.20+8-LTS on Linux 6.17.0-1020-azure +OpenJDK 64-Bit Server VM 17.0.20+8-LTS on Linux 6.17.0-1022-azure AMD EPYC 7763 64-Core Processor count a wide column: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -No encoding 4186 4215 38 0.2 4185.7 1.0X -UTF-8 is set 4448 4471 20 0.2 4448.2 0.9X +No encoding 3307 3333 22 0.3 3307.4 1.0X +UTF-8 is set 4700 4718 19 0.2 4699.9 0.7X Preparing data for benchmarking ... -OpenJDK 64-Bit Server VM 17.0.20+8-LTS on Linux 6.17.0-1020-azure +OpenJDK 64-Bit Server VM 17.0.20+8-LTS on Linux 6.17.0-1022-azure AMD EPYC 7763 64-Core Processor select wide row: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -No encoding 9034 9153 162 0.0 180685.0 1.0X -UTF-8 is set 10025 10071 57 0.0 200506.0 0.9X +No encoding 9262 9465 212 0.0 185247.3 1.0X +UTF-8 is set 10776 10800 29 0.0 215517.2 0.9X Preparing data for benchmarking ... -OpenJDK 64-Bit Server VM 17.0.20+8-LTS on Linux 6.17.0-1020-azure +OpenJDK 64-Bit Server VM 17.0.20+8-LTS on Linux 6.17.0-1022-azure AMD EPYC 7763 64-Core Processor Select a subset of 10 columns: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -Select 10 columns 1579 1586 6 0.6 1579.1 1.0X -Select 1 column 1165 1169 5 0.9 1165.5 1.4X +Select 10 columns 1714 1728 13 0.6 1713.9 1.0X +Select 1 column 1262 1266 3 0.8 1262.4 1.4X Preparing data for benchmarking ... -OpenJDK 64-Bit Server VM 17.0.20+8-LTS on Linux 6.17.0-1020-azure +OpenJDK 64-Bit Server VM 17.0.20+8-LTS on Linux 6.17.0-1022-azure AMD EPYC 7763 64-Core Processor creation of JSON parser per line: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -Short column without encoding 658 663 4 1.5 658.0 1.0X -Short column with UTF-8 1197 1197 1 0.8 1196.9 0.5X -Wide column without encoding 5188 5197 9 0.2 5188.2 0.1X -Wide column with UTF-8 6054 6079 22 0.2 6054.0 0.1X +Short column without encoding 666 669 3 1.5 665.7 1.0X +Short column with UTF-8 1218 1221 2 0.8 1218.4 0.5X +Wide column without encoding 6437 6459 19 0.2 6437.3 0.1X +Wide column with UTF-8 7465 7476 11 0.1 7465.2 0.1X Preparing data for benchmarking ... -OpenJDK 64-Bit Server VM 17.0.20+8-LTS on Linux 6.17.0-1020-azure +OpenJDK 64-Bit Server VM 17.0.20+8-LTS on Linux 6.17.0-1022-azure AMD EPYC 7763 64-Core Processor JSON functions: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -Text read 60 61 1 16.6 60.1 1.0X -from_json 1133 1135 3 0.9 1133.0 0.1X -json_tuple 1039 1045 6 1.0 1038.5 0.1X -get_json_object wholestage off 1056 1064 7 0.9 1056.4 0.1X -get_json_object wholestage on 984 989 6 1.0 984.1 0.1X +Text read 59 61 2 16.9 59.0 1.0X +from_json 1125 1143 17 0.9 1125.3 0.1X +json_tuple 1039 1041 3 1.0 1038.7 0.1X +get_json_object wholestage off 1054 1066 11 0.9 1054.2 0.1X +get_json_object wholestage on 971 972 1 1.0 971.4 0.1X Preparing data for benchmarking ... -OpenJDK 64-Bit Server VM 17.0.20+8-LTS on Linux 6.17.0-1020-azure +OpenJDK 64-Bit Server VM 17.0.20+8-LTS on Linux 6.17.0-1022-azure AMD EPYC 7763 64-Core Processor Dataset of json strings: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -Text read 252 263 13 19.9 50.3 1.0X -schema inferring 2103 2110 10 2.4 420.6 0.1X -parsing 2851 2853 2 1.8 570.2 0.1X +Text read 234 267 56 21.4 46.8 1.0X +schema inferring 1908 1921 11 2.6 381.6 0.1X +parsing 2595 2596 1 1.9 518.9 0.1X Preparing data for benchmarking ... -OpenJDK 64-Bit Server VM 17.0.20+8-LTS on Linux 6.17.0-1020-azure +OpenJDK 64-Bit Server VM 17.0.20+8-LTS on Linux 6.17.0-1022-azure AMD EPYC 7763 64-Core Processor Json files in the per-line mode: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -Text read 690 701 9 7.2 137.9 1.0X -Schema inferring 2553 2555 3 2.0 510.6 0.3X -Parsing without charset 3016 3018 3 1.7 603.2 0.2X -Parsing with UTF-8 5741 5748 8 0.9 1148.1 0.1X +Text read 577 580 4 8.7 115.4 1.0X +Schema inferring 2454 2468 13 2.0 490.8 0.2X +Parsing without charset 2973 2978 9 1.7 594.7 0.2X +Parsing with UTF-8 5773 5776 2 0.9 1154.7 0.1X -OpenJDK 64-Bit Server VM 17.0.20+8-LTS on Linux 6.17.0-1020-azure +OpenJDK 64-Bit Server VM 17.0.20+8-LTS on Linux 6.17.0-1022-azure AMD EPYC 7763 64-Core Processor Write dates and timestamps: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -Create a dataset of timestamps 105 106 1 9.5 105.1 1.0X -to_json(timestamp) 666 671 6 1.5 666.4 0.2X -write timestamps to files 666 675 12 1.5 665.9 0.2X -Create a dataset of dates 116 116 1 8.7 115.5 0.9X -to_json(date) 453 455 3 2.2 452.7 0.2X -write dates to files 442 443 1 2.3 442.3 0.2X - -OpenJDK 64-Bit Server VM 17.0.20+8-LTS on Linux 6.17.0-1020-azure +Create a dataset of timestamps 103 106 3 9.7 102.7 1.0X +to_json(timestamp) 640 641 1 1.6 640.0 0.2X +write timestamps to files 685 692 6 1.5 685.4 0.1X +Create a dataset of dates 120 125 4 8.3 120.4 0.9X +to_json(date) 439 446 7 2.3 439.2 0.2X +write dates to files 434 438 4 2.3 434.1 0.2X + +OpenJDK 64-Bit Server VM 17.0.20+8-LTS on Linux 6.17.0-1022-azure AMD EPYC 7763 64-Core Processor Read dates and timestamps: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ----------------------------------------------------------------------------------------------------------------------------------------------------- -read timestamp text from files 169 175 7 5.9 169.1 1.0X -read timestamps from files 1108 1119 11 0.9 1107.6 0.2X -infer timestamps from files 2089 2094 4 0.5 2089.2 0.1X -read date text from files 167 169 2 6.0 167.2 1.0X -read date from files 736 746 10 1.4 736.0 0.2X -timestamp strings 134 137 3 7.4 134.2 1.3X -parse timestamps from Dataset[String] 1251 1256 5 0.8 1250.6 0.1X -infer timestamps from Dataset[String] 2176 2177 1 0.5 2175.6 0.1X -date strings 206 208 2 4.9 206.1 0.8X -parse dates from Dataset[String] 1009 1010 1 1.0 1009.3 0.2X -from_json(timestamp) 1726 1732 9 0.6 1725.5 0.1X -from_json(date) 1481 1490 8 0.7 1480.7 0.1X -infer error timestamps from Dataset[String] with default format 1477 1479 3 0.7 1476.9 0.1X -infer error timestamps from Dataset[String] with user-provided format 1477 1484 11 0.7 1477.3 0.1X -infer error timestamps from Dataset[String] with legacy format 1488 1495 7 0.7 1487.9 0.1X - -OpenJDK 64-Bit Server VM 17.0.20+8-LTS on Linux 6.17.0-1020-azure +read timestamp text from files 148 153 8 6.7 148.3 1.0X +read timestamps from files 1067 1068 1 0.9 1067.2 0.1X +infer timestamps from files 2003 2013 15 0.5 2003.1 0.1X +read date text from files 141 144 4 7.1 141.2 1.1X +read date from files 750 752 2 1.3 750.4 0.2X +timestamp strings 137 137 1 7.3 136.9 1.1X +parse timestamps from Dataset[String] 1195 1201 9 0.8 1194.8 0.1X +infer timestamps from Dataset[String] 2093 2097 4 0.5 2093.0 0.1X +date strings 198 198 1 5.1 197.8 0.7X +parse dates from Dataset[String] 954 956 2 1.0 953.6 0.2X +from_json(timestamp) 1659 1677 26 0.6 1659.5 0.1X +from_json(date) 1435 1448 19 0.7 1434.6 0.1X +infer error timestamps from Dataset[String] with default format 1426 1435 12 0.7 1426.0 0.1X +infer error timestamps from Dataset[String] with user-provided format 1437 1448 19 0.7 1437.5 0.1X +infer error timestamps from Dataset[String] with legacy format 1452 1455 2 0.7 1452.1 0.1X + +OpenJDK 64-Bit Server VM 17.0.20+8-LTS on Linux 6.17.0-1022-azure AMD EPYC 7763 64-Core Processor Filters pushdown: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -w/o filters 5857 5867 14 0.0 58570.7 1.0X -pushdown disabled 5603 5612 13 0.0 56028.3 1.0X -w/ filters 532 546 17 0.2 5323.0 11.0X +w/o filters 6016 6028 10 0.0 60158.9 1.0X +pushdown disabled 5833 5857 22 0.0 58329.5 1.0X +w/ filters 667 668 1 0.1 6667.9 9.0X -OpenJDK 64-Bit Server VM 17.0.20+8-LTS on Linux 6.17.0-1020-azure +OpenJDK 64-Bit Server VM 17.0.20+8-LTS on Linux 6.17.0-1022-azure AMD EPYC 7763 64-Core Processor Partial JSON results: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -parse invalid JSON 2253 2315 93 0.0 225259.5 1.0X +parse invalid JSON 2372 2465 149 0.0 237212.6 1.0X + +OpenJDK 64-Bit Server VM 17.0.20+8-LTS on Linux 6.17.0-1022-azure +AMD EPYC 7763 64-Core Processor +Top-level JSON array with 0-byte payloads: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------- +streaming enabled: false 77 83 6 1.3 772.6 1.0X +streaming enabled: true 79 82 4 1.3 793.2 1.0X + +OpenJDK 64-Bit Server VM 17.0.20+8-LTS on Linux 6.17.0-1022-azure +AMD EPYC 7763 64-Core Processor +Top-level JSON array with 65536-byte payloads: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +----------------------------------------------------------------------------------------------------------------------------- +streaming enabled: false 160 168 13 0.0 159991.4 1.0X +streaming enabled: true 161 162 1 0.0 161295.1 1.0X diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/json/JsonDataSource.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/json/JsonDataSource.scala index 14e23a5fbdeec..432e570875a3d 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/json/JsonDataSource.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/json/JsonDataSource.scala @@ -395,7 +395,7 @@ object MultiLineJsonDataSource extends JsonDataSource { file: PartitionedFile, parser: JacksonParser, schema: StructType): Iterator[InternalRow] = { - def partitionedFileString(ignored: Any): UTF8String = { + lazy val fileLiteral: UTF8String = { Utils.tryWithResource { Utils.createResourceUninterruptiblyIfInTaskThread { CodecStreams.createInputStreamWithCloseResource(conf, file.toPath) @@ -404,6 +404,7 @@ object MultiLineJsonDataSource extends JsonDataSource { UTF8String.fromBytes(inputStream.readAllBytes()) } } + def partitionedFileString(ignored: Any): UTF8String = fileLiteral val streamParser = parser.options.encoding .map(enc => CreateJacksonParser.inputStream(enc, _: JsonFactory, _: InputStream)) .getOrElse(CreateJacksonParser.inputStream(_: JsonFactory, _: InputStream)) @@ -414,8 +415,14 @@ object MultiLineJsonDataSource extends JsonDataSource { schema, parser.options.columnNameOfCorruptRecord) - safeParser.parse( - CodecStreams.createInputStreamWithCloseResource(conf, file.toPath)) + val input = CodecStreams.createInputStreamWithCloseResource(conf, file.toPath) + if (parser.options.streamMultilineTopLevelArray) { + safeParser.parseIterator( + input, + input => parser.parseIterator[InputStream](input, streamParser, partitionedFileString)) + } else { + safeParser.parse(input) + } } override protected def readStream( @@ -425,16 +432,25 @@ object MultiLineJsonDataSource extends JsonDataSource { // The entry is a single JSON document. Buffer its bytes so the corrupt-record column can echo // the whole document on a parse failure, mirroring `readFile`'s `partitionedFileString`. val bytes = in.readAllBytes() + lazy val documentLiteral: UTF8String = UTF8String.fromBytes(bytes) val streamParser = parser.options.encoding .map(enc => CreateJacksonParser.inputStream(enc, _: JsonFactory, _: InputStream)) .getOrElse(CreateJacksonParser.inputStream(_: JsonFactory, _: InputStream)) val safeParser = new FailureSafeParser[InputStream]( - input => parser.parse[InputStream](input, streamParser, _ => UTF8String.fromBytes(bytes)), + input => parser.parse[InputStream](input, streamParser, _ => documentLiteral), parser.options.parseMode, schema, parser.options.columnNameOfCorruptRecord) - safeParser.parse(new ByteArrayInputStream(bytes)) + val input = new ByteArrayInputStream(bytes) + if (parser.options.streamMultilineTopLevelArray) { + safeParser.parseIterator( + input, + input => parser.parseIterator[InputStream]( + input, streamParser, _ => documentLiteral)) + } else { + safeParser.parse(input) + } } } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/JSONArchiveReadBase.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/JSONArchiveReadBase.scala index 70de9e5e63178..3b0d12be27917 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/JSONArchiveReadBase.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/JSONArchiveReadBase.scala @@ -21,7 +21,8 @@ import java.io.File import java.nio.charset.StandardCharsets import java.nio.file.Files -import org.apache.spark.sql.AnalysisException +import org.apache.spark.SparkException +import org.apache.spark.sql.{AnalysisException, Row} import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.{NullType, StringType} @@ -166,6 +167,66 @@ trait JSONArchiveReadBase extends ArchiveReadSuiteBase { extraOptions = Map("multiLine" -> "true")) } + test("JSON: streaming multi-line top-level arrays match a directory read") { + withSQLConf(SQLConf.JSON_STREAM_MULTILINE_TOP_LEVEL_ARRAY.key -> "true") { + assertArchiveMatchesDir( + Seq( + "a.json" -> jsonBytes("""[{"id":1,"name":"Alice"},{"id":2,"name":"Bob"}]"""), + "b.json" -> jsonBytes("""[{"id":3,"name":"Carol"}]""")), + extraOptions = Map("multiLine" -> "true")) + } + } + + gridTest("JSON: streaming archive arrays resume after a partial element")( + Seq("PERMISSIVE", "DROPMALFORMED", "FAILFAST")) { mode => + withSQLConf(SQLConf.JSON_STREAM_MULTILINE_TOP_LEVEL_ARRAY.key -> "true") { + val document = """[{"id":"bad","name":"Alice"},{"id":2,"name":"Bob"}]""" + withArchiveFile() { archive => + writeArchive(archive, Seq(entryName(0) -> jsonBytes(document))) + val df = read( + archive.getCanonicalPath, + Map("multiLine" -> "true", "mode" -> mode), + s"$readSchema, _corrupt_record STRING") + + mode match { + case "PERMISSIVE" => + checkAnswer(df, Seq(Row(null, "Alice", document), Row(2, "Bob", null))) + case "DROPMALFORMED" => + checkAnswer(df, Row(2, "Bob", null)) + case "FAILFAST" => + val error = intercept[SparkException](df.collect()) + assert(error.getCause.asInstanceOf[SparkException].getCondition === + "MALFORMED_RECORD_IN_PARSING.WITHOUT_SUGGESTION") + } + } + } + } + + gridTest("JSON: streaming archive arrays handle terminal malformed input")( + Seq("PERMISSIVE", "DROPMALFORMED", "FAILFAST")) { mode => + withSQLConf(SQLConf.JSON_STREAM_MULTILINE_TOP_LEVEL_ARRAY.key -> "true") { + val document = """[{"id":1,"name":"Alice"} {"id":2,"name":"Bob"}]""" + withArchiveFile() { archive => + writeArchive(archive, Seq(entryName(0) -> jsonBytes(document))) + val df = read( + archive.getCanonicalPath, + Map("multiLine" -> "true", "mode" -> mode), + s"$readSchema, _corrupt_record STRING") + + mode match { + case "PERMISSIVE" => + checkAnswer(df, Seq(Row(1, "Alice", null), Row(null, null, document))) + case "DROPMALFORMED" => + checkAnswer(df, Row(1, "Alice", null)) + case "FAILFAST" => + val error = intercept[SparkException](df.collect()) + assert(error.getCause.asInstanceOf[SparkException].getCondition === + "MALFORMED_RECORD_IN_PARSING.WITHOUT_SUGGESTION") + } + } + } + } + test("JSON: a malformed record in an archive entry matches a directory read (both modes)") { // Permissive mode (the default): a malformed record parses to nulls with its raw text echoed // into `_corrupt_record`. The archive path wires its own FailureSafeParser in `readStream` -- diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonBenchmark.scala index 94a2ccc41d30b..bb779b0361562 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonBenchmark.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonBenchmark.scala @@ -17,6 +17,8 @@ package org.apache.spark.sql.execution.datasources.json import java.io.File +import java.nio.charset.StandardCharsets +import java.nio.file.Files import java.time.{Instant, LocalDate} import org.apache.spark.benchmark.Benchmark @@ -37,6 +39,15 @@ import org.apache.spark.sql.types._ * 3. generate result: * SPARK_GENERATE_BENCHMARK_FILES=1 build/sbt "sql/Test/runMain " * Results will be written to "benchmarks/JSONBenchmark-results.txt". + * + * To measure only the top-level JSON array cases, run it with the "top-level-array" + * argument. It skips every other case, so it must not be combined with + * SPARK_GENERATE_BENCHMARK_FILES=1: the results file is truncated before the arguments + * are read, so it would be left holding only the top-level array sections. + * 1. without sbt: + * bin/spark-submit --class --jars , + * top-level-array + * 2. build/sbt "sql/Test/runMain top-level-array" * }}} */ object JsonBenchmark extends SqlBasedBenchmark { @@ -578,8 +589,46 @@ object JsonBenchmark extends SqlBasedBenchmark { benchmark.run() } + private def topLevelArrayBenchmark( + rowsNum: Int, + payloadSize: Int, + numIters: Int): Unit = { + val payload = "x" * payloadSize + val document = (0 until rowsNum) + .map(i => s"""{"a":$i,"payload":"$payload"}""") + .mkString("[", ",", "]") + val schema = new StructType().add("a", IntegerType).add("payload", StringType) + val benchmark = new Benchmark( + s"Top-level JSON array with $payloadSize-byte payloads", rowsNum, output = output) + + withTempPath { path => + Files.write(path.toPath, document.getBytes(StandardCharsets.UTF_8)) + + Seq(false, true).foreach { enabled => + benchmark.addCase(s"streaming enabled: $enabled", numIters) { _ => + withSQLConf(SQLConf.JSON_STREAM_MULTILINE_TOP_LEVEL_ARRAY.key -> enabled.toString) { + spark.read + .option("multiLine", true) + .schema(schema) + .json(path.getCanonicalPath) + .noop() + } + } + } + + benchmark.run() + } + } + override def runBenchmarkSuite(mainArgs: Array[String]): Unit = { val numIters = 3 + if (mainArgs.contains("top-level-array")) { + runBenchmark("Benchmark for top-level JSON array parsing") { + topLevelArrayBenchmark(rowsNum = 100000, payloadSize = 0, numIters = numIters) + topLevelArrayBenchmark(rowsNum = 1000, payloadSize = 64 * 1024, numIters = numIters) + } + return + } runBenchmark("Benchmark for performance of JSON parsing") { schemaInferring(5 * 1000 * 1000, numIters) countShortColumn(5 * 1000 * 1000, numIters) @@ -595,6 +644,8 @@ object JsonBenchmark extends SqlBasedBenchmark { // TODO (SPARK-32325): Add benchmarks for filters with nested column attributes. filtersPushdownBenchmark(rowsNum = 100 * 1000, numIters) partialResultBenchmark(rowsNum = 10000, numIters) + topLevelArrayBenchmark(rowsNum = 100000, payloadSize = 0, numIters = numIters) + topLevelArrayBenchmark(rowsNum = 1000, payloadSize = 64 * 1024, numIters = numIters) } } } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala index 4cd6783e1df60..e846fad976f3b 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala @@ -23,28 +23,32 @@ import java.nio.file.Files import java.sql.{Date, Timestamp} import java.time.{Duration, Instant, LocalDate, LocalDateTime, Period, ZoneId, ZoneOffset} import java.util.Locale -import java.util.concurrent.atomic.AtomicLong +import java.util.concurrent.atomic.{AtomicInteger, AtomicLong} -import com.fasterxml.jackson.core.JsonFactory +import com.fasterxml.jackson.core.{JsonFactory, JsonToken} +import com.fasterxml.jackson.core.util.JsonParserDelegate import org.apache.hadoop.fs.{Path, PathFilter} import org.apache.hadoop.io.SequenceFile.CompressionType import org.apache.hadoop.io.compress.{CompressionCodecFactory, GzipCodec} import org.apache.spark.{SparkConf, SparkException, SparkRuntimeException, SparkUpgradeException, TestUtils} import org.apache.spark.SparkIllegalArgumentException +import org.apache.spark.TaskContext import org.apache.spark.io.ZStdCompressionCodec +import org.apache.spark.paths.SparkPath import org.apache.spark.rdd.RDD import org.apache.spark.scheduler.{SparkListener, SparkListenerJobEnd} import org.apache.spark.sql.{functions => F, _} +import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.json._ -import org.apache.spark.sql.catalyst.util.{CharsetProvider, DateTimeTestUtils, DateTimeUtils, HadoopCompressionCodec} +import org.apache.spark.sql.catalyst.util.{BadRecordException, CharsetProvider, DateTimeTestUtils, DateTimeUtils, HadoopCompressionCodec} import org.apache.spark.sql.catalyst.util.HadoopCompressionCodec.GZIP import org.apache.spark.sql.catalyst.util.TimestampNanosTestUtils import org.apache.spark.sql.catalyst.util.TimestampNanosTestUtils.foreachNanosPrecision import org.apache.spark.sql.catalyst.util.TypeUtils.toSQLType import org.apache.spark.sql.errors.QueryExecutionErrors.toSQLId import org.apache.spark.sql.execution.ExternalRDD -import org.apache.spark.sql.execution.datasources.{CommonFileDataSourceSuite, DataSource, InMemoryFileIndex, NoopCache} +import org.apache.spark.sql.execution.datasources.{CommonFileDataSourceSuite, DataSource, InMemoryFileIndex, NoopCache, PartitionedFile} import org.apache.spark.sql.execution.datasources.v2.json.JsonScanBuilder import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.test.SharedSparkSession @@ -54,6 +58,7 @@ import org.apache.spark.sql.types.StructType.fromDDL import org.apache.spark.sql.types.TestUDT.{MyDenseVector, MyDenseVectorUDT} import org.apache.spark.sql.util.CaseInsensitiveStringMap import org.apache.spark.tags.ExtendedSQLTest +import org.apache.spark.unsafe.types.UTF8String import org.apache.spark.util.ArrayImplicits._ import org.apache.spark.util.Utils @@ -1066,6 +1071,495 @@ abstract class JsonSuite } } + gridTest("SPARK-3308 Read multiline top level JSON arrays")( + Seq(false, true)) { enabled => + withSQLConf(SQLConf.JSON_STREAM_MULTILINE_TOP_LEVEL_ARRAY.key -> enabled.toString) { + withTempPath { file => + Files.write(file.toPath, """[{"a":1},{"a":2}]""".getBytes(StandardCharsets.UTF_8)) + checkAnswer( + spark.read.option("multiLine", true).schema("a int").json(file.getCanonicalPath), + Seq(Row(1), Row(2))) + } + } + } + + gridTest("multiline top level JSON array with singleVariantColumn is not streamed")( + Seq(false, true)) { enabled => + withSQLConf(SQLConf.JSON_STREAM_MULTILINE_TOP_LEVEL_ARRAY.key -> enabled.toString) { + withTempPath { file => + Files.write(file.toPath, """[{"a":1},{"a":2}]""".getBytes(StandardCharsets.UTF_8)) + // singleVariantColumn binds the whole document to one variant, so the streaming + // predicate excludes it; without that conjunct this yields one row per element. + checkAnswer( + spark.read.option("multiLine", true).option("singleVariantColumn", "var") + .json(file.getCanonicalPath).selectExpr("to_json(var)"), + Seq(Row("""[{"a":1},{"a":2}]"""))) + } + } + } + + test("multiline top level JSON arrays are parsed lazily") { + val schema = StructType(Seq(StructField("a", IntegerType))) + val options = new JSONOptions(Map("multiLine" -> "true"), SQLConf.get.sessionLocalTimeZone) + val parser = new JacksonParser(schema, options, allowArrayAsStructs = true) + val input = new ByteArrayInputStream( + s"""[{"a":1},{"a":2,"payload":"${"x" * 200000}"}]""".getBytes(StandardCharsets.UTF_8)) + val rows = parser.parseIterator[InputStream]( + input, + CreateJacksonParser.inputStream(_: JsonFactory, _: InputStream), + stream => UTF8String.fromBytes(stream.readAllBytes())) + + assert(rows.next().getInt(0) === 1) + assert(input.available() > 0) + } + + test("non-array JSON is parsed eagerly by parseIterator") { + val schema = StructType(Seq(StructField("a", IntegerType))) + val options = new JSONOptions(Map("multiLine" -> "true"), SQLConf.get.sessionLocalTimeZone) + val parser = new JacksonParser(schema, options, allowArrayAsStructs = true) + var closed = false + val input = new ByteArrayInputStream("""{"a":1}""".getBytes(StandardCharsets.UTF_8)) { + override def close(): Unit = { + closed = true + super.close() + } + } + + val rows = parser.parseIterator[InputStream]( + input, + CreateJacksonParser.inputStream(_: JsonFactory, _: InputStream), + stream => UTF8String.fromBytes(stream.readAllBytes())) + + assert(closed) + assert(rows.next().getInt(0) === 1) + assert(!rows.hasNext) + } + + test("non-array partial results close the parser") { + val schema = StructType(Seq(StructField("a", IntegerType))) + val options = new JSONOptions(Map("multiLine" -> "true"), SQLConf.get.sessionLocalTimeZone) + val parser = new JacksonParser(schema, options, allowArrayAsStructs = true) + var closed = false + val input = new ByteArrayInputStream("""{"a":"bad"}""".getBytes(StandardCharsets.UTF_8)) { + override def close(): Unit = { + closed = true + super.close() + } + } + + val error = intercept[BadRecordException] { + parser.parseIterator[InputStream]( + input, + CreateJacksonParser.inputStream(_: JsonFactory, _: InputStream), + stream => UTF8String.fromBytes(stream.readAllBytes())) + } + + assert(closed) + assert(!error.recoverable) + } + + gridTest("multiline top level JSON array stops retaining a closed parser")( + Seq("" -> 0, "[]" -> 0, """[{"a":1},{"a":2}]""" -> 2, """[{"a":1}""" -> 1)) { + case (document, expectedRows) => + val schema = StructType(Seq(StructField("a", IntegerType))) + val options = new JSONOptions(Map("multiLine" -> "true"), SQLConf.get.sessionLocalTimeZone) + val parser = new JacksonParser(schema, options, allowArrayAsStructs = true) + val closes = new AtomicInteger(0) + val input = new ByteArrayInputStream(document.getBytes(StandardCharsets.UTF_8)) + val context = TaskContext.empty() + TaskContext.setTaskContext(context) + try { + val rows = parser.parseIterator[InputStream]( + input, + (factory: JsonFactory, stream: InputStream) => + new JsonParserDelegate(CreateJacksonParser.inputStream(factory, stream)) { + override def close(): Unit = { + closes.incrementAndGet() + super.close() + } + }, + stream => UTF8String.fromBytes(stream.readAllBytes())) + var produced = 0 + // The truncated document raises once the array runs out; that error is asserted elsewhere. + try rows.foreach(_ => produced += 1) catch { case _: BadRecordException => } + + assert(produced === expectedRows) + assert(closes.get() === 1) + context.markTaskCompleted(None) + // A second close here would mean the completion listener still held a parser that had already + // been closed, which is what keeps an archive entry's buffer alive for the rest of the task. + assert(closes.get() === 1) + } finally { + TaskContext.unset() + } + } + + gridTest("multiline top level JSON array survives a parser close failure")( + Seq("exhausted", "abandoned")) { consumption => + val schema = StructType(Seq(StructField("a", IntegerType))) + val options = new JSONOptions(Map("multiLine" -> "true"), SQLConf.get.sessionLocalTimeZone) + val parser = new JacksonParser(schema, options, allowArrayAsStructs = true) + val closes = new AtomicInteger(0) + val input = new ByteArrayInputStream("""[{"a":1},{"a":2}]""".getBytes(StandardCharsets.UTF_8)) + val context = TaskContext.empty() + TaskContext.setTaskContext(context) + try { + val rows = parser.parseIterator[InputStream]( + input, + (factory: JsonFactory, stream: InputStream) => + new JsonParserDelegate(CreateJacksonParser.inputStream(factory, stream)) { + override def close(): Unit = { + closes.incrementAndGet() + throw new IOException("close failed") + } + }, + stream => UTF8String.fromBytes(stream.readAllBytes())) + // The array's end closes eagerly, an abandoned iterator closes from the completion listener, + // and neither may surface the failure: rows already emitted cannot be withdrawn. + if (consumption == "exhausted") { + assert(rows.map(_.getInt(0)).toSeq === Seq(1, 2)) + assert(closes.get() === 1) + } else { + assert(rows.next().getInt(0) === 1) + assert(closes.get() === 0) + } + context.markTaskCompleted(None) + assert(closes.get() === 1) + } finally { + TaskContext.unset() + } + } + + gridTest("multiline top level JSON array closes the parser when a raw read fails")( + Seq("opening the array" -> 1, "advancing to an element" -> 2)) { case (_, failingToken) => + val schema = StructType(Seq(StructField("a", IntegerType))) + val options = new JSONOptions(Map("multiLine" -> "true"), SQLConf.get.sessionLocalTimeZone) + val parser = new JacksonParser(schema, options, allowArrayAsStructs = true) + val closes = new AtomicInteger(0) + val tokens = new AtomicInteger(0) + val input = new ByteArrayInputStream("""[{"a":1},{"a":2}]""".getBytes(StandardCharsets.UTF_8)) + val context = TaskContext.empty() + TaskContext.setTaskContext(context) + try { + // A raw I/O failure is none of the types `handleFailure` names, so it stays unwrapped rather + // than becoming a malformed record -- but it must still close here rather than at task + // completion, or a task skipping corrupt files holds one parser and stream per failed entry. + intercept[IOException] { + val rows = parser.parseIterator[InputStream]( + input, + (factory: JsonFactory, stream: InputStream) => + new JsonParserDelegate(CreateJacksonParser.inputStream(factory, stream)) { + override def nextToken(): JsonToken = { + if (tokens.incrementAndGet() == failingToken) throw new IOException("read failed") + super.nextToken() + } + override def close(): Unit = { + closes.incrementAndGet() + super.close() + } + }, + stream => UTF8String.fromBytes(stream.readAllBytes())) + rows.foreach(_ => ()) + } + assert(closes.get() === 1) + context.markTaskCompleted(None) + assert(closes.get() === 1) + } finally { + TaskContext.unset() + } + } + + test("multiline top level JSON array keeps the parse failure primary when the close fails") { + val schema = StructType(Seq(StructField("a", IntegerType))) + val options = new JSONOptions( + Map("multiLine" -> "true", "mode" -> "FAILFAST"), SQLConf.get.sessionLocalTimeZone) + val parser = new JacksonParser(schema, options, allowArrayAsStructs = true) + val closes = new AtomicInteger(0) + val input = new ByteArrayInputStream("""[{"a":1},42]""".getBytes(StandardCharsets.UTF_8)) + val context = TaskContext.empty() + TaskContext.setTaskContext(context) + try { + val error = intercept[BadRecordException] { + val rows = parser.parseIterator[InputStream]( + input, + (factory: JsonFactory, stream: InputStream) => + new JsonParserDelegate(CreateJacksonParser.inputStream(factory, stream)) { + override def close(): Unit = { + closes.incrementAndGet() + throw new IOException("close failed") + } + }, + stream => UTF8String.fromBytes(stream.readAllBytes())) + rows.foreach(_ => ()) + } + // The record is already lost here, so this close reports instead of swallowing -- but only + // as a detail of the parse failure, never in place of it. + assert(!error.cause.isInstanceOf[IOException]) + assert(error.cause.getSuppressed.map(_.getMessage).toSeq === Seq("close failed")) + assert(closes.get() === 1) + context.markTaskCompleted(None) + assert(closes.get() === 1) + } finally { + TaskContext.unset() + } + } + + test("multiline top level JSON array closes the parser on task completion") { + val schema = StructType(Seq(StructField("a", IntegerType))) + val options = new JSONOptions(Map("multiLine" -> "true"), SQLConf.get.sessionLocalTimeZone) + val parser = new JacksonParser(schema, options, allowArrayAsStructs = true) + var closed = false + val input = new ByteArrayInputStream( + """[{"a":1},{"a":2},{"a":3}]""".getBytes(StandardCharsets.UTF_8)) { + override def close(): Unit = { + closed = true + super.close() + } + } + val context = TaskContext.empty() + TaskContext.setTaskContext(context) + try { + val rows = parser.parseIterator[InputStream]( + input, + CreateJacksonParser.inputStream(_: JsonFactory, _: InputStream), + stream => UTF8String.fromBytes(stream.readAllBytes())) + assert(rows.next().getInt(0) === 1) + assert(!closed) + context.markTaskCompleted(None) + assert(closed) + } finally { + TaskContext.unset() + } + } + + gridTest("multiline top level JSON array keeps rows emitted before malformed input")( + Seq("PERMISSIVE", "DROPMALFORMED", "FAILFAST")) { mode => + withSQLConf(SQLConf.JSON_STREAM_MULTILINE_TOP_LEVEL_ARRAY.key -> "true") { + withTempPath { file => + val document = """[{"a":1} {"a":2}]""" + Files.write(file.toPath, document.getBytes(StandardCharsets.UTF_8)) + val actualSchema = StructType(Seq(StructField("a", IntegerType))) + val schema = StructType(Seq( + actualSchema.head, + StructField("_corrupt_record", StringType))) + val options = new JSONOptions( + Map("multiLine" -> "true", "mode" -> mode), + SQLConf.get.sessionLocalTimeZone, + SQLConf.get.columnNameOfCorruptRecord) + val parser = new JacksonParser(actualSchema, options, allowArrayAsStructs = true) + val partitionedFile = PartitionedFile( + InternalRow.empty, + SparkPath.fromPathString(file.getCanonicalPath), + 0, + file.length()) + val rows = MultiLineJsonDataSource.readFile( + spark.sessionState.newHadoopConf(), partitionedFile, parser, schema) + + assert(rows.next().getInt(0) === 1) + mode match { + case "PERMISSIVE" => + val corruptRow = rows.next() + assert(corruptRow.isNullAt(0)) + val corruptRecord = corruptRow.getUTF8String(1) + assert(corruptRecord != null, corruptRow.toString) + assert(corruptRecord.toString === document) + assert(!rows.hasNext) + case "DROPMALFORMED" => + assert(!rows.hasNext) + case "FAILFAST" => + val error = intercept[SparkException](rows.hasNext) + assert(error.getCondition === "MALFORMED_RECORD_IN_PARSING.WITHOUT_SUGGESTION") + } + } + } + } + + gridTest("multiline top level JSON array reports a truncated array as malformed")( + Seq("PERMISSIVE", "DROPMALFORMED", "FAILFAST")) { mode => + withSQLConf(SQLConf.JSON_STREAM_MULTILINE_TOP_LEVEL_ARRAY.key -> "true") { + withTempPath { file => + // No closing bracket: jackson-core throws JsonEOFException from nextToken() while the + // array context is open, so this takes the malformed-record arm, not the null branch. + val document = """[{"a":1}""" + Files.write(file.toPath, document.getBytes(StandardCharsets.UTF_8)) + val actualSchema = StructType(Seq(StructField("a", IntegerType))) + val schema = StructType(Seq( + actualSchema.head, + StructField("_corrupt_record", StringType))) + val options = new JSONOptions( + Map("multiLine" -> "true", "mode" -> mode), + SQLConf.get.sessionLocalTimeZone, + SQLConf.get.columnNameOfCorruptRecord) + val parser = new JacksonParser(actualSchema, options, allowArrayAsStructs = true) + val partitionedFile = PartitionedFile( + InternalRow.empty, + SparkPath.fromPathString(file.getCanonicalPath), + 0, + file.length()) + val rows = MultiLineJsonDataSource.readFile( + spark.sessionState.newHadoopConf(), partitionedFile, parser, schema) + + assert(rows.next().getInt(0) === 1) + mode match { + case "PERMISSIVE" => + val corruptRow = rows.next() + assert(corruptRow.isNullAt(0)) + assert(corruptRow.getUTF8String(1).toString === document) + assert(!rows.hasNext) + case "DROPMALFORMED" => + assert(!rows.hasNext) + case "FAILFAST" => + val error = intercept[SparkException](rows.hasNext) + assert(error.getCondition === "MALFORMED_RECORD_IN_PARSING.WITHOUT_SUGGESTION") + } + } + } + } + + gridTest("multiline top level JSON array parse modes with a malformed element")( + for { + element <- Seq("partial object", "scalar", "null", "nested array") + mode <- Seq("PERMISSIVE", "DROPMALFORMED", "FAILFAST") + streaming <- Seq(false, true) + } yield (element, mode, streaming)) { case (element, mode, streaming) => + withSQLConf(SQLConf.JSON_STREAM_MULTILINE_TOP_LEVEL_ARRAY.key -> streaming.toString) { + withTempPath { file => + val malformed = element match { + case "partial object" => """{"a":"bad"}""" + case "scalar" => "42" + case "null" => "null" + case "nested array" => "[1,2]" + } + val document = s"""[{"a":1},$malformed,{"a":2}]""" + Files.write(file.toPath, document.getBytes(StandardCharsets.UTF_8)) + val schema = StructType(Seq( + StructField("a", IntegerType), + StructField("_corrupt_record", StringType))) + val df = spark.read + .schema(schema) + .option("multiLine", true) + .option("mode", mode) + .json(file.getCanonicalPath) + + if (mode == "FAILFAST") { + val error = intercept[SparkException](df.collect()) + val malformed = error.getCause.asInstanceOf[SparkException] + assert(malformed.getCondition === "MALFORMED_RECORD_IN_PARSING.WITHOUT_SUGGESTION") + // Cause is the underlying parse failure, not a re-wrapped BadRecordException. + assert(!malformed.getCause.isInstanceOf[BadRecordException]) + } else { + val expected = if (streaming) { + // An array element is the record wherever the parser is left at the element boundary: a + // partial object has been consumed through END_OBJECT, and a scalar was never entered. + // A null element is a scalar too, though it is the converter's null result rather than + // the converter that fails. A nested value of the wrong shape can stop anywhere inside + // its element, so that one still ends the document. + element match { + case "nested array" if mode == "PERMISSIVE" => Seq(Row(1, null), Row(null, document)) + case "nested array" => Seq(Row(1, null)) + case _ if mode == "PERMISSIVE" => + Seq(Row(1, null), Row(null, document), Row(2, null)) + case _ => Seq(Row(1, null), Row(2, null)) + } + } else if (mode == "PERMISSIVE") { + // The whole document is the record, so this keeps only the rows the eager array had + // accumulated when it failed, each stamped with the document. Only a partial object + // accumulates any: the other two shapes abandon the array where they fail. + if (element == "partial object") { + Seq(Row(1, document), Row(null, document), Row(2, document)) + } else { + Seq(Row(null, document)) + } + } else { + Seq.empty[Row] + } + checkAnswer(df, expected) + } + } + } + } + + test("multiline top level JSON array streaming option overrides the session config") { + withSQLConf(SQLConf.JSON_STREAM_MULTILINE_TOP_LEVEL_ARRAY.key -> "false") { + withTempPath { file => + val document = """[{"a":"bad"},{"a":2}]""" + Files.write(file.toPath, document.getBytes(StandardCharsets.UTF_8)) + val schema = StructType(Seq( + StructField("a", IntegerType), + StructField("_corrupt_record", StringType))) + val df = spark.read + .schema(schema) + .option("multiLine", true) + .option("enableStreamingTopLevelArray", true) + .json(file.getCanonicalPath) + + checkAnswer(df, Seq(Row(null, document), Row(2, null))) + } + } + } + + test("multiline top level JSON array reuses the corrupt record literal") { + withSQLConf(SQLConf.JSON_STREAM_MULTILINE_TOP_LEVEL_ARRAY.key -> "true") { + withTempPath { file => + val document = """[{"a":"bad"},{"a":"also bad"},{"a":2}]""" + Files.write(file.toPath, document.getBytes(StandardCharsets.UTF_8)) + val actualSchema = StructType(Seq(StructField("a", IntegerType))) + val schema = StructType(Seq( + actualSchema.head, + StructField("_corrupt_record", StringType))) + val options = new JSONOptions( + Map("multiLine" -> "true", "mode" -> "PERMISSIVE"), + SQLConf.get.sessionLocalTimeZone, + SQLConf.get.columnNameOfCorruptRecord) + val parser = new JacksonParser(actualSchema, options, allowArrayAsStructs = true) + val partitionedFile = PartitionedFile( + InternalRow.empty, + SparkPath.fromPathString(file.getCanonicalPath), + 0, + file.length()) + val rows = MultiLineJsonDataSource.readFile( + spark.sessionState.newHadoopConf(), partitionedFile, parser, schema) + + assert(rows.next().getUTF8String(1).toString === document) + Files.delete(file.toPath) + assert(rows.next().getUTF8String(1).toString === document) + assert(rows.next().getInt(0) === 2) + assert(!rows.hasNext) + } + } + } + + gridTest("multiline JSON parser creation failures honor parse mode")( + Seq("PERMISSIVE", "DROPMALFORMED", "FAILFAST")) { mode => + withSQLConf(SQLConf.JSON_STREAM_MULTILINE_TOP_LEVEL_ARRAY.key -> "true") { + withTempPath { file => + Files.write(file.toPath, Array[Byte](0, 0, 0xff.toByte, 0xfe.toByte)) + val schema = StructType(Seq( + StructField("a", IntegerType), + StructField("_corrupt_record", StringType))) + val df = spark.read + .schema(schema) + .option("multiLine", true) + .option("mode", mode) + .json(file.getCanonicalPath) + + mode match { + case "PERMISSIVE" => + val rows = df.collect() + assert(rows.length === 1) + assert(rows.head.isNullAt(0)) + assert(!rows.head.isNullAt(1)) + case "DROPMALFORMED" => + checkAnswer(df, Nil) + case "FAILFAST" => + val error = intercept[SparkException](df.collect()) + assert(error.getCause.asInstanceOf[SparkException].getCondition === + "MALFORMED_RECORD_IN_PARSING.WITHOUT_SUGGESTION") + } + } + } + } + test("Corrupt records: FAILFAST mode") { // `FAILFAST` mode should throw an exception for corrupt records. checkError( @@ -2008,100 +2502,117 @@ abstract class JsonSuite } } - test("SPARK-18352: Handle multi-line corrupt documents (PERMISSIVE)") { - withTempPath { dir => - val path = dir.getCanonicalPath - val corruptRecordCount = additionalCorruptRecords.count().toInt - assert(corruptRecordCount === 5) + gridTest("SPARK-18352: Handle multi-line corrupt documents (PERMISSIVE)")( + Seq(false, true)) { enabled => + withSQLConf(SQLConf.JSON_STREAM_MULTILINE_TOP_LEVEL_ARRAY.key -> enabled.toString) { + withTempPath { dir => + val path = dir.getCanonicalPath + val corruptRecordCount = additionalCorruptRecords.count().toInt + assert(corruptRecordCount === 5) - additionalCorruptRecords - .toDF("value") - // this is the minimum partition count that avoids hash collisions - .repartition(corruptRecordCount * 4, F.hash($"value")) - .write - .text(path) + additionalCorruptRecords + .toDF("value") + // Each record must land in its own file for multiLine to read one document per record. + // This count hashes the five records apart; a larger one need not -- 21 collides. + .repartition(corruptRecordCount * 4, F.hash($"value")) + .write + .text(path) + + val jsonDF = spark.read.option("multiLine", true).option("mode", "PERMISSIVE").json(path) + // `[1,2,3]` is a top-level array of scalars. Streaming makes each element its own record, + // so it reports one corrupt record per element where the whole document reports one. + assert(jsonDF.count() === (if (enabled) corruptRecordCount + 2 else corruptRecordCount)) + assert(jsonDF.schema === new StructType() + .add("_corrupt_record", StringType) + .add("dummy", StringType)) + val counts = jsonDF + .join( + additionalCorruptRecords.toDF("value"), + F.regexp_replace($"_corrupt_record", "(^\\s+|\\s+$)", "") === F.trim($"value"), + "outer") + .agg( + F.count($"dummy").as("valid"), + F.count($"_corrupt_record").as("corrupt"), + F.count("*").as("count")) + checkAnswer(counts, if (enabled) Row(1, 6, 8) else Row(1, 4, 6)) + } + } + } - val jsonDF = spark.read.option("multiLine", true).option("mode", "PERMISSIVE").json(path) - assert(jsonDF.count() === corruptRecordCount) - assert(jsonDF.schema === new StructType() - .add("_corrupt_record", StringType) - .add("dummy", StringType)) - val counts = jsonDF - .join( - additionalCorruptRecords.toDF("value"), - F.regexp_replace($"_corrupt_record", "(^\\s+|\\s+$)", "") === F.trim($"value"), - "outer") - .agg( - F.count($"dummy").as("valid"), - F.count($"_corrupt_record").as("corrupt"), - F.count("*").as("count")) - checkAnswer(counts, Row(1, 4, 6)) - } - } - - test("SPARK-19641: Handle multi-line corrupt documents (DROPMALFORMED)") { - withTempPath { dir => - val path = dir.getCanonicalPath - val corruptRecordCount = additionalCorruptRecords.count().toInt - assert(corruptRecordCount === 5) + gridTest("SPARK-19641: Handle multi-line corrupt documents (DROPMALFORMED)")( + Seq(false, true)) { enabled => + withSQLConf(SQLConf.JSON_STREAM_MULTILINE_TOP_LEVEL_ARRAY.key -> enabled.toString) { + withTempPath { dir => + val path = dir.getCanonicalPath + val corruptRecordCount = additionalCorruptRecords.count().toInt + assert(corruptRecordCount === 5) - additionalCorruptRecords - .toDF("value") - // this is the minimum partition count that avoids hash collisions - .repartition(corruptRecordCount * 4, F.hash($"value")) - .write - .text(path) + additionalCorruptRecords + .toDF("value") + // Each record must land in its own file for multiLine to read one document per record. + // This count hashes the five records apart; a larger one need not -- 21 collides. + .repartition(corruptRecordCount * 4, F.hash($"value")) + .write + .text(path) - val jsonDF = spark.read.option("multiLine", true).option("mode", "DROPMALFORMED").json(path) - checkAnswer(jsonDF, Seq(Row("test"))) + val jsonDF = spark.read + .option("multiLine", true) + .option("mode", "DROPMALFORMED") + .json(path) + checkAnswer(jsonDF, Seq(Row("test"))) + } } } - test("SPARK-18352: Handle multi-line corrupt documents (FAILFAST)") { - withTempPath { dir => - val path = dir.getCanonicalPath - val corruptRecordCount = additionalCorruptRecords.count().toInt - assert(corruptRecordCount === 5) + gridTest("SPARK-18352: Handle multi-line corrupt documents (FAILFAST)")( + Seq(false, true)) { enabled => + withSQLConf(SQLConf.JSON_STREAM_MULTILINE_TOP_LEVEL_ARRAY.key -> enabled.toString) { + withTempPath { dir => + val path = dir.getCanonicalPath + val corruptRecordCount = additionalCorruptRecords.count().toInt + assert(corruptRecordCount === 5) - additionalCorruptRecords - .toDF("value") - // this is the minimum partition count that avoids hash collisions - .repartition(corruptRecordCount * 4, F.hash($"value")) - .write - .text(path) + additionalCorruptRecords + .toDF("value") + // Each record must land in its own file for multiLine to read one document per record. + // This count hashes the five records apart; a larger one need not -- 21 collides. + .repartition(corruptRecordCount * 4, F.hash($"value")) + .write + .text(path) - val schema = new StructType().add("dummy", StringType) + val schema = new StructType().add("dummy", StringType) + + // `FAILFAST` mode should throw an exception for corrupt records. + checkErrorMatchPVals( + exception = intercept[SparkException] { + spark.read + .option("multiLine", true) + .option("mode", "FAILFAST") + .json(path) + }, + condition = "INVALID_JSON_RECORD_TYPE", + parameters = Map("failFastMode" -> "FAILFAST", "invalidType" -> "\"STRING\"|\"BIGINT\"")) - // `FAILFAST` mode should throw an exception for corrupt records. - checkErrorMatchPVals( - exception = intercept[SparkException] { + val ex = intercept[SparkException] { spark.read .option("multiLine", true) .option("mode", "FAILFAST") + .schema(schema) .json(path) - }, - condition = "INVALID_JSON_RECORD_TYPE", - parameters = Map("failFastMode" -> "FAILFAST", "invalidType" -> "\"STRING\"|\"BIGINT\"")) - - val ex = intercept[SparkException] { - spark.read - .option("multiLine", true) - .option("mode", "FAILFAST") - .schema(schema) - .json(path) - .collect() + .collect() + } + checkErrorMatchPVals( + exception = ex, + condition = "FAILED_READ_FILE.NO_HINT", + parameters = Map("path" -> s".*$path.*")) + checkError( + exception = ex.getCause.asInstanceOf[SparkException], + condition = "MALFORMED_RECORD_IN_PARSING.WITHOUT_SUGGESTION", + parameters = Map( + "badRecord" -> "[null]", + "failFastMode" -> "FAILFAST") + ) } - checkErrorMatchPVals( - exception = ex, - condition = "FAILED_READ_FILE.NO_HINT", - parameters = Map("path" -> s".*$path.*")) - checkError( - exception = ex.getCause.asInstanceOf[SparkException], - condition = "MALFORMED_RECORD_IN_PARSING.WITHOUT_SUGGESTION", - parameters = Map( - "badRecord" -> "[null]", - "failFastMode" -> "FAILFAST") - ) } } @@ -3908,7 +4419,7 @@ abstract class JsonSuite } test("SPARK-40667: validate JSON Options") { - assert(JSONOptions.getAllOptions.size == 33) + assert(JSONOptions.getAllOptions.size == 34) // Please add validation on any new Json options here assert(JSONOptions.isValidOption("samplingRatio")) assert(JSONOptions.isValidOption("primitivesAsString")) @@ -3941,6 +4452,7 @@ abstract class JsonSuite assert(JSONOptions.isValidOption("singleVariantColumn")) assert(JSONOptions.isValidOption("explodeEmbeddedArray")) assert(JSONOptions.isValidOption("useUnsafeRow")) + assert(JSONOptions.isValidOption("enableStreamingTopLevelArray")) assert(JSONOptions.isValidOption("encoding")) assert(JSONOptions.isValidOption("charset")) // Please add validation on any new Json options with alternative here