diff --git a/client-v2/pom.xml b/client-v2/pom.xml index 0c6409cdf..57ff943a8 100644 --- a/client-v2/pom.xml +++ b/client-v2/pom.xml @@ -88,8 +88,26 @@ com.fasterxml.jackson.core jackson-databind - test ${jackson.version} + provided + + + com.fasterxml.jackson.core + jackson-core + ${jackson.version} + provided + + + com.fasterxml.jackson.core + jackson-annotations + ${jackson.version} + provided + + + com.google.code.gson + gson + ${gson.version} + provided ${project.parent.groupId} diff --git a/client-v2/src/main/java/com/clickhouse/client/api/Client.java b/client-v2/src/main/java/com/clickhouse/client/api/Client.java index d4f979026..d4e43ef22 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/Client.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/Client.java @@ -3,11 +3,14 @@ import com.clickhouse.client.api.command.CommandResponse; import com.clickhouse.client.api.command.CommandSettings; import com.clickhouse.client.api.data_formats.ClickHouseBinaryFormatReader; +import com.clickhouse.client.api.data_formats.JSONEachRowFormatReader; import com.clickhouse.client.api.data_formats.NativeFormatReader; import com.clickhouse.client.api.data_formats.RowBinaryFormatReader; import com.clickhouse.client.api.data_formats.RowBinaryWithNamesAndTypesFormatReader; import com.clickhouse.client.api.data_formats.RowBinaryWithNamesFormatReader; import com.clickhouse.client.api.data_formats.internal.BinaryStreamReader; +import com.clickhouse.client.api.data_formats.internal.JsonParser; +import com.clickhouse.client.api.data_formats.internal.JsonParserFactory; import com.clickhouse.client.api.data_formats.internal.MapBackedRecord; import com.clickhouse.client.api.data_formats.internal.ProcessParser; import com.clickhouse.client.api.enums.Protocol; @@ -2077,6 +2080,11 @@ public ClickHouseBinaryFormatReader newBinaryFormatReader(QueryResponse response reader = new RowBinaryFormatReader(response.getInputStream(), response.getSettings(), schema, byteBufferPool, typeHintMapping); break; + case JSONEachRow: + String jsonProcessor = ClientConfigProperties.JSON_PROCESSOR.getOrDefault(configuration); + JsonParser parser = JsonParserFactory.createParser(jsonProcessor, response.getInputStream()); + reader = new JSONEachRowFormatReader(parser); + break; default: throw new IllegalArgumentException("Binary readers doesn't support format: " + response.getFormat()); } diff --git a/client-v2/src/main/java/com/clickhouse/client/api/ClientConfigProperties.java b/client-v2/src/main/java/com/clickhouse/client/api/ClientConfigProperties.java index e548a90f9..9c40e48dd 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/ClientConfigProperties.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/ClientConfigProperties.java @@ -196,6 +196,15 @@ public Object parseValue(String value) { * See ClickHouse Docs */ CUSTOM_SETTINGS_PREFIX("custom_settings_prefix", String.class, "custom_"), + + /** + * Configures what JSON processor will be used for JSON formats. Choices: + * + */ + JSON_PROCESSOR("json_processor", String.class, "JACKSON"), ; private static final Logger LOG = LoggerFactory.getLogger(ClientConfigProperties.class); diff --git a/client-v2/src/main/java/com/clickhouse/client/api/data_formats/JSONEachRowFormatReader.java b/client-v2/src/main/java/com/clickhouse/client/api/data_formats/JSONEachRowFormatReader.java new file mode 100644 index 000000000..d0615e526 --- /dev/null +++ b/client-v2/src/main/java/com/clickhouse/client/api/data_formats/JSONEachRowFormatReader.java @@ -0,0 +1,528 @@ +package com.clickhouse.client.api.data_formats; + +import com.clickhouse.client.api.data_formats.internal.JsonParser; +import com.clickhouse.client.api.metadata.TableSchema; +import com.clickhouse.data.ClickHouseColumn; +import com.clickhouse.data.ClickHouseDataType; +import com.clickhouse.data.value.ClickHouseBitmap; +import com.clickhouse.data.value.ClickHouseGeoMultiPolygonValue; +import com.clickhouse.data.value.ClickHouseGeoPointValue; +import com.clickhouse.data.value.ClickHouseGeoPolygonValue; +import com.clickhouse.data.value.ClickHouseGeoRingValue; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.net.Inet4Address; +import java.net.Inet6Address; +import java.time.Duration; +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.time.OffsetDateTime; +import java.time.ZonedDateTime; +import java.time.temporal.TemporalAmount; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.UUID; + +public class JSONEachRowFormatReader implements ClickHouseBinaryFormatReader { + private final JsonParser parser; + private TableSchema schema; + private Map currentRow; + private Map firstRow; + private boolean firstRowRead = false; + + public JSONEachRowFormatReader(JsonParser parser) { + this.parser = parser; + try { + this.firstRow = parser.nextRow(); + if (firstRow != null) { + List columns = new ArrayList<>(); + for (String key : firstRow.keySet()) { + // For JSONEachRow we don't know the exact ClickHouse type, so we use a reasonable default. + // We can try to guess based on the value type in the first row. + columns.add(ClickHouseColumn.of(key, guessDataType(firstRow.get(key)), false)); + } + this.schema = new TableSchema(columns); + } else { + this.schema = new TableSchema(new ArrayList<>()); + } + } catch (Exception e) { + throw new RuntimeException("Failed to initialize JSON reader", e); + } + } + + private ClickHouseDataType guessDataType(Object value) { + if (value instanceof Number) { + if (value instanceof Integer || value instanceof Long || value instanceof BigInteger) { + return ClickHouseDataType.Int64; + } else if (value instanceof Double || value instanceof Float || value instanceof BigDecimal) { + double d = ((Number) value).doubleValue(); + if (d == Math.floor(d) && !Double.isInfinite(d) && d <= Long.MAX_VALUE && d >= Long.MIN_VALUE) { + return ClickHouseDataType.Int64; + } + return ClickHouseDataType.Float64; + } else { + return ClickHouseDataType.Float64; + } + } else if (value instanceof Boolean) { + return ClickHouseDataType.Bool; + } else { + return ClickHouseDataType.String; + } + } + + @Override + public T readValue(int colIndex) { + return (T) currentRow.get(schema.columnIndexToName(colIndex)); + } + + @Override + public T readValue(String colName) { + return (T) currentRow.get(colName); + } + + @Override + public boolean hasValue(String colName) { + return currentRow.containsKey(colName) && currentRow.get(colName) != null; + } + + @Override + public boolean hasValue(int colIndex) { + return hasValue(schema.columnIndexToName(colIndex)); + } + + @Override + public boolean hasNext() { + if (!firstRowRead) { + return firstRow != null; + } + return true; // We'll find out in next() + } + + @Override + public Map next() { + if (!firstRowRead) { + firstRowRead = true; + currentRow = firstRow; + return currentRow; + } + try { + currentRow = parser.nextRow(); + return currentRow; + } catch (Exception e) { + throw new RuntimeException("Failed to read next JSON row", e); + } + } + + @Override + public String getString(String colName) { + Object val = currentRow.get(colName); + return val == null ? null : val.toString(); + } + + @Override + public byte getByte(String colName) { + return ((Number) currentRow.get(colName)).byteValue(); + } + + @Override + public short getShort(String colName) { + return ((Number) currentRow.get(colName)).shortValue(); + } + + @Override + public int getInteger(String colName) { + return ((Number) currentRow.get(colName)).intValue(); + } + + @Override + public long getLong(String colName) { + return ((Number) currentRow.get(colName)).longValue(); + } + + @Override + public float getFloat(String colName) { + return ((Number) currentRow.get(colName)).floatValue(); + } + + @Override + public double getDouble(String colName) { + return ((Number) currentRow.get(colName)).doubleValue(); + } + + @Override + public boolean getBoolean(String colName) { + Object val = currentRow.get(colName); + if (val instanceof Boolean) return (Boolean) val; + if (val instanceof Number) return ((Number) val).intValue() != 0; + return Boolean.parseBoolean(val.toString()); + } + + @Override + public BigInteger getBigInteger(String colName) { + Object val = currentRow.get(colName); + if (val == null) return null; + if (val instanceof BigInteger) return (BigInteger) val; + return new BigDecimal(val.toString()).toBigInteger(); + } + + @Override + public BigDecimal getBigDecimal(String colName) { + Object val = currentRow.get(colName); + if (val instanceof BigDecimal) return (BigDecimal) val; + return new BigDecimal(val.toString()); + } + + @Override + public Instant getInstant(String colName) { + throw new UnsupportedOperationException(); + } + + @Override + public ZonedDateTime getZonedDateTime(String colName) { + throw new UnsupportedOperationException(); + } + + @Override + public Duration getDuration(String colName) { + throw new UnsupportedOperationException(); + } + + @Override + public Inet4Address getInet4Address(String colName) { + throw new UnsupportedOperationException(); + } + + @Override + public Inet6Address getInet6Address(String colName) { + throw new UnsupportedOperationException(); + } + + @Override + public UUID getUUID(String colName) { + return UUID.fromString(currentRow.get(colName).toString()); + } + + @Override + public ClickHouseGeoPointValue getGeoPoint(String colName) { + throw new UnsupportedOperationException(); + } + + @Override + public ClickHouseGeoRingValue getGeoRing(String colName) { + throw new UnsupportedOperationException(); + } + + @Override + public ClickHouseGeoPolygonValue getGeoPolygon(String colName) { + throw new UnsupportedOperationException(); + } + + @Override + public ClickHouseGeoMultiPolygonValue getGeoMultiPolygon(String colName) { + throw new UnsupportedOperationException(); + } + + @Override + public List getList(String colName) { + return (List) currentRow.get(colName); + } + + @Override + public byte[] getByteArray(String colName) { + throw new UnsupportedOperationException(); + } + + @Override + public int[] getIntArray(String colName) { + throw new UnsupportedOperationException(); + } + + @Override + public long[] getLongArray(String colName) { + throw new UnsupportedOperationException(); + } + + @Override + public float[] getFloatArray(String colName) { + throw new UnsupportedOperationException(); + } + + @Override + public double[] getDoubleArray(String colName) { + throw new UnsupportedOperationException(); + } + + @Override + public boolean[] getBooleanArray(String colName) { + throw new UnsupportedOperationException(); + } + + @Override + public short[] getShortArray(String colName) { + throw new UnsupportedOperationException(); + } + + @Override + public String[] getStringArray(String colName) { + throw new UnsupportedOperationException(); + } + + @Override + public Object[] getObjectArray(String colName) { + throw new UnsupportedOperationException(); + } + + @Override + public String getString(int index) { + return getString(schema.columnIndexToName(index)); + } + + @Override + public byte getByte(int index) { + return getByte(schema.columnIndexToName(index)); + } + + @Override + public short getShort(int index) { + return getShort(schema.columnIndexToName(index)); + } + + @Override + public int getInteger(int index) { + return getInteger(schema.columnIndexToName(index)); + } + + @Override + public long getLong(int index) { + return getLong(schema.columnIndexToName(index)); + } + + @Override + public float getFloat(int index) { + return getFloat(schema.columnIndexToName(index)); + } + + @Override + public double getDouble(int index) { + return getDouble(schema.columnIndexToName(index)); + } + + @Override + public boolean getBoolean(int index) { + return getBoolean(schema.columnIndexToName(index)); + } + + @Override + public BigInteger getBigInteger(int index) { + return getBigInteger(schema.columnIndexToName(index)); + } + + @Override + public BigDecimal getBigDecimal(int index) { + return getBigDecimal(schema.columnIndexToName(index)); + } + + @Override + public Instant getInstant(int index) { + return getInstant(schema.columnIndexToName(index)); + } + + @Override + public ZonedDateTime getZonedDateTime(int index) { + return getZonedDateTime(schema.columnIndexToName(index)); + } + + @Override + public Duration getDuration(int index) { + return getDuration(schema.columnIndexToName(index)); + } + + @Override + public Inet4Address getInet4Address(int index) { + return getInet4Address(schema.columnIndexToName(index)); + } + + @Override + public Inet6Address getInet6Address(int index) { + return getInet6Address(schema.columnIndexToName(index)); + } + + @Override + public UUID getUUID(int index) { + return getUUID(schema.columnIndexToName(index)); + } + + @Override + public ClickHouseGeoPointValue getGeoPoint(int index) { + return getGeoPoint(schema.columnIndexToName(index)); + } + + @Override + public ClickHouseGeoRingValue getGeoRing(int index) { + return getGeoRing(schema.columnIndexToName(index)); + } + + @Override + public ClickHouseGeoPolygonValue getGeoPolygon(int index) { + return getGeoPolygon(schema.columnIndexToName(index)); + } + + @Override + public ClickHouseGeoMultiPolygonValue getGeoMultiPolygon(int index) { + return getGeoMultiPolygon(schema.columnIndexToName(index)); + } + + @Override + public List getList(int index) { + return getList(schema.columnIndexToName(index)); + } + + @Override + public byte[] getByteArray(int index) { + return getByteArray(schema.columnIndexToName(index)); + } + + @Override + public int[] getIntArray(int index) { + return getIntArray(schema.columnIndexToName(index)); + } + + @Override + public long[] getLongArray(int index) { + return getLongArray(schema.columnIndexToName(index)); + } + + @Override + public float[] getFloatArray(int index) { + return getFloatArray(schema.columnIndexToName(index)); + } + + @Override + public double[] getDoubleArray(int index) { + return getDoubleArray(schema.columnIndexToName(index)); + } + + @Override + public boolean[] getBooleanArray(int index) { + return getBooleanArray(schema.columnIndexToName(index)); + } + + @Override + public short[] getShortArray(int index) { + return getShortArray(schema.columnIndexToName(index)); + } + + @Override + public String[] getStringArray(int index) { + return getStringArray(schema.columnIndexToName(index)); + } + + @Override + public Object[] getObjectArray(int index) { + return getObjectArray(schema.columnIndexToName(index)); + } + + @Override + public Object[] getTuple(int index) { + return getTuple(schema.columnIndexToName(index)); + } + + @Override + public Object[] getTuple(String colName) { + return (Object[]) currentRow.get(colName); + } + + @Override + public byte getEnum8(String colName) { + return getByte(colName); + } + + @Override + public byte getEnum8(int index) { + return getByte(index); + } + + @Override + public short getEnum16(String colName) { + return getShort(colName); + } + + @Override + public short getEnum16(int index) { + return getShort(index); + } + + @Override + public LocalDate getLocalDate(String colName) { + return LocalDate.parse(currentRow.get(colName).toString()); + } + + @Override + public LocalDate getLocalDate(int index) { + return getLocalDate(schema.columnIndexToName(index)); + } + + @Override + public LocalTime getLocalTime(String colName) { + return LocalTime.parse(currentRow.get(colName).toString()); + } + + @Override + public LocalTime getLocalTime(int index) { + return getLocalTime(schema.columnIndexToName(index)); + } + + @Override + public LocalDateTime getLocalDateTime(String colName) { + return LocalDateTime.parse(currentRow.get(colName).toString()); + } + + @Override + public LocalDateTime getLocalDateTime(int index) { + return getLocalDateTime(schema.columnIndexToName(index)); + } + + @Override + public OffsetDateTime getOffsetDateTime(String colName) { + return OffsetDateTime.parse(currentRow.get(colName).toString()); + } + + @Override + public OffsetDateTime getOffsetDateTime(int index) { + return getOffsetDateTime(schema.columnIndexToName(index)); + } + + @Override + public TableSchema getSchema() { + return schema; + } + + @Override + public ClickHouseBitmap getClickHouseBitmap(String colName) { + throw new UnsupportedOperationException(); + } + + @Override + public ClickHouseBitmap getClickHouseBitmap(int index) { + throw new UnsupportedOperationException(); + } + + @Override + public TemporalAmount getTemporalAmount(int index) { + throw new UnsupportedOperationException(); + } + + @Override + public TemporalAmount getTemporalAmount(String colName) { + throw new UnsupportedOperationException(); + } + + @Override + public void close() throws Exception { + parser.close(); + } +} diff --git a/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/GsonJsonParser.java b/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/GsonJsonParser.java new file mode 100644 index 000000000..fe70ec93f --- /dev/null +++ b/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/GsonJsonParser.java @@ -0,0 +1,39 @@ +package com.clickhouse.client.api.data_formats.internal; + +import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; +import com.google.gson.stream.JsonReader; +import com.google.gson.stream.JsonToken; + +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.util.Map; + +public class GsonJsonParser implements JsonParser { + private final Gson gson; + private final JsonReader reader; + + public GsonJsonParser(InputStream inputStream) { + this.gson = new Gson(); + this.reader = new JsonReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8)); + this.reader.setLenient(true); // JSONEachRow needs lenient reader for multiple root objects + } + + @Override + public Map nextRow() throws Exception { + try { + if (reader.peek() == JsonToken.END_DOCUMENT) { + return null; + } + } catch (java.io.EOFException e) { + return null; + } + return gson.fromJson(reader, new TypeToken>() {}.getType()); + } + + @Override + public void close() throws Exception { + reader.close(); + } +} diff --git a/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/JacksonJsonParser.java b/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/JacksonJsonParser.java new file mode 100644 index 000000000..e406ba5cc --- /dev/null +++ b/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/JacksonJsonParser.java @@ -0,0 +1,49 @@ +package com.clickhouse.client.api.data_formats.internal; + +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonToken; +import com.fasterxml.jackson.databind.ObjectMapper; + +import java.io.InputStream; +import java.util.Map; + +public class JacksonJsonParser implements com.clickhouse.client.api.data_formats.internal.JsonParser { + private final ObjectMapper mapper; + private final JsonFactory factory; + private com.fasterxml.jackson.core.JsonParser parser; + + public JacksonJsonParser(InputStream inputStream) { + this.mapper = new ObjectMapper(); + this.factory = new JsonFactory(); + try { + this.parser = factory.createParser(inputStream); + } catch (Exception e) { + throw new RuntimeException("Failed to create Jackson parser", e); + } + } + + @Override + public Map nextRow() throws Exception { + if (parser.nextToken() == null) { + return null; + } + if (parser.currentToken() != JsonToken.START_OBJECT) { + // Handle cases where there might be extra characters between objects, + // like newlines in JSONEachRow. + while (parser.nextToken() != null && parser.currentToken() != JsonToken.START_OBJECT) { + // skip + } + if (parser.currentToken() == null) { + return null; + } + } + return mapper.readValue(parser, Map.class); + } + + @Override + public void close() throws Exception { + if (parser != null) { + parser.close(); + } + } +} diff --git a/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/JsonParser.java b/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/JsonParser.java new file mode 100644 index 000000000..2d02dabb3 --- /dev/null +++ b/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/JsonParser.java @@ -0,0 +1,15 @@ +package com.clickhouse.client.api.data_formats.internal; + +import java.util.Map; + +/** + * Interface for JSON row processors. + */ +public interface JsonParser extends AutoCloseable { + /** + * Reads next row from the input stream. + * @return map of column names to values, or null if no more rows + * @throws Exception if an error occurs during parsing + */ + Map nextRow() throws Exception; +} diff --git a/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/JsonParserFactory.java b/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/JsonParserFactory.java new file mode 100644 index 000000000..0e86fd70a --- /dev/null +++ b/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/JsonParserFactory.java @@ -0,0 +1,27 @@ +package com.clickhouse.client.api.data_formats.internal; + +import java.io.InputStream; +import java.lang.reflect.Constructor; + +public class JsonParserFactory { + public static JsonParser createParser(String type, InputStream inputStream) { + String className; + if ("JACKSON".equalsIgnoreCase(type)) { + className = "com.clickhouse.client.api.data_formats.internal.JacksonJsonParser"; + } else if ("GSON".equalsIgnoreCase(type)) { + className = "com.clickhouse.client.api.data_formats.internal.GsonJsonParser"; + } else { + throw new IllegalArgumentException("Unsupported JSON processor: " + type + ". Supported: JACKSON, GSON"); + } + + try { + Class clazz = Class.forName(className); + Constructor constructor = clazz.getConstructor(InputStream.class); + return (JsonParser) constructor.newInstance(inputStream); + } catch (ClassNotFoundException e) { + throw new RuntimeException("JSON processor class not found: " + className + ". Make sure you have the required library (Jackson or Gson) on your classpath.", e); + } catch (Exception e) { + throw new RuntimeException("Failed to instantiate JSON processor: " + type, e); + } + } +} diff --git a/client-v2/src/test/java/com/clickhouse/client/api/data_formats/AbstractJSONEachRowFormatReaderTests.java b/client-v2/src/test/java/com/clickhouse/client/api/data_formats/AbstractJSONEachRowFormatReaderTests.java new file mode 100644 index 000000000..41c1398ae --- /dev/null +++ b/client-v2/src/test/java/com/clickhouse/client/api/data_formats/AbstractJSONEachRowFormatReaderTests.java @@ -0,0 +1,128 @@ +package com.clickhouse.client.api.data_formats; + +import com.clickhouse.client.BaseIntegrationTest; +import com.clickhouse.client.ClickHouseNode; +import com.clickhouse.client.ClickHouseProtocol; +import com.clickhouse.client.ClickHouseServerForTest; +import com.clickhouse.client.api.Client; +import com.clickhouse.client.api.ClientConfigProperties; +import com.clickhouse.client.api.enums.Protocol; +import com.clickhouse.client.api.query.QueryResponse; +import com.clickhouse.client.api.query.QuerySettings; +import com.clickhouse.data.ClickHouseDataType; +import com.clickhouse.data.ClickHouseFormat; +import org.testng.Assert; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +import java.util.Map; + +public abstract class AbstractJSONEachRowFormatReaderTests extends BaseIntegrationTest { + + protected Client client; + + protected abstract String getProcessor(); + + @BeforeMethod(groups = {"integration"}) + public void setUp() { + ClickHouseNode node = getServer(ClickHouseProtocol.HTTP); + client = new Client.Builder() + .addEndpoint(Protocol.HTTP, node.getHost(), node.getPort(), isCloud()) + .setUsername("default") + .setPassword(ClickHouseServerForTest.getPassword()) + .setOption(ClientConfigProperties.JSON_PROCESSOR.getKey(), getProcessor()) + .build(); + } + + @AfterMethod(groups = {"integration"}) + public void tearDown() { + if (client != null) { + client.close(); + } + } + + @Test(groups = {"integration"}) + public void testBasicParsing() throws Exception { + String sql = "SELECT 1 as id, 'test' as name, true as active " + + "UNION ALL SELECT 2, 'clickhouse', false " + + "FORMAT JSONEachRow"; + + try (QueryResponse response = client.query(sql).get()) { + ClickHouseBinaryFormatReader reader = client.newBinaryFormatReader(response); + + // First row + Assert.assertTrue(reader.hasNext()); + Map row1 = reader.next(); + Assert.assertNotNull(row1); + Assert.assertEquals(reader.getInteger("id"), 1); + Assert.assertEquals(reader.getString("name"), "test"); + Assert.assertEquals(reader.getBoolean("active"), true); + + // Second row + Assert.assertTrue(reader.hasNext()); + Map row2 = reader.next(); + Assert.assertNotNull(row2); + Assert.assertEquals(reader.getInteger("id"), 2); + Assert.assertEquals(reader.getString("name"), "clickhouse"); + Assert.assertEquals(reader.getBoolean("active"), false); + + // No more rows + Assert.assertNull(reader.next()); + } + } + + @Test(groups = {"integration"}) + public void testSchemaInference() throws Exception { + String sql = "SELECT toInt64(42) as col_int, toFloat64(3.14) as col_float, " + + "true as col_bool, 'val' as col_str " + + "FORMAT JSONEachRow"; + + try (QueryResponse response = client.query(sql).get()) { + ClickHouseBinaryFormatReader reader = client.newBinaryFormatReader(response); + + Assert.assertNotNull(reader.getSchema()); + Assert.assertEquals(reader.getSchema().getColumns().size(), 4); + + Assert.assertEquals(reader.getSchema().getColumnByIndex(1).getDataType(), ClickHouseDataType.Int64); + Assert.assertEquals(reader.getSchema().getColumnByIndex(2).getDataType(), ClickHouseDataType.Float64); + Assert.assertEquals(reader.getSchema().getColumnByIndex(3).getDataType(), ClickHouseDataType.Bool); + Assert.assertEquals(reader.getSchema().getColumnByIndex(4).getDataType(), ClickHouseDataType.String); + } + } + + @Test(groups = {"integration"}) + public void testDataTypes() throws Exception { + String sql = "SELECT toInt8(120) as b, toInt16(30000) as s, toInt32(1000000) as i, " + + "toInt64(10000000000) as l, toFloat32(1.23) as f, toFloat64(1.23456789) as d, " + + "true as bool, 'hello' as str " + + "FORMAT JSONEachRow"; + + try (QueryResponse response = client.query(sql).get()) { + ClickHouseBinaryFormatReader reader = client.newBinaryFormatReader(response); + + reader.next(); + Assert.assertEquals(reader.getByte("b"), (byte) 120); + Assert.assertEquals(reader.getShort("s"), (short) 30000); + Assert.assertEquals(reader.getInteger("i"), 1000000); + Assert.assertEquals(reader.getLong("l"), 10000000000L); + Assert.assertEquals(reader.getFloat("f"), 1.23f, 0.001f); + Assert.assertEquals(reader.getDouble("d"), 1.23456789d, 0.00000001d); + Assert.assertEquals(reader.getBoolean("bool"), true); + Assert.assertEquals(reader.getString("str"), "hello"); + } + } + + @Test(groups = {"integration"}) + public void testEmptyData() throws Exception { + String sql = "SELECT * FROM remote('127.0.0.1', system.one) WHERE dummy > 1 FORMAT JSONEachRow"; + + try (QueryResponse response = client.query(sql).get()) { + ClickHouseBinaryFormatReader reader = client.newBinaryFormatReader(response); + + Assert.assertFalse(reader.hasNext()); + Assert.assertNull(reader.next()); + Assert.assertEquals(reader.getSchema().getColumns().size(), 0); + } + } +} diff --git a/client-v2/src/test/java/com/clickhouse/client/api/data_formats/GsonJSONEachRowFormatReaderTests.java b/client-v2/src/test/java/com/clickhouse/client/api/data_formats/GsonJSONEachRowFormatReaderTests.java new file mode 100644 index 000000000..58573da0c --- /dev/null +++ b/client-v2/src/test/java/com/clickhouse/client/api/data_formats/GsonJSONEachRowFormatReaderTests.java @@ -0,0 +1,11 @@ +package com.clickhouse.client.api.data_formats; + +import org.testng.annotations.Test; + +@Test(groups = {"integration"}) +public class GsonJSONEachRowFormatReaderTests extends AbstractJSONEachRowFormatReaderTests { + @Override + protected String getProcessor() { + return "GSON"; + } +} diff --git a/client-v2/src/test/java/com/clickhouse/client/api/data_formats/JacksonJSONEachRowFormatReaderTests.java b/client-v2/src/test/java/com/clickhouse/client/api/data_formats/JacksonJSONEachRowFormatReaderTests.java new file mode 100644 index 000000000..8689504bb --- /dev/null +++ b/client-v2/src/test/java/com/clickhouse/client/api/data_formats/JacksonJSONEachRowFormatReaderTests.java @@ -0,0 +1,13 @@ +package com.clickhouse.client.api.data_formats; + +import org.testng.Assert; +import org.testng.annotations.Test; + +@Test(groups = {"integration"}) +public class JacksonJSONEachRowFormatReaderTests extends AbstractJSONEachRowFormatReaderTests { + + @Override + protected String getProcessor() { + return "JACKSON"; + } +} diff --git a/jdbc-v2/pom.xml b/jdbc-v2/pom.xml index a8f02f2e9..05b035694 100644 --- a/jdbc-v2/pom.xml +++ b/jdbc-v2/pom.xml @@ -50,14 +50,35 @@ ${guava.version} - com.fasterxml.jackson.core jackson-databind - test ${jackson.version} + provided + + + + com.fasterxml.jackson.core + jackson-core + ${jackson.version} + provided + + com.fasterxml.jackson.core + jackson-annotations + ${jackson.version} + provided + + + + com.google.code.gson + gson + ${gson.version} + provided + + + ${project.parent.groupId} clickhouse-client @@ -89,18 +110,6 @@ test - - com.fasterxml.jackson.core - jackson-core - ${jackson.version} - test - - - com.fasterxml.jackson.core - jackson-databind - ${jackson.version} - test - com.fasterxml.jackson.dataformat jackson-dataformat-yaml @@ -211,4 +220,4 @@ - \ No newline at end of file + diff --git a/jdbc-v2/src/main/java/com/clickhouse/jdbc/DriverProperties.java b/jdbc-v2/src/main/java/com/clickhouse/jdbc/DriverProperties.java index 7027c95c1..bb6d4ce08 100644 --- a/jdbc-v2/src/main/java/com/clickhouse/jdbc/DriverProperties.java +++ b/jdbc-v2/src/main/java/com/clickhouse/jdbc/DriverProperties.java @@ -78,9 +78,17 @@ public enum DriverProperties { */ QUERY_ID_GENERATOR("jdbc_query_id_generator", null), + /** + * Configures what JSON processor will be used for JSON formats. Choices: + * + */ + JSON_PROCESSOR("json_processor", "JACKSON", Arrays.asList("JACKSON", "GSON")), + /** * Controls logic of saving roles that were set using {@code SET } statement. - * Default: true - save roles */ REMEMBER_LAST_SET_ROLES("remember_last_set_roles", String.valueOf(Boolean.TRUE)), diff --git a/jdbc-v2/src/main/java/com/clickhouse/jdbc/StatementImpl.java b/jdbc-v2/src/main/java/com/clickhouse/jdbc/StatementImpl.java index f50546393..59949c6f5 100644 --- a/jdbc-v2/src/main/java/com/clickhouse/jdbc/StatementImpl.java +++ b/jdbc-v2/src/main/java/com/clickhouse/jdbc/StatementImpl.java @@ -6,6 +6,7 @@ import com.clickhouse.client.api.query.QueryResponse; import com.clickhouse.client.api.query.QuerySettings; import com.clickhouse.client.api.sql.SQLUtils; +import com.clickhouse.data.ClickHouseFormat; import com.clickhouse.jdbc.internal.ExceptionUtils; import com.clickhouse.jdbc.internal.FeatureManager; import com.clickhouse.jdbc.internal.ParsedStatement; @@ -177,11 +178,14 @@ protected ResultSetImpl executeQueryImpl(String sql, QuerySettings settings) thr response = connection.getClient().query(lastStatementSql, mergedSettings).get(queryTimeout, TimeUnit.SECONDS); } - if (response.getFormat().isText()) { - throw new SQLException("Only RowBinaryWithNameAndTypes is supported for output format. Please check your query.", + ClickHouseBinaryFormatReader reader; + if (response.getFormat() == ClickHouseFormat.JSONEachRow || !response.getFormat().isText()) { + reader = connection.getClient().newBinaryFormatReader(response); + } else { + throw new SQLException("Only RowBinaryWithNameAndTypes and JSONEachRow are supported for output format. Please check your query.", ExceptionUtils.SQL_STATE_CLIENT_ERROR); } - ClickHouseBinaryFormatReader reader = connection.getClient().newBinaryFormatReader(response); + if (reader.getSchema() == null) { long writtenRows = 0L; try { diff --git a/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/JdbcConfiguration.java b/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/JdbcConfiguration.java index 9aa5ce61a..6fd387297 100644 --- a/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/JdbcConfiguration.java +++ b/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/JdbcConfiguration.java @@ -303,6 +303,8 @@ private void initProperties(Map urlProperties, Properties provid prop.getKey().equalsIgnoreCase(DriverProperties.CUSTOM_SETTINGS.getKey())) { ClientConfigProperties.toKeyValuePairs(prop.getValue()) .forEach((k, v) -> clientProperties.put(ClientConfigProperties.serverSetting(k), v)); + } else if (prop.getKey().equalsIgnoreCase(DriverProperties.JSON_PROCESSOR.getKey())) { + clientProperties.put(prop.getKey(), prop.getValue()); } driverProperties.put(prop.getKey(), prop.getValue()); } else { diff --git a/jdbc-v2/src/test/java/com/clickhouse/jdbc/StatementTest.java b/jdbc-v2/src/test/java/com/clickhouse/jdbc/StatementTest.java index 575dde388..f7fdb1295 100644 --- a/jdbc-v2/src/test/java/com/clickhouse/jdbc/StatementTest.java +++ b/jdbc-v2/src/test/java/com/clickhouse/jdbc/StatementTest.java @@ -619,6 +619,38 @@ public void testTextFormatInResponse() throws Exception { } } + @Test(groups = {"integration"}) + public void testJSONEachRowFormat() throws Exception { + Properties properties = new Properties(); + properties.setProperty(DriverProperties.JSON_PROCESSOR.getKey(), "JACKSON"); + try (Connection conn = getJdbcConnection(properties)) { + try (Statement stmt = conn.createStatement()) { + try (ResultSet rs = stmt.executeQuery("SELECT 1 AS num, 'test' AS str FORMAT JSONEachRow")) { + assertTrue(rs.next()); + assertEquals(rs.getInt("num"), 1); + assertEquals(rs.getString("str"), "test"); + assertFalse(rs.next()); + } + } + } + } + + @Test(groups = {"integration"}) + public void testJSONEachRowFormatGson() throws Exception { + Properties properties = new Properties(); + properties.setProperty(DriverProperties.JSON_PROCESSOR.getKey(), "GSON"); + try (Connection conn = getJdbcConnection(properties)) { + try (Statement stmt = conn.createStatement()) { + try (ResultSet rs = stmt.executeQuery("SELECT 2 AS num, 'gson' AS str FORMAT JSONEachRow")) { + assertTrue(rs.next()); + assertEquals(rs.getInt("num"), 2); + assertEquals(rs.getString("str"), "gson"); + assertFalse(rs.next()); + } + } + } + } + @Test(groups = "integration") void testWithClause() throws Exception { int count = 0;