From e7a09cfdb8de667a2a04fe7ab827131a1a0c20a7 Mon Sep 17 00:00:00 2001 From: minleejae Date: Mon, 9 Nov 2026 10:25:45 +0900 Subject: [PATCH] feat: parse MySQL LOAD DATA and LOAD XML LOAD DATA INFILE failed at INFILE, since LOAD DATA only read BigQuery's LOAD DATA INTO, and LOAD XML was read as DuckDB's LOAD of an extension named xml. Both now parse to MySqlLoadDataStatement with every clause of the MySQL 8.4 grammar, in the MySQL and MariaDB dialects and without a dialect. The other LOAD readings and the other dialects are unchanged. The FIELDS and LINES clauses become MySqlFileFormat, one model and one production shared with SELECT ... INTO OUTFILE, whose getters and setters now read and write it. As in MySQL, each clause needs a subclause and the subclauses may come in any order, so a bare FIELDS or LINES after INTO OUTFILE is now rejected. INTO OUTFILE also takes CHARSET and a quoted character set name. BigQuery's LOAD DATA now deparses its options through the expression deparser, and LOAD DATA and EXPORT DATA share one WITH CONNECTION production. Signed-off-by: minleejae --- README.md | 1 + .../sf/jsqlparser/parser/feature/Feature.java | 4 +- .../jsqlparser/statement/MySqlFileFormat.java | 196 ++++++++ .../statement/StatementFeatureVisitor.java | 8 + .../statement/StatementVisitor.java | 9 + .../statement/StatementVisitorAdapter.java | 9 + .../statement/load/LoadDataStatement.java | 24 +- .../load/MySqlLoadDataStatement.java | 370 ++++++++++++++ .../select/MySqlSelectIntoClause.java | 136 +++--- .../sf/jsqlparser/util/TablesNamesFinder.java | 17 + .../util/deparser/StatementDeParser.java | 10 +- .../validation/feature/MariaDbVersion.java | 4 + .../util/validation/feature/MySqlVersion.java | 4 + .../validator/StatementValidator.java | 14 + .../net/sf/jsqlparser/parser/JSqlParserCC.jjt | 341 +++++++++---- src/site/sphinx/index.rst | 2 + src/site/sphinx/usage.rst | 42 ++ .../parser/ASTNodeCoverageTest.java | 9 +- .../statement/MySqlFileFormatTest.java | 207 ++++++++ .../load/MySqlLoadDataStatementTest.java | 454 ++++++++++++++++++ .../StructuredStatementVisitorTest.java | 21 +- .../validator/LoadDataValidatorTest.java | 45 ++ .../validator/StatementValidatorTest.java | 1 + 23 files changed, 1748 insertions(+), 180 deletions(-) create mode 100644 src/main/java/net/sf/jsqlparser/statement/MySqlFileFormat.java create mode 100644 src/main/java/net/sf/jsqlparser/statement/load/MySqlLoadDataStatement.java create mode 100644 src/test/java/net/sf/jsqlparser/statement/MySqlFileFormatTest.java create mode 100644 src/test/java/net/sf/jsqlparser/statement/load/MySqlLoadDataStatementTest.java create mode 100644 src/test/java/net/sf/jsqlparser/util/validation/validator/LoadDataValidatorTest.java diff --git a/README.md b/README.md index e7a8db4521..d19272f2b5 100644 --- a/README.md +++ b/README.md @@ -150,6 +150,7 @@ and missing syntax gets added on demand — [open an issue](https://github.com/J | **MySQL table maintenance** | `ANALYZE` · `CHECK` · `CHECKSUM` · `OPTIMIZE` · `REPAIR TABLE` | | **PostgreSQL maintenance** | `VACUUM` · `ANALYZE` · `CLUSTER` · `CHECKPOINT` | | **MySQL locks** | `LOCK TABLES` · `UNLOCK TABLES` · `LOCK INSTANCE FOR BACKUP` · `UNLOCK INSTANCE` | +| **MySQL data loading** | `LOAD DATA` · `LOAD XML` · `SELECT … INTO OUTFILE` with the same `FIELDS` and `LINES` clauses | | **MySQL account management** | `CREATE`/`ALTER`/`RENAME USER` · `CREATE ROLE` · `SET ROLE` · `SET DEFAULT ROLE` · `SET PASSWORD` | | **Tablespaces** | MySQL and PostgreSQL `CREATE TABLESPACE` · `ALTER TABLESPACE` | | **PostgreSQL RLS** | `CREATE POLICY` · `ALTER TABLE … ENABLE`/`DISABLE`/`FORCE`/`NO FORCE ROW LEVEL SECURITY` | diff --git a/src/main/java/net/sf/jsqlparser/parser/feature/Feature.java b/src/main/java/net/sf/jsqlparser/parser/feature/Feature.java index 447245f4aa..1e7ca0b975 100644 --- a/src/main/java/net/sf/jsqlparser/parser/feature/Feature.java +++ b/src/main/java/net/sf/jsqlparser/parser/feature/Feature.java @@ -1153,10 +1153,12 @@ public enum Feature { export, /** - * PostgreSQL and DuckDB "COPY" and BigQuery "LOAD DATA" statements are allowed + * PostgreSQL and DuckDB "COPY", BigQuery and MySQL "LOAD DATA" and MySQL "LOAD XML" statements + * are allowed * * @see net.sf.jsqlparser.statement.CopyStatement * @see net.sf.jsqlparser.statement.load.LoadDataStatement + * @see net.sf.jsqlparser.statement.load.MySqlLoadDataStatement */ copy, loadData, diff --git a/src/main/java/net/sf/jsqlparser/statement/MySqlFileFormat.java b/src/main/java/net/sf/jsqlparser/statement/MySqlFileFormat.java new file mode 100644 index 0000000000..1460604ec0 --- /dev/null +++ b/src/main/java/net/sf/jsqlparser/statement/MySqlFileFormat.java @@ -0,0 +1,196 @@ +/*- + * #%L + * JSQLParser library + * %% + * Copyright (C) 2004 - 2026 JSQLParser + * %% + * Dual licensed under GNU LGPL 2.1 or Apache License 2.0 + * #L% + */ +package net.sf.jsqlparser.statement; + +import java.io.Serializable; +import net.sf.jsqlparser.expression.StringValue; +import net.sf.jsqlparser.parser.ASTNodeAccessImpl; + +/** + * The {@code {FIELDS | COLUMNS}} and {@code LINES} clauses that describe the text file of a MySQL + * {@code LOAD DATA} or {@code SELECT ... INTO OUTFILE}: + * + * 
+ * [{FIELDS | COLUMNS} [TERMINATED BY 'string'] [[OPTIONALLY] ENCLOSED BY 'char'] [ESCAPED BY 'char']]
+ * [LINES [STARTING BY 'string'] [TERMINATED BY 'string']]
+ *
+ * + * Each clause has at least one subclause. MySQL accepts the subclauses of a clause in any order and + * more than once; like the server, the model keeps the last value of each and prints them in the + * order above. {@code OPTIONALLY} stays set once a subclause has written it. + * + * @see LOAD DATA + */ +public class MySqlFileFormat extends ASTNodeAccessImpl implements Serializable { + + /** The spelling of the fields clause. */ + public enum FieldsKeyword { + FIELDS, COLUMNS + } + + private FieldsKeyword fieldsKeyword; + private StringValue fieldsTerminatedBy; + private boolean fieldsOptionallyEnclosed; + private StringValue fieldsEnclosedBy; + private StringValue fieldsEscapedBy; + private StringValue linesStartingBy; + private StringValue linesTerminatedBy; + + /** The spelling of the fields clause; null prints FIELDS when a subclause is set. */ + public FieldsKeyword getFieldsKeyword() { + return fieldsKeyword; + } + + public void setFieldsKeyword(FieldsKeyword fieldsKeyword) { + this.fieldsKeyword = fieldsKeyword; + } + + public MySqlFileFormat withFieldsKeyword(FieldsKeyword fieldsKeyword) { + setFieldsKeyword(fieldsKeyword); + return this; + } + + public StringValue getFieldsTerminatedBy() { + return fieldsTerminatedBy; + } + + public void setFieldsTerminatedBy(StringValue fieldsTerminatedBy) { + this.fieldsTerminatedBy = fieldsTerminatedBy; + } + + public MySqlFileFormat withFieldsTerminatedBy(StringValue fieldsTerminatedBy) { + setFieldsTerminatedBy(fieldsTerminatedBy); + return this; + } + + public boolean isFieldsOptionallyEnclosed() { + return fieldsOptionallyEnclosed; + } + + public void setFieldsOptionallyEnclosed(boolean fieldsOptionallyEnclosed) { + this.fieldsOptionallyEnclosed = fieldsOptionallyEnclosed; + } + + public MySqlFileFormat withFieldsOptionallyEnclosed(boolean fieldsOptionallyEnclosed) { + setFieldsOptionallyEnclosed(fieldsOptionallyEnclosed); + return this; + } + + public StringValue getFieldsEnclosedBy() { + return fieldsEnclosedBy; + } + + public void setFieldsEnclosedBy(StringValue fieldsEnclosedBy) { + this.fieldsEnclosedBy = fieldsEnclosedBy; + } + + public MySqlFileFormat withFieldsEnclosedBy(StringValue fieldsEnclosedBy) { + setFieldsEnclosedBy(fieldsEnclosedBy); + return this; + } + + public StringValue getFieldsEscapedBy() { + return fieldsEscapedBy; + } + + public void setFieldsEscapedBy(StringValue fieldsEscapedBy) { + this.fieldsEscapedBy = fieldsEscapedBy; + } + + public MySqlFileFormat withFieldsEscapedBy(StringValue fieldsEscapedBy) { + setFieldsEscapedBy(fieldsEscapedBy); + return this; + } + + public StringValue getLinesStartingBy() { + return linesStartingBy; + } + + public void setLinesStartingBy(StringValue linesStartingBy) { + this.linesStartingBy = linesStartingBy; + } + + public MySqlFileFormat withLinesStartingBy(StringValue linesStartingBy) { + setLinesStartingBy(linesStartingBy); + return this; + } + + public StringValue getLinesTerminatedBy() { + return linesTerminatedBy; + } + + public void setLinesTerminatedBy(StringValue linesTerminatedBy) { + this.linesTerminatedBy = linesTerminatedBy; + } + + public MySqlFileFormat withLinesTerminatedBy(StringValue linesTerminatedBy) { + setLinesTerminatedBy(linesTerminatedBy); + return this; + } + + /** Whether the format prints a fields clause: a spelling or a subclause is set. */ + public boolean hasFieldsClause() { + return fieldsKeyword != null || fieldsTerminatedBy != null || fieldsEnclosedBy != null + || fieldsEscapedBy != null; + } + + /** Whether the format prints a lines clause: a subclause is set. */ + public boolean hasLinesClause() { + return linesStartingBy != null || linesTerminatedBy != null; + } + + /** Whether the format prints nothing. */ + public boolean isEmpty() { + return !hasFieldsClause() && !hasLinesClause(); + } + + /** Appends the fields clause and then the lines clause, separated by a space. */ + public StringBuilder appendTo(StringBuilder builder) { + if (hasFieldsClause()) { + appendFieldsClause(builder); + if (hasLinesClause()) { + builder.append(' '); + } + } + if (hasLinesClause()) { + appendLinesClause(builder); + } + return builder; + } + + private void appendFieldsClause(StringBuilder builder) { + builder.append(fieldsKeyword != null ? fieldsKeyword : FieldsKeyword.FIELDS); + if (fieldsTerminatedBy != null) { + builder.append(" TERMINATED BY ").append(fieldsTerminatedBy); + } + if (fieldsEnclosedBy != null) { + builder.append(fieldsOptionallyEnclosed ? " OPTIONALLY ENCLOSED BY " : " ENCLOSED BY ") + .append(fieldsEnclosedBy); + } + if (fieldsEscapedBy != null) { + builder.append(" ESCAPED BY ").append(fieldsEscapedBy); + } + } + + private void appendLinesClause(StringBuilder builder) { + builder.append("LINES"); + if (linesStartingBy != null) { + builder.append(" STARTING BY ").append(linesStartingBy); + } + if (linesTerminatedBy != null) { + builder.append(" TERMINATED BY ").append(linesTerminatedBy); + } + } + + @Override + public String toString() { + return appendTo(new StringBuilder()).toString(); + } +} diff --git a/src/main/java/net/sf/jsqlparser/statement/StatementFeatureVisitor.java b/src/main/java/net/sf/jsqlparser/statement/StatementFeatureVisitor.java index f91ea876a6..e6f26f11a9 100644 --- a/src/main/java/net/sf/jsqlparser/statement/StatementFeatureVisitor.java +++ b/src/main/java/net/sf/jsqlparser/statement/StatementFeatureVisitor.java @@ -98,6 +98,7 @@ import net.sf.jsqlparser.statement.insert.Insert; import net.sf.jsqlparser.statement.insert.InsertBulk; import net.sf.jsqlparser.statement.load.LoadDataStatement; +import net.sf.jsqlparser.statement.load.MySqlLoadDataStatement; import net.sf.jsqlparser.statement.lock.BackupLockStatement; import net.sf.jsqlparser.statement.lock.LockStatement; import net.sf.jsqlparser.statement.lock.UnlockStatement; @@ -680,6 +681,13 @@ public Void visit(LoadDataStatement loadDataStatement, S context) { return null; } + @Override + public Void visit(MySqlLoadDataStatement loadData, S context) { + analysis.claimTopLevel(); + analysis.certain(StmtFeature.MODIFIES_DATA); + return super.visit(loadData, context); + } + @Override public Void visit(PurgeStatement purgeStatement, S context) { analysis.claimTopLevel(); diff --git a/src/main/java/net/sf/jsqlparser/statement/StatementVisitor.java b/src/main/java/net/sf/jsqlparser/statement/StatementVisitor.java index bd9119530e..18a534accb 100644 --- a/src/main/java/net/sf/jsqlparser/statement/StatementVisitor.java +++ b/src/main/java/net/sf/jsqlparser/statement/StatementVisitor.java @@ -113,6 +113,7 @@ import net.sf.jsqlparser.statement.create.extension.CreateExtensionRepository; import net.sf.jsqlparser.statement.export.ExportDataStatement; import net.sf.jsqlparser.statement.load.LoadDataStatement; +import net.sf.jsqlparser.statement.load.MySqlLoadDataStatement; public interface StatementVisitor { @@ -660,6 +661,14 @@ default void visit(LoadDataStatement loadDataStatement) { this.visit(loadDataStatement, null); } + default T visit(MySqlLoadDataStatement loadData, S context) { + return null; + } + + default void visit(MySqlLoadDataStatement loadData) { + this.visit(loadData, null); + } + T visit(PurgeStatement purgeStatement, S context); default void visit(PurgeStatement purgeStatement) { diff --git a/src/main/java/net/sf/jsqlparser/statement/StatementVisitorAdapter.java b/src/main/java/net/sf/jsqlparser/statement/StatementVisitorAdapter.java index 32a947382e..1cb164a7c0 100644 --- a/src/main/java/net/sf/jsqlparser/statement/StatementVisitorAdapter.java +++ b/src/main/java/net/sf/jsqlparser/statement/StatementVisitorAdapter.java @@ -130,6 +130,7 @@ import net.sf.jsqlparser.statement.create.extension.CreateExtensionRepository; import net.sf.jsqlparser.statement.export.ExportDataStatement; import net.sf.jsqlparser.statement.load.LoadDataStatement; +import net.sf.jsqlparser.statement.load.MySqlLoadDataStatement; @SuppressWarnings({"PMD.UncommentedEmptyMethodBody"}) public class StatementVisitorAdapter implements StatementVisitor { @@ -850,6 +851,14 @@ public T visit(LoadDataStatement loadDataStatement, S context) { return null; } + @Override + public T visit(MySqlLoadDataStatement loadData, S context) { + fromItemVisitor.visitFromItem(loadData.getTable(), context); + expressionVisitor.visitExpressions(loadData.getColumns(), context); + expressionVisitor.visitUpdateSets(loadData.getUpdateSets(), context); + return null; + } + @Override public T visit(PurgeStatement purgeStatement, S context) { return null; diff --git a/src/main/java/net/sf/jsqlparser/statement/load/LoadDataStatement.java b/src/main/java/net/sf/jsqlparser/statement/load/LoadDataStatement.java index 16c8a42b0f..545e6f50c8 100644 --- a/src/main/java/net/sf/jsqlparser/statement/load/LoadDataStatement.java +++ b/src/main/java/net/sf/jsqlparser/statement/load/LoadDataStatement.java @@ -9,6 +9,7 @@ */ package net.sf.jsqlparser.statement.load; +import net.sf.jsqlparser.expression.Expression; import net.sf.jsqlparser.expression.operators.relational.ExpressionList; import net.sf.jsqlparser.parser.ASTNodeAccessImpl; import net.sf.jsqlparser.schema.Table; @@ -18,6 +19,7 @@ import net.sf.jsqlparser.statement.select.PlainSelect; import java.util.List; +import java.util.function.Consumer; /** * BigQuery's {@code LOAD DATA {INTO | OVERWRITE} table [(column_definitions)] [OPTIONS @@ -128,14 +130,19 @@ public LoadDataStatement withConnectionName(String connectionName) { } public StringBuilder appendTo(StringBuilder builder) { + return appendTo(builder, builder::append); + } + + /** Renders the options and file options through the caller's expression visitor. */ + public StringBuilder appendTo(StringBuilder builder, Consumer expressionRenderer) { builder.append("LOAD DATA ").append(overwrite ? "OVERWRITE " : "INTO ").append(table); if (columnDefinitions != null && !columnDefinitions.isEmpty()) { builder.append(" ").append(PlainSelect.getStringList(columnDefinitions, true, true)); } if (options != null) { - builder.append(" OPTIONS (").append(options).append(")"); + appendOptions(builder.append(" OPTIONS ("), options, expressionRenderer).append(")"); } - builder.append(" FROM FILES (").append(fileOptions).append(")"); + appendOptions(builder.append(" FROM FILES ("), fileOptions, expressionRenderer).append(")"); if (withPartitionColumns) { builder.append(" WITH PARTITION COLUMNS"); } @@ -145,6 +152,19 @@ public StringBuilder appendTo(StringBuilder builder) { return builder; } + private static StringBuilder appendOptions(StringBuilder builder, ExpressionList options, + Consumer expressionRenderer) { + if (options != null) { + for (int i = 0; i < options.size(); i++) { + if (i > 0) { + builder.append(", "); + } + expressionRenderer.accept(options.get(i)); + } + } + return builder; + } + @Override public String toString() { return appendTo(new StringBuilder()).toString(); diff --git a/src/main/java/net/sf/jsqlparser/statement/load/MySqlLoadDataStatement.java b/src/main/java/net/sf/jsqlparser/statement/load/MySqlLoadDataStatement.java new file mode 100644 index 0000000000..e6c632a6b4 --- /dev/null +++ b/src/main/java/net/sf/jsqlparser/statement/load/MySqlLoadDataStatement.java @@ -0,0 +1,370 @@ +/*- + * #%L + * JSQLParser library + * %% + * Copyright (C) 2004 - 2026 JSQLParser + * %% + * Dual licensed under GNU LGPL 2.1 or Apache License 2.0 + * #L% + */ +package net.sf.jsqlparser.statement.load; + +import java.util.List; +import java.util.function.Consumer; +import net.sf.jsqlparser.expression.Expression; +import net.sf.jsqlparser.expression.StringValue; +import net.sf.jsqlparser.expression.operators.relational.ExpressionList; +import net.sf.jsqlparser.parser.ASTNodeAccessImpl; +import net.sf.jsqlparser.schema.Table; +import net.sf.jsqlparser.statement.MySqlFileFormat; +import net.sf.jsqlparser.statement.Statement; +import net.sf.jsqlparser.statement.StatementVisitor; +import net.sf.jsqlparser.statement.create.table.CreateTable; +import net.sf.jsqlparser.statement.update.UpdateSet; + +/** + * MySQL's {@code LOAD DATA} and {@code LOAD XML}: + * + * 
+ * LOAD {DATA | XML} [LOW_PRIORITY | CONCURRENT] [LOCAL] INFILE 'file_name' [REPLACE | IGNORE]
+ *     INTO TABLE tbl_name [PARTITION (partition_name [, partition_name] ...)]
+ *     [CHARACTER SET charset_name] [ROWS IDENTIFIED BY '<tagname>']
+ *     [{FIELDS | COLUMNS} ...] [LINES ...] [IGNORE number {LINES | ROWS}]
+ *     [(col_name_or_user_var [, col_name_or_user_var] ...)]
+ *     [SET col_name = {expr | DEFAULT} [, col_name = {expr | DEFAULT}] ...]
+ *
+ * + * Both forms share one grammar in MySQL, so every clause is available to both. The FIELDS and LINES + * clauses are the {@link MySqlFileFormat} of {@code SELECT ... INTO OUTFILE}. BigQuery's + * {@code LOAD DATA INTO} is {@link LoadDataStatement}. + * + * @see LOAD DATA + * @see LOAD XML + */ +public class MySqlLoadDataStatement extends ASTNodeAccessImpl implements Statement { + + /** The word after LOAD: a delimited text file or an XML file. */ + public enum Format { + DATA, XML + } + + /** When the rows are written while other sessions use the table. */ + public enum LockOption { + LOW_PRIORITY, CONCURRENT + } + + /** The word of {@code IGNORE number {LINES | ROWS}}. */ + public enum IgnoreUnit { + LINES, ROWS + } + + private Format format = Format.DATA; + private LockOption lockOption; + private boolean local; + private StringValue fileName; + private CreateTable.DuplicateHandling duplicateHandling; + private Table table; + private List partitions; + private String characterSet; + private boolean useCharsetKeyword; + private StringValue rowsIdentifiedBy; + private MySqlFileFormat fileFormat; + private Long ignoreCount; + private IgnoreUnit ignoreUnit = IgnoreUnit.LINES; + private ExpressionList columns; + private List updateSets; + + public Format getFormat() { + return format; + } + + public void setFormat(Format format) { + this.format = format; + } + + public MySqlLoadDataStatement withFormat(Format format) { + setFormat(format); + return this; + } + + public LockOption getLockOption() { + return lockOption; + } + + public void setLockOption(LockOption lockOption) { + this.lockOption = lockOption; + } + + public MySqlLoadDataStatement withLockOption(LockOption lockOption) { + setLockOption(lockOption); + return this; + } + + /** Whether the client reads the file ({@code LOCAL}) rather than the server. */ + public boolean isLocal() { + return local; + } + + public void setLocal(boolean local) { + this.local = local; + } + + public MySqlLoadDataStatement withLocal(boolean local) { + setLocal(local); + return this; + } + + public StringValue getFileName() { + return fileName; + } + + public void setFileName(StringValue fileName) { + this.fileName = fileName; + } + + public MySqlLoadDataStatement withFileName(StringValue fileName) { + setFileName(fileName); + return this; + } + + /** {@code REPLACE} or {@code IGNORE} for rows that duplicate a unique key, or null. */ + public CreateTable.DuplicateHandling getDuplicateHandling() { + return duplicateHandling; + } + + public void setDuplicateHandling(CreateTable.DuplicateHandling duplicateHandling) { + this.duplicateHandling = duplicateHandling; + } + + public MySqlLoadDataStatement withDuplicateHandling( + CreateTable.DuplicateHandling duplicateHandling) { + setDuplicateHandling(duplicateHandling); + return this; + } + + public Table getTable() { + return table; + } + + public void setTable(Table table) { + this.table = table; + } + + public MySqlLoadDataStatement withTable(Table table) { + setTable(table); + return this; + } + + /** The names of {@code PARTITION (p0, p1)}, or null without the clause. */ + public List getPartitions() { + return partitions; + } + + public void setPartitions(List partitions) { + this.partitions = partitions; + } + + public MySqlLoadDataStatement withPartitions(List partitions) { + setPartitions(partitions); + return this; + } + + /** The character set name as written, a quoted name keeping its quotes. */ + public String getCharacterSet() { + return characterSet; + } + + public void setCharacterSet(String characterSet) { + this.characterSet = characterSet; + } + + public MySqlLoadDataStatement withCharacterSet(String characterSet) { + setCharacterSet(characterSet); + return this; + } + + /** Whether the character set clause is spelled CHARSET rather than CHARACTER SET. */ + public boolean isUseCharsetKeyword() { + return useCharsetKeyword; + } + + public void setUseCharsetKeyword(boolean useCharsetKeyword) { + this.useCharsetKeyword = useCharsetKeyword; + } + + public MySqlLoadDataStatement withUseCharsetKeyword(boolean useCharsetKeyword) { + setUseCharsetKeyword(useCharsetKeyword); + return this; + } + + /** The XML tag of {@code ROWS IDENTIFIED BY ''}, or null. */ + public StringValue getRowsIdentifiedBy() { + return rowsIdentifiedBy; + } + + public void setRowsIdentifiedBy(StringValue rowsIdentifiedBy) { + this.rowsIdentifiedBy = rowsIdentifiedBy; + } + + public MySqlLoadDataStatement withRowsIdentifiedBy(StringValue rowsIdentifiedBy) { + setRowsIdentifiedBy(rowsIdentifiedBy); + return this; + } + + /** The FIELDS and LINES clauses, or null without either. */ + public MySqlFileFormat getFileFormat() { + return fileFormat; + } + + public void setFileFormat(MySqlFileFormat fileFormat) { + this.fileFormat = fileFormat; + } + + public MySqlLoadDataStatement withFileFormat(MySqlFileFormat fileFormat) { + setFileFormat(fileFormat); + return this; + } + + /** The number of {@code IGNORE number {LINES | ROWS}}, or null without the clause. */ + public Long getIgnoreCount() { + return ignoreCount; + } + + public void setIgnoreCount(Long ignoreCount) { + this.ignoreCount = ignoreCount; + } + + public MySqlLoadDataStatement withIgnoreCount(Long ignoreCount) { + setIgnoreCount(ignoreCount); + return this; + } + + public IgnoreUnit getIgnoreUnit() { + return ignoreUnit; + } + + public void setIgnoreUnit(IgnoreUnit ignoreUnit) { + this.ignoreUnit = ignoreUnit; + } + + public MySqlLoadDataStatement withIgnoreUnit(IgnoreUnit ignoreUnit) { + setIgnoreUnit(ignoreUnit); + return this; + } + + /** + * The columns and user variables the input fields go to: {@code Column} and + * {@code UserVariable} elements. Null without a list, empty for {@code ()}. + */ + public ExpressionList getColumns() { + return columns; + } + + public void setColumns(ExpressionList columns) { + this.columns = columns; + } + + public MySqlLoadDataStatement withColumns(ExpressionList columns) { + setColumns(columns); + return this; + } + + /** The assignments of the SET clause, each one column and one value, or null. */ + public List getUpdateSets() { + return updateSets; + } + + public void setUpdateSets(List updateSets) { + this.updateSets = updateSets; + } + + public MySqlLoadDataStatement withUpdateSets(List updateSets) { + setUpdateSets(updateSets); + return this; + } + + public StringBuilder appendTo(StringBuilder builder) { + return appendTo(builder, builder::append); + } + + /** Renders the column list and the SET clause through the caller's expression visitor. */ + public StringBuilder appendTo(StringBuilder builder, Consumer expressionRenderer) { + appendSource(builder); + appendTarget(builder); + appendFileOptions(builder); + if (columns != null) { + builder.append(" ("); + appendList(builder, columns, expressionRenderer); + builder.append(")"); + } + if (updateSets != null && !updateSets.isEmpty()) { + builder.append(" SET "); + for (int i = 0; i < updateSets.size(); i++) { + if (i > 0) { + builder.append(", "); + } + appendList(builder, updateSets.get(i).getColumns(), expressionRenderer); + builder.append(" = "); + appendList(builder, updateSets.get(i).getValues(), expressionRenderer); + } + } + return builder; + } + + private void appendSource(StringBuilder builder) { + builder.append("LOAD ").append(format); + if (lockOption != null) { + builder.append(' ').append(lockOption); + } + if (local) { + builder.append(" LOCAL"); + } + builder.append(" INFILE ").append(fileName); + if (duplicateHandling != null) { + builder.append(' ').append(duplicateHandling); + } + } + + private void appendTarget(StringBuilder builder) { + builder.append(" INTO TABLE ").append(table); + if (partitions != null) { + builder.append(" PARTITION (").append(String.join(", ", partitions)).append(")"); + } + if (characterSet != null) { + builder.append(useCharsetKeyword ? " CHARSET " : " CHARACTER SET ") + .append(characterSet); + } + } + + private void appendFileOptions(StringBuilder builder) { + if (rowsIdentifiedBy != null) { + builder.append(" ROWS IDENTIFIED BY ").append(rowsIdentifiedBy); + } + if (fileFormat != null && !fileFormat.isEmpty()) { + fileFormat.appendTo(builder.append(' ')); + } + if (ignoreCount != null) { + builder.append(" IGNORE ").append(ignoreCount).append(' ').append(ignoreUnit); + } + } + + private static void appendList(StringBuilder builder, List expressions, + Consumer expressionRenderer) { + for (int i = 0; i < expressions.size(); i++) { + if (i > 0) { + builder.append(", "); + } + expressionRenderer.accept(expressions.get(i)); + } + } + + @Override + public String toString() { + return appendTo(new StringBuilder()).toString(); + } + + @Override + public T accept(StatementVisitor statementVisitor, S context) { + return statementVisitor.visit(this, context); + } +} diff --git a/src/main/java/net/sf/jsqlparser/statement/select/MySqlSelectIntoClause.java b/src/main/java/net/sf/jsqlparser/statement/select/MySqlSelectIntoClause.java index 16a0755fc0..dbb0c2dea5 100644 --- a/src/main/java/net/sf/jsqlparser/statement/select/MySqlSelectIntoClause.java +++ b/src/main/java/net/sf/jsqlparser/statement/select/MySqlSelectIntoClause.java @@ -16,6 +16,7 @@ import net.sf.jsqlparser.expression.UserVariable; import net.sf.jsqlparser.expression.operators.relational.ExpressionList; import net.sf.jsqlparser.parser.ASTNodeAccessImpl; +import net.sf.jsqlparser.statement.MySqlFileFormat; public class MySqlSelectIntoClause extends ASTNodeAccessImpl implements Serializable { @@ -27,6 +28,10 @@ public enum Type { OUTFILE, DUMPFILE, VARIABLES } + /** + * The spelling of the fields clause, as {@link #getFieldsKeyword()} reports it from + * {@link MySqlFileFormat.FieldsKeyword}. + */ public enum FieldsKeyword { FIELDS, COLUMNS } @@ -36,13 +41,8 @@ public enum FieldsKeyword { private StringValue fileName; private ExpressionList variables; private String characterSet; - private FieldsKeyword fieldsKeyword; - private StringValue fieldsTerminatedBy; - private boolean fieldsOptionallyEnclosed; - private StringValue fieldsEnclosedBy; - private StringValue fieldsEscapedBy; - private StringValue linesStartingBy; - private StringValue linesTerminatedBy; + private boolean useCharsetKeyword; + private MySqlFileFormat fileFormat; public Position getPosition() { return position; @@ -89,69 +89,105 @@ public void setCharacterSet(String characterSet) { this.characterSet = characterSet; } + /** Whether the character set clause is spelled CHARSET rather than CHARACTER SET. */ + public boolean isUseCharsetKeyword() { + return useCharsetKeyword; + } + + public void setUseCharsetKeyword(boolean useCharsetKeyword) { + this.useCharsetKeyword = useCharsetKeyword; + } + + /** + * The FIELDS and LINES clauses of INTO OUTFILE, shared with LOAD DATA. The getters and setters + * of the separate subclauses below read and write this format. + */ + public MySqlFileFormat getFileFormat() { + return fileFormat; + } + + public void setFileFormat(MySqlFileFormat fileFormat) { + this.fileFormat = fileFormat; + } + + public MySqlSelectIntoClause withFileFormat(MySqlFileFormat fileFormat) { + setFileFormat(fileFormat); + return this; + } + + private MySqlFileFormat fileFormat() { + if (fileFormat == null) { + fileFormat = new MySqlFileFormat(); + } + return fileFormat; + } + public FieldsKeyword getFieldsKeyword() { - return fieldsKeyword; + MySqlFileFormat.FieldsKeyword keyword = + fileFormat != null ? fileFormat.getFieldsKeyword() : null; + return keyword != null ? FieldsKeyword.valueOf(keyword.name()) : null; } public void setFieldsKeyword(FieldsKeyword fieldsKeyword) { - this.fieldsKeyword = fieldsKeyword; + fileFormat().setFieldsKeyword(fieldsKeyword != null + ? MySqlFileFormat.FieldsKeyword.valueOf(fieldsKeyword.name()) + : null); } public StringValue getFieldsTerminatedBy() { - return fieldsTerminatedBy; + return fileFormat != null ? fileFormat.getFieldsTerminatedBy() : null; } public void setFieldsTerminatedBy(StringValue fieldsTerminatedBy) { - this.fieldsTerminatedBy = fieldsTerminatedBy; + fileFormat().setFieldsTerminatedBy(fieldsTerminatedBy); } public boolean isFieldsOptionallyEnclosed() { - return fieldsOptionallyEnclosed; + return fileFormat != null && fileFormat.isFieldsOptionallyEnclosed(); } public void setFieldsOptionallyEnclosed(boolean fieldsOptionallyEnclosed) { - this.fieldsOptionallyEnclosed = fieldsOptionallyEnclosed; + fileFormat().setFieldsOptionallyEnclosed(fieldsOptionallyEnclosed); } public StringValue getFieldsEnclosedBy() { - return fieldsEnclosedBy; + return fileFormat != null ? fileFormat.getFieldsEnclosedBy() : null; } public void setFieldsEnclosedBy(StringValue fieldsEnclosedBy) { - this.fieldsEnclosedBy = fieldsEnclosedBy; + fileFormat().setFieldsEnclosedBy(fieldsEnclosedBy); } public StringValue getFieldsEscapedBy() { - return fieldsEscapedBy; + return fileFormat != null ? fileFormat.getFieldsEscapedBy() : null; } public void setFieldsEscapedBy(StringValue fieldsEscapedBy) { - this.fieldsEscapedBy = fieldsEscapedBy; + fileFormat().setFieldsEscapedBy(fieldsEscapedBy); } public StringValue getLinesStartingBy() { - return linesStartingBy; + return fileFormat != null ? fileFormat.getLinesStartingBy() : null; } public void setLinesStartingBy(StringValue linesStartingBy) { - this.linesStartingBy = linesStartingBy; + fileFormat().setLinesStartingBy(linesStartingBy); } public StringValue getLinesTerminatedBy() { - return linesTerminatedBy; + return fileFormat != null ? fileFormat.getLinesTerminatedBy() : null; } public void setLinesTerminatedBy(StringValue linesTerminatedBy) { - this.linesTerminatedBy = linesTerminatedBy; + fileFormat().setLinesTerminatedBy(linesTerminatedBy); } public boolean hasFieldsClause() { - return fieldsKeyword != null || fieldsTerminatedBy != null || fieldsEnclosedBy != null - || fieldsEscapedBy != null; + return fileFormat != null && fileFormat.hasFieldsClause(); } public boolean hasLinesClause() { - return linesStartingBy != null || linesTerminatedBy != null; + return fileFormat != null && fileFormat.hasLinesClause(); } public StringBuilder appendTo(StringBuilder builder) { @@ -168,59 +204,17 @@ public StringBuilder appendTo(StringBuilder builder, Consumer expres return builder; } builder.append(type); - appendFileName(builder); - appendCharacterSet(builder); - appendFieldsClause(builder); - appendLinesClause(builder); - return builder; - } - - private void appendFileName(StringBuilder builder) { if (fileName != null) { builder.append(" ").append(fileName); } - } - - private void appendCharacterSet(StringBuilder builder) { if (characterSet != null) { - builder.append(" CHARACTER SET ").append(characterSet); - } - } - - private void appendFieldsClause(StringBuilder builder) { - if (!hasFieldsClause()) { - return; + builder.append(useCharsetKeyword ? " CHARSET " : " CHARACTER SET ") + .append(characterSet); } - - builder.append(" ").append(fieldsKeyword != null ? fieldsKeyword : FieldsKeyword.FIELDS); - - if (fieldsTerminatedBy != null) { - builder.append(" TERMINATED BY ").append(fieldsTerminatedBy); - } - if (fieldsEnclosedBy != null) { - builder.append(" "); - if (fieldsOptionallyEnclosed) { - builder.append("OPTIONALLY "); - } - builder.append("ENCLOSED BY ").append(fieldsEnclosedBy); - } - if (fieldsEscapedBy != null) { - builder.append(" ESCAPED BY ").append(fieldsEscapedBy); - } - } - - private void appendLinesClause(StringBuilder builder) { - if (!hasLinesClause()) { - return; - } - - builder.append(" LINES"); - if (linesStartingBy != null) { - builder.append(" STARTING BY ").append(linesStartingBy); - } - if (linesTerminatedBy != null) { - builder.append(" TERMINATED BY ").append(linesTerminatedBy); + if (fileFormat != null && !fileFormat.isEmpty()) { + fileFormat.appendTo(builder.append(" ")); } + return builder; } @Override diff --git a/src/main/java/net/sf/jsqlparser/util/TablesNamesFinder.java b/src/main/java/net/sf/jsqlparser/util/TablesNamesFinder.java index 87444c8e5f..a5c0945a9d 100644 --- a/src/main/java/net/sf/jsqlparser/util/TablesNamesFinder.java +++ b/src/main/java/net/sf/jsqlparser/util/TablesNamesFinder.java @@ -196,6 +196,7 @@ import net.sf.jsqlparser.statement.insert.OracleMultiInsertClause; import net.sf.jsqlparser.statement.insert.ParenthesedInsert; import net.sf.jsqlparser.statement.load.LoadDataStatement; +import net.sf.jsqlparser.statement.load.MySqlLoadDataStatement; import net.sf.jsqlparser.statement.lock.BackupLockStatement; import net.sf.jsqlparser.statement.lock.LockStatement; import net.sf.jsqlparser.statement.lock.UnlockStatement; @@ -2912,6 +2913,17 @@ public Void visit(LoadDataStatement loadDataStatement, S context) { return null; } + @Override + public Void visit(MySqlLoadDataStatement loadData, S context) { + visitTargetTable(loadData.getTable(), context); + if (loadData.getUpdateSets() != null) { + for (UpdateSet updateSet : loadData.getUpdateSets()) { + updateSet.getValues().accept(this, context); + } + } + return null; + } + @Override public void visit(AttachStatement attachStatement) { StatementVisitor.super.visit(attachStatement); @@ -3018,6 +3030,11 @@ public void visit(LoadDataStatement loadDataStatement) { StatementVisitor.super.visit(loadDataStatement); } + @Override + public void visit(MySqlLoadDataStatement loadData) { + StatementVisitor.super.visit(loadData); + } + @Override public Void visit(PurgeStatement purgeStatement, S context) { if (purgeStatement.getPurgeObjectType() == PurgeObjectType.TABLE) { diff --git a/src/main/java/net/sf/jsqlparser/util/deparser/StatementDeParser.java b/src/main/java/net/sf/jsqlparser/util/deparser/StatementDeParser.java index c7c959128b..4e9bed5719 100644 --- a/src/main/java/net/sf/jsqlparser/util/deparser/StatementDeParser.java +++ b/src/main/java/net/sf/jsqlparser/util/deparser/StatementDeParser.java @@ -158,6 +158,7 @@ import net.sf.jsqlparser.statement.AssertStatement; import net.sf.jsqlparser.statement.export.ExportDataStatement; import net.sf.jsqlparser.statement.load.LoadDataStatement; +import net.sf.jsqlparser.statement.load.MySqlLoadDataStatement; public class StatementDeParser extends AbstractDeParser implements StatementVisitor { @@ -787,10 +788,17 @@ public StringBuilder visit(CreateExtensionRepository createExtensionReposito @Override public StringBuilder visit(LoadDataStatement loadDataStatement, S context) { - loadDataStatement.appendTo(builder); + loadDataStatement.appendTo(builder, + expression -> expression.accept(expressionDeParser, context)); return builder; } + @Override + public StringBuilder visit(MySqlLoadDataStatement loadData, S context) { + return loadData.appendTo(builder, + expression -> expression.accept(expressionDeParser, context)); + } + @Override public StringBuilder visit(PurgeStatement purgeStatement, S context) { purgeStatement.appendTo(builder); diff --git a/src/main/java/net/sf/jsqlparser/util/validation/feature/MariaDbVersion.java b/src/main/java/net/sf/jsqlparser/util/validation/feature/MariaDbVersion.java index f4ff88e851..3f89a41dc9 100644 --- a/src/main/java/net/sf/jsqlparser/util/validation/feature/MariaDbVersion.java +++ b/src/main/java/net/sf/jsqlparser/util/validation/feature/MariaDbVersion.java @@ -86,6 +86,10 @@ public enum MariaDbVersion implements Version { // https://mariadb.com/kb/en/lock-tables/ Feature.lock, Feature.unlockTables, + // https://mariadb.com/kb/en/load-data-infile/ + // https://mariadb.com/kb/en/load-xml/ + Feature.loadData, + // https://mariadb.com/kb/en/account-management-sql-statements/ Feature.createRole, Feature.createUser, Feature.alterUser, Feature.renameUser, Feature.setRole, diff --git a/src/main/java/net/sf/jsqlparser/util/validation/feature/MySqlVersion.java b/src/main/java/net/sf/jsqlparser/util/validation/feature/MySqlVersion.java index 26c630ea34..d75f1b1de2 100644 --- a/src/main/java/net/sf/jsqlparser/util/validation/feature/MySqlVersion.java +++ b/src/main/java/net/sf/jsqlparser/util/validation/feature/MySqlVersion.java @@ -85,6 +85,10 @@ public enum MySqlVersion implements Version { // https://dev.mysql.com/doc/refman/8.0/en/lock-instance-for-backup.html Feature.backupLock, + // https://dev.mysql.com/doc/refman/8.0/en/load-data.html + // https://dev.mysql.com/doc/refman/8.0/en/load-xml.html + Feature.loadData, + // https://dev.mysql.com/doc/refman/8.0/en/account-management-statements.html Feature.createRole, Feature.createUser, Feature.alterUser, Feature.renameUser, Feature.setRole, diff --git a/src/main/java/net/sf/jsqlparser/util/validation/validator/StatementValidator.java b/src/main/java/net/sf/jsqlparser/util/validation/validator/StatementValidator.java index ee7cfd848f..e4e3cf5377 100644 --- a/src/main/java/net/sf/jsqlparser/util/validation/validator/StatementValidator.java +++ b/src/main/java/net/sf/jsqlparser/util/validation/validator/StatementValidator.java @@ -166,6 +166,7 @@ import net.sf.jsqlparser.statement.AssertStatement; import net.sf.jsqlparser.statement.export.ExportDataStatement; import net.sf.jsqlparser.statement.load.LoadDataStatement; +import net.sf.jsqlparser.statement.load.MySqlLoadDataStatement; import java.util.List; @@ -834,6 +835,15 @@ public Void visit(LoadDataStatement loadDataStatement, S context) { return null; } + @Override + public Void visit(MySqlLoadDataStatement loadData, S context) { + validateFeature(Feature.loadData); + validateOptionalFromItem(loadData.getTable()); + validateOptionalExpressions(loadData.getColumns()); + validateOptionalUpdateSets(loadData.getUpdateSets()); + return null; + } + @Override public Void visit(PurgeStatement purgeStatement, S context) { validateFeature(Feature.purge); @@ -1178,6 +1188,10 @@ public void visit(LoadDataStatement loadDataStatement) { visit(loadDataStatement, null); } + public void visit(MySqlLoadDataStatement loadData) { + visit(loadData, null); + } + public void visit(PurgeStatement purgeStatement) { visit(purgeStatement, null); } diff --git a/src/main/jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt b/src/main/jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt index 13f4f1b62f..55c367f913 100644 --- a/src/main/jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt +++ b/src/main/jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt @@ -2573,6 +2573,50 @@ public class CCJSqlParser extends AbstractJSqlParser { && isMySqlLockDialect(false); } + /** + * Whether MySQL's LOAD {DATA | XML} [LOW_PRIORITY | CONCURRENT] [LOCAL] INFILE starts here, + * without a dialect or in MySQL and MariaDB. BigQuery's LOAD DATA continues with INTO or + * OVERWRITE and DuckDB's or PostgreSQL's LOAD with a name, so their readings, including an + * extension named xml, stay unchanged. + */ + private boolean isMySqlLoadDataAhead() { + int format = getToken(2).kind; + String dialect = getAsString(Feature.dialect); + if (getToken(1).kind != K_LOAD || format != K_DATA && format != K_XML + || dialect != null && !isMySqlDialect()) { + return false; + } + int offset = 3; + if (getToken(offset).kind == K_LOW_PRIORITY || isWordAt(offset, "CONCURRENT")) { + offset++; + } + if (getToken(offset).kind == K_LOCAL) { + offset++; + } + return isWordAt(offset, "INFILE"); + } + + private boolean isMySqlCharacterSetClauseAhead() { + return getToken(1).kind == K_CHARACTER && getToken(2).kind == K_SET + || isWordAt(1, "CHARSET"); + } + + /** + * The number of MySQL's IGNORE number {LINES | ROWS}. MySQL reads it as an integer of at most + * 2147483647, leading zeros aside, and reports a larger one as a syntax error. + */ + private static Long mySqlIgnoreCount(Token token) throws ParseException { + String digits = token.image.replaceFirst("^0+(?=.)", ""); + if (digits.length() > 10 || Long.parseLong(digits) > Integer.MAX_VALUE) { + throw new ParseException("IGNORE takes a number of at most 2147483647"); + } + return Long.valueOf(digits); + } + + private boolean isBigQueryWithConnectionAhead() { + return getToken(1).kind == K_WITH && "CONNECTION".equalsIgnoreCase(getToken(2).image); + } + /** * Whether NO_WRITE_TO_BINLOG or LOCAL modifies a MySQL partition operation. Followed by a * comma, INTO or the end of the statement, the word remains a partition name. @@ -5469,6 +5513,8 @@ Statement SingleStatement() : | LOOKAHEAD({ getToken(1).kind == K_PRAGMA }) stm = PragmaStatement() | + LOOKAHEAD({ isMySqlLoadDataAhead() }) stm = MySqlLoadDataStatement() + | LOOKAHEAD({ getToken(1).kind == K_INSTALL || (getToken(1).kind == K_LOAD && getToken(2).kind != K_DATA) || (getToken(1).kind == K_FORCE && getToken(2).kind == K_INSTALL) }) @@ -7469,7 +7515,7 @@ LoadDataStatement LoadDataStatement() #LoadDataStatement: List columnDefinitions = new ArrayList(); ColumnDefinition columnDefinition; ExpressionList options; - ObjectNames connectionName; + String connectionName; Table table; } { @@ -7494,13 +7540,181 @@ LoadDataStatement LoadDataStatement() #LoadDataStatement: { loadDataStatement.setWithPartitionColumns(true); } ] [ - LOOKAHEAD({ getToken(1).kind == K_WITH && "CONNECTION".equalsIgnoreCase(getToken(2).image) }) - connectionName=RelObjectNames() - { loadDataStatement.setConnectionName(String.join(".", connectionName.getNames())); } + LOOKAHEAD({ isBigQueryWithConnectionAhead() }) + connectionName=BigQueryWithConnection() { loadDataStatement.setConnectionName(connectionName); } ] { return loadDataStatement; } } +/** BigQuery's WITH CONNECTION connection_name of LOAD DATA and EXPORT DATA. */ +String BigQueryWithConnection(): +{ ObjectNames connectionName; } +{ + ContextualKeyword("CONNECTION") connectionName=RelObjectNames() + { return String.join(".", connectionName.getNames()); } +} + +/** + * MySQL's LOAD {DATA | XML} [LOW_PRIORITY | CONCURRENT] [LOCAL] INFILE 'file_name' + * [REPLACE | IGNORE] INTO TABLE tbl_name [PARTITION (names)] [CHARACTER SET charset_name] + * [ROWS IDENTIFIED BY 'tag'] [FIELDS ...] [LINES ...] [IGNORE number {LINES | ROWS}] + * [(col_name_or_user_var, ...)] [SET col_name = {expr | DEFAULT}, ...]. One grammar serves + * DATA and XML, as in the server. + * + * https://dev.mysql.com/doc/refman/8.4/en/load-data.html + * https://dev.mysql.com/doc/refman/8.4/en/load-xml.html + */ +MySqlLoadDataStatement MySqlLoadDataStatement(): +{ + MySqlLoadDataStatement statement = new MySqlLoadDataStatement(); + Token token; + Table table; + List partitions; + boolean charsetKeyword; + MySqlFileFormat fileFormat; + ExpressionList columns; + List updateSets; +} +{ + ( | { statement.setFormat(MySqlLoadDataStatement.Format.XML); } ) + [ + { statement.setLockOption(MySqlLoadDataStatement.LockOption.LOW_PRIORITY); } + | + LOOKAHEAD({ isKeywordAhead("CONCURRENT") }) ContextualKeyword("CONCURRENT") + { statement.setLockOption(MySqlLoadDataStatement.LockOption.CONCURRENT); } + ] + [ { statement.setLocal(true); } ] + ContextualKeyword("INFILE") token= { statement.setFileName(stringValue(token)); } + [ + { statement.setDuplicateHandling(CreateTable.DuplicateHandling.REPLACE); } + | + { statement.setDuplicateHandling(CreateTable.DuplicateHandling.IGNORE); } + ] + table=Table() { statement.setTable(table); } + [ "(" partitions=PartitionNamesList() ")" { statement.setPartitions(partitions); } ] + [ + LOOKAHEAD({ isMySqlCharacterSetClauseAhead() }) + charsetKeyword=MySqlCharacterSetKeyword() token=CharacterSetName() + { statement.setCharacterSet(token.image); statement.setUseCharsetKeyword(charsetKeyword); } + ] + [ token= { statement.setRowsIdentifiedBy(stringValue(token)); } ] + [ fileFormat=MySqlFileFormat() { statement.setFileFormat(fileFormat); } ] + [ + token= { statement.setIgnoreCount(mySqlIgnoreCount(token)); } + ( | { statement.setIgnoreUnit(MySqlLoadDataStatement.IgnoreUnit.ROWS); } ) + ] + [ LOOKAHEAD(1, "(") columns=MySqlLoadDataColumns() { statement.setColumns(columns); } ] + [ LOOKAHEAD(1, ) updateSets=MySqlLoadDataSets() { statement.setUpdateSets(updateSets); } ] + { return statement; } +} + +/** The column list of LOAD DATA: column names and user variables, possibly none. */ +ExpressionList MySqlLoadDataColumns(): +{ + ExpressionList columns = new ExpressionList(); + Expression column; +} +{ + "(" + [ + ( column=MySqlIntoVariable() | column=Column() ) { columns.add(column); } + ( "," ( column=MySqlIntoVariable() | column=Column() ) { columns.add(column); } )* + ] + ")" + { return linkElements(columns); } +} + +/** The SET clause of LOAD DATA: each assignment sets one column to an expression or DEFAULT. */ +List MySqlLoadDataSets(): +{ + List updateSets = new ArrayList(); + UpdateSet updateSet; + Column column; + Expression value; +} +{ + column=Column() "=" value=Expression() + { + updateSet = new UpdateSet(column, value); + linkElements(updateSet.getColumns()); + linkElements(updateSet.getValues()); + updateSets.add(updateSet); + } + ( + "," column=Column() "=" value=Expression() + { + updateSet = new UpdateSet(column, value); + linkElements(updateSet.getColumns()); + linkElements(updateSet.getValues()); + updateSets.add(updateSet); + } + )* + { return updateSets; } +} + +/** MySQL's CHARACTER SET or CHARSET before a character set name: true for CHARSET. */ +boolean MySqlCharacterSetKeyword(): +{ boolean charsetKeyword = false; } +{ + ( | ContextualKeyword("CHARSET") { charsetKeyword = true; } ) + { return charsetKeyword; } +} + +/** + * The FIELDS (or COLUMNS) and LINES clauses of MySQL's LOAD DATA and SELECT ... INTO OUTFILE. + * Each clause takes at least one subclause, in any order and possibly repeated, as in the + * server's grammar; the last value of a subclause wins. + */ +MySqlFileFormat MySqlFileFormat(): +{ + MySqlFileFormat fileFormat = new MySqlFileFormat(); + Token first = getToken(1); +} +{ + ( + ( + { fileFormat.setFieldsKeyword(MySqlFileFormat.FieldsKeyword.FIELDS); } + | + { fileFormat.setFieldsKeyword(MySqlFileFormat.FieldsKeyword.COLUMNS); } + ) + ( LOOKAHEAD(1) MySqlFieldsSubclause(fileFormat) )+ + [ LOOKAHEAD(1, ) MySqlLinesClause(fileFormat) ] + | + MySqlLinesClause(fileFormat) + ) + { linkAST(fileFormat, first); return fileFormat; } +} + +void MySqlFieldsSubclause(MySqlFileFormat fileFormat): +{ Token token; } +{ + ( + token= { fileFormat.setFieldsTerminatedBy(stringValue(token)); } + | + // OPTIONALLY stays set when a later ENCLOSED BY replaces the character, as in the server + token= + { fileFormat.setFieldsOptionallyEnclosed(true); fileFormat.setFieldsEnclosedBy(stringValue(token)); } + | + token= { fileFormat.setFieldsEnclosedBy(stringValue(token)); } + | + token= { fileFormat.setFieldsEscapedBy(stringValue(token)); } + ) +} + +void MySqlLinesClause(MySqlFileFormat fileFormat): +{ Token token; } +{ + + ( + LOOKAHEAD(1) + ( + token= { fileFormat.setLinesStartingBy(stringValue(token)); } + | + token= { fileFormat.setLinesTerminatedBy(stringValue(token)); } + ) + )+ +} + /** * BigQuery's EXPORT DATA [WITH CONNECTION connection] OPTIONS (option_list) AS query. */ @@ -7508,15 +7722,14 @@ ExportDataStatement ExportDataStatement() #ExportDataStatement: { ExportDataStatement exportDataStatement = new ExportDataStatement(); ExpressionList options; - ObjectNames connectionName; + String connectionName; Select select; } { [ - LOOKAHEAD({ getToken(1).kind == K_WITH && "CONNECTION".equalsIgnoreCase(getToken(2).image) }) - connectionName=RelObjectNames() - { exportDataStatement.setConnectionName(String.join(".", connectionName.getNames())); } + LOOKAHEAD({ isBigQueryWithConnectionAhead() }) + connectionName=BigQueryWithConnection() { exportDataStatement.setConnectionName(connectionName); } ] [ LOOKAHEAD({ isKeywordAhead("OPTIONS") }) @@ -11273,6 +11486,7 @@ List IntoClause(): MySqlSelectIntoClause MySqlSelectIntoClause(MySqlSelectIntoClause.Position position): { MySqlSelectIntoClause intoClause = new MySqlSelectIntoClause().withPosition(position); + Token first = getToken(1); Token token; ExpressionList variables = new ExpressionList(); UserVariable variable; @@ -11295,6 +11509,7 @@ MySqlSelectIntoClause MySqlSelectIntoClause(MySqlSelectIntoClause.Position posit { intoClause.setType(MySqlSelectIntoClause.Type.VARIABLES); intoClause.setVariables(linkElements(variables)); } ) { + linkAST(intoClause, first); return intoClause; } } @@ -11342,106 +11557,26 @@ MySqlProcedureAnalyse MySqlProcedureAnalyse(): } } +/** + * The rest of INTO OUTFILE 'file_name': [CHARACTER SET charset_name] and the FIELDS and LINES + * clauses that LOAD DATA shares. + */ void MySqlSelectIntoOutfileTail(MySqlSelectIntoClause intoClause): { Token token; + boolean charsetKeyword; + MySqlFileFormat fileFormat; } { - ( - LOOKAHEAD( ) - - (token= | token=) - { intoClause.setCharacterSet(token.image); } - ( - LOOKAHEAD(1, ( | )) - MySqlSelectIntoFieldsClause(intoClause) - [ LOOKAHEAD(1, ) MySqlSelectIntoLinesClause(intoClause) ] - | - LOOKAHEAD(1, ) - MySqlSelectIntoLinesClause(intoClause) - | - { } - ) - | - LOOKAHEAD(1, ( | )) - MySqlSelectIntoFieldsClause(intoClause) - [ LOOKAHEAD(1, ) MySqlSelectIntoLinesClause(intoClause) ] - | - LOOKAHEAD(1, ) - MySqlSelectIntoLinesClause(intoClause) - | - { } - ) -} - -void MySqlSelectIntoFieldsClause(MySqlSelectIntoClause intoClause): -{ - Token token; -} -{ - ( - { intoClause.setFieldsKeyword(MySqlSelectIntoClause.FieldsKeyword.FIELDS); } - | { intoClause.setFieldsKeyword(MySqlSelectIntoClause.FieldsKeyword.COLUMNS); } - ) - ( - LOOKAHEAD( ) - token= - { intoClause.setFieldsTerminatedBy(stringValue(token)); } - ( - LOOKAHEAD(1, ( | )) - [ { intoClause.setFieldsOptionallyEnclosed(true); } ] - token= - { intoClause.setFieldsEnclosedBy(stringValue(token)); } - [ LOOKAHEAD( ) - token= - { intoClause.setFieldsEscapedBy(stringValue(token)); } - ] - | - LOOKAHEAD( ) - token= - { intoClause.setFieldsEscapedBy(stringValue(token)); } - | - { } - ) - | - LOOKAHEAD(1, ( | )) - [ { intoClause.setFieldsOptionallyEnclosed(true); } ] - token= - { intoClause.setFieldsEnclosedBy(stringValue(token)); } - [ LOOKAHEAD( ) - token= - { intoClause.setFieldsEscapedBy(stringValue(token)); } - ] - | - LOOKAHEAD( ) - token= - { intoClause.setFieldsEscapedBy(stringValue(token)); } - | - { } - ) -} - -void MySqlSelectIntoLinesClause(MySqlSelectIntoClause intoClause): -{ - Token token; -} -{ - - ( - LOOKAHEAD( ) - token= - { intoClause.setLinesStartingBy(stringValue(token)); } - [ LOOKAHEAD( ) - token= - { intoClause.setLinesTerminatedBy(stringValue(token)); } - ] - | - LOOKAHEAD( ) - token= - { intoClause.setLinesTerminatedBy(stringValue(token)); } - | - { } - ) + [ + LOOKAHEAD({ isMySqlCharacterSetClauseAhead() }) + charsetKeyword=MySqlCharacterSetKeyword() token=CharacterSetName() + { intoClause.setCharacterSet(token.image); intoClause.setUseCharsetKeyword(charsetKeyword); } + ] + [ + LOOKAHEAD(1, ( | | )) + fileFormat=MySqlFileFormat() { intoClause.setFileFormat(fileFormat); } + ] } FromItem ParenthesedFromItem(): diff --git a/src/site/sphinx/index.rst b/src/site/sphinx/index.rst index e0ff2d8a4d..e83bc837ef 100644 --- a/src/site/sphinx/index.rst +++ b/src/site/sphinx/index.rst @@ -164,6 +164,8 @@ One grammar covers the SQL standard plus all major RDBMS. Missing syntax gets ad - ``VACUUM`` · ``ANALYZE`` · ``CLUSTER`` · ``CHECKPOINT`` * - **MySQL locks** - ``LOCK TABLES`` · ``UNLOCK TABLES`` · ``LOCK INSTANCE FOR BACKUP`` · ``UNLOCK INSTANCE`` + * - **MySQL data loading** + - ``LOAD DATA`` · ``LOAD XML`` · ``SELECT … INTO OUTFILE`` with the same ``FIELDS`` and ``LINES`` clauses * - **MySQL account management** - ``CREATE``/``ALTER``/``RENAME USER`` · ``CREATE ROLE`` · ``SET ROLE`` · ``SET DEFAULT ROLE`` · ``SET PASSWORD`` * - **Tablespaces** diff --git a/src/site/sphinx/usage.rst b/src/site/sphinx/usage.rst index 1fe9a87183..5350c24aa6 100644 --- a/src/site/sphinx/usage.rst +++ b/src/site/sphinx/usage.rst @@ -2606,6 +2606,48 @@ Table visitors report the locked tables but not their aliases. Validators report three, MariaDB the first two, and PostgreSQL and Oracle ``lock``. Feature analysis reports ``MODIFIES_TRANSACTION``. +MySQL LOAD DATA and LOAD XML +---------------------------- + +MySQL and MariaDB `LOAD DATA `_ and +`LOAD XML `_ parse to +``MySqlLoadDataStatement``; BigQuery's ``LOAD DATA {INTO | OVERWRITE}`` remains +``LoadDataStatement``. ``getFormat()`` is ``DATA`` or ``XML``. As in the server's grammar, +both forms take every clause: ``getLockOption()`` (``LOW_PRIORITY`` or ``CONCURRENT``), +``isLocal()``, ``getFileName()``, ``getDuplicateHandling()`` (``REPLACE`` or ``IGNORE``, the +enum of ``CREATE TABLE … SELECT``), ``getTable()``, ``getPartitions()``, +``getCharacterSet()`` with ``isUseCharsetKeyword()`` for the ``CHARSET`` spelling, +``getRowsIdentifiedBy()``, ``getFileFormat()``, ``getIgnoreCount()`` with ``getIgnoreUnit()`` +(``LINES`` or ``ROWS``), ``getColumns()`` (columns and ``UserVariable`` targets, empty for +``()``) and ``getUpdateSets()`` for the ``SET`` clause. + +.. code-block:: java + + MySqlLoadDataStatement load = (MySqlLoadDataStatement) CCJSqlParserUtil.parse( + "LOAD DATA LOCAL INFILE 'a.csv' INTO TABLE t FIELDS TERMINATED BY ',' " + + "IGNORE 1 LINES (a, @b) SET c = @b + 1"); + load.getFileFormat().getFieldsTerminatedBy().getValue(); // , + load.getUpdateSets().get(0).getValue(0); // @b + 1 + +``MySqlFileFormat`` holds the ``{FIELDS | COLUMNS}`` and ``LINES`` clauses, which +``SELECT … INTO OUTFILE`` shares: ``MySqlSelectIntoClause.getFileFormat()`` returns it, and the +clause's earlier getters and setters, such as ``getFieldsTerminatedBy()``, read and write it. +Each clause needs at least one subclause, so a bare ``FIELDS`` or ``LINES``, which MySQL +rejects, is now a parse error. The subclauses may come in any order and repeat, as MySQL +allows; like the server, the format keeps the last value of each and prints them in the +documented order. ``INTO OUTFILE`` also takes ``CHARSET`` and a quoted character set name. + +The MySQL and MariaDB dialects read these forms, and so does the parser without a dialect: +``INFILE`` after ``LOAD DATA`` or ``LOAD XML`` and the optional ``LOW_PRIORITY``, +``CONCURRENT`` and ``LOCAL`` tells them from BigQuery's ``LOAD DATA INTO`` and from DuckDB's +and PostgreSQL's ``LOAD name``, so ``LOAD xml`` still loads an extension named ``xml``. The +other dialects keep rejecting the MySQL forms. ``INFILE`` and ``CONCURRENT`` remain ordinary +identifiers. + +Table visitors report the target table and the tables of subqueries in the ``SET`` clause. +Validators report the ``loadData`` feature, which the MySQL and MariaDB capabilities now +enable. Feature analysis reports ``MODIFIES_DATA``. + MySQL account management statements ----------------------------------- diff --git a/src/test/java/net/sf/jsqlparser/parser/ASTNodeCoverageTest.java b/src/test/java/net/sf/jsqlparser/parser/ASTNodeCoverageTest.java index 25dd26d38d..810510beb0 100644 --- a/src/test/java/net/sf/jsqlparser/parser/ASTNodeCoverageTest.java +++ b/src/test/java/net/sf/jsqlparser/parser/ASTNodeCoverageTest.java @@ -149,7 +149,14 @@ public void testStatementsTheCorpusLacks() throws JSQLParserException { "CHECKPOINT", "DROP CAST IF EXISTS (int[] AS public.t) CASCADE", "DROP OPERATOR public.=== (integer, integer), ~ (NONE, bigint)", - "DROP TRANSFORM FOR public.hstore LANGUAGE plpython3u"}) { + "DROP TRANSFORM FOR public.hstore LANGUAGE plpython3u", + "LOAD DATA LOCAL INFILE 'a' REPLACE INTO TABLE db1.t PARTITION (p0) CHARSET latin1" + + " FIELDS TERMINATED BY ',' LINES TERMINATED BY ';' IGNORE 1 LINES" + + " (a, @b) SET c = @b + 1, d = DEFAULT", + "LOAD XML INFILE 'a' INTO TABLE t ROWS IDENTIFIED BY '' LINES STARTING BY 'x' ()", + "SELECT a INTO OUTFILE '/tmp/o' COLUMNS ESCAPED BY '^' FROM t", + "SELECT a FROM t INTO OUTFILE '/tmp/o' CHARACTER SET 'utf8mb4'" + + " LINES TERMINATED BY ';'"}) { new Walk(sql, sql, problems).check(CCJSqlParserUtil.parseStatements(sql)); } String block = "BEGIN x := 1; NULL; foo(1); END;"; diff --git a/src/test/java/net/sf/jsqlparser/statement/MySqlFileFormatTest.java b/src/test/java/net/sf/jsqlparser/statement/MySqlFileFormatTest.java new file mode 100644 index 0000000000..68e8fad265 --- /dev/null +++ b/src/test/java/net/sf/jsqlparser/statement/MySqlFileFormatTest.java @@ -0,0 +1,207 @@ +/*- + * #%L + * JSQLParser library + * %% + * Copyright (C) 2004 - 2026 JSQLParser + * %% + * Dual licensed under GNU LGPL 2.1 or Apache License 2.0 + * #L% + */ +package net.sf.jsqlparser.statement; + +import static net.sf.jsqlparser.test.TestUtils.assertSqlCanBeParsedAndDeparsed; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.List; +import java.util.function.Consumer; +import net.sf.jsqlparser.JSQLParserException; +import net.sf.jsqlparser.expression.StringValue; +import net.sf.jsqlparser.parser.ASTNodeAccess; +import net.sf.jsqlparser.parser.AbstractJSqlParser.Dialect; +import net.sf.jsqlparser.parser.CCJSqlParser; +import net.sf.jsqlparser.parser.CCJSqlParserUtil; +import net.sf.jsqlparser.parser.Node; +import net.sf.jsqlparser.statement.load.MySqlLoadDataStatement; +import net.sf.jsqlparser.statement.select.MySqlSelectIntoClause; +import net.sf.jsqlparser.statement.select.Select; +import net.sf.jsqlparser.util.deparser.StatementDeParser; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +class MySqlFileFormatTest { + private static final Consumer MYSQL = parser -> parser.withDialect(Dialect.MYSQL); + private static final Consumer NO_DIALECT = parser -> { + }; + + @ParameterizedTest + @ValueSource(strings = { + "SELECT a FROM t INTO OUTFILE '/tmp/o' CHARSET utf8mb4 FIELDS TERMINATED BY ','", + "SELECT a FROM t INTO OUTFILE '/tmp/o' CHARACTER SET 'utf8mb4' " + + "LINES TERMINATED BY '\\n'", + "SELECT a FROM t INTO OUTFILE '/tmp/o' CHARACTER SET binary", + "SELECT a FROM t INTO OUTFILE '/tmp/o' COLUMNS ENCLOSED BY '\"' LINES STARTING BY 'x'", + "SELECT a INTO OUTFILE '/tmp/o' FIELDS ESCAPED BY '^' LINES STARTING BY '>' " + + "TERMINATED BY ';' FROM t"}) + void parsesIntoOutfileClauses(String sql) throws JSQLParserException { + for (Consumer dialect : List.of(MYSQL, NO_DIALECT)) { + Statement statement = assertSqlCanBeParsedAndDeparsed(sql, false, dialect); + assertEquals(sql, deparse(statement)); + } + } + + @Test + void intoOutfileAcceptsSubclausesInAnyOrder() throws JSQLParserException { + String sql = "SELECT a INTO OUTFILE '/tmp/o' FIELDS ESCAPED BY '^' TERMINATED BY ',' " + + "TERMINATED BY ';' LINES TERMINATED BY '\\n' STARTING BY 'x' FROM t"; + String expected = "SELECT a INTO OUTFILE '/tmp/o' FIELDS TERMINATED BY ';' ESCAPED BY '^' " + + "LINES STARTING BY 'x' TERMINATED BY '\\n' FROM t"; + for (Consumer dialect : List.of(MYSQL, NO_DIALECT)) { + Statement statement = CCJSqlParserUtil.parse(sql, dialect); + assertEquals(expected, statement.toString()); + assertEquals(expected, deparse(statement)); + } + } + + @ParameterizedTest + @ValueSource(strings = { + "SELECT a INTO OUTFILE '/tmp/o' FIELDS FROM t", + "SELECT a INTO OUTFILE '/tmp/o' LINES FROM t", + "SELECT a INTO OUTFILE '/tmp/o' COLUMNS LINES TERMINATED BY 'x' FROM t", + "SELECT a FROM t INTO OUTFILE '/tmp/o' FIELDS", + "SELECT a FROM t INTO OUTFILE '/tmp/o' LINES", + "SELECT a FROM t INTO OUTFILE '/tmp/o' FIELDS TERMINATED BY ',' CHARSET latin1", + "SELECT a FROM t INTO OUTFILE '/tmp/o' LINES TERMINATED BY 'x' FIELDS ESCAPED BY '^'"}) + void rejectsBareOrMisplacedClauses(String sql) { + assertThrows(JSQLParserException.class, () -> CCJSqlParserUtil.parse(sql, MYSQL)); + assertThrows(JSQLParserException.class, () -> CCJSqlParserUtil.parse(sql)); + } + + @Test + void outfileGettersReadTheSharedFormat() throws JSQLParserException { + MySqlSelectIntoClause into = into("SELECT a FROM t INTO OUTFILE '/tmp/o' CHARSET latin1 " + + "COLUMNS TERMINATED BY ',' OPTIONALLY ENCLOSED BY '\"' ESCAPED BY '^' " + + "LINES STARTING BY 'x' TERMINATED BY ';'"); + MySqlFileFormat format = into.getFileFormat(); + + assertEquals("latin1", into.getCharacterSet()); + assertTrue(into.isUseCharsetKeyword()); + assertEquals(MySqlSelectIntoClause.FieldsKeyword.COLUMNS, into.getFieldsKeyword()); + assertEquals(MySqlFileFormat.FieldsKeyword.COLUMNS, format.getFieldsKeyword()); + assertSame(format.getFieldsTerminatedBy(), into.getFieldsTerminatedBy()); + assertTrue(into.isFieldsOptionallyEnclosed()); + assertSame(format.getFieldsEnclosedBy(), into.getFieldsEnclosedBy()); + assertSame(format.getFieldsEscapedBy(), into.getFieldsEscapedBy()); + assertSame(format.getLinesStartingBy(), into.getLinesStartingBy()); + assertSame(format.getLinesTerminatedBy(), into.getLinesTerminatedBy()); + assertTrue(into.hasFieldsClause()); + assertTrue(into.hasLinesClause()); + + MySqlSelectIntoClause plain = into("SELECT a FROM t INTO OUTFILE '/tmp/o'"); + assertNull(plain.getFileFormat()); + assertNull(plain.getFieldsKeyword()); + assertNull(plain.getFieldsTerminatedBy()); + assertFalse(plain.isFieldsOptionallyEnclosed()); + assertNull(plain.getLinesTerminatedBy()); + assertFalse(plain.hasFieldsClause()); + assertFalse(plain.hasLinesClause()); + } + + @Test + void loadDataAndIntoOutfileShareTheFormat() throws JSQLParserException { + String clauses = "FIELDS TERMINATED BY ',' LINES TERMINATED BY '\\n'"; + MySqlFileFormat outfile = + into("SELECT a FROM t INTO OUTFILE '/tmp/o' " + clauses).getFileFormat(); + MySqlFileFormat load = ((MySqlLoadDataStatement) CCJSqlParserUtil + .parse("LOAD DATA INFILE '/tmp/o' INTO TABLE t " + clauses)).getFileFormat(); + assertEquals(clauses, outfile.toString()); + assertEquals(clauses, load.toString()); + } + + @Test + void legacySettersWriteTheSharedFormat() { + MySqlSelectIntoClause into = new MySqlSelectIntoClause(); + into.setType(MySqlSelectIntoClause.Type.OUTFILE); + into.setFileName(new StringValue("/tmp/o")); + assertNull(into.getFileFormat()); + assertEquals("INTO OUTFILE '/tmp/o'", into.toString()); + + // OPTIONALLY set before the character it applies to still renders + into.setFieldsOptionallyEnclosed(true); + into.setFieldsEnclosedBy(new StringValue("\"")); + into.setFieldsKeyword(MySqlSelectIntoClause.FieldsKeyword.COLUMNS); + into.setLinesTerminatedBy(new StringValue(";")); + into.setCharacterSet("utf8mb4"); + assertEquals(MySqlFileFormat.FieldsKeyword.COLUMNS, + into.getFileFormat().getFieldsKeyword()); + assertEquals("INTO OUTFILE '/tmp/o' CHARACTER SET utf8mb4 COLUMNS OPTIONALLY ENCLOSED BY " + + "'\"' LINES TERMINATED BY ';'", into.toString()); + + into.setFieldsKeyword(null); + into.setUseCharsetKeyword(true); + assertNull(into.getFieldsKeyword()); + assertEquals("INTO OUTFILE '/tmp/o' CHARSET utf8mb4 FIELDS OPTIONALLY ENCLOSED BY '\"' " + + "LINES TERMINATED BY ';'", into.toString()); + + into.withFileFormat(new MySqlFileFormat().withFieldsEscapedBy(new StringValue("^"))); + assertEquals("^", into.getFieldsEscapedBy().getValue()); + assertNull(into.getFieldsEnclosedBy()); + assertEquals("INTO OUTFILE '/tmp/o' CHARSET utf8mb4 FIELDS ESCAPED BY '^'", + into.toString()); + } + + @Test + void rendersEachClauseOnlyWhenSet() { + MySqlFileFormat format = new MySqlFileFormat(); + assertTrue(format.isEmpty()); + assertEquals("", format.toString()); + + format.setLinesStartingBy(new StringValue("x")); + assertFalse(format.hasFieldsClause()); + assertEquals("LINES STARTING BY 'x'", format.toString()); + + format.withFieldsEnclosedBy(new StringValue("\"")).withFieldsOptionallyEnclosed(false) + .withLinesStartingBy(null).withLinesTerminatedBy(new StringValue(";")); + assertEquals("FIELDS ENCLOSED BY '\"' LINES TERMINATED BY ';'", format.toString()); + + // a spelling without a subclause renders as before, though MySQL rejects it + MySqlFileFormat keywordOnly = + new MySqlFileFormat().withFieldsKeyword(MySqlFileFormat.FieldsKeyword.FIELDS); + assertTrue(keywordOnly.hasFieldsClause()); + assertEquals("FIELDS", keywordOnly.toString()); + } + + @Test + void nodesSpanTheIntoClauseAndItsFormat() throws JSQLParserException { + String sql = "SELECT a INTO OUTFILE '/tmp/o' CHARSET latin1 FIELDS TERMINATED BY ',' " + + "LINES TERMINATED BY ';' FROM t"; + MySqlSelectIntoClause into = into(sql); + assertEquals("INTO OUTFILE '/tmp/o' CHARSET latin1 FIELDS TERMINATED BY ',' " + + "LINES TERMINATED BY ';'", text(sql, into)); + assertEquals("FIELDS TERMINATED BY ',' LINES TERMINATED BY ';'", + text(sql, into.getFileFormat())); + assertEquals("';'", text(sql, into.getLinesTerminatedBy())); + } + + private static MySqlSelectIntoClause into(String sql) throws JSQLParserException { + Select select = (Select) assertSqlCanBeParsedAndDeparsed(sql, false, MYSQL); + return select.getPlainSelect().getMySqlSelectIntoClause(); + } + + private static String deparse(Statement statement) { + StringBuilder builder = new StringBuilder(); + statement.accept(new StatementDeParser(builder), null); + return builder.toString(); + } + + private static String text(String sql, ASTNodeAccess access) { + Node node = access.getASTNode(); + return sql.substring(node.jjtGetFirstToken().absoluteBegin - 1, + node.jjtGetLastToken().absoluteEnd - 1); + } +} diff --git a/src/test/java/net/sf/jsqlparser/statement/load/MySqlLoadDataStatementTest.java b/src/test/java/net/sf/jsqlparser/statement/load/MySqlLoadDataStatementTest.java new file mode 100644 index 0000000000..b8f2df58f0 --- /dev/null +++ b/src/test/java/net/sf/jsqlparser/statement/load/MySqlLoadDataStatementTest.java @@ -0,0 +1,454 @@ +/*- + * #%L + * JSQLParser library + * %% + * Copyright (C) 2004 - 2026 JSQLParser + * %% + * Dual licensed under GNU LGPL 2.1 or Apache License 2.0 + * #L% + */ +package net.sf.jsqlparser.statement.load; + +import static net.sf.jsqlparser.test.TestUtils.assertSqlCanBeParsedAndDeparsed; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.function.Consumer; +import java.util.stream.Collectors; +import net.sf.jsqlparser.JSQLParserException; +import net.sf.jsqlparser.expression.Expression; +import net.sf.jsqlparser.expression.ExpressionVisitorAdapter; +import net.sf.jsqlparser.expression.LongValue; +import net.sf.jsqlparser.expression.StringValue; +import net.sf.jsqlparser.expression.UserVariable; +import net.sf.jsqlparser.expression.operators.relational.ExpressionList; +import net.sf.jsqlparser.parser.ASTNodeAccess; +import net.sf.jsqlparser.parser.AbstractJSqlParser.Dialect; +import net.sf.jsqlparser.parser.CCJSqlParser; +import net.sf.jsqlparser.parser.CCJSqlParserUtil; +import net.sf.jsqlparser.parser.Node; +import net.sf.jsqlparser.schema.Column; +import net.sf.jsqlparser.schema.Table; +import net.sf.jsqlparser.statement.ExtensionStatement; +import net.sf.jsqlparser.statement.MySqlFileFormat; +import net.sf.jsqlparser.statement.Statement; +import net.sf.jsqlparser.statement.StatementVisitorAdapter; +import net.sf.jsqlparser.statement.Statements; +import net.sf.jsqlparser.statement.StmtFeature; +import net.sf.jsqlparser.statement.UnsupportedStatement; +import net.sf.jsqlparser.statement.create.table.CreateTable; +import net.sf.jsqlparser.statement.select.FromItemVisitorAdapter; +import net.sf.jsqlparser.statement.update.UpdateSet; +import net.sf.jsqlparser.util.TablesNamesFinder; +import net.sf.jsqlparser.util.deparser.StatementDeParser; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +class MySqlLoadDataStatementTest { + private static final Consumer MYSQL = parser -> parser.withDialect(Dialect.MYSQL); + private static final Consumer MARIADB = + parser -> parser.withDialect(Dialect.MARIADB); + private static final Consumer NO_DIALECT = parser -> { + }; + + @ParameterizedTest + @ValueSource(strings = { + "LOAD DATA INFILE '/tmp/a.csv' INTO TABLE t", + "LOAD DATA LOCAL INFILE 'a.csv' REPLACE INTO TABLE db1.t PARTITION (p0, p1) " + + "CHARACTER SET utf8mb4 FIELDS TERMINATED BY ',' OPTIONALLY ENCLOSED BY '\"' " + + "LINES STARTING BY 'x' TERMINATED BY '\\n' IGNORE 1 LINES (a, @b) " + + "SET c = @b + 1, d = DEFAULT", + "LOAD DATA LOW_PRIORITY INFILE 'a' IGNORE INTO TABLE t COLUMNS TERMINATED BY '\\t' " + + "IGNORE 2 ROWS", + "LOAD DATA CONCURRENT INFILE 'a' INTO TABLE t LINES TERMINATED BY '\\r\\n'", + "LOAD DATA CONCURRENT LOCAL INFILE 'a' INTO TABLE t", + "LOAD DATA LOW_PRIORITY LOCAL INFILE 'a' IGNORE INTO TABLE t", + "LOAD DATA INFILE 'a' INTO TABLE t FIELDS ENCLOSED BY '\"' ESCAPED BY '\\\\'", + "LOAD DATA INFILE 'a' INTO TABLE t FIELDS ESCAPED BY '' LINES STARTING BY ''", + "LOAD DATA INFILE 'a' INTO TABLE t LINES STARTING BY 'x'", + "LOAD DATA INFILE 'a' INTO TABLE t CHARSET latin1", + "LOAD DATA INFILE 'a' INTO TABLE t CHARACTER SET 'utf8mb4'", + "LOAD DATA INFILE 'a' INTO TABLE t CHARACTER SET binary", + "LOAD DATA INFILE 'a' INTO TABLE t ()", + "LOAD DATA INFILE 'a' INTO TABLE db1.t (t.a, db1.t.b, @c, @`d e`)", + "LOAD DATA INFILE 'a' INTO TABLE t SET a = (SELECT MAX(b) FROM u), " + + "c = CURRENT_TIMESTAMP", + "LOAD DATA INFILE 'a' INTO TABLE t IGNORE 0 LINES", + "LOAD DATA INFILE 'a' INTO TABLE t IGNORE 2147483647 LINES", + "LOAD DATA INFILE 'a' INTO TABLE t ROWS IDENTIFIED BY ''", + "LOAD XML LOCAL INFILE 'a.xml' INTO TABLE t ROWS IDENTIFIED BY ''", + "LOAD XML INFILE 'a.xml' REPLACE INTO TABLE t CHARACTER SET latin1 " + + "ROWS IDENTIFIED BY '' IGNORE 1 ROWS (a, @x) SET b = @x", + "LOAD XML LOW_PRIORITY INFILE 'a.xml' IGNORE INTO TABLE t PARTITION (p0)", + "LOAD XML INFILE 'a.xml' INTO TABLE t FIELDS TERMINATED BY ',' " + + "LINES TERMINATED BY '\\n'"}) + void parsesMySqlFormsWithAndWithoutDialect(String sql) throws JSQLParserException { + for (Consumer dialect : List.of(MYSQL, MARIADB, NO_DIALECT)) { + Statement statement = assertSqlCanBeParsedAndDeparsed(sql, false, dialect); + assertInstanceOf(MySqlLoadDataStatement.class, statement); + assertEquals(sql, deparse(statement)); + assertEquals(sql, CCJSqlParserUtil.parse(statement.toString(), dialect).toString()); + } + } + + @Test + void keepsEveryClause() throws JSQLParserException { + MySqlLoadDataStatement load = parse("LOAD DATA LOW_PRIORITY LOCAL INFILE 'a.csv' " + + "REPLACE INTO TABLE db1.t PARTITION (p0, p1) CHARSET 'utf8mb4' " + + "COLUMNS TERMINATED BY ',' OPTIONALLY ENCLOSED BY '\"' ESCAPED BY '^' " + + "LINES STARTING BY 'x' TERMINATED BY ';' IGNORE 3 ROWS (a, @b, t.c) " + + "SET d = @b + 1, e = DEFAULT", MYSQL); + + assertEquals(MySqlLoadDataStatement.Format.DATA, load.getFormat()); + assertEquals(MySqlLoadDataStatement.LockOption.LOW_PRIORITY, load.getLockOption()); + assertTrue(load.isLocal()); + assertEquals("a.csv", load.getFileName().getValue()); + assertEquals(CreateTable.DuplicateHandling.REPLACE, load.getDuplicateHandling()); + assertEquals("db1.t", load.getTable().getFullyQualifiedName()); + assertEquals(Arrays.asList("p0", "p1"), load.getPartitions()); + assertEquals("'utf8mb4'", load.getCharacterSet()); + assertTrue(load.isUseCharsetKeyword()); + assertNull(load.getRowsIdentifiedBy()); + + MySqlFileFormat format = load.getFileFormat(); + assertEquals(MySqlFileFormat.FieldsKeyword.COLUMNS, format.getFieldsKeyword()); + assertEquals(",", format.getFieldsTerminatedBy().getValue()); + assertTrue(format.isFieldsOptionallyEnclosed()); + assertEquals("\"", format.getFieldsEnclosedBy().getValue()); + assertEquals("^", format.getFieldsEscapedBy().getValue()); + assertEquals("x", format.getLinesStartingBy().getValue()); + assertEquals(";", format.getLinesTerminatedBy().getValue()); + + assertEquals(3L, load.getIgnoreCount()); + assertEquals(MySqlLoadDataStatement.IgnoreUnit.ROWS, load.getIgnoreUnit()); + assertEquals(3, load.getColumns().size()); + assertInstanceOf(Column.class, load.getColumns().get(0)); + assertEquals("b", ((UserVariable) load.getColumns().get(1)).getName()); + assertEquals("t.c", ((Column) load.getColumns().get(2)).getFullyQualifiedName()); + assertEquals(2, load.getUpdateSets().size()); + assertEquals("d", load.getUpdateSets().get(0).getColumn(0).getColumnName()); + assertEquals("@b + 1", load.getUpdateSets().get(0).getValue(0).toString()); + assertEquals("DEFAULT", load.getUpdateSets().get(1).getValue(0).toString()); + } + + @Test + void readsLoadXmlAndItsRowTag() throws JSQLParserException { + MySqlLoadDataStatement load = parse( + "LOAD XML CONCURRENT INFILE 'a.xml' IGNORE INTO TABLE t ROWS IDENTIFIED BY ''", + MYSQL); + assertEquals(MySqlLoadDataStatement.Format.XML, load.getFormat()); + assertEquals(MySqlLoadDataStatement.LockOption.CONCURRENT, load.getLockOption()); + assertFalse(load.isLocal()); + assertEquals(CreateTable.DuplicateHandling.IGNORE, load.getDuplicateHandling()); + assertEquals("", load.getRowsIdentifiedBy().getValue()); + assertNull(load.getPartitions()); + assertNull(load.getCharacterSet()); + assertNull(load.getFileFormat()); + assertNull(load.getIgnoreCount()); + assertNull(load.getColumns()); + assertNull(load.getUpdateSets()); + } + + @Test + void keepsAnEmptyColumnListApartFromNone() throws JSQLParserException { + assertTrue(parse("LOAD DATA INFILE 'a' INTO TABLE t ()", MYSQL).getColumns().isEmpty()); + assertNull(parse("LOAD DATA INFILE 'a' INTO TABLE t", MYSQL).getColumns()); + } + + @Test + void acceptsSubclausesInAnyOrderAndKeepsTheLastValue() throws JSQLParserException { + // the server merges repeated subclauses, the last one winning, and keeps OPTIONALLY + assertNormalized( + "LOAD DATA INFILE 'a' INTO TABLE t COLUMNS OPTIONALLY ENCLOSED BY '\"' " + + "TERMINATED BY ','", + "LOAD DATA INFILE 'a' INTO TABLE t COLUMNS TERMINATED BY ',' " + + "OPTIONALLY ENCLOSED BY '\"'"); + assertNormalized( + "LOAD DATA INFILE 'a' INTO TABLE t FIELDS ESCAPED BY '^' TERMINATED BY ',' " + + "TERMINATED BY ';' LINES TERMINATED BY '\\n' STARTING BY '>'", + "LOAD DATA INFILE 'a' INTO TABLE t FIELDS TERMINATED BY ';' ESCAPED BY '^' " + + "LINES STARTING BY '>' TERMINATED BY '\\n'"); + assertNormalized( + "LOAD DATA INFILE 'a' INTO TABLE t FIELDS OPTIONALLY ENCLOSED BY '\"' " + + "ENCLOSED BY '|'", + "LOAD DATA INFILE 'a' INTO TABLE t FIELDS OPTIONALLY ENCLOSED BY '|'"); + assertNormalized("load data local infile 'a' replace into table t fields terminated by " + + "',' ignore 1 lines (a) set b = 1", + "LOAD DATA LOCAL INFILE 'a' REPLACE INTO TABLE t FIELDS TERMINATED BY ',' " + + "IGNORE 1 LINES (a) SET b = 1"); + assertNormalized("LOAD DATA INFILE 'a' INTO TABLE t IGNORE 007 LINES", + "LOAD DATA INFILE 'a' INTO TABLE t IGNORE 7 LINES"); + } + + @Test + void readsDoubleQuotedStringsInMySql() throws JSQLParserException { + MySqlLoadDataStatement load = parse( + "LOAD DATA INFILE \"a.csv\" INTO TABLE t FIELDS ENCLOSED BY \"'\"", MYSQL); + assertEquals("a.csv", load.getFileName().getValue()); + assertEquals("'", load.getFileFormat().getFieldsEnclosedBy().getValue()); + } + + @ParameterizedTest + @ValueSource(strings = { + "LOAD DATA INFILE 'a' INTO TABLE t FIELDS", + "LOAD DATA INFILE 'a' INTO TABLE t LINES", + "LOAD DATA INFILE 'a' INTO TABLE t COLUMNS", + "LOAD DATA INFILE 'a' INTO TABLE t FIELDS LINES TERMINATED BY '\\n'", + "LOAD DATA INFILE 'a' INTO TABLE t LINES TERMINATED BY '\\n' FIELDS TERMINATED BY ','", + "LOAD DATA INFILE 'a' INTO TABLE t CHARACTER SET utf8mb4 PARTITION (p0)", + "LOAD DATA INFILE 'a' INTO TABLE t FIELDS TERMINATED BY ',' ROWS IDENTIFIED BY ''", + "LOAD DATA INFILE 'a' INTO TABLE t IGNORE 1 LINES FIELDS TERMINATED BY ','", + "LOAD DATA INFILE 'a' INTO TABLE t SET a = 1 (a)", + "LOAD DATA INFILE 'a' INTO t", + "LOAD DATA 'a' INTO TABLE t", + "LOAD DATA LOCAL LOW_PRIORITY INFILE 'a' INTO TABLE t", + "LOAD DATA LOW_PRIORITY CONCURRENT INFILE 'a' INTO TABLE t", + "LOAD DATA INFILE 'a' REPLACE IGNORE INTO TABLE t", + "LOAD DATA INFILE 'a' INTO TABLE t IGNORE 1", + "LOAD DATA INFILE 'a' INTO TABLE t IGNORE 2147483648 LINES", + "LOAD DATA INFILE 'a' INTO TABLE t IGNORE 99999999999999999999 LINES", + "LOAD DATA INFILE 'a' INTO TABLE t IGNORE -1 LINES", + "LOAD DATA INFILE 'a' INTO TABLE t (a, @@b)", + "LOAD DATA INFILE 'a' INTO TABLE t SET @x = 1", + "LOAD DATA INFILE 'a' INTO TABLE t SET (a, b) = (1, 2)", + "LOAD DATA INFILE a INTO TABLE t", + "LOAD DATA INFILE 'a' INTO TABLE t FIELDS TERMINATED ','", + "LOAD DATA INFILE 'a' INTO TABLE t PARTITION ()", + "LOAD XML INFILE 'a' INTO TABLE t ROWS IDENTIFIED ''", + "LOAD DATA INFILE 'a' INTO TABLE t SET a = 1,", + "LOAD DATA INFILE 'a' INTO TABLE t (a,)"}) + void rejectsWhatMySqlRejects(String sql) { + assertThrows(JSQLParserException.class, () -> CCJSqlParserUtil.parse(sql, MYSQL)); + assertThrows(JSQLParserException.class, () -> CCJSqlParserUtil.parse(sql)); + } + + @ParameterizedTest + @ValueSource(strings = {"LOAD DATA INFILE 'a' INTO TABLE t", + "LOAD XML INFILE 'a.xml' INTO TABLE t", + "LOAD DATA LOCAL INFILE 'a' INTO TABLE t"}) + void otherDialectsKeepRejectingMySqlForms(String sql) { + for (Dialect dialect : new Dialect[] {Dialect.POSTGRESQL, Dialect.ORACLE, + Dialect.SQLSERVER, Dialect.BIGQUERY}) { + assertThrows(JSQLParserException.class, + () -> CCJSqlParserUtil.parse(sql, parser -> parser.withDialect(dialect)), + dialect.name()); + } + } + + @Test + void otherLoadStatementsKeepTheirReading() throws JSQLParserException { + for (Consumer dialect : List.of(MYSQL, NO_DIALECT)) { + // DuckDB's LOAD of an extension named xml or data stays an extension statement + ExtensionStatement xml = (ExtensionStatement) CCJSqlParserUtil.parse("LOAD xml", + dialect); + assertEquals("xml", xml.getExtensionName()); + assertInstanceOf(ExtensionStatement.class, + CCJSqlParserUtil.parse("LOAD httpfs", dialect)); + assertInstanceOf(LoadDataStatement.class, CCJSqlParserUtil.parse( + "LOAD DATA INTO ds.t FROM FILES (format = 'CSV')", dialect)); + } + ExtensionStatement library = (ExtensionStatement) CCJSqlParserUtil.parse( + "LOAD 'auto_explain'", parser -> parser.withDialect(Dialect.POSTGRESQL)); + assertEquals("'auto_explain'", library.getExtensionName()); + assertInstanceOf(LoadDataStatement.class, + CCJSqlParserUtil.parse("LOAD DATA OVERWRITE ds.t FROM FILES (format = 'CSV') " + + "WITH CONNECTION p.us.c", + parser -> parser.withDialect(Dialect.BIGQUERY))); + } + + @Test + void wordsRemainUsableAsNames() throws JSQLParserException { + // CONCURRENT, XML and CHARSET are names in MySQL too + for (String sql : new String[] {"CREATE TABLE concurrent (xml INT, charset INT)", + "SELECT concurrent, xml, charset FROM t"}) { + assertSqlCanBeParsedAndDeparsed(sql, false, MYSQL); + } + MySqlLoadDataStatement load = + parse("LOAD DATA INFILE 'a' INTO TABLE concurrent (xml, charset)", MYSQL); + assertEquals("concurrent", load.getTable().getName()); + + // MySQL reserves INFILE, but the parser adds no reserved word + for (String sql : new String[] {"CREATE TABLE infile (a INT)", "SELECT infile FROM t", + "LOAD DATA INFILE 'a' INTO TABLE infile (infile)"}) { + assertSqlCanBeParsedAndDeparsed(sql, false, NO_DIALECT); + } + } + + @Test + void endsTheStatementsAroundIt() throws JSQLParserException { + for (Consumer dialect : List.of(MYSQL, NO_DIALECT)) { + Statements statements = CCJSqlParserUtil.parseStatements( + "SELECT 1; LOAD DATA INFILE 'a' INTO TABLE t (a) SET b = 1; " + + "LOAD XML INFILE 'b' INTO TABLE u;\nLOAD xml; SELECT 2", + dialect); + assertEquals(List.of("PlainSelect", "MySqlLoadDataStatement", + "MySqlLoadDataStatement", "ExtensionStatement", "PlainSelect"), + statements.stream().map(statement -> statement.getClass().getSimpleName()) + .collect(Collectors.toList())); + } + } + + @Test + void isAStatementStartForUnsupportedStatements() throws JSQLParserException { + Statements statements = CCJSqlParserUtil.parseStatements( + "FOO BAR; LOAD DATA INFILE 'a' INTO TABLE t; BAZ", + parser -> parser.withDialect(Dialect.MYSQL).withUnsupportedStatements(true)); + assertEquals(3, statements.size()); + assertInstanceOf(UnsupportedStatement.class, statements.get(0)); + assertInstanceOf(MySqlLoadDataStatement.class, statements.get(1)); + assertInstanceOf(UnsupportedStatement.class, statements.get(2)); + } + + @Test + void findsTheTargetAndTheTablesOfSubqueries() throws JSQLParserException { + Statement load = CCJSqlParserUtil.parse("LOAD DATA INFILE 'a' INTO TABLE db1.t (a, @b) " + + "SET c = (SELECT MAX(x) FROM u WHERE u.k = @b)"); + assertThat(new TablesNamesFinder().getTables(load)) + .containsExactlyInAnyOrder("db1.t", "u"); + assertThat(new TablesNamesFinder().getTables( + CCJSqlParserUtil.parse("LOAD XML INFILE 'a' INTO TABLE t"))).containsExactly("t"); + } + + @Test + void reportsDataFeatures() throws JSQLParserException { + Statement plain = CCJSqlParserUtil.parse("LOAD DATA INFILE 'a' INTO TABLE t"); + assertThat(plain.getFeatures().getCertain()).containsExactly(StmtFeature.MODIFIES_DATA); + + Statement subquery = CCJSqlParserUtil.parse( + "LOAD DATA INFILE 'a' INTO TABLE t SET c = (SELECT MAX(x) FROM u)"); + assertThat(subquery.getFeatures().getCertain()) + .contains(StmtFeature.MODIFIES_DATA, StmtFeature.READS_DATA); + } + + @Test + void adapterVisitsTheTableColumnsAndAssignments() throws JSQLParserException { + Statement load = CCJSqlParserUtil.parse( + "LOAD DATA INFILE 'a' INTO TABLE t (a, @b) SET c = @b + 1"); + Object context = new Object(); + List visited = new ArrayList<>(); + ExpressionVisitorAdapter expressions = new ExpressionVisitorAdapter() { + @Override + public Void visit(Column column, S value) { + visited.add(column.getColumnName()); + return null; + } + + @Override + public Void visit(UserVariable variable, S value) { + visited.add(variable.toString()); + return null; + } + + @Override + public Void visit(LongValue longValue, S value) { + visited.add(longValue.getStringValue()); + return null; + } + }; + FromItemVisitorAdapter tables = new FromItemVisitorAdapter() { + @Override + public Void visit(Table table, S value) { + visited.add("table " + table.getName()); + return null; + } + }; + load.accept(new StatementVisitorAdapter(expressions, null, null, tables, null, null), + context); + assertEquals(List.of("table t", "a", "@b", "c", "@b", "1"), visited); + assertNull(load.accept(new StatementVisitorAdapter<>(), null)); + } + + @Test + void nodesSpanTheStatementAndItsClauses() throws JSQLParserException { + String sql = "LOAD DATA INFILE 'a' INTO TABLE t CHARSET latin1 " + + "FIELDS TERMINATED BY ',' LINES TERMINATED BY ';' IGNORE 1 LINES (a, @b) " + + "SET c = @b"; + MySqlLoadDataStatement load = parse(sql, MYSQL); + assertEquals(sql, text(sql, load)); + assertEquals("'a'", text(sql, load.getFileName())); + assertEquals("t", text(sql, load.getTable())); + assertEquals("FIELDS TERMINATED BY ',' LINES TERMINATED BY ';'", + text(sql, load.getFileFormat())); + assertEquals("','", text(sql, load.getFileFormat().getFieldsTerminatedBy())); + assertEquals("a, @b", text(sql, load.getColumns())); + assertEquals("@b", text(sql, (ASTNodeAccess) load.getUpdateSets().get(0).getValues())); + + String xml = "LOAD XML INFILE 'a' INTO TABLE t LINES STARTING BY '<'"; + MySqlLoadDataStatement lines = parse(xml, MYSQL); + assertEquals("LINES STARTING BY '<'", text(xml, lines.getFileFormat())); + } + + @Test + void buildsStatementsInCode() { + MySqlLoadDataStatement load = new MySqlLoadDataStatement() + .withFileName(new StringValue("data.csv")) + .withTable(new Table("db1", "t")); + assertEquals("LOAD DATA INFILE 'data.csv' INTO TABLE db1.t", load.toString()); + + ExpressionList columns = new ExpressionList<>(); + columns.add(new Column("a")); + columns.add(new UserVariable("b")); + load.withFormat(MySqlLoadDataStatement.Format.XML) + .withLockOption(MySqlLoadDataStatement.LockOption.CONCURRENT) + .withLocal(true) + .withDuplicateHandling(CreateTable.DuplicateHandling.IGNORE) + .withPartitions(Arrays.asList("p0", "p1")) + .withCharacterSet("utf8mb4") + .withUseCharsetKeyword(true) + .withRowsIdentifiedBy(new StringValue("")) + .withFileFormat(new MySqlFileFormat() + .withFieldsTerminatedBy(new StringValue(",")) + .withLinesTerminatedBy(new StringValue("\\n"))) + .withIgnoreCount(1L) + .withIgnoreUnit(MySqlLoadDataStatement.IgnoreUnit.ROWS) + .withColumns(columns) + .withUpdateSets(List.of(new UpdateSet(new Column("c"), new UserVariable("b")))); + String expected = "LOAD XML CONCURRENT LOCAL INFILE 'data.csv' IGNORE INTO TABLE db1.t " + + "PARTITION (p0, p1) CHARSET utf8mb4 ROWS IDENTIFIED BY '' " + + "FIELDS TERMINATED BY ',' LINES TERMINATED BY '\\n' IGNORE 1 ROWS (a, @b) " + + "SET c = @b"; + assertEquals(expected, load.toString()); + assertEquals(expected, deparse(load)); + + load.setFileFormat(new MySqlFileFormat()); + load.setUseCharsetKeyword(false); + assertEquals("LOAD XML CONCURRENT LOCAL INFILE 'data.csv' IGNORE INTO TABLE db1.t " + + "PARTITION (p0, p1) CHARACTER SET utf8mb4 ROWS IDENTIFIED BY '' " + + "IGNORE 1 ROWS (a, @b) SET c = @b", load.toString()); + } + + private static void assertNormalized(String sql, String expected) throws JSQLParserException { + for (Consumer dialect : List.of(MYSQL, NO_DIALECT)) { + Statement statement = CCJSqlParserUtil.parse(sql, dialect); + assertEquals(expected, statement.toString()); + assertEquals(expected, deparse(statement)); + assertEquals(expected, CCJSqlParserUtil.parse(expected, dialect).toString()); + } + } + + private static MySqlLoadDataStatement parse(String sql, Consumer dialect) + throws JSQLParserException { + return (MySqlLoadDataStatement) assertSqlCanBeParsedAndDeparsed(sql, false, dialect); + } + + private static String deparse(Statement statement) { + StringBuilder builder = new StringBuilder(); + statement.accept(new StatementDeParser(builder), null); + return builder.toString(); + } + + private static String text(String sql, ASTNodeAccess access) { + Node node = access.getASTNode(); + return sql.substring(node.jjtGetFirstToken().absoluteBegin - 1, + node.jjtGetLastToken().absoluteEnd - 1); + } +} diff --git a/src/test/java/net/sf/jsqlparser/util/deparser/StructuredStatementVisitorTest.java b/src/test/java/net/sf/jsqlparser/util/deparser/StructuredStatementVisitorTest.java index b4f4232e11..76450f7a5b 100644 --- a/src/test/java/net/sf/jsqlparser/util/deparser/StructuredStatementVisitorTest.java +++ b/src/test/java/net/sf/jsqlparser/util/deparser/StructuredStatementVisitorTest.java @@ -30,7 +30,9 @@ class StructuredStatementVisitorTest { "ASSERT (1 = 1) AS 'expected'", "ASSERT EXISTS (SELECT 7 FROM users WHERE id = 8)", "EXPORT DATA OPTIONS(uri='gs://example/*.csv',format='CSV') AS SELECT 7 FROM users", - "EXPORT DATA OPTIONS(uri='gs://example/*.csv',format='CSV') AS WITH c AS (SELECT 7 AS id) SELECT id FROM c UNION ALL SELECT 8" + "EXPORT DATA OPTIONS(uri='gs://example/*.csv',format='CSV') AS WITH c AS (SELECT 7 AS id) SELECT id FROM c UNION ALL SELECT 8", + "LOAD DATA INTO ds.t OPTIONS (description = 'x') FROM FILES (format = 'CSV', skip_leading_rows = 1)", + "LOAD DATA INFILE 'a' INTO TABLE t FIELDS TERMINATED BY ',' (a, @b) SET c = @b + 1, d = 'x'" }) void defaultDeparserMatchesStatementOutput(String sql) throws Exception { Statement statement = CCJSqlParserUtil.parse(sql); @@ -68,6 +70,23 @@ void visitsExportOptionsAndQuery() throws Exception { "EXPORT DATA OPTIONS (uri = 'changed', format = 'changed') AS SELECT 107 FROM users WHERE id = 108"); } + @Test + void visitsBigQueryLoadOptions() throws Exception { + assertVisited( + "LOAD DATA INTO ds.t OPTIONS (description = 'x') FROM FILES (format = 'CSV', skip_leading_rows = 1)", + List.of("x", "CSV", "1"), + "LOAD DATA INTO ds.t OPTIONS (description = 'changed') FROM FILES (format = 'changed', skip_leading_rows = 101)"); + } + + @Test + void visitsMySqlLoadColumnsAndAssignments() throws Exception { + // the file name and the separators are literal syntax, as in SELECT ... INTO OUTFILE + assertVisited( + "LOAD DATA INFILE 'a' INTO TABLE t FIELDS TERMINATED BY ',' (a, @b) SET c = @b + 1, d = 'x'", + List.of("1", "x"), + "LOAD DATA INFILE 'a' INTO TABLE t FIELDS TERMINATED BY ',' (a, @b) SET c = @b + 101, d = 'changed'"); + } + private void assertVisited(String sql, List expectedVisits, String expectedOutput) throws Exception { List visited = new ArrayList<>(); diff --git a/src/test/java/net/sf/jsqlparser/util/validation/validator/LoadDataValidatorTest.java b/src/test/java/net/sf/jsqlparser/util/validation/validator/LoadDataValidatorTest.java new file mode 100644 index 0000000000..7da35b40d4 --- /dev/null +++ b/src/test/java/net/sf/jsqlparser/util/validation/validator/LoadDataValidatorTest.java @@ -0,0 +1,45 @@ +/*- + * #%L + * JSQLParser library + * %% + * Copyright (C) 2004 - 2026 JSQLParser + * %% + * Dual licensed under GNU LGPL 2.1 or Apache License 2.0 + * #L% + */ +package net.sf.jsqlparser.util.validation.validator; + +import net.sf.jsqlparser.JSQLParserException; +import net.sf.jsqlparser.parser.feature.Feature; +import net.sf.jsqlparser.util.validation.ValidationTestAsserts; +import net.sf.jsqlparser.util.validation.feature.DatabaseType; +import net.sf.jsqlparser.util.validation.feature.FeaturesAllowed; +import net.sf.jsqlparser.util.validation.feature.MariaDbVersion; +import net.sf.jsqlparser.util.validation.feature.MySqlVersion; +import net.sf.jsqlparser.util.validation.feature.PostgresqlVersion; +import org.junit.jupiter.api.Test; + +public class LoadDataValidatorTest extends ValidationTestAsserts { + + @Test + public void testLoadDataAllowedForMySqlAndMariaDb() throws JSQLParserException { + for (String sql : new String[] { + "LOAD DATA LOCAL INFILE 'a.csv' INTO TABLE t FIELDS TERMINATED BY ',' (a, @b) " + + "SET c = @b + 1", + "LOAD XML INFILE 'a.xml' INTO TABLE t ROWS IDENTIFIED BY ''"}) { + validateNoErrors(sql, 1, MySqlVersion.V8_0, MariaDbVersion.V10_5_4, + DatabaseType.MYSQL, DatabaseType.MARIADB); + } + } + + @Test + public void testLoadDataNotAllowedForPostgresql() throws JSQLParserException { + validate("LOAD DATA INFILE 'a.csv' INTO TABLE t", 1, 1, PostgresqlVersion.V14); + } + + @Test + public void testFeatureNotAllowed() throws JSQLParserException { + validateNotAllowed("LOAD DATA INFILE 'a.csv' INTO TABLE t", 1, 1, FeaturesAllowed.SELECT, + Feature.loadData); + } +} diff --git a/src/test/java/net/sf/jsqlparser/util/validation/validator/StatementValidatorTest.java b/src/test/java/net/sf/jsqlparser/util/validation/validator/StatementValidatorTest.java index 51d4717635..1fe76e8c7d 100644 --- a/src/test/java/net/sf/jsqlparser/util/validation/validator/StatementValidatorTest.java +++ b/src/test/java/net/sf/jsqlparser/util/validation/validator/StatementValidatorTest.java @@ -32,6 +32,7 @@ static Stream statementsOutsideSelect() { return Stream.of( Arguments.of("COPY t TO '/tmp/t.csv'", Feature.copy), Arguments.of("LOAD DATA INTO ds.t FROM FILES (format = 'CSV')", Feature.loadData), + Arguments.of("LOAD XML INFILE 'a.xml' INTO TABLE t", Feature.loadData), Arguments.of("EXPORT DATA OPTIONS (uri = 'gs://b/*.csv', format = 'CSV') " + "AS SELECT a FROM t", Feature.export), Arguments.of("ATTACH DATABASE '/tmp/x.db' AS x", Feature.attach),