Skip to content

Commit 956e109

Browse files
kyy-logskyy-logs
authored
Fix parsing of ClickHouse transformers after '*' (#2653)
AllColumns modelled the trailing modifiers itself and only accepted a single EXCEPT/EXCLUDE followed by a single REPLACE, so '* APPLY(sum)' was parsed as a wildcard carrying the alias "APPLY(sum)" — a wrong AST that round-trips without error. AllColumns now holds a List<ColumnsTransformer>, mirroring ColumnsExpression, and the production loops over ColumnsTransformer(), so the modifiers may repeat and combine in any order. ColumnsTransformerType gains EXCLUDE (DuckDB), which AllColumns previously carried as a plain string. The transformer lookahead is position specific: reusing ColumnsTransformerAhead() verbatim would turn '* REPLACE (x)' into an alias.
1 parent 521a27f commit 956e109

6 files changed

Lines changed: 100 additions & 94 deletions

File tree

‎src/main/java/net/sf/jsqlparser/expression/ColumnsTransformer.java‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,15 +17,15 @@
1717
import net.sf.jsqlparser.statement.select.SelectItem;
1818

1919
/**
20-
* A ClickHouse transformer following a {@code COLUMNS(...)} matcher, for example the
20+
* A ClickHouse transformer following a {@code COLUMNS(...)} matcher or a {@code *}, for example the
2121
* {@code APPLY(x -> round(x, 2))} in {@code SELECT COLUMNS('^m') APPLY(x -> round(x, 2))}.
2222
*
2323
* ClickHouse parses its transformers in a loop, so they may repeat and combine in any order.
2424
*/
2525
public class ColumnsTransformer extends ASTNodeAccessImpl {
2626

2727
public enum ColumnsTransformerType {
28-
APPLY, EXCEPT, REPLACE
28+
APPLY, EXCEPT, EXCLUDE, REPLACE
2929
}
3030

3131
private ColumnsTransformerType type;
@@ -81,6 +81,7 @@ void collectExpressions(List<Expression> expressions) {
8181
}
8282
break;
8383
case EXCEPT:
84+
case EXCLUDE:
8485
if (exceptColumns != null) {
8586
expressions.addAll(exceptColumns);
8687
}
@@ -105,6 +106,9 @@ public StringBuilder appendTo(StringBuilder builder) {
105106
case EXCEPT:
106107
builder.append("EXCEPT ").append(exceptColumns);
107108
break;
109+
case EXCLUDE:
110+
builder.append("EXCLUDE ").append(exceptColumns);
111+
break;
108112
case REPLACE:
109113
builder.append("REPLACE(").append(Select.getStringList(replaceItems)).append(")");
110114
break;

‎src/main/java/net/sf/jsqlparser/statement/select/AllColumns.java‎

Lines changed: 17 additions & 64 deletions
Original file line numberDiff line numberDiff line change
@@ -9,92 +9,45 @@
99
*/
1010
package net.sf.jsqlparser.statement.select;
1111

12+
import java.util.ArrayList;
13+
import java.util.List;
14+
import net.sf.jsqlparser.expression.ColumnsTransformer;
1215
import net.sf.jsqlparser.expression.Expression;
1316
import net.sf.jsqlparser.expression.ExpressionVisitor;
14-
import net.sf.jsqlparser.expression.operators.relational.ExpressionList;
1517
import net.sf.jsqlparser.parser.ASTNodeAccessImpl;
16-
import net.sf.jsqlparser.schema.Column;
17-
18-
import java.util.ArrayList;
19-
import java.util.List;
2018

2119
public class AllColumns extends ASTNodeAccessImpl implements Expression {
22-
protected ExpressionList<Column> exceptColumns;
23-
protected List<SelectItem<?>> replaceExpressions;
24-
private String exceptKeyword;
25-
26-
public AllColumns(ExpressionList<Column> exceptColumns,
27-
List<SelectItem<?>> replaceExpressions) {
28-
this.exceptColumns = exceptColumns;
29-
this.replaceExpressions = replaceExpressions;
30-
this.exceptKeyword = exceptColumns != null ? "Except" : null;
31-
}
32-
33-
public AllColumns(ExpressionList<Column> exceptColumns,
34-
List<SelectItem<?>> replaceExpressions, String exceptKeyword) {
35-
this.exceptColumns = exceptColumns;
36-
this.replaceExpressions = replaceExpressions;
37-
this.exceptKeyword = exceptKeyword;
38-
}
20+
private List<ColumnsTransformer> transformers;
3921

4022
public AllColumns() {
41-
this(null, null);
42-
}
43-
44-
public ExpressionList<Column> getExceptColumns() {
45-
return exceptColumns;
23+
this(null);
4624
}
4725

48-
public AllColumns setExceptColumns(ExpressionList<Column> exceptColumns) {
49-
this.exceptColumns = exceptColumns;
50-
return this;
26+
public AllColumns(List<ColumnsTransformer> transformers) {
27+
this.transformers = transformers;
5128
}
5229

53-
public ExpressionList<Column> addExceptColumn(Column column) {
54-
if (exceptColumns == null) {
55-
exceptColumns = new ExpressionList<>();
30+
public List<ColumnsTransformer> getTransformers() {
31+
if (transformers == null) {
32+
transformers = new ArrayList<>();
5633
}
57-
exceptColumns.add(column);
58-
return exceptColumns;
34+
return transformers;
5935
}
6036

61-
public List<SelectItem<?>> getReplaceExpressions() {
62-
return replaceExpressions;
63-
}
64-
65-
public AllColumns setReplaceExpressions(List<SelectItem<?>> replaceExpressions) {
66-
this.replaceExpressions = replaceExpressions;
37+
public AllColumns setTransformers(List<ColumnsTransformer> transformers) {
38+
this.transformers = transformers;
6739
return this;
6840
}
6941

70-
public List<SelectItem<?>> addReplaceExpression(SelectItem<?> selectItem) {
71-
if (replaceExpressions == null) {
72-
replaceExpressions = new ArrayList<>();
73-
}
74-
replaceExpressions.add(selectItem);
75-
return replaceExpressions;
76-
}
77-
78-
public String getExceptKeyword() {
79-
return exceptKeyword;
80-
}
81-
82-
public AllColumns setExceptKeyword(String exceptKeyword) {
83-
this.exceptKeyword = exceptKeyword;
42+
public AllColumns addTransformer(ColumnsTransformer transformer) {
43+
getTransformers().add(transformer);
8444
return this;
8545
}
8646

8747
public StringBuilder appendTo(StringBuilder builder) {
8848
builder.append("*");
89-
if (exceptColumns != null && !exceptColumns.isEmpty()) {
90-
builder.append(" ").append(exceptKeyword).append("( ");
91-
exceptColumns.appendTo(builder);
92-
builder.append(" )");
93-
}
94-
if (replaceExpressions != null && !replaceExpressions.isEmpty()) {
95-
builder.append(" REPLACE( ");
96-
builder.append(Select.getStringList(replaceExpressions));
97-
builder.append(" )");
49+
for (ColumnsTransformer transformer : getTransformers()) {
50+
builder.append(" ").append(transformer);
9851
}
9952
return builder;
10053
}

‎src/main/java/net/sf/jsqlparser/statement/select/AllTableColumns.java‎

Lines changed: 5 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -10,9 +10,8 @@
1010
package net.sf.jsqlparser.statement.select;
1111

1212
import java.util.List;
13+
import net.sf.jsqlparser.expression.ColumnsTransformer;
1314
import net.sf.jsqlparser.expression.ExpressionVisitor;
14-
import net.sf.jsqlparser.expression.operators.relational.ExpressionList;
15-
import net.sf.jsqlparser.schema.Column;
1615
import net.sf.jsqlparser.schema.Table;
1716
import net.sf.jsqlparser.statement.ReturningReferenceType;
1817

@@ -22,24 +21,17 @@ public class AllTableColumns extends AllColumns {
2221
private ReturningReferenceType returningReferenceType = null;
2322
private String returningQualifier = null;
2423

25-
public AllTableColumns(Table table, ExpressionList<Column> exceptColumns,
26-
List<SelectItem<?>> replaceExpressions, String exceptKeyword) {
27-
super(exceptColumns, replaceExpressions, exceptKeyword);
24+
public AllTableColumns(Table table, List<ColumnsTransformer> transformers) {
25+
super(transformers);
2826
this.table = table;
2927
}
3028

31-
public AllTableColumns(Table table, ExpressionList<Column> exceptColumns,
32-
List<SelectItem<?>> replaceExpressions) {
33-
this(table, exceptColumns, replaceExpressions, "EXCEPT");
34-
}
35-
3629
public AllTableColumns(Table table) {
37-
this(table, null, null);
30+
this(table, (List<ColumnsTransformer>) null);
3831
}
3932

4033
public AllTableColumns(Table table, AllColumns allColumns) {
41-
this(table, allColumns.exceptColumns, allColumns.replaceExpressions,
42-
allColumns.getExceptKeyword());
34+
this(table, allColumns.getTransformers());
4335
}
4436

4537
public Table getTable() {

‎src/main/java/net/sf/jsqlparser/statement/select/FunctionAllColumns.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ public class FunctionAllColumns extends AllColumns {
1717
private Function function;
1818

1919
public FunctionAllColumns(Function function) {
20-
super(null, null, null);
20+
super(null);
2121
this.function = function;
2222
}
2323

‎src/main/jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt‎

Lines changed: 48 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -7882,7 +7882,27 @@ boolean ColumnsTransformerAhead() #void:
78827882
{
78837883
LOOKAHEAD( <K_APPLY> "(" RelObjectName() ")" ) { return true; }
78847884
| LOOKAHEAD( ( <K_APPLY> | <K_REPLACE> ) AliasColumns() ) { return false; }
7885-
| LOOKAHEAD( <K_APPLY> "(" | <K_REPLACE> "(" | <K_EXCEPT> ParenthesedColumnList() )
7885+
| LOOKAHEAD( <K_APPLY> "(" | <K_REPLACE> "("
7886+
| ( <K_EXCEPT> | <K_EXCLUDE> ) ParenthesedColumnList() )
7887+
{ return true; }
7888+
| { return false; }
7889+
}
7890+
7891+
/**
7892+
* Recognizes a transformer after the AllColumns-Pattern '*'.
7893+
*
7894+
* This differs from ColumnsTransformerAhead() in one point: after '*' an alias is not a
7895+
* meaningful reading of REPLACE, so REPLACE (col) stays a transformer here whereas
7896+
* APPLY (col1, col2) remains an alias. Reusing the COLUMNS(...) rules verbatim would turn
7897+
* '* REPLACE (x)' into an alias.
7898+
*/
7899+
boolean AllColumnsTransformerAhead() #void:
7900+
{}
7901+
{
7902+
LOOKAHEAD( <K_APPLY> "(" RelObjectName() ")" ) { return true; }
7903+
| LOOKAHEAD( <K_APPLY> AliasColumns() ) { return false; }
7904+
| LOOKAHEAD( <K_APPLY> "(" | <K_REPLACE> "("
7905+
| ( <K_EXCEPT> | <K_EXCLUDE> ) ParenthesedColumnList() )
78867906
{ return true; }
78877907
| { return false; }
78887908
}
@@ -7950,6 +7970,7 @@ ColumnsTransformer ColumnsTransformer():
79507970
ParenthesedExpressionList<Column> exceptColumns;
79517971
List<SelectItem<?>> replaceItems;
79527972
ColumnsTransformer transformer;
7973+
Token tk;
79537974
}
79547975
{
79557976
(
@@ -7963,8 +7984,11 @@ ColumnsTransformer ColumnsTransformer():
79637984
{ transformer = new ColumnsTransformer(ColumnsTransformer.ColumnsTransformerType.APPLY)
79647985
.setApplyExpression(applyExpression); }
79657986

7966-
| <K_EXCEPT> exceptColumns = ParenthesedColumnList()
7967-
{ transformer = new ColumnsTransformer(ColumnsTransformer.ColumnsTransformerType.EXCEPT)
7987+
| ( tk=<K_EXCEPT> | tk=<K_EXCLUDE> ) exceptColumns = ParenthesedColumnList()
7988+
{ transformer = new ColumnsTransformer(
7989+
tk.kind == K_EXCLUDE
7990+
? ColumnsTransformer.ColumnsTransformerType.EXCLUDE
7991+
: ColumnsTransformer.ColumnsTransformerType.EXCEPT)
79687992
.setExceptColumns(exceptColumns); }
79697993

79707994
| <K_REPLACE> "(" replaceItems = SelectItemsList() ")"
@@ -7981,20 +8005,32 @@ ColumnsTransformer ColumnsTransformer():
79818005
*/
79828006
AllColumns AllColumns(boolean allowAdditions):
79838007
{
7984-
ParenthesedExpressionList<Column> exceptColumns = null;
7985-
List<SelectItem<?>> replaceExpressions = null;
7986-
String exceptKeyword=null;
7987-
Token tk;
8008+
ColumnsTransformer transformer;
8009+
List<ColumnsTransformer> transformers = null;
79888010
}
79898011
{
79908012
"*"
7991-
// BigData allows EXCEPT, DuckDB allows EXCLUDE
7992-
[ LOOKAHEAD(2, { allowAdditions }) ( tk=<K_EXCEPT> | tk=<K_EXCLUDE> ) exceptColumns = ParenthesedColumnList() { exceptKeyword=tk.image; } ]
7993-
// BigData allows REPLACE
7994-
[ LOOKAHEAD(2, { allowAdditions }) <K_REPLACE> "(" replaceExpressions = SelectItemsList() ")" ]
8013+
// BigData allows EXCEPT, DuckDB allows EXCLUDE, ClickHouse allows APPLY;
8014+
// they may repeat and combine in any order
8015+
(
8016+
LOOKAHEAD( <K_APPLY> "(" | <K_REPLACE> "("
8017+
| ( <K_EXCEPT> | <K_EXCLUDE> ) ParenthesedColumnList() )
8018+
{
8019+
if (!allowAdditions || !AllColumnsTransformerAhead()) {
8020+
break;
8021+
}
8022+
}
8023+
transformer = ColumnsTransformer()
8024+
{
8025+
if (transformers == null) {
8026+
transformers = new ArrayList<ColumnsTransformer>();
8027+
}
8028+
transformers.add(transformer);
8029+
}
8030+
)*
79958031

79968032
{
7997-
return new AllColumns(exceptColumns, replaceExpressions, exceptKeyword);
8033+
return new AllColumns(transformers);
79988034
}
79998035
}
80008036

‎src/test/java/net/sf/jsqlparser/statement/select/ClickHouseTest.java‎

Lines changed: 23 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,13 +14,12 @@
1414
import java.util.ArrayList;
1515
import java.util.List;
1616
import java.util.Set;
17-
1817
import net.sf.jsqlparser.JSQLParserException;
1918
import net.sf.jsqlparser.expression.Alias;
2019
import net.sf.jsqlparser.expression.ColumnsExpression;
2120
import net.sf.jsqlparser.expression.ColumnsTransformer;
22-
import net.sf.jsqlparser.expression.Function;
2321
import net.sf.jsqlparser.expression.ExpressionVisitorAdapter;
22+
import net.sf.jsqlparser.expression.Function;
2423
import net.sf.jsqlparser.expression.LambdaExpression;
2524
import net.sf.jsqlparser.expression.operators.relational.ParenthesedExpressionList;
2625
import net.sf.jsqlparser.parser.CCJSqlParserUtil;
@@ -337,6 +336,28 @@ public void testColumnsCombinedTransformers() throws JSQLParserException {
337336
assertSqlCanBeParsedAndDeparsed(sql, true);
338337
}
339338

339+
@Test
340+
public void testAllColumnsApplyIssue2636() throws JSQLParserException {
341+
// ClickHouse accepts the same transformers after '*':
342+
// https://github.com/JSQLParser/JSqlParser/issues/2636
343+
String sql = "SELECT * APPLY(sum) FROM t";
344+
Select select = (Select) CCJSqlParserUtil.parse(sql);
345+
PlainSelect plainSelect = (PlainSelect) select.getSelectBody();
346+
SelectItem<?> selectItem = plainSelect.getSelectItems().get(0);
347+
348+
Assertions.assertNull(selectItem.getAlias(),
349+
"APPLY must not fall back to an alias");
350+
Assertions.assertInstanceOf(AllColumns.class, selectItem.getExpression());
351+
assertSqlCanBeParsedAndDeparsed(sql, true);
352+
}
353+
354+
@Test
355+
public void testAllColumnsChainedTransformersIssue2636() throws JSQLParserException {
356+
// ClickHouse parses the transformers in a loop, so they may repeat after '*'
357+
String sql = "SELECT * APPLY(sum) EXCEPT (a) FROM t";
358+
assertSqlCanBeParsedAndDeparsed(sql, true);
359+
}
360+
340361
@Test
341362
public void testColumnsApplyMultiParamLambda() throws JSQLParserException {
342363
String sql = "SELECT COLUMNS('m') APPLY((k, v) -> v > 5) FROM metrics";

0 commit comments

Comments
 (0)