feat: support accentuated identifiers as column in all db

Author: taozhi8833998

pegjs/athena.pegjs

@@ -1097,7 +1097,7 @@ column_list_item
 
 value_alias_clause
   = KW_AS? __ i:(func_call / alias_ident) { return i; }
-  
+
 alias_clause
   = KW_AS __ i:(func_call / alias_ident) { return i; }
   / KW_AS? __ i:ident { return i; }
@@ -2015,7 +2015,7 @@ ident_start = [A-Za-z_]
 ident_part  = [A-Za-z0-9_]
 
 // to support column name like `cf1:name` in hbase
-column_part  = [A-Za-z0-9_:]
+column_part  = [A-Za-z0-9_:\u4e00-\u9fa5\u00C0-\u017F]
 
 param
   = l:(':' ident_name) {
@@ -2114,7 +2114,7 @@ distinct_args
     }
     return { distinct: d, expr: result, orderby: or };
   }
-  
+
 count_arg
   = e:star_expr { return { expr: e }; }
   / distinct_args
@@ -2172,7 +2172,7 @@ trim_func_clause
         args,
     };
   }
-  
+
 func_call
   = name:scalar_func __ LPAREN __ l:expr_list? __ RPAREN __ bc:over_partition? {
       return {

pegjs/bigquery.pegjs

@@ -195,7 +195,7 @@
     columnList.clear()
     columns.forEach(col => columnList.add(col))
   }
-  
+
   function getSurroundFromLiteralType(literal) {
     switch (literal.type) {
       case 'double_quote_string':
@@ -2484,7 +2484,7 @@ column_ref
       return {
         type: 'column_ref',
         table: null,
-        column, 
+        column,
         collate: ce && ce[1],
         ...getLocationObject()
       };
@@ -2579,7 +2579,7 @@ ident_start = [A-Za-z_]
 ident_part  = [A-Za-z0-9_-]
 
 // to support column name like `cf1:name` in hbase
-column_part  = [A-Za-z0-9_:]
+column_part  = [A-Za-z0-9_:\u4e00-\u9fa5\u00C0-\u017F]
 
 param
   = s:(':'/'@') n:ident_name {

pegjs/db2.pegjs

@@ -1047,7 +1047,7 @@ select_stmt_nake
     g:group_by_clause?  __
     h:having_clause?    __
     o:order_by_clause?  __
-    l:limit_clause? 
+    l:limit_clause?
     iso:isolation_clause? __ {
       if(f) f.forEach(info => info.table && tableList.add(`select::${info.db}::${info.table}`));
       return {
@@ -1972,7 +1972,7 @@ ident_start = [A-Za-z_\u4e00-\u9fa5]
 ident_part  = [A-Za-z0-9_$\u4e00-\u9fa5\u00C0-\u017F]
 
 // to support column name like `cf1:name` in hbase
-column_part  = [A-Za-z0-9_:]
+column_part  = [A-Za-z0-9_:\u4e00-\u9fa5\u00C0-\u017F]
 
 param
   = l:(':' ident_name) {

pegjs/hive.pegjs

@@ -1929,7 +1929,7 @@ ident_start = [A-Za-z_\u4e00-\u9fa5]
 ident_part  = [A-Za-z0-9_$\u4e00-\u9fa5\u00C0-\u017F]
 
 // to support column name like `cf1:name` in hbase
-column_part  = [A-Za-z0-9_:]
+column_part  = [A-Za-z0-9_:\u4e00-\u9fa5\u00C0-\u017F]
 
 param
   = l:(':' ident_name) {

pegjs/mariadb.pegjs

@@ -1094,7 +1094,7 @@ ALTER_OPERATE_PARTITION
       }
       return expr
   }
-    
+
 ALTER_ADD_COLUMN
   = KW_ADD __
     kc:KW_COLUMN? __
@@ -1745,7 +1745,7 @@ transaction_mode_isolation_level
       value: `read ${e.toLowerCase()}`
     }
   }
-  
+
 transaction_mode
   = 'ISOLATION'i __ 'LEVEL'i __ l:transaction_mode_isolation_level {
     return {
@@ -1808,7 +1808,7 @@ transaction_stmt
       }
     }
   }
-  
+
 load_data_field
   = k:('FIELDS'i / 'COLUMNS'i) __ t:('TERMINATED'i __ 'BY'i __ ident_without_kw_type)? __ en:(('OPTIONALLY'i)? __ 'ENCLOSED'i __ 'BY'i __ ident_without_kw_type)? __ es:('ESCAPED'i __ 'BY'i __ ident_without_kw_type)? {
     if (t) t[4].prefix = 'TERMINATED BY'
@@ -3203,7 +3203,7 @@ ident_start = [A-Za-z_\u4e00-\u9fa5]
 ident_part  = [A-Za-z0-9_$$\u4e00-\u9fa5\u00C0-\u017F]
 
 // to support column name like `cf1:name` in hbase
-column_part  = [A-Za-z0-9_:]
+column_part  = [A-Za-z0-9_:\u4e00-\u9fa5\u00C0-\u017F]
 
 param
   = l:(':' ident_name) {
@@ -3612,7 +3612,7 @@ literal_basic
   / literal_bool
   / literal_null
   / literal_datetime
-  
+
 literal
   = literal_basic / literal_numeric
 

pegjs/mysql.pegjs

@@ -1297,7 +1297,7 @@ ALTER_OPERATE_PARTITION
       }
       return expr
   }
-  
+
 ALTER_ADD_COLUMN
   = KW_ADD __
     kc:KW_COLUMN __
@@ -2028,7 +2028,7 @@ transaction_mode_isolation_level
       value: `read ${e.toLowerCase()}`
     }
   }
-  
+
 transaction_mode
   = 'ISOLATION'i __ 'LEVEL'i __ l:transaction_mode_isolation_level {
     return {
@@ -3483,7 +3483,7 @@ ident_start = [A-Za-z_\u4e00-\u9fa5]
 ident_part  = [A-Za-z0-9_$\u0080-\uffff]
 
 // to support column name like `cf1:name` in hbase
-column_part  = [A-Za-z0-9_:]
+column_part  = [A-Za-z0-9_:\u4e00-\u9fa5\u00C0-\u017F]
 
 param
   = l:(':' ident_name) {
@@ -3893,10 +3893,10 @@ literal_basic
   / literal_bool
   / literal_null
   / literal_datetime
-  
+
 literal
   = literal_basic / literal_numeric
-  
+
 
 literal_list
   = head:literal tail:(__ COMMA __ literal)* {
@@ -3965,7 +3965,7 @@ literal_string
         value: ca[1].join('')
       };
     }
-  
+
 
 literal_datetime
   = type:(KW_TIME / KW_DATE / KW_TIMESTAMP / KW_DATETIME) __ ca:("'" single_char* "'") {

pegjs/sqlite.pegjs

@@ -2375,7 +2375,7 @@ ident_start = [A-Za-z_]
 ident_part  = [A-Za-z0-9_]
 
 // to support column name like `cf1:name` in hbase
-column_part  = [A-Za-z0-9_:]
+column_part  = [A-Za-z0-9_:\u4e00-\u9fa5\u00C0-\u017F]
 
 param
   = l:(':' ident_name) {

pegjs/transactsql.pegjs

@@ -1104,7 +1104,7 @@ create_constraint_default
         with_values: w && { type: 'origin', value: 'with values' },
       }
   }
-  
+
 create_constraint_foreign
   = kc:constraint_name? __
   p:('FOREIGN KEY'i) __
@@ -2603,7 +2603,7 @@ ident_start = [A-Za-z_@#\u4e00-\u9fa5]
 ident_part  = [A-Za-z0-9_\-@$$\u4e00-\u9fa5\u00C0-\u017F]
 
 // to support column name like `cf1:name` in hbase
-column_part  = [A-Za-z0-9_:]
+column_part  = [A-Za-z0-9_:\u4e00-\u9fa5\u00C0-\u017F]
 
 param
   = l:(':' ident_name) {
@@ -2794,7 +2794,7 @@ distinct_args
   / d:KW_DISTINCT? __ c:or_and_expr __ s:concat_separator? __ or:order_by_clause? {
     return { distinct: d, expr: c, orderby: or, separator: s };
   }
-  
+
 count_arg
   = e:star_expr { return { expr: e }; }
   / distinct_args
@@ -2816,7 +2816,7 @@ within_group
   = 'WITHIN'i __ KW_GROUP __ LPAREN __ or:order_by_clause __ RPAREN {
       return { type: 'within', keyword: 'group', orderby: or };
   }
-  
+
 func_call
   = name:scalar_func __ LPAREN __ l:expr_list? __ RPAREN __ bc:over_partition? {
       return {
@@ -3541,4 +3541,3 @@ uniqueidentifier_type
   = lb:LBRAKE? __ t:(KW_UNIQUEIDENTIFIER) __ rb:RBRAKE? !{ return (lb && !rb) || (!lb && rb) } {
     return { dataType: t }
   }
-

pegjs/trino.pegjs

@@ -3558,7 +3558,7 @@ ident_start = [A-Za-z_\u4e00-\u9fa5]
 ident_part  = [A-Za-z0-9_$\u0080-\uffff]
 
 // to support column name like `cf1:name` in hbase
-column_part  = [A-Za-z0-9_\u4e00-\u9fa5]
+column_part  = [A-Za-z0-9_\u4e00-\u9fa5\u00C0-\u017F]
 
 param
   = l:(':' ident_name) {
@@ -3970,7 +3970,7 @@ cast_data_type
     if (p && s) t.quoted = '"'
     return t
   }
-  
+
 cast_double_colon
   = s:(KW_DOUBLE_COLON __ cast_data_type)+ __ alias:alias_clause? {
     return {

test/mysql-mariadb.spec.js

@@ -382,7 +382,7 @@ describe('mysql', () => {
           'SELECT * FROM (`t1` AS `eti` INNER JOIN `bagel` ON `bagel`.`id` = `eti`.`id`) ; SELECT * FROM ((`t1`))'
         ]
       },
-      
+
       {
         title: 'blob data type',
         sql: [
@@ -1229,8 +1229,8 @@ describe('mysql', () => {
             inner join salaries using (emp_no)
           order by emp_no desc;
 
-          SELECT * FROM \`employees\` 
-          INNER JOIN \`salaries\` USING (\`emp_no\`) 
+          SELECT * FROM \`employees\`
+          INNER JOIN \`salaries\` USING (\`emp_no\`)
           ORDER BY \`emp_no\` DESC;`,
           'SELECT * FROM `employees` INNER JOIN `salaries` USING (emp_no) ORDER BY `emp_no` DESC ; SELECT * FROM `employees` INNER JOIN `salaries` USING (`emp_no`) ORDER BY `emp_no` DESC'
         ]
@@ -1312,6 +1312,13 @@ describe('mysql', () => {
           "SELECT * FROM `T` WHERE `a` LIKE 'foobar%' XOR `c` = `d`"
         ]
       },
+      {
+        title: 'support accentuated identifiers',
+        sql: [
+          "SELECT crème AS brûlée FROM café WHERE théâtre = 'Molière'",
+          "SELECT `crème` AS `brûlée` FROM `café` WHERE `théâtre` = 'Molière'"
+        ]
+      },
     ]
     SQL_LIST.forEach(sqlInfo => {
       const { title, sql } = sqlInfo
@@ -1322,13 +1329,13 @@ describe('mysql', () => {
         expect(getParsedSql(sql[0], mariadb)).to.equal(sql[1])
       })
     })
-    
+
     it('should throw error when args is not right', () => {
       let sql = `select convert(json_unquote(json_extract('{"thing": "252"}', "$.thing")));`
       expect(parser.astify.bind(parser, sql)).to.throw('Expected "!=", "#", "#-", "#>", "#>>", "%", "&", "&&", "*", "+", ",", "-", "--", "->", "->>", "/", "/*", "<", "<<", "<=", "<>", "<@", "=", ">", ">=", ">>", "?", "?&", "?|", "@>", "AND", "BETWEEN", "IN", "IS", "LIKE", "NOT", "ON", "OR", "OVER", "REGEXP", "RLIKE", "USING", "XOR", "^", "div", "mod", "|", "||", or [ \\t\\n\\r] but ")" found.')
       expect(parser.astify.bind(parser, 'select convert("");')).to.throw('Expected "!=", "#", "#-", "#>", "#>>", "%", "&", "&&", "*", "+", ",", "-", "--", "->", "->>", "/", "/*", "<", "<<", "<=", "<>", "<@", "=", ">", ">=", ">>", "?", "?&", "?|", "@>", "AND", "BETWEEN", "COLLATE", "IN", "IS", "LIKE", "NOT", "OR", "REGEXP", "RLIKE", "USING", "XOR", "^", "div", "mod", "|", "||", or [ \\t\\n\\r] but ")" found.')
       sql = 'SELECT AVG(Quantity,age) FROM table1;'
-      expect(parser.astify.bind(parser, sql)).to.throw('Expected "!=", "#", "#-", "#>", "#>>", "%", "&", "&&", "(", ")", "*", "+", "-", "--", "->", "->>", ".", "/", "/*", "<", "<<", "<=", "<>", "<@", "=", ">", ">=", ">>", "?", "?&", "?|", "@>", "BETWEEN", "COLLATE", "IN", "IS", "LIKE", "NOT", "REGEXP", "RLIKE", "XOR", "^", "div", "mod", "|", "||", [ \\t\\n\\r], [A-Za-z0-9_$\\x80-], or [A-Za-z0-9_:] but "," found')
+      expect(parser.astify.bind(parser, sql)).to.throw('Expected "!=", "#", "#-", "#>", "#>>", "%", "&", "&&", "(", ")", "*", "+", "-", "--", "->", "->>", ".", "/", "/*", "<", "<<", "<=", "<>", "<@", "=", ">", ">=", ">>", "?", "?&", "?|", "@>", "BETWEEN", "COLLATE", "IN", "IS", "LIKE", "NOT", "REGEXP", "RLIKE", "XOR", "^", "div", "mod", "|", "||", [ \\t\\n\\r], [A-Za-z0-9_$\\x80-], or [A-Za-z0-9_:一-龥À-ſ] but "," found')
     })
 
     it('should join multiple table with comma', () => {
@@ -1481,4 +1488,4 @@ describe('mysql', () => {
     })
   })
 
-})
+})