From 4027b790668c735a75d58fcf1fb7d1f49d2163ca Mon Sep 17 00:00:00 2001 From: git-hulk Date: Mon, 14 Sep 2026 16:42:37 +0800 Subject: [PATCH] Fix contextual keyword parsing Fix DATE and TIMESTAMP literals by consuming the type keyword before parsing the following string. Disambiguate projection EXCEPT modifiers from EXCEPT set operations using SELECT lookahead, without changing the AST or global keyword rules. Before applying this PR: - SELECT DATE '2024-01-01' and SELECT TIMESTAMP '2024-01-01 00:00:00' failed because the type keyword was still current when the string was parsed. - SELECT 1 EXCEPT SELECT 2 failed because EXCEPT was treated as a projection modifier and expected modifier arguments. After this PR: - Typed date literals, keyword identifiers and aliases, projection modifiers, and EXCEPT set operations parse in their production-specific contexts. - Invalid forms such as SELECT * EXCEPT FROM t and SELECT 1 EXCEPT 2 remain rejected. Add focused parser tests and golden fixtures for the supported forms. Verification: make test; pinned golangci-lint v1.53.3; ClickHouse local syntax checks for positive and negative cases. --- parser/parser_column.go | 14 +- parser/parser_test.go | 19 ++ parser/testdata/query/contextual_keywords.sql | 3 + .../format/beautify/contextual_keywords.sql | 20 ++ .../query/format/contextual_keywords.sql | 10 + .../contextual_keywords.sql.golden.json | 220 ++++++++++++++++++ 6 files changed, 279 insertions(+), 7 deletions(-) create mode 100644 parser/testdata/query/contextual_keywords.sql create mode 100644 parser/testdata/query/format/beautify/contextual_keywords.sql create mode 100644 parser/testdata/query/format/contextual_keywords.sql create mode 100644 parser/testdata/query/output/contextual_keywords.sql.golden.json diff --git a/parser/parser_column.go b/parser/parser_column.go index a9c39e1..e806972 100644 --- a/parser/parser_column.go +++ b/parser/parser_column.go @@ -580,13 +580,13 @@ func (p *Parser) parseColumnExpr(pos Pos) (Expr, error) { //nolint:funlen return interval, nil case p.matchKeyword(KeywordDate), p.matchKeyword(KeywordTimestamp): - nextToken, err := p.lexer.peekToken() - if err != nil { - return nil, err - } - if nextToken != nil && nextToken.Kind == TokenKindString { - return p.parseString(p.Pos()) + literalPos := p.Pos() + savedState := p.lexer.saveState() + _ = p.lexer.consumeToken() + if p.matchTokenKind(TokenKindString) { + return p.parseString(literalPos) } + p.lexer.restoreState(savedState) return p.parseIdentOrFunction(pos) case p.matchKeyword(KeywordCast): return p.parseColumnCastExpr(pos) @@ -1172,7 +1172,7 @@ func (p *Parser) parseSelectItem() (*SelectItem, error) { modifiers := make([]*FunctionExpr, 0) for { - if p.matchKeyword(KeywordExcept) || p.matchKeyword(KeywordApply) || p.matchKeyword(KeywordReplace) { + if (p.matchKeyword(KeywordExcept) && !p.peekKeyword(KeywordSelect)) || p.matchKeyword(KeywordApply) || p.matchKeyword(KeywordReplace) { modifier, err := p.parseFunctionExpr(p.Pos()) if err != nil { return nil, err diff --git a/parser/parser_test.go b/parser/parser_test.go index 17ab8e9..3daecf2 100644 --- a/parser/parser_test.go +++ b/parser/parser_test.go @@ -31,6 +31,23 @@ func TestParser_TableSettingsFunctionExpression(t *testing.T) { } } +func TestParser_ContextualKeywords(t *testing.T) { + tests := []string{ + "SELECT DATE '2024-01-01'", + "SELECT TIMESTAMP '2024-01-01 00:00:00'", + "SELECT date AS timestamp", + "SELECT * EXCEPT (a) FROM t", + "SELECT 1 EXCEPT SELECT 2", + } + + for _, sql := range tests { + t.Run(sql, func(t *testing.T) { + _, err := NewParser(sql).ParseStmts() + require.NoError(t, err) + }) + } +} + func TestParser_Compatible(t *testing.T) { if !*runCompatible { t.Skip("Compatible test runs only if -compatible is set") @@ -285,6 +302,8 @@ func TestParser_InvalidSyntax(t *testing.T) { // ALL or DISTINCT "(SELECT 1", "(SELECT 1) UNION SELECT 2", + "SELECT * EXCEPT FROM t", + "SELECT 1 EXCEPT 2", // ClickHouse rejects a set operator once SETTINGS is bound to a // parenthesized group "(SELECT 1) SETTINGS max_threads=1 UNION ALL SELECT 2", diff --git a/parser/testdata/query/contextual_keywords.sql b/parser/testdata/query/contextual_keywords.sql new file mode 100644 index 0000000..ddec97f --- /dev/null +++ b/parser/testdata/query/contextual_keywords.sql @@ -0,0 +1,3 @@ +SELECT DATE '2024-01-01', TIMESTAMP '2024-01-01 00:00:00', date AS timestamp; +SELECT * EXCEPT (a) FROM t; +SELECT 1 EXCEPT SELECT 2; diff --git a/parser/testdata/query/format/beautify/contextual_keywords.sql b/parser/testdata/query/format/beautify/contextual_keywords.sql new file mode 100644 index 0000000..63c7e54 --- /dev/null +++ b/parser/testdata/query/format/beautify/contextual_keywords.sql @@ -0,0 +1,20 @@ +-- Origin SQL: +SELECT DATE '2024-01-01', TIMESTAMP '2024-01-01 00:00:00', date AS timestamp; +SELECT * EXCEPT (a) FROM t; +SELECT 1 EXCEPT SELECT 2; + + +-- Beautify SQL: +SELECT + '2024-01-01', + '2024-01-01 00:00:00', + date AS timestamp; +SELECT + * EXCEPT(a) +FROM + t; +SELECT + 1 +EXCEPT +SELECT + 2; diff --git a/parser/testdata/query/format/contextual_keywords.sql b/parser/testdata/query/format/contextual_keywords.sql new file mode 100644 index 0000000..a1f4a6a --- /dev/null +++ b/parser/testdata/query/format/contextual_keywords.sql @@ -0,0 +1,10 @@ +-- Origin SQL: +SELECT DATE '2024-01-01', TIMESTAMP '2024-01-01 00:00:00', date AS timestamp; +SELECT * EXCEPT (a) FROM t; +SELECT 1 EXCEPT SELECT 2; + + +-- Format SQL: +SELECT '2024-01-01', '2024-01-01 00:00:00', date AS timestamp; +SELECT * EXCEPT(a) FROM t; +SELECT 1 EXCEPT SELECT 2; diff --git a/parser/testdata/query/output/contextual_keywords.sql.golden.json b/parser/testdata/query/output/contextual_keywords.sql.golden.json new file mode 100644 index 0000000..fe7aeac --- /dev/null +++ b/parser/testdata/query/output/contextual_keywords.sql.golden.json @@ -0,0 +1,220 @@ +[ + { + "SelectPos": 0, + "StatementEnd": 76, + "With": null, + "Top": null, + "HasDistinct": false, + "DistinctOn": null, + "SelectItems": [ + { + "Expr": { + "LiteralPos": 7, + "LiteralEnd": 23, + "Literal": "2024-01-01" + }, + "Modifiers": [], + "Alias": null + }, + { + "Expr": { + "LiteralPos": 26, + "LiteralEnd": 56, + "Literal": "2024-01-01 00:00:00" + }, + "Modifiers": [], + "Alias": null + }, + { + "Expr": { + "Name": "date", + "QuoteType": 1, + "NamePos": 59, + "NameEnd": 63 + }, + "Modifiers": [], + "Alias": { + "Name": "timestamp", + "QuoteType": 1, + "NamePos": 67, + "NameEnd": 76 + } + } + ], + "From": null, + "Window": null, + "Prewhere": null, + "Where": null, + "GroupBy": null, + "WithTotal": false, + "Having": null, + "OrderBy": null, + "LimitBy": null, + "Limit": null, + "Settings": null, + "Format": null, + "UnionAll": null, + "UnionDistinct": null, + "Except": null, + "Intersect": null + }, + { + "SelectPos": 78, + "StatementEnd": 104, + "With": null, + "Top": null, + "HasDistinct": false, + "DistinctOn": null, + "SelectItems": [ + { + "Expr": { + "Name": "*", + "QuoteType": 0, + "NamePos": 85, + "NameEnd": 85 + }, + "Modifiers": [ + { + "Name": { + "Name": "EXCEPT", + "QuoteType": 1, + "NamePos": 87, + "NameEnd": 93 + }, + "Params": { + "LeftParenPos": 94, + "RightParenPos": 96, + "Items": { + "ListPos": 95, + "ListEnd": 96, + "HasDistinct": false, + "Items": [ + { + "Expr": { + "Name": "a", + "QuoteType": 1, + "NamePos": 95, + "NameEnd": 96 + }, + "Alias": null + } + ] + }, + "ColumnArgList": null + } + } + ], + "Alias": null + } + ], + "From": { + "FromPos": 98, + "Expr": { + "Table": { + "TablePos": 103, + "TableEnd": 104, + "Alias": null, + "Expr": { + "Database": null, + "Table": { + "Name": "t", + "QuoteType": 1, + "NamePos": 103, + "NameEnd": 104 + } + }, + "HasFinal": false + }, + "StatementEnd": 104, + "SampleRatio": null, + "HasFinal": false + } + }, + "Window": null, + "Prewhere": null, + "Where": null, + "GroupBy": null, + "WithTotal": false, + "Having": null, + "OrderBy": null, + "LimitBy": null, + "Limit": null, + "Settings": null, + "Format": null, + "UnionAll": null, + "UnionDistinct": null, + "Except": null, + "Intersect": null + }, + { + "SelectPos": 106, + "StatementEnd": 114, + "With": null, + "Top": null, + "HasDistinct": false, + "DistinctOn": null, + "SelectItems": [ + { + "Expr": { + "NumPos": 113, + "NumEnd": 114, + "Literal": "1", + "Base": 10 + }, + "Modifiers": [], + "Alias": null + } + ], + "From": null, + "Window": null, + "Prewhere": null, + "Where": null, + "GroupBy": null, + "WithTotal": false, + "Having": null, + "OrderBy": null, + "LimitBy": null, + "Limit": null, + "Settings": null, + "Format": null, + "UnionAll": null, + "UnionDistinct": null, + "Except": { + "SelectPos": 122, + "StatementEnd": 130, + "With": null, + "Top": null, + "HasDistinct": false, + "DistinctOn": null, + "SelectItems": [ + { + "Expr": { + "NumPos": 129, + "NumEnd": 130, + "Literal": "2", + "Base": 10 + }, + "Modifiers": [], + "Alias": null + } + ], + "From": null, + "Window": null, + "Prewhere": null, + "Where": null, + "GroupBy": null, + "WithTotal": false, + "Having": null, + "OrderBy": null, + "LimitBy": null, + "Limit": null, + "Settings": null, + "Format": null, + "UnionAll": null, + "UnionDistinct": null, + "Except": null, + "Intersect": null + }, + "Intersect": null + } +] \ No newline at end of file