diff --git a/sqlparse/engine/filter_stack.py b/sqlparse/engine/filter_stack.py index 415d3fc9..7685484a 100644 --- a/sqlparse/engine/filter_stack.py +++ b/sqlparse/engine/filter_stack.py @@ -20,6 +20,7 @@ def __init__(self, strip_semicolon=False): self.stmtprocess = [] self.postprocess = [] self._grouping = False + self.lexer = lexer.Lexer.get_default_instance() if strip_semicolon: self.stmtprocess.append(StripTrailingSemicolonFilter()) @@ -28,7 +29,7 @@ def enable_grouping(self): def run(self, sql, encoding=None): try: - stream = lexer.tokenize(sql, encoding) + stream = self.lexer.get_tokens(sql, encoding) # Process token stream for filter_ in self.preprocess: stream = filter_.process(stream) diff --git a/tests/test_filter_stack.py b/tests/test_filter_stack.py new file mode 100644 index 00000000..251b2a01 --- /dev/null +++ b/tests/test_filter_stack.py @@ -0,0 +1,15 @@ +from sqlparse import engine, tokens as T + + +class CustomLexer: + def get_tokens(self, sql, encoding=None): + yield T.Keyword, "CUSTOM" + + +def test_filter_stack_custom_lexer(): + stack = engine.FilterStack() + stack.lexer = CustomLexer() + + statement = next(stack.run("ignored input")) + + assert str(statement) == "CUSTOM"