From 7cbbaad25299f67b5e33a66a9d12f38bcf7db6c8 Mon Sep 17 00:00:00 2001 From: Daniel Harding Date: Tue, 28 Mar 2023 16:07:23 +0300 Subject: [PATCH 1/3] Strip comments from a token list before sublists. Avoid stripping T.Comment tokens contained within an sql.Comment before stripping the sql.Comment itself. Now an sql.Comment token will be stripped first along with any contained T.Comment tokens. --- sqlparse/filters/others.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqlparse/filters/others.py b/sqlparse/filters/others.py index 95bc436c..cd2ef210 100644 --- a/sqlparse/filters/others.py +++ b/sqlparse/filters/others.py @@ -76,8 +76,8 @@ def _get_insert_token(token): tidx, token = get_next_comment(idx=tidx) def process(self, stmt): - [self.process(sgroup) for sgroup in stmt.get_sublists()] StripCommentsFilter._process(stmt) + [self.process(sgroup) for sgroup in stmt.get_sublists()] return stmt From 97747846f91a508cdaa5f6f9cf7fa578be493b52 Mon Sep 17 00:00:00 2001 From: Daniel Harding Date: Wed, 12 May 2021 14:42:11 +0300 Subject: [PATCH 2/3] Restructure token class hierarchy. Rename Token to TokenBase and make it a superclass for TokenList and a new Token class. Move some of the functionality of TokenBase into Token and TokenList. This will make it easier to maintain separate functionality for Token versus TokenList. --- sqlparse/sql.py | 107 ++++++++++++++++++++++++++++++------------------ 1 file changed, 67 insertions(+), 40 deletions(-) diff --git a/sqlparse/sql.py b/sqlparse/sql.py index ec44a6da..e578f373 100644 --- a/sqlparse/sql.py +++ b/sqlparse/sql.py @@ -39,38 +39,17 @@ def get_alias(self): return self._get_first_name(reverse=True) -class Token: - """Base class for all other classes in this module. +class TokenBase: + """Base class for ``Token`` and ``TokenList``. - It represents a single token and has two instance attributes: - ``value`` is the unchanged value of the token and ``ttype`` is - the type of the token. + It has a single instance attribute, ``parent``, which if not ``None`` + represents the ``TokenList`` that contains this token. """ - __slots__ = ( - 'is_group', - 'is_keyword', - 'is_newline', - 'is_whitespace', - 'normalized', - 'parent', - 'ttype', - 'value', - ) + __slots__ = 'parent' - def __init__(self, ttype, value): - value = str(value) - self.value = value - self.ttype = ttype + def __init__(self): self.parent = None - self.is_group = False - self.is_keyword = ttype in T.Keyword - self.is_whitespace = self.ttype in T.Whitespace - self.is_newline = self.ttype in T.Newline - self.normalized = value.upper() if self.is_keyword else value - - def __str__(self): - return self.value # Pending tokenlist __len__ bug fix # def __len__(self): @@ -84,19 +63,12 @@ def __repr__(self): return "<{cls} {q}{value}{q} at 0x{id:2X}>".format( id=id(self), **locals()) - def _get_repr_name(self): - return str(self.ttype).split('.')[-1] - def _get_repr_value(self): raw = str(self) if len(raw) > 7: raw = raw[:6] + '...' return re.sub(r'\s+', ' ', raw) - def flatten(self): - """Resolve subgroups.""" - yield self - def match(self, ttype, values, regex=False): """Checks whether the token matches the given arguments. @@ -155,24 +127,79 @@ def has_ancestor(self, other): return False -class TokenList(Token): +class Token(TokenBase): + """"A single token. + + It has several additional instance attributes: + ``value`` is the unchanged value of the token + ``ttype`` is the type of the token + ``normalized`` is the value of the token, converted to uppercase if it + is a keyword + ``is_keyword`` is a boolean indicating if the token is a keyword + ``is_whitespace`` is a boolean indicating if the token is whitespace + ``is_newline`` is a boolean indicating if the token is a newline + character + """ + __slots__ = ( + 'is_keyword', + 'is_newline', + 'is_whitespace', + 'normalized', + 'ttype', + 'value', + ) + + is_group = False + + def __init__(self, ttype, value): + super().__init__() + value = str(value) + self.value = value + self.ttype = ttype + self.is_keyword = ttype in T.Keyword + self.is_whitespace = ttype in T.Whitespace + self.is_newline = self.ttype in T.Newline + self.normalized = value.upper() if self.is_keyword else value + + def __str__(self): + return self.value + + def _get_repr_name(self): + return str(self.ttype).split('.')[-1] + + def flatten(self): + """Resolve subgroups.""" + yield self + + +class TokenList(TokenBase): """A group of tokens. - It has an additional instance attribute ``tokens`` which holds a - list of child-tokens. + It has two additional instance attributes, ``value``, which is the value + of the token list, and ``tokens``, which holds a list of child-tokens. """ - __slots__ = 'tokens' + __slots__ = ('tokens', 'value') + + is_group = True + ttype = None + is_keyword = False + is_whitespace = False + is_newline = False def __init__(self, tokens=None): + super().__init__() self.tokens = tokens or [] + self.value = str(self) [setattr(token, 'parent', self) for token in self.tokens] - super().__init__(None, ''.join(token.value for token in self.tokens)) - self.is_group = True def __str__(self): return ''.join(token.value for token in self.flatten()) + @property + def normalized(self): + return self.value + # weird bug # def __len__(self): # return len(self.tokens) From 14abdd603f00080c774052fd609c7f75d8b56d80 Mon Sep 17 00:00:00 2001 From: Daniel Harding Date: Wed, 12 May 2021 14:56:38 +0300 Subject: [PATCH 3/3] Make TokenList.value a property not an attribute. The fact that a new value was being computed each time TokenList.group_tokens() was called caused supralinear runtime when token grouping was enabled. Address by making TokenList.value a dynamically-computed property rather than a static attribute. --- sqlparse/filters/output.py | 4 ++-- sqlparse/sql.py | 17 ++++++++--------- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/sqlparse/filters/output.py b/sqlparse/filters/output.py index d4e20f29..db062fac 100644 --- a/sqlparse/filters/output.py +++ b/sqlparse/filters/output.py @@ -65,7 +65,7 @@ def _process(self, stream, varname, has_nl): # Escape backslashes before quotes so a backslash preceding a # quote cannot break out of the generated string literal # (GHSA-3496-9g83-7v6x). - else: + elif not token.is_group: token.value = token.value.replace('\\', '\\\\').replace("'", "\\'") # Put the token @@ -116,7 +116,7 @@ def _process(self, stream, varname, has_nl): # Escape backslashes before quotes so a backslash preceding a # quote cannot break out of the generated string literal # (GHSA-3496-9g83-7v6x). - else: + elif not token.is_group: token.value = token.value.replace('\\', '\\\\').replace('"', '\\"') # Put the token diff --git a/sqlparse/sql.py b/sqlparse/sql.py index e578f373..ce05ab5c 100644 --- a/sqlparse/sql.py +++ b/sqlparse/sql.py @@ -51,6 +51,9 @@ class TokenBase: def __init__(self): self.parent = None + def __str__(self): + return self.value + # Pending tokenlist __len__ bug fix # def __len__(self): # return len(self.value) @@ -161,9 +164,6 @@ def __init__(self, ttype, value): self.is_newline = self.ttype in T.Newline self.normalized = value.upper() if self.is_keyword else value - def __str__(self): - return self.value - def _get_repr_name(self): return str(self.ttype).split('.')[-1] @@ -175,11 +175,11 @@ def flatten(self): class TokenList(TokenBase): """A group of tokens. - It has two additional instance attributes, ``value``, which is the value - of the token list, and ``tokens``, which holds a list of child-tokens. + It has an additional instance attribute, ``tokens``, which holds a + list of child-tokens. """ - __slots__ = ('tokens', 'value') + __slots__ = 'tokens' is_group = True ttype = None @@ -190,10 +190,10 @@ class TokenList(TokenBase): def __init__(self, tokens=None): super().__init__() self.tokens = tokens or [] - self.value = str(self) [setattr(token, 'parent', self) for token in self.tokens] - def __str__(self): + @property + def value(self): return ''.join(token.value for token in self.flatten()) @property @@ -357,7 +357,6 @@ def group_tokens(self, grp_cls, start, end, include_end=True, grp = start grp.tokens.extend(subtokens) del self.tokens[start_idx + 1:end_idx] - grp.value += ''.join(token.value for token in subtokens) else: subtokens = self.tokens[start_idx:end_idx] grp = grp_cls(subtokens)