From b5d9badf0e4817955fe50ee80f7765a6b1870430 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Slanina?= Date: Tue, 14 Jan 2025 19:52:22 +0100 Subject: [PATCH 1/2] Fix encoding crashes when reading files --- cohesion/filesystem.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cohesion/filesystem.py b/cohesion/filesystem.py index cbaabd8..8ac991d 100644 --- a/cohesion/filesystem.py +++ b/cohesion/filesystem.py @@ -7,7 +7,7 @@ def get_file_contents(filename): """ Return contents of a file """ - with open(filename) as fd: + with open(filename, encoding="utf-8") as fd: return fd.read() From d59723c442be8fb30eb93f208b691c32abc0cf6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Slanina?= Date: Thu, 16 Jan 2025 18:41:37 +0100 Subject: [PATCH 2/2] Use bytes instead of str when reading files --- cohesion/filesystem.py | 2 +- cohesion/module.py | 2 +- cohesion/parser.py | 4 +- tests/test_filesystem.py | 2 +- tests/test_flake8_extension.py | 10 ++--- tests/test_parser.py | 78 +++++++++++++++++----------------- 6 files changed, 49 insertions(+), 49 deletions(-) diff --git a/cohesion/filesystem.py b/cohesion/filesystem.py index 8ac991d..ef3d5ed 100644 --- a/cohesion/filesystem.py +++ b/cohesion/filesystem.py @@ -7,7 +7,7 @@ def get_file_contents(filename): """ Return contents of a file """ - with open(filename, encoding="utf-8") as fd: + with open(filename, "rb") as fd: return fd.read() diff --git a/cohesion/module.py b/cohesion/module.py index af1d77c..7b72289 100644 --- a/cohesion/module.py +++ b/cohesion/module.py @@ -36,7 +36,7 @@ def from_file(cls, filename): @classmethod def from_string(cls, python_string): - module_ast_node = parser.get_ast_node_from_string(python_string) + module_ast_node = parser.get_ast_node_from_str_or_bytes(python_string) return cls(module_ast_node) diff --git a/cohesion/parser.py b/cohesion/parser.py index fafafc9..0d4faab 100644 --- a/cohesion/parser.py +++ b/cohesion/parser.py @@ -162,8 +162,8 @@ def get_module_classes(node): ] -def get_ast_node_from_string(string): +def get_ast_node_from_str_or_bytes(source): """ Return an AST node from a string """ - return ast.parse(string) + return ast.parse(source) diff --git a/tests/test_filesystem.py b/tests/test_filesystem.py index 1592098..2ea63ac 100644 --- a/tests/test_filesystem.py +++ b/tests/test_filesystem.py @@ -24,7 +24,7 @@ def test_get_file_contents(self): contents = textwrap.dedent(""" class Cls(object): pass - """) + """).encode("utf-8") self.fs.create_file( filename, diff --git a/tests/test_flake8_extension.py b/tests/test_flake8_extension.py index f3c82fc..f6ab5d1 100644 --- a/tests/test_flake8_extension.py +++ b/tests/test_flake8_extension.py @@ -15,7 +15,7 @@ def assertEmpty(self, iterable): def test_flake8_extension_empty(self): python_string = textwrap.dedent("") - ast_node = parser.get_ast_node_from_string(python_string) + ast_node = parser.get_ast_node_from_str_or_bytes(python_string) checker = flake8_extension.CohesionChecker(ast_node, "unused") checker.cohesion_below = 0.0 @@ -29,7 +29,7 @@ class Cls(object): pass """) - ast_node = parser.get_ast_node_from_string(python_string) + ast_node = parser.get_ast_node_from_str_or_bytes(python_string) checker = flake8_extension.CohesionChecker(ast_node, "unused") checker.cohesion_below = 0.0 @@ -52,7 +52,7 @@ def func(self): self.variable = 'foo' """) - ast_node = parser.get_ast_node_from_string(python_string) + ast_node = parser.get_ast_node_from_str_or_bytes(python_string) checker = flake8_extension.CohesionChecker(ast_node, "unused") checker.cohesion_below = 50.0 @@ -69,7 +69,7 @@ def func(self): self.variable2 = 'baz' """) - ast_node = parser.get_ast_node_from_string(python_string) + ast_node = parser.get_ast_node_from_str_or_bytes(python_string) checker = flake8_extension.CohesionChecker(ast_node, "unused") checker.cohesion_below = 75.0 @@ -92,7 +92,7 @@ def func(self): self.variable = 'foo' """) - ast_node = parser.get_ast_node_from_string(python_string) + ast_node = parser.get_ast_node_from_str_or_bytes(python_string) checker = flake8_extension.CohesionChecker(ast_node, "unused") checker.cohesion_below = str(0.0) diff --git a/tests/test_parser.py b/tests/test_parser.py index 48df317..446671e 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -17,7 +17,7 @@ def test_valid_syntax(self): a = 5 """) - result = parser.get_ast_node_from_string(python_string) + result = parser.get_ast_node_from_str_or_bytes(python_string) expected = ast.Module self.assertIsInstance(result, expected) @@ -28,7 +28,7 @@ def test_invalid_syntax(self): """) with self.assertRaises(SyntaxError): - parser.get_ast_node_from_string(python_string) + parser.get_ast_node_from_str_or_bytes(python_string) def test_get_module_classes_empty(self): python_string = textwrap.dedent(""" @@ -36,7 +36,7 @@ def func(arg1): print("Hi") """) - node = parser.get_ast_node_from_string(python_string) + node = parser.get_ast_node_from_str_or_bytes(python_string) result = parser.get_module_classes(node) self.assertEmpty(result) @@ -47,7 +47,7 @@ class Cls(object): pass """) - node = parser.get_ast_node_from_string(python_string) + node = parser.get_ast_node_from_str_or_bytes(python_string) classes = parser.get_module_classes(node) result = [cls.name for cls in classes] expected = ["Cls"] @@ -62,7 +62,7 @@ class Cls2(object): pass """) - node = parser.get_ast_node_from_string(python_string) + node = parser.get_ast_node_from_str_or_bytes(python_string) classes = parser.get_module_classes(node) result = [cls.name for cls in classes] expected = ["Cls1", "Cls2"] @@ -75,7 +75,7 @@ class Cls(object): pass """) - node = parser.get_ast_node_from_string(python_string) + node = parser.get_ast_node_from_str_or_bytes(python_string) result = [ method for cls in parser.get_module_classes(node) @@ -91,7 +91,7 @@ def func1(self, arg1): pass """) - node = parser.get_ast_node_from_string(python_string) + node = parser.get_ast_node_from_str_or_bytes(python_string) methods = [ method for cls in parser.get_module_classes(node) @@ -114,7 +114,7 @@ def func3(self, arg1): pass """) - node = parser.get_ast_node_from_string(python_string) + node = parser.get_ast_node_from_str_or_bytes(python_string) methods = [ method for cls in parser.get_module_classes(node) @@ -134,7 +134,7 @@ def func2(arg2): print("Hi") """) - node = parser.get_ast_node_from_string(python_string) + node = parser.get_ast_node_from_str_or_bytes(python_string) methods = [ method for cls in parser.get_module_classes(node) @@ -153,7 +153,7 @@ def func1(self, arg1): pass """) - node = parser.get_ast_node_from_string(python_string) + node = parser.get_ast_node_from_str_or_bytes(python_string) methods = [ method for cls in parser.get_module_classes(node) @@ -171,7 +171,7 @@ def func(self, arg1): pass """) - node = parser.get_ast_node_from_string(python_string) + node = parser.get_ast_node_from_str_or_bytes(python_string) methods = [ method for cls in parser.get_module_classes(node) @@ -191,7 +191,7 @@ def func(self, arg1): pass """) - node = parser.get_ast_node_from_string(python_string) + node = parser.get_ast_node_from_str_or_bytes(python_string) methods = [ method for cls in parser.get_module_classes(node) @@ -210,7 +210,7 @@ def func(self, arg1): pass """) - node = parser.get_ast_node_from_string(python_string) + node = parser.get_ast_node_from_str_or_bytes(python_string) methods = [ method for cls in parser.get_module_classes(node) @@ -229,7 +229,7 @@ def func(self, arg1): pass """) - node = parser.get_ast_node_from_string(python_string) + node = parser.get_ast_node_from_str_or_bytes(python_string) methods = [ method for cls in parser.get_module_classes(node) @@ -248,7 +248,7 @@ def func(self, arg1): pass """) - node = parser.get_ast_node_from_string(python_string) + node = parser.get_ast_node_from_str_or_bytes(python_string) methods = [ method for cls in parser.get_module_classes(node) @@ -267,7 +267,7 @@ def func(self, arg1): pass """) - node = parser.get_ast_node_from_string(python_string) + node = parser.get_ast_node_from_str_or_bytes(python_string) methods = [ method for cls in parser.get_module_classes(node) @@ -286,7 +286,7 @@ def func(self, arg1): pass """) - node = parser.get_ast_node_from_string(python_string) + node = parser.get_ast_node_from_str_or_bytes(python_string) methods = [ method for cls in parser.get_module_classes(node) @@ -305,7 +305,7 @@ def func(self, arg1): pass """) - node = parser.get_ast_node_from_string(python_string) + node = parser.get_ast_node_from_str_or_bytes(python_string) methods = [ method for cls in parser.get_module_classes(node) @@ -324,7 +324,7 @@ def func(self, arg1): pass """) - node = parser.get_ast_node_from_string(python_string) + node = parser.get_ast_node_from_str_or_bytes(python_string) methods = [ method for cls in parser.get_module_classes(node) @@ -342,7 +342,7 @@ def func(this, arg1): pass """) - node = parser.get_ast_node_from_string(python_string) + node = parser.get_ast_node_from_str_or_bytes(python_string) methods = [ method for cls in parser.get_module_classes(node) @@ -362,7 +362,7 @@ def func(arg1): pass """) - node = parser.get_ast_node_from_string(python_string) + node = parser.get_ast_node_from_str_or_bytes(python_string) methods = [ method for cls in parser.get_module_classes(node) @@ -381,7 +381,7 @@ def func(): pass """) - node = parser.get_ast_node_from_string(python_string) + node = parser.get_ast_node_from_str_or_bytes(python_string) methods = [ method for cls in parser.get_module_classes(node) @@ -399,7 +399,7 @@ def func(arg1): pass """) - node = parser.get_ast_node_from_string(python_string) + node = parser.get_ast_node_from_str_or_bytes(python_string) methods = [ method for cls in parser.get_module_classes(node) @@ -417,7 +417,7 @@ def __init__(self): self.attr = 5 """) - node = parser.get_ast_node_from_string(python_string) + node = parser.get_ast_node_from_str_or_bytes(python_string) instance_variables = parser.get_instance_variables(node) result = [instance_variable.attr for instance_variable in instance_variables] expected = ["attr"] @@ -432,7 +432,7 @@ def __init__(self): self.attr2 = 6 """) - node = parser.get_ast_node_from_string(python_string) + node = parser.get_ast_node_from_str_or_bytes(python_string) instance_variables = parser.get_instance_variables(node) result = [instance_variable.attr for instance_variable in instance_variables] expected = ["attr1", "attr2"] @@ -446,7 +446,7 @@ def __init__(self): self.attr1 = self.attr2 = 5 """) - node = parser.get_ast_node_from_string(python_string) + node = parser.get_ast_node_from_str_or_bytes(python_string) instance_variables = parser.get_instance_variables(node) result = [instance_variable.attr for instance_variable in instance_variables] expected = ["attr1", "attr2"] @@ -459,7 +459,7 @@ class Cls(object): attr = 5 """) - node = parser.get_ast_node_from_string(python_string) + node = parser.get_ast_node_from_str_or_bytes(python_string) instance_variables = parser.get_instance_variables(node) result = [instance_variable.attr for instance_variable in instance_variables] @@ -471,7 +471,7 @@ class Cls(object): attr = 5 """) - node = parser.get_ast_node_from_string(python_string) + node = parser.get_ast_node_from_str_or_bytes(python_string) classes = parser.get_module_classes(node) class_variables = [ class_variable @@ -489,7 +489,7 @@ class Cls(object): attr1 = attr2 = 5 """) - node = parser.get_ast_node_from_string(python_string) + node = parser.get_ast_node_from_str_or_bytes(python_string) classes = parser.get_module_classes(node) class_variables = [ class_variable @@ -508,7 +508,7 @@ def __init__(self): self.attr = 5 """) - node = parser.get_ast_node_from_string(python_string) + node = parser.get_ast_node_from_str_or_bytes(python_string) classes = parser.get_module_classes(node) class_variables = [ class_variable @@ -527,7 +527,7 @@ def __init__(self): self.attr2 = 6 """) - node = parser.get_ast_node_from_string(python_string) + node = parser.get_ast_node_from_str_or_bytes(python_string) classes = parser.get_module_classes(node) result = [ name @@ -545,7 +545,7 @@ def __init__(self): self.attr = 6 """) - node = parser.get_ast_node_from_string(python_string) + node = parser.get_ast_node_from_str_or_bytes(python_string) classes = parser.get_module_classes(node) result = [ name @@ -564,7 +564,7 @@ def __init__(self): pass """) - node = parser.get_ast_node_from_string(python_string) + node = parser.get_ast_node_from_str_or_bytes(python_string) classes = parser.get_module_classes(node) result = [ name @@ -583,7 +583,7 @@ def func(self): pass """) - node = parser.get_ast_node_from_string(python_string) + node = parser.get_ast_node_from_str_or_bytes(python_string) classes = parser.get_module_classes(node) result = [ name @@ -603,7 +603,7 @@ def func(self): self.attr2 = 6 """) - node = parser.get_ast_node_from_string(python_string) + node = parser.get_ast_node_from_str_or_bytes(python_string) class_methods = [ method for cls in parser.get_module_classes(node) @@ -629,7 +629,7 @@ def func2(self): self.attr2 = self.attr1 """) - node = parser.get_ast_node_from_string(python_string) + node = parser.get_ast_node_from_str_or_bytes(python_string) class_methods = [ method for cls in parser.get_module_classes(node) @@ -653,7 +653,7 @@ def func(self): self.attr1 = 5 """) - node = parser.get_ast_node_from_string(python_string) + node = parser.get_ast_node_from_str_or_bytes(python_string) class_methods = [ method for cls in parser.get_module_classes(node) @@ -677,7 +677,7 @@ def func(self): otherclass.attr2 = 6 """) - node = parser.get_ast_node_from_string(python_string) + node = parser.get_ast_node_from_str_or_bytes(python_string) class_methods = [ method for cls in parser.get_module_classes(node) @@ -711,7 +711,7 @@ def f(self, choice): c.f(False) """) - node = parser.get_ast_node_from_string(python_string) + node = parser.get_ast_node_from_str_or_bytes(python_string) classes = parser.get_module_classes(node) result = [ name