From 9f8940cdb80c4f4f2bcc10d451ff7e9929d69a20 Mon Sep 17 00:00:00 2001 From: James Wilkinson Date: Mon, 8 Jun 2026 20:31:31 +0100 Subject: [PATCH 1/4] feat: raise TM1pyNetworkException for HTML/Cloudflare responses - Added TM1pyNetworkException to handle upstream HTML/WAF/Cloudflare error pages - Updated RestService.verify_response to detect non-JSON HTML responses - Added comprehensive unit tests for exception behaviour and verify_response integration --- TM1py/Exceptions/Exceptions.py | 64 +++++++++- TM1py/Services/RestService.py | 16 ++- Tests/NetworkException_test.py | 215 +++++++++++++++++++++++++++++++++ 3 files changed, 292 insertions(+), 3 deletions(-) create mode 100644 Tests/NetworkException_test.py diff --git a/TM1py/Exceptions/Exceptions.py b/TM1py/Exceptions/Exceptions.py index 4b3132fb..7ab662c0 100644 --- a/TM1py/Exceptions/Exceptions.py +++ b/TM1py/Exceptions/Exceptions.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- # TM1py Exceptions are defined here +import re from typing import List, Mapping @@ -172,6 +173,67 @@ def __str__(self): self.message, self._status_code, self._reason, self._headers ) +class TM1pyNetworkException(TM1pyException): + """Exception for network/upstream errors where the request never reached TM1. + + Raised when a non-JSON response is received (e.g. an HTML block page from + Cloudflare or another WAF/proxy), indicating the issue is infrastructure-level + rather than a TM1 REST API error. + """ + + def __init__(self, response: str, status_code: int, reason: str, headers: Mapping): + """ + :param response: Response text (e.g. raw HTML from a block page) + :param status_code: HTTP status code + :param reason: Reason phrase + :param headers: HTTP headers + """ + super().__init__(response) + self._status_code = status_code + self._reason = reason + self._headers = headers + self._ray_id = self._extract_cloudflare_ray_id(response) + + @staticmethod + def _extract_cloudflare_ray_id(text: str) -> str: + """Extract Cloudflare Ray ID from an HTML block page, if present.""" + match = re.search(r"Ray ID[:\s]+([a-f0-9]+)", text, re.IGNORECASE) + return match.group(1) if match else None + + @property + def status_code(self) -> int: + """HTTP status code.""" + return self._status_code + + @property + def reason(self) -> str: + """Reason phrase.""" + return self._reason + + @property + def response(self) -> str: + """Response text.""" + return self.message + + @property + def headers(self) -> Mapping: + """HTTP headers.""" + return self._headers + + @property + def ray_id(self): + """Cloudflare Ray ID if the block page contained one, else None.""" + return self._ray_id + + def __str__(self) -> str: + msg = ( + f"Network/upstream error — request was likely blocked before reaching TM1. " + f"Status Code: {self._status_code} - Reason: '{self._reason}'" + ) + if self._ray_id: + msg += f" - Cloudflare Ray ID: {self._ray_id}" + return msg + class TM1pyWriteFailureException(TM1pyException): """Exception for complete failure of write operations.""" @@ -205,4 +267,4 @@ def __init__(self, statuses: List[str], error_log_files: List[str], attempts: in f"{len(self.statuses)} out of {self.attempts} write operations failed partially. " f"Details: {self.error_log_files}" ) - super(TM1pyWritePartialFailureException, self).__init__(message) + super(TM1pyWritePartialFailureException, self).__init__(message) \ No newline at end of file diff --git a/TM1py/Services/RestService.py b/TM1py/Services/RestService.py index d14b4656..c8f74144 100644 --- a/TM1py/Services/RestService.py +++ b/TM1py/Services/RestService.py @@ -20,7 +20,7 @@ from requests.auth import HTTPBasicAuth from urllib3._collections import HTTPHeaderDict -from TM1py.Exceptions.Exceptions import TM1pyTimeout, TM1pyVersionDeprecationException +from TM1py.Exceptions.Exceptions import TM1pyTimeout, TM1pyVersionDeprecationException, TM1pyNetworkException from TM1py.Utils import ( CaseAndSpaceInsensitiveSet, HTTPAdapterWithSocketOptions, @@ -1152,9 +1152,21 @@ def verify_response(response: Response): :Exceptions: TM1pyException, raises TM1pyException when Code is not 200, 204 etc. """ + if not response.ok: + content_type = response.headers.get("Content-Type", "") + if "text/html" in content_type or response.text.lstrip().startswith(" + +Attention Required! | Cloudflare + +

You have been blocked

+

Sorry, you have been blocked from accessing this resource.

+

Cloudflare Ray ID: 9ef02047ab37aac9

+ +""" + +CLOUDFLARE_HTML_COLON_FORMAT = """ + + +

Ray ID: abc123def456

+ +""" + +CLOUDFLARE_HTML_SPACE_FORMAT = """ + + +

Ray ID abc123def456

+ +""" + +GENERIC_HTML_NO_RAY_ID = """ + +502 Bad Gateway + +

502 Bad Gateway

+

nginx

+ +""" + +SAMPLE_HEADERS = {"Content-Type": "text/html", "Connection": "keep-alive"} + +class TestNetworkException(unittest.TestCase): + + # Instantiation & properties + def test_status_code_property(self): + exc = TM1pyNetworkException( + response=GENERIC_HTML_NO_RAY_ID, + status_code=403, + reason="Forbidden", + headers=SAMPLE_HEADERS, + ) + self.assertEqual(exc.status_code, 403) + + def test_reason_property(self): + exc = TM1pyNetworkException( + response=GENERIC_HTML_NO_RAY_ID, + status_code=403, + reason="Forbidden", + headers=SAMPLE_HEADERS, + ) + self.assertEqual(exc.reason, "Forbidden") + + def test_headers_property(self): + exc = TM1pyNetworkException( + response=GENERIC_HTML_NO_RAY_ID, + status_code=403, + reason="Forbidden", + headers=SAMPLE_HEADERS, + ) + self.assertEqual(exc.headers, SAMPLE_HEADERS) + + def test_response_property_mirrors_message(self): + exc = TM1pyNetworkException( + response=GENERIC_HTML_NO_RAY_ID, + status_code=403, + reason="Forbidden", + headers=SAMPLE_HEADERS, + ) + self.assertEqual(exc.response, exc.message) + self.assertEqual(exc.response, GENERIC_HTML_NO_RAY_ID) + + # __str__test + def test_str_without_ray_id(self): + exc = TM1pyNetworkException( + response=GENERIC_HTML_NO_RAY_ID, + status_code=502, + reason="Bad Gateway", + headers=SAMPLE_HEADERS, + ) + result = str(exc) + self.assertIn("502", result) + self.assertIn("Bad Gateway", result) + self.assertNotIn("Cloudflare Ray ID", result) + + def test_str_with_ray_id(self): + exc = TM1pyNetworkException( + response=CLOUDFLARE_HTML_WITH_RAY_ID, + status_code=403, + reason="Forbidden", + headers=SAMPLE_HEADERS, + ) + result = str(exc) + self.assertIn("403", result) + self.assertIn("Forbidden", result) + self.assertIn("9ef02047ab37aac9", result) + self.assertIn("Cloudflare Ray ID", result) + + + # Ray ID extraction + def test_ray_id_extracted_with_colon_format(self): + exc = TM1pyNetworkException( + response=CLOUDFLARE_HTML_COLON_FORMAT, + status_code=403, + reason="Forbidden", + headers=SAMPLE_HEADERS, + ) + self.assertEqual(exc.ray_id, "abc123def456") + + def test_ray_id_extracted_with_space_format(self): + exc = TM1pyNetworkException( + response=CLOUDFLARE_HTML_SPACE_FORMAT, + status_code=403, + reason="Forbidden", + headers=SAMPLE_HEADERS, + ) + self.assertEqual(exc.ray_id, "abc123def456") + + def test_ray_id_extracted_from_real_cloudflare_html(self): + exc = TM1pyNetworkException( + response=CLOUDFLARE_HTML_WITH_RAY_ID, + status_code=403, + reason="Forbidden", + headers=SAMPLE_HEADERS, + ) + self.assertEqual(exc.ray_id, "9ef02047ab37aac9") + + def test_ray_id_is_none_for_generic_html(self): + exc = TM1pyNetworkException( + response=GENERIC_HTML_NO_RAY_ID, + status_code=502, + reason="Bad Gateway", + headers=SAMPLE_HEADERS, + ) + self.assertIsNone(exc.ray_id) + + def test_ray_id_is_none_for_empty_response(self): + exc = TM1pyNetworkException( + response="", + status_code=403, + reason="Forbidden", + headers=SAMPLE_HEADERS, + ) + self.assertIsNone(exc.ray_id) + + + # Inheritance test + def test_is_instance_of_tm1py_exception(self): + exc = TM1pyNetworkException( + response=GENERIC_HTML_NO_RAY_ID, + status_code=403, + reason="Forbidden", + headers=SAMPLE_HEADERS, + ) + self.assertIsInstance(exc, TM1pyException) + + def test_is_not_instance_of_tm1py_rest_exception(self): + exc = TM1pyNetworkException( + response=GENERIC_HTML_NO_RAY_ID, + status_code=403, + reason="Forbidden", + headers=SAMPLE_HEADERS, + ) + self.assertNotIsInstance(exc, TM1pyRestException) + + def test_can_be_raised_and_caught_as_tm1py_exception(self): + with self.assertRaises(TM1pyException): + raise TM1pyNetworkException( + response=GENERIC_HTML_NO_RAY_ID, + status_code=403, + reason="Forbidden", + headers=SAMPLE_HEADERS, + ) + + def test_not_caught_by_tm1py_rest_exception_handler(self): + """Existing except TM1pyRestException blocks must NOT swallow this exception.""" + caught_as_rest = False + try: + raise TM1pyNetworkException( + response=GENERIC_HTML_NO_RAY_ID, + status_code=403, + reason="Forbidden", + headers=SAMPLE_HEADERS, + ) + except TM1pyRestException: + caught_as_rest = True + except TM1pyException: + pass + + self.assertFalse(caught_as_rest, "TM1pyNetworkException must not be caught by TM1pyRestException handler") + + def test_verify_response_raises_network_exception_for_html(self): + response = Response() + response.status_code = 403 + response._content = CLOUDFLARE_HTML_WITH_RAY_ID.encode("utf-8") + response.headers = {"Content-Type": "text/html"} + response.reason = "Forbidden" + + with self.assertRaises(TM1pyNetworkException): + RestService.verify_response(response) + +if __name__ == "__main__": + unittest.main() \ No newline at end of file From 1c35535146bf3bbd3c04cea3fa983b2948035432 Mon Sep 17 00:00:00 2001 From: James Wilkinson Date: Tue, 9 Jun 2026 11:47:35 +0100 Subject: [PATCH 2/4] fix: address PR review feedback - Merge exception imports onto single line using package path in RestService.py - Add TM1pyNetworkException to retry loop alongside TM1pyRestException - Add headers paramater to _extract_cloudflare_ray_id to first check CF-RAY header --- TM1py/Exceptions/Exceptions.py | 7 +++++-- TM1py/Exceptions/__init__.py | 1 + TM1py/Services/RestService.py | 7 +++---- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/TM1py/Exceptions/Exceptions.py b/TM1py/Exceptions/Exceptions.py index 7ab662c0..1079b100 100644 --- a/TM1py/Exceptions/Exceptions.py +++ b/TM1py/Exceptions/Exceptions.py @@ -192,11 +192,14 @@ def __init__(self, response: str, status_code: int, reason: str, headers: Mappin self._status_code = status_code self._reason = reason self._headers = headers - self._ray_id = self._extract_cloudflare_ray_id(response) + self._ray_id = self._extract_cloudflare_ray_id(response, headers) @staticmethod - def _extract_cloudflare_ray_id(text: str) -> str: + def _extract_cloudflare_ray_id(text: str, headers: Mapping) -> str: """Extract Cloudflare Ray ID from an HTML block page, if present.""" + ray_id = headers.get("CF-RAY", "") + if ray_id: + return ray_id match = re.search(r"Ray ID[:\s]+([a-f0-9]+)", text, re.IGNORECASE) return match.group(1) if match else None diff --git a/TM1py/Exceptions/__init__.py b/TM1py/Exceptions/__init__.py index d84d59a2..c42d49a3 100644 --- a/TM1py/Exceptions/__init__.py +++ b/TM1py/Exceptions/__init__.py @@ -7,4 +7,5 @@ TM1pyVersionException, TM1pyWriteFailureException, TM1pyWritePartialFailureException, + TM1pyNetworkException, ) diff --git a/TM1py/Services/RestService.py b/TM1py/Services/RestService.py index c8f74144..1072bdb8 100644 --- a/TM1py/Services/RestService.py +++ b/TM1py/Services/RestService.py @@ -20,7 +20,6 @@ from requests.auth import HTTPBasicAuth from urllib3._collections import HTTPHeaderDict -from TM1py.Exceptions.Exceptions import TM1pyTimeout, TM1pyVersionDeprecationException, TM1pyNetworkException from TM1py.Utils import ( CaseAndSpaceInsensitiveSet, HTTPAdapterWithSocketOptions, @@ -35,7 +34,7 @@ import http.client as http_client -from TM1py.Exceptions import TM1pyRestException +from TM1py.Exceptions import TM1pyTimeout, TM1pyVersionDeprecationException, TM1pyNetworkException, TM1pyRestException class AuthenticationMode(Enum): @@ -507,7 +506,7 @@ def _handle_remote_disconnect( except TM1pyTimeout: # Re-raise timeout exceptions as-is raise - except TM1pyRestException: + except (TM1pyRestException, TM1pyNetworkException): # Re-raise TM1 exceptions as-is raise except Exception as retry_error: @@ -1155,7 +1154,7 @@ def verify_response(response: Response): if not response.ok: content_type = response.headers.get("Content-Type", "") - if "text/html" in content_type or response.text.lstrip().startswith(" Date: Tue, 9 Jun 2026 10:11:26 -0300 Subject: [PATCH 3/4] fix: address review nits + export regression on TM1pyNetworkException - Export TM1pyVersionDeprecationException from TM1py.Exceptions (the package-path import in RestService introduced in the prior commit referenced a name that was never exported, breaking `import TM1py`) - Broaden HTML detection: case-insensitive, also match bare - Type hints: Optional[str] for _extract_cloudflare_ray_id and ray_id - Document the intentional non-subclassing of TM1pyRestException - Add regression tests: JSON error still raises TM1pyRestException, isolate each HTML detection branch, OK response raises nothing - Cleanup: trailing whitespace, missing EOF newlines, import sorting Co-Authored-By: Claude Opus 4.8 (1M context) --- TM1py/Exceptions/Exceptions.py | 15 ++++++--- TM1py/Exceptions/__init__.py | 3 +- TM1py/Services/RestService.py | 11 ++++-- Tests/NetworkException_test.py | 61 +++++++++++++++++++++++++++++++--- 4 files changed, 77 insertions(+), 13 deletions(-) diff --git a/TM1py/Exceptions/Exceptions.py b/TM1py/Exceptions/Exceptions.py index 1079b100..ed9c71aa 100644 --- a/TM1py/Exceptions/Exceptions.py +++ b/TM1py/Exceptions/Exceptions.py @@ -2,7 +2,7 @@ # TM1py Exceptions are defined here import re -from typing import List, Mapping +from typing import List, Mapping, Optional class TM1pyTimeout(Exception): @@ -173,12 +173,17 @@ def __str__(self): self.message, self._status_code, self._reason, self._headers ) + class TM1pyNetworkException(TM1pyException): """Exception for network/upstream errors where the request never reached TM1. Raised when a non-JSON response is received (e.g. an HTML block page from Cloudflare or another WAF/proxy), indicating the issue is infrastructure-level rather than a TM1 REST API error. + + Intentionally does NOT subclass TM1pyRestException so that existing + `except TM1pyRestException` handlers do not swallow infrastructure-level + failures. Both remain siblings under TM1pyException. """ def __init__(self, response: str, status_code: int, reason: str, headers: Mapping): @@ -192,10 +197,10 @@ def __init__(self, response: str, status_code: int, reason: str, headers: Mappin self._status_code = status_code self._reason = reason self._headers = headers - self._ray_id = self._extract_cloudflare_ray_id(response, headers) + self._ray_id = self._extract_cloudflare_ray_id(response, headers) @staticmethod - def _extract_cloudflare_ray_id(text: str, headers: Mapping) -> str: + def _extract_cloudflare_ray_id(text: str, headers: Mapping) -> Optional[str]: """Extract Cloudflare Ray ID from an HTML block page, if present.""" ray_id = headers.get("CF-RAY", "") if ray_id: @@ -224,7 +229,7 @@ def headers(self) -> Mapping: return self._headers @property - def ray_id(self): + def ray_id(self) -> Optional[str]: """Cloudflare Ray ID if the block page contained one, else None.""" return self._ray_id @@ -270,4 +275,4 @@ def __init__(self, statuses: List[str], error_log_files: List[str], attempts: in f"{len(self.statuses)} out of {self.attempts} write operations failed partially. " f"Details: {self.error_log_files}" ) - super(TM1pyWritePartialFailureException, self).__init__(message) \ No newline at end of file + super(TM1pyWritePartialFailureException, self).__init__(message) diff --git a/TM1py/Exceptions/__init__.py b/TM1py/Exceptions/__init__.py index c42d49a3..b2150408 100644 --- a/TM1py/Exceptions/__init__.py +++ b/TM1py/Exceptions/__init__.py @@ -1,11 +1,12 @@ # ruff: noqa: F401 from TM1py.Exceptions.Exceptions import ( TM1pyException, + TM1pyNetworkException, TM1pyNotAdminException, TM1pyRestException, TM1pyTimeout, + TM1pyVersionDeprecationException, TM1pyVersionException, TM1pyWriteFailureException, TM1pyWritePartialFailureException, - TM1pyNetworkException, ) diff --git a/TM1py/Services/RestService.py b/TM1py/Services/RestService.py index 1072bdb8..520151d2 100644 --- a/TM1py/Services/RestService.py +++ b/TM1py/Services/RestService.py @@ -34,7 +34,12 @@ import http.client as http_client -from TM1py.Exceptions import TM1pyTimeout, TM1pyVersionDeprecationException, TM1pyNetworkException, TM1pyRestException +from TM1py.Exceptions import ( + TM1pyNetworkException, + TM1pyRestException, + TM1pyTimeout, + TM1pyVersionDeprecationException, +) class AuthenticationMode(Enum): @@ -1151,10 +1156,10 @@ def verify_response(response: Response): :Exceptions: TM1pyException, raises TM1pyException when Code is not 200, 204 etc. """ - if not response.ok: content_type = response.headers.get("Content-Type", "") - if "text/html" in content_type.lower() or response.text.lstrip().startswith(" @@ -207,9 +211,58 @@ def test_verify_response_raises_network_exception_for_html(self): response._content = CLOUDFLARE_HTML_WITH_RAY_ID.encode("utf-8") response.headers = {"Content-Type": "text/html"} response.reason = "Forbidden" - + + with self.assertRaises(TM1pyNetworkException): + RestService.verify_response(response) + + def test_verify_response_raises_network_exception_for_html_body_without_html_content_type(self): + """HTML body detected via the blocked" + response.headers = {"Content-Type": "text/html; charset=utf-8"} + response.reason = "Forbidden" + + with self.assertRaises(TM1pyNetworkException): + RestService.verify_response(response) + + def test_verify_response_raises_rest_exception_for_json_error(self): + """Genuine TM1 REST errors (JSON, non-HTML) must still raise TM1pyRestException.""" + response = Response() + response.status_code = 400 + response._content = b'{"error":{"message":"Invalid MDX statement"}}' + response.headers = {"Content-Type": "application/json"} + response.reason = "Bad Request" + + with self.assertRaises(TM1pyRestException): + RestService.verify_response(response) + + try: + RestService.verify_response(response) + except TM1pyRestException as exc: + self.assertNotIsInstance(exc, TM1pyNetworkException) + + def test_verify_response_does_not_raise_for_ok_response(self): + """A successful response must pass through without raising, even with an HTML-ish body.""" + response = Response() + response.status_code = 200 + response._content = b"" + response.headers = {"Content-Type": "text/html"} + response.reason = "OK" + + self.assertIsNone(RestService.verify_response(response)) + + if __name__ == "__main__": - unittest.main() \ No newline at end of file + unittest.main() From e7186488a61f0620c40bc1693590cb768d9c1c10 Mon Sep 17 00:00:00 2001 From: nicolasbisurgi Date: Wed, 10 Jun 2026 10:01:14 -0300 Subject: [PATCH 4/4] style: apply black formatting (line-length 120) Collapse multi-line exception calls and fix blank-line/comment whitespace to satisfy the PR-validation black --check step. Co-Authored-By: Claude Opus 4.8 (1M context) --- TM1py/Services/RestService.py | 10 ++-------- Tests/NetworkException_test.py | 9 ++++----- 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/TM1py/Services/RestService.py b/TM1py/Services/RestService.py index 520151d2..fcab731f 100644 --- a/TM1py/Services/RestService.py +++ b/TM1py/Services/RestService.py @@ -1161,16 +1161,10 @@ def verify_response(response: Response): body_start = response.text.lstrip().lower() if "text/html" in content_type.lower() or body_start.startswith(("