From 05f33b59a2fefbd51303bde74368c73b2317549a Mon Sep 17 00:00:00 2001 From: Olivier Lambert Date: Tue, 7 Jul 2026 17:46:48 +0200 Subject: [PATCH 1/2] Retry the xapi connection instead of latching it broken forever Auth sets masterConnectionBroken when a XenAPI call times out, and OpenSession never attempts another connection once the flag is set. A long-lived xsconsole (e.g. the one agetty spawns on the physical console) that experiences a single 15-second xapi hang therefore never talks to xapi again: the status screen shows 'Pool master is unreachable.' forever, while newly launched xsconsole instances on the same host work fine. This is easy to hit during an HA pool master failover, where xapi on the surviving host can be unresponsive for a while, and was reported after exactly that scenario. Keep the flag as a cooldown rather than a latch: after a timeout, skip connection attempts for CONNECTION_RETRY_SECONDS (so an unresponsive xapi doesn't block the UI for the socket timeout on every update cycle), then try again, and clear the flag on the next successful login. Also log the transitions, since previously the process went quiet with no trace of why. Reproduced and verified on XCP-ng 8.3: freeze xapi (SIGSTOP) for 60s while pressing F5 in xsconsole; before this change the console never recovers, with it the connection recovers within a minute of xapi returning. --- XSConsoleAuth.py | 77 ++++++++++++++++++++++++++++++------------------ 1 file changed, 49 insertions(+), 28 deletions(-) diff --git a/XSConsoleAuth.py b/XSConsoleAuth.py index 5c65636..b5f0ed9 100644 --- a/XSConsoleAuth.py +++ b/XSConsoleAuth.py @@ -35,6 +35,11 @@ class Auth: instance = None + # Seconds to wait before retrying to connect to xapi after a connection + # timeout, so that a long-lived xsconsole recovers once xapi is responsive + # again, e.g. after an HA pool master failover. + CONNECTION_RETRY_SECONDS = 60 + def __init__(self): self.isAuthenticated = False self.loggedInUsername = '' @@ -43,6 +48,7 @@ def __init__(self): self.testingHost = None self.authTimestampSeconds = None self.masterConnectionBroken = False + self.connectionBrokenTimestamp = None socket.setdefaulttimeout(15) self.testMode = False @@ -107,8 +113,7 @@ def TCPAuthenticate(self, inUsername, inPassword): session.logout() except socket.timeout: session = None - self.masterConnectionBroken = True - self.error = 'The master connection has timed out.' + self.ConnectionTimedOut() finally: session.close() @@ -158,43 +163,59 @@ def LogOut(self): self.isAuthenticated = False self.loggedInUsername = None + def ConnectionTimedOut(self): + self.masterConnectionBroken = True + self.connectionBrokenTimestamp = getTimeStamp() + self.error = 'The master connection has timed out.' + XSLog('XenAPI connection timed out - retrying in ' + str(self.CONNECTION_RETRY_SECONDS) + ' seconds at most') + def OpenSession(self): session = None - if not self.masterConnectionBroken: + if self.masterConnectionBroken: + if getTimeStamp() - self.connectionBrokenTimestamp < self.CONNECTION_RETRY_SECONDS: + # Don't hammer an unresponsive xapi with blocking connection attempts + return None + # Restart the cooldown before retrying, so a failure of any kind below + # doesn't lead to a retry on every call + self.connectionBrokenTimestamp = getTimeStamp() + + try: + # Try the local Unix domain socket first + session = XenAPI.xapi_local() + if not session is None: + session.login_with_password('root', '', '', 'XSConsole') + if session._session is None: + session = None + except socket.timeout: + session = None + self.ConnectionTimedOut() + except Exception as e: + session = None + # pylint: disable-next=redefined-variable-type # ToString may handle it + self.error = e + + if session is None and self.testingHost is not None: + # Local session couldn't connect, so try remote. + session = XenAPI.Session("https://" + self.testingHost) try: - # Try the local Unix domain socket first - session = XenAPI.xapi_local() - if not session is None: - session.login_with_password('root','','','XSConsole') - if session._session is None: - session = None + session.login_with_password('root', self.defaultPassword, '', 'XSConsole') + + except XenAPI.Failure as e: + if e.details[0] != 'HOST_IS_SLAVE': # Ignore slave errors when testing + session = None + self.error = e except socket.timeout: session = None - self.masterConnectionBroken = True - self.error = 'The master connection has timed out.' + self.ConnectionTimedOut() except Exception as e: session = None - # pylint: disable-next=redefined-variable-type # ToString may handle it self.error = e - if session is None and self.testingHost is not None: - # Local session couldn't connect, so try remote. - session = XenAPI.Session("https://"+self.testingHost) - try: - session.login_with_password('root', self.defaultPassword,'','XSConsole') + if session is not None and self.masterConnectionBroken: + XSLog('XenAPI connection recovered') + self.masterConnectionBroken = False - except XenAPI.Failure as e: - if e.details[0] != 'HOST_IS_SLAVE': # Ignore slave errors when testing - session = None - self.error = e - except socket.timeout: - session = None - self.masterConnectionBroken = True - self.error = 'The master connection has timed out.' - except Exception as e: - session = None - self.error = e return session def NewSession(self): From e18a9de384770b2e0d5d058a848da391a1a032e0 Mon Sep 17 00:00:00 2001 From: Olivier Lambert Date: Fri, 31 Jul 2026 13:37:30 +0200 Subject: [PATCH 2/2] Remove the write-only Auth.error attribute Nothing has ever read Auth.error: it was only assigned, with types flip-flopping between str, Exception and XenAPI.Failure depending on the failure path. That flip-flop is what pylint's redefined-variable-type check trips over in CI, and the previous disable-next suppression no longer covers the reordered code. Drop the dead attribute and the now-unneeded suppression; the connection timeout path already reports through XSLog. --- XSConsoleAuth.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/XSConsoleAuth.py b/XSConsoleAuth.py index b5f0ed9..ba5c25e 100644 --- a/XSConsoleAuth.py +++ b/XSConsoleAuth.py @@ -166,7 +166,6 @@ def LogOut(self): def ConnectionTimedOut(self): self.masterConnectionBroken = True self.connectionBrokenTimestamp = getTimeStamp() - self.error = 'The master connection has timed out.' XSLog('XenAPI connection timed out - retrying in ' + str(self.CONNECTION_RETRY_SECONDS) + ' seconds at most') def OpenSession(self): @@ -190,10 +189,8 @@ def OpenSession(self): except socket.timeout: session = None self.ConnectionTimedOut() - except Exception as e: + except Exception: session = None - # pylint: disable-next=redefined-variable-type # ToString may handle it - self.error = e if session is None and self.testingHost is not None: # Local session couldn't connect, so try remote. @@ -204,13 +201,11 @@ def OpenSession(self): except XenAPI.Failure as e: if e.details[0] != 'HOST_IS_SLAVE': # Ignore slave errors when testing session = None - self.error = e except socket.timeout: session = None self.ConnectionTimedOut() - except Exception as e: + except Exception: session = None - self.error = e if session is not None and self.masterConnectionBroken: XSLog('XenAPI connection recovered')