From 666877784509be1c53ac3da4ae11ca8ca47c76d0 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 17 Nov 2026 17:40:45 +0000 Subject: [PATCH] Retry hosted verification soon after a failed fetch; warn once per outage The desktop bot reads verification from the hosted list. When the first fetch at boot failed, nothing retried for 30 minutes, so every relayed chat line went out as plain text and logged "Chat relay identity lookup unavailable". The periodic refresh also ran exactly at the 30-minute freshness limit, leaving a brief stale window every cycle. - Failed hosted refresh retries after 30, 60, 120, then 300 seconds; success resets the backoff and cancels the pending retry. - Hosted read-only refresh runs every 15 minutes, inside the freshness window. - The chat relay warning is logged once per outage, not per line. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016WTP2UbZ584w2CXPdx7Vin --- src/Civ13/GameServer.php | 9 ++++- src/Civ13/Verifier.php | 41 ++++++++++++++++++++-- tests/local/admin-verification-refresh.php | 19 ++++++++++ tests/local/chat-moderation.php | 19 +++++++++- 4 files changed, 84 insertions(+), 4 deletions(-) diff --git a/src/Civ13/GameServer.php b/src/Civ13/GameServer.php index c63ab7bb..5cd75261 100644 --- a/src/Civ13/GameServer.php +++ b/src/Civ13/GameServer.php @@ -67,6 +67,8 @@ class GameServer public Logger $logger; public StreamSelectLoop $loop; public bool $ready = false; + /** True while the chat relay is sending unverified text because hosted verification is unavailable; limits the warning to once per outage. */ + private bool $relayIdentityUnavailable = false; private ?NativeLifecycle $nativeLifecycle = null; /** Opt-in confirmed `manage` controls for a shard_group region (docs/launcher-management.md). */ public ?\Civ13\Local\LauncherManagement $launcherManagement = null; @@ -671,10 +673,15 @@ private function __gameChatRelay(Channel|Thread|string $channel, array $array, ? } try { $item = $this->civ13->verifier->get('ss13', Civ13::sanitizeInput($array['ckey'])); + $this->relayIdentityUnavailable = false; } catch (VerifierException) { // The hosted list is not a current identity proof. Keep chat visible // as plain text without using retained or stale verification data. - $this->logger->warning('Chat relay identity lookup unavailable; sending unverified display text'); + // Warn once per outage instead of once per chat line. + if (! $this->relayIdentityUnavailable) { + $this->relayIdentityUnavailable = true; + $this->logger->warning('Chat relay identity lookup unavailable; sending unverified display text until hosted verification refreshes'); + } $item = null; } if (! $item) { diff --git a/src/Civ13/Verifier.php b/src/Civ13/Verifier.php index c5e0afe3..704ec7f8 100644 --- a/src/Civ13/Verifier.php +++ b/src/Civ13/Verifier.php @@ -42,6 +42,11 @@ class Verifier private ?HostedVerificationSnapshot $hosted = null; private ?\Closure $hostedTransportFetch = null; private ?\React\EventLoop\LoopInterface $hostedTransportLoop = null; + /** Hosted refresh period; well inside HostedVerificationSnapshot::FRESH_SECONDS so a normal refresh never lets the list lapse. */ + public const HOSTED_REFRESH_SECONDS = 900; + /** Backoff after a failed hosted refresh, so one failed fetch (often at boot) does not leave the list unavailable until the next period. */ + public const HOSTED_RETRY_SECONDS = [30, 60, 120, 300]; + private int $hostedRetryAttempt = 0; private const READ_ONLY_MESSAGE = 'Desktop uses the hosted Civilizationbot verification source. Complete verification with the primary bot.'; public CollectionInterface $verified; // This probably needs a default value for Collection, maybe make it a Repository instead? public CollectionInterface $pending; @@ -1271,11 +1276,11 @@ public function getVerifiedUser(Member|User|array|string|null $input): ?User return null; } - /** The 30-minute periodic timer that re-checks verifier connectivity and replays provisional registrations when it comes back online. */ + /** The periodic timer (15 minutes for hosted read-only, otherwise 30) that re-checks verifier connectivity and replays provisional registrations when it comes back online. */ public function verifierStatusTimer(): TimerInterface { if (! isset($this->timers['verifier_status_timer'])) { - $this->timers['verifier_status_timer'] = $this->discord->getLoop()->addPeriodicTimer(1800, function () { + $this->timers['verifier_status_timer'] = $this->discord->getLoop()->addPeriodicTimer($this->readOnly ? self::HOSTED_REFRESH_SECONDS : 1800, function () { if ($this->readOnly) { $this->refreshHosted(); return; @@ -1373,9 +1378,41 @@ public function refreshHosted(): PromiseInterface } } $this->logger->info('Hosted verification refresh', $status); + if (($status['coalesced'] ?? true) === false) { + // Retry scheduling is best effort and must not turn a finished refresh into a rejection. + try { $status['available'] === true ? $this->hostedRetryReset() : $this->scheduleHostedRetry(); } + catch (\Throwable) { + try { $this->logger->warning('Hosted verification retry could not be scheduled'); } catch (\Throwable) {} + } + } return $status; }); } + + /** Schedules one bounded retry after a failed hosted refresh; never throws into the loop. */ + private function scheduleHostedRetry(): void + { + if (isset($this->timers['hosted_retry'])) { return; } + $delays = self::HOSTED_RETRY_SECONDS; + $delay = $delays[min($this->hostedRetryAttempt, count($delays) - 1)]; + $this->hostedRetryAttempt++; + $this->timers['hosted_retry'] = $this->civ13->loop->addTimer($delay, function (): void { + unset($this->timers['hosted_retry']); + try { $this->refreshHosted(); } + catch (\Throwable) { + try { $this->logger->warning('Hosted verification retry failed to start'); } catch (\Throwable) {} + } + }); + } + + private function hostedRetryReset(): void + { + $this->hostedRetryAttempt = 0; + if (isset($this->timers['hosted_retry'])) { + $this->civ13->loop->cancelTimer($this->timers['hosted_retry']); + unset($this->timers['hosted_retry']); + } + } // Magic Methods /** * Cancels every timer this verifier registered. diff --git a/tests/local/admin-verification-refresh.php b/tests/local/admin-verification-refresh.php index 6be52d8b..343d7d10 100644 --- a/tests/local/admin-verification-refresh.php +++ b/tests/local/admin-verification-refresh.php @@ -72,6 +72,7 @@ public function warning(string|\Stringable $message,array $context=[]): void { }; $owner=(new \ReflectionClass(RefreshObserver::class))->newInstanceWithoutConstructor(); $owner->browser=(new \ReflectionClass(\React\Http\Browser::class))->newInstanceWithoutConstructor(); + $owner->loop=new \React\EventLoop\StreamSelectLoop(); // Failed refreshes schedule a retry timer; this loop never runs. $verifier=(new \ReflectionClass(Verifier::class))->newInstanceWithoutConstructor(); $verifier->civ13=$owner; $owner->verifier=$verifier; (new \ReflectionProperty($verifier,'verify_url'))->setValue($verifier,'http://civ13.com:55555/verified'); @@ -148,6 +149,24 @@ public function warning(string|\Stringable $message,array $context=[]): void { 'Failed source invoked optional synchronization, retried, or lost its ordinary status log'); $owner->notificationHook=null; + // A failed refresh (for example the first one at boot) retries on a short backoff instead of waiting a full period. + refreshCheck(Verifier::HOSTED_REFRESH_SECONDS < HostedVerificationSnapshot::FRESH_SECONDS,'Periodic hosted refresh lets the list lapse before the next refresh'); + $retry=(new \ReflectionClass(Verifier::class))->newInstanceWithoutConstructor(); + $retry->civ13=$owner;$retry->logger=$logger;$retry->verified=new Collection([], 'discord'); + (new \ReflectionProperty($retry,'verify_url'))->setValue($retry,'http://civ13.com:55555/verified'); + (new \ReflectionProperty($retry,'readOnly'))->setValue($retry,true); + (new \ReflectionProperty(Verifier::class,'hosted'))->setValue($retry,new HostedVerificationSnapshot(static fn()=>$f->now)); + $f->pending=null;$f->now+=20;$f->reply=['http_status'=>503,'json'=>'[]']; + $fire=static function () use ($retry,$f): void { $timer=$retry->timers['hosted_retry']; ($timer->getCallback())(); }; + refreshCheck(refreshResult($retry->refreshHosted())['available']===false && ($retry->timers['hosted_retry'] ?? null)?->getInterval()===30.0,'Failed refresh did not schedule a 30 s retry'); + $calls=$f->calls;$f->now+=30;$fire(); + refreshCheck($f->calls===$calls+1 && $retry->timers['hosted_retry']->getInterval()===60.0,'Second failure did not back off to 60 s'); + $f->now+=60;$f->reply=['http_status'=>200,'json'=>json_encode($records,JSON_THROW_ON_ERROR)];$fire(); + refreshCheck($f->calls===$calls+2 && !isset($retry->timers['hosted_retry']) && iterator_to_array($retry->verified,false)===$records,'Recovered retry left a timer or lost the list'); + $f->now+=20;$f->reply=['http_status'=>503,'json'=>'[]'];refreshResult($retry->refreshHosted()); + refreshCheck($retry->timers['hosted_retry']->getInterval()===30.0,'Backoff was not reset after recovery'); + $owner->loop->cancelTimer($retry->timers['hosted_retry']); + // Real owner bridge is inert when default-disabled/unattached and does not autoload the feature. $plain=(new \ReflectionClass(Civ13::class))->newInstanceWithoutConstructor(); $plain->verifier=$verifier; $autoloads=0; diff --git a/tests/local/chat-moderation.php b/tests/local/chat-moderation.php index 3d6cc67e..7a5722e7 100644 --- a/tests/local/chat-moderation.php +++ b/tests/local/chat-moderation.php @@ -30,8 +30,9 @@ public function __destruct() {} } final class ModerationVerifier extends \Civ13\Verifier { public array $discords = []; + public bool $unavailable = false; public function getVerifiedItem(Member|User|array|string $input): ?array { return null; } - public function get(string $discrim, $key): mixed { return isset($this->discords[$key]) ? ['ss13' => $key, 'discord' => $this->discords[$key]] : null; } + public function get(string $discrim, $key): mixed { if ($this->unavailable) { throw new \Civ13\Exceptions\VerifierException('Hosted verification unavailable'); } return isset($this->discords[$key]) ? ['ss13' => $key, 'discord' => $this->discords[$key]] : null; } } final class ModerationBot extends Civ13 { public array $sent = []; @@ -106,6 +107,22 @@ public function react($emoticon): PromiseInterface { $this->reactions[] = $emoti // The bot's own relayed text is never counted. $say('CivBot', 'badword', true); check(! isset($bot->ooc_badwords_warnings['civbot']) && ! isset($bot->ooc_badwords_warnings['CivBot']), 'Bot was moderated'); + // Hosted verification outage: chat still relays as plain text, and the warning is logged once per outage, not per line. + $identityWarnings = static fn (): int => count(array_filter($log->getRecords(), static fn ($r) => str_contains((string) $r['message'], 'Chat relay identity lookup unavailable'))); + $sentBefore = count($bot->sent); + $bot->verifier->unavailable = true; + $say('speaker', 'hello one', true); + $say('speaker', 'hello two', true); + $say('other', 'hello three', true); + check(count($bot->sent) === $sentBefore + 3, 'Chat was dropped while hosted verification was unavailable'); + check($identityWarnings() === 1, 'Identity outage warning was not logged exactly once per outage'); + $bot->verifier->unavailable = false; + $say('speaker', 'hello four', true); + $bot->verifier->unavailable = true; + $say('speaker', 'hello five', true); + check($identityWarnings() === 2, 'A new outage after recovery did not warn again'); + $bot->verifier->unavailable = false; + // Without a file the counters still update in memory (callers outside the relay). $memory = []; check($bot->moderator->moderate($server, 'other', 'badword', [$rule], $memory) && $memory === ['other' => ['test' => 1]], 'In-memory counter failed');