diff --git a/src/Civ13/GameServer.php b/src/Civ13/GameServer.php index c63ab7bb..5cd75261 100644 --- a/src/Civ13/GameServer.php +++ b/src/Civ13/GameServer.php @@ -67,6 +67,8 @@ class GameServer public Logger $logger; public StreamSelectLoop $loop; public bool $ready = false; + /** True while the chat relay is sending unverified text because hosted verification is unavailable; limits the warning to once per outage. */ + private bool $relayIdentityUnavailable = false; private ?NativeLifecycle $nativeLifecycle = null; /** Opt-in confirmed `manage` controls for a shard_group region (docs/launcher-management.md). */ public ?\Civ13\Local\LauncherManagement $launcherManagement = null; @@ -671,10 +673,15 @@ private function __gameChatRelay(Channel|Thread|string $channel, array $array, ? } try { $item = $this->civ13->verifier->get('ss13', Civ13::sanitizeInput($array['ckey'])); + $this->relayIdentityUnavailable = false; } catch (VerifierException) { // The hosted list is not a current identity proof. Keep chat visible // as plain text without using retained or stale verification data. - $this->logger->warning('Chat relay identity lookup unavailable; sending unverified display text'); + // Warn once per outage instead of once per chat line. + if (! $this->relayIdentityUnavailable) { + $this->relayIdentityUnavailable = true; + $this->logger->warning('Chat relay identity lookup unavailable; sending unverified display text until hosted verification refreshes'); + } $item = null; } if (! $item) { diff --git a/src/Civ13/Verifier.php b/src/Civ13/Verifier.php index c5e0afe3..704ec7f8 100644 --- a/src/Civ13/Verifier.php +++ b/src/Civ13/Verifier.php @@ -42,6 +42,11 @@ class Verifier private ?HostedVerificationSnapshot $hosted = null; private ?\Closure $hostedTransportFetch = null; private ?\React\EventLoop\LoopInterface $hostedTransportLoop = null; + /** Hosted refresh period; well inside HostedVerificationSnapshot::FRESH_SECONDS so a normal refresh never lets the list lapse. */ + public const HOSTED_REFRESH_SECONDS = 900; + /** Backoff after a failed hosted refresh, so one failed fetch (often at boot) does not leave the list unavailable until the next period. */ + public const HOSTED_RETRY_SECONDS = [30, 60, 120, 300]; + private int $hostedRetryAttempt = 0; private const READ_ONLY_MESSAGE = 'Desktop uses the hosted Civilizationbot verification source. Complete verification with the primary bot.'; public CollectionInterface $verified; // This probably needs a default value for Collection, maybe make it a Repository instead? public CollectionInterface $pending; @@ -1271,11 +1276,11 @@ public function getVerifiedUser(Member|User|array|string|null $input): ?User return null; } - /** The 30-minute periodic timer that re-checks verifier connectivity and replays provisional registrations when it comes back online. */ + /** The periodic timer (15 minutes for hosted read-only, otherwise 30) that re-checks verifier connectivity and replays provisional registrations when it comes back online. */ public function verifierStatusTimer(): TimerInterface { if (! isset($this->timers['verifier_status_timer'])) { - $this->timers['verifier_status_timer'] = $this->discord->getLoop()->addPeriodicTimer(1800, function () { + $this->timers['verifier_status_timer'] = $this->discord->getLoop()->addPeriodicTimer($this->readOnly ? self::HOSTED_REFRESH_SECONDS : 1800, function () { if ($this->readOnly) { $this->refreshHosted(); return; @@ -1373,9 +1378,41 @@ public function refreshHosted(): PromiseInterface } } $this->logger->info('Hosted verification refresh', $status); + if (($status['coalesced'] ?? true) === false) { + // Retry scheduling is best effort and must not turn a finished refresh into a rejection. + try { $status['available'] === true ? $this->hostedRetryReset() : $this->scheduleHostedRetry(); } + catch (\Throwable) { + try { $this->logger->warning('Hosted verification retry could not be scheduled'); } catch (\Throwable) {} + } + } return $status; }); } + + /** Schedules one bounded retry after a failed hosted refresh; never throws into the loop. */ + private function scheduleHostedRetry(): void + { + if (isset($this->timers['hosted_retry'])) { return; } + $delays = self::HOSTED_RETRY_SECONDS; + $delay = $delays[min($this->hostedRetryAttempt, count($delays) - 1)]; + $this->hostedRetryAttempt++; + $this->timers['hosted_retry'] = $this->civ13->loop->addTimer($delay, function (): void { + unset($this->timers['hosted_retry']); + try { $this->refreshHosted(); } + catch (\Throwable) { + try { $this->logger->warning('Hosted verification retry failed to start'); } catch (\Throwable) {} + } + }); + } + + private function hostedRetryReset(): void + { + $this->hostedRetryAttempt = 0; + if (isset($this->timers['hosted_retry'])) { + $this->civ13->loop->cancelTimer($this->timers['hosted_retry']); + unset($this->timers['hosted_retry']); + } + } // Magic Methods /** * Cancels every timer this verifier registered. diff --git a/tests/local/admin-verification-refresh.php b/tests/local/admin-verification-refresh.php index 6be52d8b..343d7d10 100644 --- a/tests/local/admin-verification-refresh.php +++ b/tests/local/admin-verification-refresh.php @@ -72,6 +72,7 @@ public function warning(string|\Stringable $message,array $context=[]): void { }; $owner=(new \ReflectionClass(RefreshObserver::class))->newInstanceWithoutConstructor(); $owner->browser=(new \ReflectionClass(\React\Http\Browser::class))->newInstanceWithoutConstructor(); + $owner->loop=new \React\EventLoop\StreamSelectLoop(); // Failed refreshes schedule a retry timer; this loop never runs. $verifier=(new \ReflectionClass(Verifier::class))->newInstanceWithoutConstructor(); $verifier->civ13=$owner; $owner->verifier=$verifier; (new \ReflectionProperty($verifier,'verify_url'))->setValue($verifier,'http://civ13.com:55555/verified'); @@ -148,6 +149,24 @@ public function warning(string|\Stringable $message,array $context=[]): void { 'Failed source invoked optional synchronization, retried, or lost its ordinary status log'); $owner->notificationHook=null; + // A failed refresh (for example the first one at boot) retries on a short backoff instead of waiting a full period. + refreshCheck(Verifier::HOSTED_REFRESH_SECONDS < HostedVerificationSnapshot::FRESH_SECONDS,'Periodic hosted refresh lets the list lapse before the next refresh'); + $retry=(new \ReflectionClass(Verifier::class))->newInstanceWithoutConstructor(); + $retry->civ13=$owner;$retry->logger=$logger;$retry->verified=new Collection([], 'discord'); + (new \ReflectionProperty($retry,'verify_url'))->setValue($retry,'http://civ13.com:55555/verified'); + (new \ReflectionProperty($retry,'readOnly'))->setValue($retry,true); + (new \ReflectionProperty(Verifier::class,'hosted'))->setValue($retry,new HostedVerificationSnapshot(static fn()=>$f->now)); + $f->pending=null;$f->now+=20;$f->reply=['http_status'=>503,'json'=>'[]']; + $fire=static function () use ($retry,$f): void { $timer=$retry->timers['hosted_retry']; ($timer->getCallback())(); }; + refreshCheck(refreshResult($retry->refreshHosted())['available']===false && ($retry->timers['hosted_retry'] ?? null)?->getInterval()===30.0,'Failed refresh did not schedule a 30 s retry'); + $calls=$f->calls;$f->now+=30;$fire(); + refreshCheck($f->calls===$calls+1 && $retry->timers['hosted_retry']->getInterval()===60.0,'Second failure did not back off to 60 s'); + $f->now+=60;$f->reply=['http_status'=>200,'json'=>json_encode($records,JSON_THROW_ON_ERROR)];$fire(); + refreshCheck($f->calls===$calls+2 && !isset($retry->timers['hosted_retry']) && iterator_to_array($retry->verified,false)===$records,'Recovered retry left a timer or lost the list'); + $f->now+=20;$f->reply=['http_status'=>503,'json'=>'[]'];refreshResult($retry->refreshHosted()); + refreshCheck($retry->timers['hosted_retry']->getInterval()===30.0,'Backoff was not reset after recovery'); + $owner->loop->cancelTimer($retry->timers['hosted_retry']); + // Real owner bridge is inert when default-disabled/unattached and does not autoload the feature. $plain=(new \ReflectionClass(Civ13::class))->newInstanceWithoutConstructor(); $plain->verifier=$verifier; $autoloads=0; diff --git a/tests/local/chat-moderation.php b/tests/local/chat-moderation.php index 3d6cc67e..7a5722e7 100644 --- a/tests/local/chat-moderation.php +++ b/tests/local/chat-moderation.php @@ -30,8 +30,9 @@ public function __destruct() {} } final class ModerationVerifier extends \Civ13\Verifier { public array $discords = []; + public bool $unavailable = false; public function getVerifiedItem(Member|User|array|string $input): ?array { return null; } - public function get(string $discrim, $key): mixed { return isset($this->discords[$key]) ? ['ss13' => $key, 'discord' => $this->discords[$key]] : null; } + public function get(string $discrim, $key): mixed { if ($this->unavailable) { throw new \Civ13\Exceptions\VerifierException('Hosted verification unavailable'); } return isset($this->discords[$key]) ? ['ss13' => $key, 'discord' => $this->discords[$key]] : null; } } final class ModerationBot extends Civ13 { public array $sent = []; @@ -106,6 +107,22 @@ public function react($emoticon): PromiseInterface { $this->reactions[] = $emoti // The bot's own relayed text is never counted. $say('CivBot', 'badword', true); check(! isset($bot->ooc_badwords_warnings['civbot']) && ! isset($bot->ooc_badwords_warnings['CivBot']), 'Bot was moderated'); + // Hosted verification outage: chat still relays as plain text, and the warning is logged once per outage, not per line. + $identityWarnings = static fn (): int => count(array_filter($log->getRecords(), static fn ($r) => str_contains((string) $r['message'], 'Chat relay identity lookup unavailable'))); + $sentBefore = count($bot->sent); + $bot->verifier->unavailable = true; + $say('speaker', 'hello one', true); + $say('speaker', 'hello two', true); + $say('other', 'hello three', true); + check(count($bot->sent) === $sentBefore + 3, 'Chat was dropped while hosted verification was unavailable'); + check($identityWarnings() === 1, 'Identity outage warning was not logged exactly once per outage'); + $bot->verifier->unavailable = false; + $say('speaker', 'hello four', true); + $bot->verifier->unavailable = true; + $say('speaker', 'hello five', true); + check($identityWarnings() === 2, 'A new outage after recovery did not warn again'); + $bot->verifier->unavailable = false; + // Without a file the counters still update in memory (callers outside the relay). $memory = []; check($bot->moderator->moderate($server, 'other', 'badword', [$rule], $memory) && $memory === ['other' => ['test' => 1]], 'In-memory counter failed');