Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 8 additions & 1 deletion src/Civ13/GameServer.php
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,8 @@ class GameServer
public Logger $logger;
public StreamSelectLoop $loop;
public bool $ready = false;
/** True while the chat relay is sending unverified text because hosted verification is unavailable; limits the warning to once per outage. */
private bool $relayIdentityUnavailable = false;
private ?NativeLifecycle $nativeLifecycle = null;
/** Opt-in confirmed `<key>manage` controls for a shard_group region (docs/launcher-management.md). */
public ?\Civ13\Local\LauncherManagement $launcherManagement = null;
Expand Down Expand Up @@ -671,10 +673,15 @@ private function __gameChatRelay(Channel|Thread|string $channel, array $array, ?
}
try {
$item = $this->civ13->verifier->get('ss13', Civ13::sanitizeInput($array['ckey']));
$this->relayIdentityUnavailable = false;
} catch (VerifierException) {
// The hosted list is not a current identity proof. Keep chat visible
// as plain text without using retained or stale verification data.
$this->logger->warning('Chat relay identity lookup unavailable; sending unverified display text');
// Warn once per outage instead of once per chat line.
if (! $this->relayIdentityUnavailable) {
$this->relayIdentityUnavailable = true;
$this->logger->warning('Chat relay identity lookup unavailable; sending unverified display text until hosted verification refreshes');
}
$item = null;
}
if (! $item) {
Expand Down
41 changes: 39 additions & 2 deletions src/Civ13/Verifier.php
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,11 @@ class Verifier
private ?HostedVerificationSnapshot $hosted = null;
private ?\Closure $hostedTransportFetch = null;
private ?\React\EventLoop\LoopInterface $hostedTransportLoop = null;
/** Hosted refresh period; well inside HostedVerificationSnapshot::FRESH_SECONDS so a normal refresh never lets the list lapse. */
public const HOSTED_REFRESH_SECONDS = 900;
/** Backoff after a failed hosted refresh, so one failed fetch (often at boot) does not leave the list unavailable until the next period. */
public const HOSTED_RETRY_SECONDS = [30, 60, 120, 300];
private int $hostedRetryAttempt = 0;
private const READ_ONLY_MESSAGE = 'Desktop uses the hosted Civilizationbot verification source. Complete verification with the primary bot.';
public CollectionInterface $verified; // This probably needs a default value for Collection, maybe make it a Repository instead?
public CollectionInterface $pending;
Expand Down Expand Up @@ -1271,11 +1276,11 @@ public function getVerifiedUser(Member|User|array|string|null $input): ?User
return null;
}

/** The 30-minute periodic timer that re-checks verifier connectivity and replays provisional registrations when it comes back online. */
/** The periodic timer (15 minutes for hosted read-only, otherwise 30) that re-checks verifier connectivity and replays provisional registrations when it comes back online. */
public function verifierStatusTimer(): TimerInterface
{
if (! isset($this->timers['verifier_status_timer'])) {
$this->timers['verifier_status_timer'] = $this->discord->getLoop()->addPeriodicTimer(1800, function () {
$this->timers['verifier_status_timer'] = $this->discord->getLoop()->addPeriodicTimer($this->readOnly ? self::HOSTED_REFRESH_SECONDS : 1800, function () {
if ($this->readOnly) {
$this->refreshHosted();
return;
Expand Down Expand Up @@ -1373,9 +1378,41 @@ public function refreshHosted(): PromiseInterface
}
}
$this->logger->info('Hosted verification refresh', $status);
if (($status['coalesced'] ?? true) === false) {
// Retry scheduling is best effort and must not turn a finished refresh into a rejection.
try { $status['available'] === true ? $this->hostedRetryReset() : $this->scheduleHostedRetry(); }
catch (\Throwable) {
try { $this->logger->warning('Hosted verification retry could not be scheduled'); } catch (\Throwable) {}
}
}
return $status;
});
}

/** Schedules one bounded retry after a failed hosted refresh; never throws into the loop. */
private function scheduleHostedRetry(): void
{
if (isset($this->timers['hosted_retry'])) { return; }
$delays = self::HOSTED_RETRY_SECONDS;
$delay = $delays[min($this->hostedRetryAttempt, count($delays) - 1)];
$this->hostedRetryAttempt++;
$this->timers['hosted_retry'] = $this->civ13->loop->addTimer($delay, function (): void {
unset($this->timers['hosted_retry']);
try { $this->refreshHosted(); }
catch (\Throwable) {
try { $this->logger->warning('Hosted verification retry failed to start'); } catch (\Throwable) {}
}
});
}

private function hostedRetryReset(): void
{
$this->hostedRetryAttempt = 0;
if (isset($this->timers['hosted_retry'])) {
$this->civ13->loop->cancelTimer($this->timers['hosted_retry']);
unset($this->timers['hosted_retry']);
}
}
// Magic Methods
/**
* Cancels every timer this verifier registered.
Expand Down
19 changes: 19 additions & 0 deletions tests/local/admin-verification-refresh.php
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,7 @@ public function warning(string|\Stringable $message,array $context=[]): void {
};
$owner=(new \ReflectionClass(RefreshObserver::class))->newInstanceWithoutConstructor();
$owner->browser=(new \ReflectionClass(\React\Http\Browser::class))->newInstanceWithoutConstructor();
$owner->loop=new \React\EventLoop\StreamSelectLoop(); // Failed refreshes schedule a retry timer; this loop never runs.
$verifier=(new \ReflectionClass(Verifier::class))->newInstanceWithoutConstructor();
$verifier->civ13=$owner; $owner->verifier=$verifier;
(new \ReflectionProperty($verifier,'verify_url'))->setValue($verifier,'http://civ13.com:55555/verified');
Expand Down Expand Up @@ -148,6 +149,24 @@ public function warning(string|\Stringable $message,array $context=[]): void {
'Failed source invoked optional synchronization, retried, or lost its ordinary status log');
$owner->notificationHook=null;

// A failed refresh (for example the first one at boot) retries on a short backoff instead of waiting a full period.
refreshCheck(Verifier::HOSTED_REFRESH_SECONDS < HostedVerificationSnapshot::FRESH_SECONDS,'Periodic hosted refresh lets the list lapse before the next refresh');
$retry=(new \ReflectionClass(Verifier::class))->newInstanceWithoutConstructor();
$retry->civ13=$owner;$retry->logger=$logger;$retry->verified=new Collection([], 'discord');
(new \ReflectionProperty($retry,'verify_url'))->setValue($retry,'http://civ13.com:55555/verified');
(new \ReflectionProperty($retry,'readOnly'))->setValue($retry,true);
(new \ReflectionProperty(Verifier::class,'hosted'))->setValue($retry,new HostedVerificationSnapshot(static fn()=>$f->now));
$f->pending=null;$f->now+=20;$f->reply=['http_status'=>503,'json'=>'[]'];
$fire=static function () use ($retry,$f): void { $timer=$retry->timers['hosted_retry']; ($timer->getCallback())(); };
refreshCheck(refreshResult($retry->refreshHosted())['available']===false && ($retry->timers['hosted_retry'] ?? null)?->getInterval()===30.0,'Failed refresh did not schedule a 30 s retry');
$calls=$f->calls;$f->now+=30;$fire();
refreshCheck($f->calls===$calls+1 && $retry->timers['hosted_retry']->getInterval()===60.0,'Second failure did not back off to 60 s');
$f->now+=60;$f->reply=['http_status'=>200,'json'=>json_encode($records,JSON_THROW_ON_ERROR)];$fire();
refreshCheck($f->calls===$calls+2 && !isset($retry->timers['hosted_retry']) && iterator_to_array($retry->verified,false)===$records,'Recovered retry left a timer or lost the list');
$f->now+=20;$f->reply=['http_status'=>503,'json'=>'[]'];refreshResult($retry->refreshHosted());
refreshCheck($retry->timers['hosted_retry']->getInterval()===30.0,'Backoff was not reset after recovery');
$owner->loop->cancelTimer($retry->timers['hosted_retry']);

// Real owner bridge is inert when default-disabled/unattached and does not autoload the feature.
$plain=(new \ReflectionClass(Civ13::class))->newInstanceWithoutConstructor();
$plain->verifier=$verifier; $autoloads=0;
Expand Down
19 changes: 18 additions & 1 deletion tests/local/chat-moderation.php
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,9 @@ public function __destruct() {}
}
final class ModerationVerifier extends \Civ13\Verifier {
public array $discords = [];
public bool $unavailable = false;
public function getVerifiedItem(Member|User|array|string $input): ?array { return null; }
public function get(string $discrim, $key): mixed { return isset($this->discords[$key]) ? ['ss13' => $key, 'discord' => $this->discords[$key]] : null; }
public function get(string $discrim, $key): mixed { if ($this->unavailable) { throw new \Civ13\Exceptions\VerifierException('Hosted verification unavailable'); } return isset($this->discords[$key]) ? ['ss13' => $key, 'discord' => $this->discords[$key]] : null; }
}
final class ModerationBot extends Civ13 {
public array $sent = [];
Expand Down Expand Up @@ -106,6 +107,22 @@ public function react($emoticon): PromiseInterface { $this->reactions[] = $emoti
// The bot's own relayed text is never counted.
$say('CivBot', 'badword', true);
check(! isset($bot->ooc_badwords_warnings['civbot']) && ! isset($bot->ooc_badwords_warnings['CivBot']), 'Bot was moderated');
// Hosted verification outage: chat still relays as plain text, and the warning is logged once per outage, not per line.
$identityWarnings = static fn (): int => count(array_filter($log->getRecords(), static fn ($r) => str_contains((string) $r['message'], 'Chat relay identity lookup unavailable')));
$sentBefore = count($bot->sent);
$bot->verifier->unavailable = true;
$say('speaker', 'hello one', true);
$say('speaker', 'hello two', true);
$say('other', 'hello three', true);
check(count($bot->sent) === $sentBefore + 3, 'Chat was dropped while hosted verification was unavailable');
check($identityWarnings() === 1, 'Identity outage warning was not logged exactly once per outage');
$bot->verifier->unavailable = false;
$say('speaker', 'hello four', true);
$bot->verifier->unavailable = true;
$say('speaker', 'hello five', true);
check($identityWarnings() === 2, 'A new outage after recovery did not warn again');
$bot->verifier->unavailable = false;

// Without a file the counters still update in memory (callers outside the relay).
$memory = [];
check($bot->moderator->moderate($server, 'other', 'badword', [$rule], $memory) && $memory === ['other' => ['test' => 1]], 'In-memory counter failed');
Expand Down
Loading