From 2bf41cb15b61fe6916dab7cff8ed72612f606e92 Mon Sep 17 00:00:00 2001 From: tintinhamans <5984296+tintinhamans@users.noreply.github.com> Date: Sat, 26 Sep 2026 23:15:25 +0200 Subject: [PATCH 1/8] feat(lobby): Add JoinSequence to lobby members for a stable join order --- GenOnlineService/LobbyManager.cs | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/GenOnlineService/LobbyManager.cs b/GenOnlineService/LobbyManager.cs index 055afc3..b9210a5 100644 --- a/GenOnlineService/LobbyManager.cs +++ b/GenOnlineService/LobbyManager.cs @@ -205,6 +205,11 @@ public async Task RunExclusiveAsync(Func> action) private static Int64 s_NextFullMeshCheckID = 0; + // Backing counter for LobbyMember.JoinSequence: per-lobby, starts at 1, only ever assigned + // from within AddMember (which already runs under m_LobbyGate), so a plain Interlocked + // increment is enough without adding another lock. + private Int64 m_NextJoinSequence = 0; + [JsonIgnore] public ConcurrentDictionary> FullMeshConnectivityChecks { get; set; } = new(); @@ -1163,6 +1168,9 @@ public async Task AddMember(UserSession playerSession, string strDisplayNa strDisplayName = String.Format("{0} ({1})", strDisplayName, dupesSeen); } + // AddMember only runs inside RunExclusiveAsync, so this increment is already serialized. + Int64 joinSequence = Interlocked.Increment(ref m_NextJoinSequence); + // only apply lobby prefs if not QM LobbyMember? newMember = null; if (LobbyType == ELobbyType.CustomGame) @@ -1188,7 +1196,7 @@ public async Task AddMember(UserSession playerSession, string strDisplayNa } } - newMember = new LobbyMember(this, playerSession, playerSession.m_UserID, strDisplayName, strOriginalDisplayName, userPreferredPort, sideToUse, colorToUse, -1, EPlayerType.SLOT_PLAYER, slotIndex, bHasMap); + newMember = new LobbyMember(this, playerSession, playerSession.m_UserID, strDisplayName, strOriginalDisplayName, userPreferredPort, sideToUse, colorToUse, -1, EPlayerType.SLOT_PLAYER, slotIndex, bHasMap, joinSequence); } else { @@ -1212,7 +1220,7 @@ public async Task AddMember(UserSession playerSession, string strDisplayNa int sideToUse = allowedTeams[Random.Shared.Next(0, allowedTeams.Length)]; // team is random for now, matchmaker will assign teams on start - newMember = new LobbyMember(this, playerSession, playerSession.m_UserID, strDisplayName, strOriginalDisplayName, userPreferredPort, sideToUse, -1, -1, EPlayerType.SLOT_PLAYER, slotIndex, bHasMap); + newMember = new LobbyMember(this, playerSession, playerSession.m_UserID, strDisplayName, strOriginalDisplayName, userPreferredPort, sideToUse, -1, -1, EPlayerType.SLOT_PLAYER, slotIndex, bHasMap, joinSequence); } Members[slotIndex] = newMember; @@ -1706,6 +1714,13 @@ public void UpdateSlotIndex(UInt16 index) public string Region { get; private set; } = "Unknown"; public string MiddlewareUserID { get; private set; } = String.Empty; + // Per-lobby, monotonically increasing, assigned once when a human player is added + // (AddMember/CreateLobby) and never reassigned afterwards: host migration and slot moves + // only change SlotIndex/Owner on this same object, so JoinSequence is untouched. AI and + // open/closed placeholder slots keep the default of 0. A player who leaves and rejoins gets + // a new LobbyMember instance with a new, higher value. + public Int64 JoinSequence { get; private set; } = 0; + [JsonIgnore] // cant serialize refs private WeakReference CurrentLobby = new(null); @@ -1717,7 +1732,7 @@ public void UpdateSlotIndex(UInt16 index) return PlayerSession; } - public LobbyMember(Lobby owningLobby, UserSession? owningSession, Int64 UserID_in, string DisplayName_in, string strUndedupedDisplayName, UInt16 Port_in, int Side_in, int Color_in, int StartingPosition_in, EPlayerType SlotState_in, UInt16 SlotIndex_in, bool bHasMap_in) + public LobbyMember(Lobby owningLobby, UserSession? owningSession, Int64 UserID_in, string DisplayName_in, string strUndedupedDisplayName, UInt16 Port_in, int Side_in, int Color_in, int StartingPosition_in, EPlayerType SlotState_in, UInt16 SlotIndex_in, bool bHasMap_in, Int64 JoinSequence_in = 0) { CurrentLobby = new WeakReference(owningLobby); PlayerSession = new WeakReference(owningSession); @@ -1732,6 +1747,7 @@ public LobbyMember(Lobby owningLobby, UserSession? owningSession, Int64 UserID_i HasMap = bHasMap_in; SlotState = SlotState_in; SlotIndex = SlotIndex_in; + JoinSequence = JoinSequence_in; // default slots are created with null if (owningSession != null) From 2834f99a7b848c83b6377f55057e25737c32b134 Mon Sep 17 00:00:00 2001 From: tintinhamans <5984296+tintinhamans@users.noreply.github.com> Date: Sat, 26 Sep 2026 23:41:01 +0200 Subject: [PATCH 2/8] feat(session): Remember the player's game CRCs and check them on custom lobby join --- GenOnlineService/Constants.cs | 15 ++- .../CheckLogin/CheckLoginController.cs | 3 +- .../Controllers/Lobby/LobbyController.cs | 29 +++++ .../LoginWithTokenController.cs | 3 +- GenOnlineService/Discord.cs | 101 +++++++++++++++++- GenOnlineService/Program.cs | 4 + 6 files changed, 146 insertions(+), 9 deletions(-) diff --git a/GenOnlineService/Constants.cs b/GenOnlineService/Constants.cs index d717c00..f20cc49 100644 --- a/GenOnlineService/Constants.cs +++ b/GenOnlineService/Constants.cs @@ -1256,10 +1256,19 @@ public UserSession(Int64 ownerID, EUserSessionType sessionType, KnownClients.EKn m_UserID = ownerID; // store the exe CRC (this is actually the .CODE section, for AC) - if (Helpers.g_dictInitialExeCRCs.ContainsKey(ownerID)) + if (Helpers.g_dictInitialExeCRCs.TryRemove(ownerID, out (string ExeCrcHash, Int64 RegisteredAtTicks) acExeCrcEntry)) { - ACExeCRC = Helpers.g_dictInitialExeCRCs[ownerID].ToUpper(); - Helpers.g_dictInitialExeCRCs.Remove(ownerID, out string removedCRC); + ACExeCRC = acExeCrcEntry.ExeCrcHash.ToUpper(); + } + + // store the game exe/ini CRCs registered at login (new clients only - see + // Helpers.RegisterInitialPlayerCRCsFromLoginPayload). Matchmaking registration + // (MatchmakingManager.RegisterPlayer) may still overwrite these from its own request + // body; that's fine as long as the values agree. + if (Helpers.g_dictInitialGameCRCs.TryRemove(ownerID, out (UInt32 ExeCRC, UInt32 IniCRC, Int64 RegisteredAtTicks) gameCRCs)) + { + ExeCRC = gameCRCs.ExeCRC; + IniCRC = gameCRCs.IniCRC; } } diff --git a/GenOnlineService/Controllers/CheckLogin/CheckLoginController.cs b/GenOnlineService/Controllers/CheckLogin/CheckLoginController.cs index 24954bb..5f42c9e 100644 --- a/GenOnlineService/Controllers/CheckLogin/CheckLoginController.cs +++ b/GenOnlineService/Controllers/CheckLogin/CheckLoginController.cs @@ -205,8 +205,7 @@ public async Task Post_InternalHandler(string jsonData, string ipAddr await Database.UserDevices.RegisterUserDevice(db, user_id, hwid_0, hwid_1, hwid_2, ipAddr); } - string exe_crc = data.ContainsKey("exe_crc") ? data["exe_crc"].ToString() : "NONE"; - Helpers.RegisterInitialPlayerExeCRC(user_id, exe_crc); + Helpers.RegisterInitialPlayerCRCsFromLoginPayload(user_id, data); var sessiontoken = Program.g_tokenGenerator.GenerateToken(strDisplayName, user_id, ipAddr, Program.JwtTokenGenerator.ETokenType.Session, knownClientID, sessionType, bIsAdmin); var refreshtoken = Program.g_tokenGenerator.GenerateToken(strDisplayName, user_id, ipAddr, Program.JwtTokenGenerator.ETokenType.Refresh, knownClientID, sessionType, false, out string refreshJti); diff --git a/GenOnlineService/Controllers/Lobby/LobbyController.cs b/GenOnlineService/Controllers/Lobby/LobbyController.cs index 62fb454..d35b4c5 100644 --- a/GenOnlineService/Controllers/Lobby/LobbyController.cs +++ b/GenOnlineService/Controllers/Lobby/LobbyController.cs @@ -514,6 +514,22 @@ internal static bool TryParseLobbyUpdateField(int rawValue, out ELobbyUpdateFiel return Enum.IsDefined(typeof(ELobbyUpdateField), field); } + // Pure decision for the join-time CRC check: only rejects when the joining session actually + // knows BOTH of its own game CRCs (non-zero) and at least one differs from the lobby's. A + // session with either CRC still at its default of 0 - the currently released client, or any + // client that hasn't gone through a path that reports them - is never rejected, since there + // is no real CRC data to compare. + internal static bool ShouldRejectJoinForCrcMismatch(UInt32 sessionExeCrc, UInt32 sessionIniCrc, UInt32 lobbyExeCrc, UInt32 lobbyIniCrc) + { + bool bSessionCrcsKnown = sessionExeCrc != 0 && sessionIniCrc != 0; + if (!bSessionCrcsKnown) + { + return false; + } + + return sessionExeCrc != lobbyExeCrc || sessionIniCrc != lobbyIniCrc; + } + // Runs the per-field lobby update dispatch. Must only be called from inside // lobby.RunExclusiveAsync: every branch here mutates this lobby's Members, slot // state/fields, or ready state, and previously ran completely unguarded. @@ -894,6 +910,19 @@ public async Task Put(Int64 lobbyID) if (playerSession != null) { + // Defense in depth: the released client already refuses client-side to join a + // lobby built from a different exe/ini, so a legitimate player never hits + // this. Only enforced when the joining session actually knows both of its own + // game CRCs (set at login by a new client, or by matchmaking registration - + // see Helpers.RegisterInitialPlayerCRCsFromLoginPayload); the currently + // released client never populates them, so it is never rejected here. + if (ShouldRejectJoinForCrcMismatch(playerSession.ExeCRC, playerSession.IniCRC, lobby.ExeCRC, lobby.IniCRC)) + { + Response.StatusCode = (int)HttpStatusCode.Conflict; + result.success = false; + return result; + } + // leave any lobby await _lobbyManager.LeaveAnyLobby(user_id); diff --git a/GenOnlineService/Controllers/LoginWithToken/LoginWithTokenController.cs b/GenOnlineService/Controllers/LoginWithToken/LoginWithTokenController.cs index 97adc87..be0992b 100644 --- a/GenOnlineService/Controllers/LoginWithToken/LoginWithTokenController.cs +++ b/GenOnlineService/Controllers/LoginWithToken/LoginWithTokenController.cs @@ -141,8 +141,7 @@ public async Task Post_InternalHandler(string jsonData, string ipAddr return result; } - string exe_crc = data.ContainsKey("exe_crc") ? data["exe_crc"].ToString() : "NONE"; - Helpers.RegisterInitialPlayerExeCRC(user_id, exe_crc); + Helpers.RegisterInitialPlayerCRCsFromLoginPayload(user_id, data); string strDisplayName = await Database.Users.GetDisplayName(db, user_id); await SessionHelpers.SetUsedLoggedIn(user_id, clientID, sessionType); diff --git a/GenOnlineService/Discord.cs b/GenOnlineService/Discord.cs index 29c5728..5506f50 100644 --- a/GenOnlineService/Discord.cs +++ b/GenOnlineService/Discord.cs @@ -56,10 +56,107 @@ public enum DiscordCommandParsingFlags public static class Helpers { - public static ConcurrentDictionary g_dictInitialExeCRCs = new(); + // A user who logs in but never opens a websocket (dropped connection, abandoned launcher, ...) + // would otherwise leak an entry here forever. RegisteredAtTicks (Environment.TickCount64) lets + // PruneExpiredLoginCRCs remove anything nobody ever consumed within c_LoginCrcEntryExpiryMS. + internal const Int64 c_LoginCrcEntryExpiryMS = 10 * 60 * 1000; // 10 minutes + + public static ConcurrentDictionary g_dictInitialExeCRCs = new(); public static void RegisterInitialPlayerExeCRC(Int64 user_id, string exe_crc) { - g_dictInitialExeCRCs[user_id] = exe_crc; + g_dictInitialExeCRCs[user_id] = (exe_crc, Environment.TickCount64); + } + + // Handoff for the UInt32 game CRCs (the same ExeCRC/IniCRC meaning as lobby create/matchmaking + // bodies), separate from the AC .text-section hash above. Set at login, consumed once by + // UserSession's constructor when the websocket session is actually created. + public static ConcurrentDictionary g_dictInitialGameCRCs = new(); + public static void RegisterInitialPlayerGameCRCs(Int64 user_id, UInt32 exeCrc, UInt32 iniCrc) + { + g_dictInitialGameCRCs[user_id] = (exeCrc, iniCrc, Environment.TickCount64); + } + + // Pure so the expiry boundary is unit-testable without touching the static dictionaries. + internal static bool IsLoginCrcEntryExpired(Int64 registeredAtTicks, Int64 nowTicks) + { + return nowTicks - registeredAtTicks >= c_LoginCrcEntryExpiryMS; + } + + // Called from the existing 5s WebSocketManager.CheckForTimeouts sweep (Program.cs's + // timerCleanup), so entries for users who log in but never open a websocket don't leak forever. + public static void PruneExpiredLoginCRCs() + { + Int64 nowTicks = Environment.TickCount64; + + foreach (var kvPair in g_dictInitialExeCRCs) + { + if (IsLoginCrcEntryExpired(kvPair.Value.RegisteredAtTicks, nowTicks)) + { + g_dictInitialExeCRCs.TryRemove(kvPair.Key, out _); + } + } + + foreach (var kvPair in g_dictInitialGameCRCs) + { + if (IsLoginCrcEntryExpired(kvPair.Value.RegisteredAtTicks, nowTicks)) + { + g_dictInitialGameCRCs.TryRemove(kvPair.Key, out _); + } + } + } + + // exe_crc/ini_crc may arrive as a JSON number or a numeric string depending on the client; + // never throw on a malformed login payload. + public static bool TryParseUInt32Flexible(JsonElement element, out UInt32 value) + { + if (element.ValueKind == JsonValueKind.Number && element.TryGetUInt32(out value)) + { + return true; + } + + if (element.ValueKind == JsonValueKind.String && UInt32.TryParse(element.GetString(), out value)) + { + return true; + } + + value = 0; + return false; + } + + // Login payload CRC handling. + // ac_exe_crc - SHA-256 hex hash of the exe's .text section, for anti-cheat. + // exe_crc - meaning depends on whether ac_exe_crc is present: + // - present (new client): exe_crc is the UInt32 game CRC (same value/meaning + // as the exe_crc field in lobby create / matchmaking bodies). + // - absent (currently released client): exe_crc IS the AC hash, exactly as + // before this was split into two fields, and no game exe CRC is known. + // ini_crc - UInt32 game CRC, sent by both old and new clients, always with the same meaning. + // TODO: Remove the "ac_exe_crc absent" branch once every released client sends ac_exe_crc. + public static void RegisterInitialPlayerCRCsFromLoginPayload(Int64 user_id, Dictionary data) + { + string acExeCrc; + bool bHasGameExeCrc = false; + UInt32 gameExeCrc = 0; + + if (data.ContainsKey("ac_exe_crc")) + { + acExeCrc = data["ac_exe_crc"].ToString(); + bHasGameExeCrc = data.ContainsKey("exe_crc") && TryParseUInt32Flexible(data["exe_crc"], out gameExeCrc); + } + else + { + acExeCrc = data.ContainsKey("exe_crc") ? data["exe_crc"].ToString() : "NONE"; + } + + RegisterInitialPlayerExeCRC(user_id, acExeCrc); + + UInt32 iniCrc = 0; + bool bHasIniCrc = data.ContainsKey("ini_crc") && TryParseUInt32Flexible(data["ini_crc"], out iniCrc); + + if (bHasGameExeCrc || bHasIniCrc) + { + RegisterInitialPlayerGameCRCs(user_id, bHasGameExeCrc ? gameExeCrc : 0, bHasIniCrc ? iniCrc : 0); + } } public static string ComputeMD5Hash(string input) diff --git a/GenOnlineService/Program.cs b/GenOnlineService/Program.cs index 7de732d..750bfe9 100644 --- a/GenOnlineService/Program.cs +++ b/GenOnlineService/Program.cs @@ -1632,6 +1632,10 @@ public static async Task Main(string[] args) await lobbyManager.Cleanup(); PendingLoginManager.CleanupExpiredLogins(); + + // Users who log in but never open a websocket would otherwise leak their + // registered login CRC entries forever. + Helpers.PruneExpiredLoginCRCs(); } catch (Exception ex) { From 0d0a51874ba04bc4a488734b3f90bb7abfa23df6 Mon Sep 17 00:00:00 2001 From: tintinhamans <5984296+tintinhamans@users.noreply.github.com> Date: Sat, 26 Sep 2026 23:24:54 +0200 Subject: [PATCH 3/8] feat(lobby): Give mesh-check outcomes a reason and ignore stale ones --- GenOnlineService/Constants.cs | 17 +++ GenOnlineService/LobbyManager.cs | 183 ++++++++++++++++++++++--- GenOnlineService/MatchmakingManager.cs | 44 ++++++ 3 files changed, 222 insertions(+), 22 deletions(-) diff --git a/GenOnlineService/Constants.cs b/GenOnlineService/Constants.cs index f20cc49..fb1b503 100644 --- a/GenOnlineService/Constants.cs +++ b/GenOnlineService/Constants.cs @@ -658,6 +658,11 @@ public static async Task DeleteSession(Int64 user_id, EUserSessionType sessionTy { sourceData.MarkAbandoned(); + // A quick match must never start believing this player is still connected. If + // they're in a QuickMatch lobby that's mid setup/countdown, invalidate its bucket's + // auto-start the same way a lobby-level leave would. + MatchmakingManager.InvalidateAutoStartForLobby(sourceData.currentLobbyID); + // If the player was in an active game when their connection dropped, record the // abandon time NOW (before any lobby-structure cleanup runs). This timestamp is // the authoritative "who quit first" signal used by DetermineLobbyWinnerIfNotPresent, @@ -3032,10 +3037,22 @@ public class WebSocketMessage_Social_NewFriendRequest : WebSocketMessage public string display_name { get; set; } = String.Empty; } + // Sent as FULL_MESH_CONNECTIVITY_CHECK_RESPONSE_COMPLETE_TO_HOST. The released client keeps a + // single callback slot for this and consumes the FIRST one it receives, so the server guarantees + // exactly one of these for every check the CURRENT owner is still waiting on - never more than + // one "in flight" answer. A check that gets superseded by a newer one, or whose requester is no + // longer the lobby's owner (e.g. they left and host migration promoted someone else), sends + // nothing at all: see Lobby.CompleteFullMeshConnectivityCheckLocked. public class WebSocketMessage_FullMeshConnectivityCheckOutcome: WebSocketMessage { public bool mesh_complete { get; set; } public List missing_connections { get; set; } = new(); + + // "" when mesh_complete is true. Otherwise one of (see GenOnlineService.FullMeshCheckOutcomeReason): + // "missing_connections" - the check ran to completion with a real connection still missing + // "timeout" - nobody ever reported a connectivity snapshot before the window closed + // "member_left" - a member left the lobby while this check was pending + public string reason { get; set; } = string.Empty; } public class WebSocketMessage_FullMeshConnectivityCheckOutcomeForHost : WebSocketMessage diff --git a/GenOnlineService/LobbyManager.cs b/GenOnlineService/LobbyManager.cs index b9210a5..8ead040 100644 --- a/GenOnlineService/LobbyManager.cs +++ b/GenOnlineService/LobbyManager.cs @@ -62,6 +62,34 @@ internal static bool ShouldRetry(bool meshComplete, bool hasLegacyResponse, int } } + // The set of "reason" values FULL_MESH_CONNECTIVITY_CHECK_RESPONSE_COMPLETE_TO_HOST can carry. + // Empty string ("") always means mesh_complete was true; every other outcome sets exactly one + // of these: + // - MissingConnections: the check ran to completion (attempts exhausted) with at least one + // real peer-to-peer connection still missing between members who are still in the lobby. + // - Timeout: the final attempt's window elapsed without a single member ever reporting a + // connectivity snapshot, so nothing could be judged connected. + // - MemberLeft: a member left the lobby while this check was pending, so the outcome may be + // explained by that departure rather than a real connectivity failure. + // + // A check that gets superseded (a newer StartFullMeshConnectivityCheck call arrives before this + // one finishes) or whose requester is no longer the lobby's owner (they left and host migration + // promoted someone else) never sends a COMPLETE_TO_HOST at all - see + // Lobby.CompleteFullMeshConnectivityCheckLocked. The released client keeps a single callback + // slot and consumes the FIRST completion it receives, so sending a stale/superseded answer + // would be consumed as the answer to whatever check the client is actually still waiting on and + // the real answer would then be silently dropped. The guarantee is therefore scoped to "the + // CURRENT owner gets exactly one completion for the check they are currently waiting on", not + // "every StartFullMeshConnectivityCheck call produces a message": a superseded check's own + // question is answered by the newer check's eventual completion instead. + internal static class FullMeshCheckOutcomeReason + { + internal const string None = ""; + internal const string MissingConnections = "missing_connections"; + internal const string Timeout = "timeout"; + internal const string MemberLeft = "member_left"; + } + // Core:full_mesh_check_* in appsettings.json, read on use internal static class FullMeshCheckSettings { @@ -105,6 +133,19 @@ public class Lobby [JsonIgnore] public Int64 TimeStartFullMeshChecks { get; private set; } = -1; + // Incremented under m_LobbyGate every time a human member is added or removed. A quick match + // must never start on the strength of a mesh check that passed for a different set of + // players than the lobby currently holds, so this - plus MembershipVersionAtLastCheckStart - + // lets the check outcome be tied to the exact membership it was run against. Any membership + // change also clears LastFullMeshConnectivityCheckOutcome outright, as a second, simpler line + // of defense. + [JsonIgnore] + public int MembershipVersion { get; private set; } = 0; + + // MembershipVersion as of the moment the current/most recently started check began. + [JsonIgnore] + public int MembershipVersionAtLastCheckStart { get; private set; } = -1; + // Single per-lobby exclusive gate for every mutation of this lobby's Members/Owner, slot // state/fields, ready state, and mesh-check state. Everything that used to be split across // g_SlotLock (slots) and m_FullMeshCheckLock (mesh-check fields) now goes through this one @@ -203,6 +244,20 @@ public async Task RunExclusiveAsync(Func> action) [JsonIgnore] private bool m_bCurrentAttemptHasLegacyResponse = false; + // Set while a check is pending if a member leaves the lobby before it completes, so the + // eventual outcome can report FullMeshCheckOutcomeReason.MemberLeft instead of a generic + // connectivity failure. Reset each time a new check starts. + [JsonIgnore] + private bool m_bMemberLeftDuringCurrentCheck = false; + + // The user who was Owner when the current check started (both real call sites - the + // websocket host-requests-begin handler and quickmatch's TriggerFullMeshConnectivityChecks - + // only ever start a check while its requester is the current owner). The eventual outcome is + // only ever sent to this user, and only if they are still Owner: see + // CompleteFullMeshConnectivityCheckLocked. + [JsonIgnore] + private Int64 m_MeshCheckRequestingUserID = -1; + private static Int64 s_NextFullMeshCheckID = 0; // Backing counter for LobbyMember.JoinSequence: per-lobby, starts at 1, only ever assigned @@ -271,15 +326,42 @@ public async Task StartFullMeshConnectivityCheck() { await RunExclusiveAsync(() => { + if (PendingFullMeshConnectivityChecks) + { + // A new check preempts whatever was still in flight. The old check's own question + // is answered by the new check's eventual completion instead of sending a stale + // COMPLETE_TO_HOST here: the released client has only one callback slot and would + // consume whichever answer arrives first, dropping the real one. So this discards + // the old check's state without sending anything. + DiscardPendingFullMeshCheckLocked(); + } + FullMeshCheckID = Interlocked.Increment(ref s_NextFullMeshCheckID); FullMeshCheckAttempt = 1; m_TimeToRetryFullMeshChecks = -1; LastFullMeshConnectivityCheckOutcome = null; + m_bMemberLeftDuringCurrentCheck = false; + MembershipVersionAtLastCheckStart = MembershipVersion; + // Both real callers only ever start a check while they are the current owner (the + // websocket handler checks this explicitly; quickmatch's dummy host is set as Owner at + // lobby creation), so Owner at this instant is the requester the eventual outcome + // belongs to. + m_MeshCheckRequestingUserID = Owner; BeginFullMeshConnectivityCheckAttempt(); return Task.CompletedTask; }); } + // Resets pending-check state without sending a COMPLETE_TO_HOST. Must only be called while + // holding m_LobbyGate. + private void DiscardPendingFullMeshCheckLocked() + { + PendingFullMeshConnectivityChecks = false; + TimeStartFullMeshChecks = -1; + m_TimeToRetryFullMeshChecks = -1; + LastFullMeshConnectivityCheckOutcome = null; + } + private void BeginFullMeshConnectivityCheckAttempt() { PendingFullMeshConnectivityChecks = true; @@ -549,39 +631,70 @@ private void ProcessPendingFullMeshConnectivityChecksInternal() } // inform host that we are done - // start full mesh connectivity checks - WebSocketMessage_FullMeshConnectivityCheckOutcome outcome = new WebSocketMessage_FullMeshConnectivityCheckOutcome(); - outcome.msg_id = (int)EWebSocketMessageID.FULL_MESH_CONNECTIVITY_CHECK_RESPONSE_COMPLETE_TO_HOST; - - if (bDisableMeshCheck) + bool bMembershipChangedDuringCheck = MembershipVersion != MembershipVersionAtLastCheckStart; + bool bMeshCompleteFinal = !bMembershipChangedDuringCheck && (bDisableMeshCheck || lstMissingConnections.Count == 0); + List lstFinalMissingConnections = bDisableMeshCheck + ? new List() + : lstMissingConnections; + + string reason; + if (bMeshCompleteFinal) + { + reason = FullMeshCheckOutcomeReason.None; + } + else if (bMembershipChangedDuringCheck || m_bMemberLeftDuringCurrentCheck) { - outcome.mesh_complete = true; - outcome.missing_connections = new List(); + // A join also invalidates the check (not just a leave): either way the set of + // players this check was judged against is no longer the lobby's actual membership. + reason = FullMeshCheckOutcomeReason.MemberLeft; + } + else if (FullMeshConnectivityChecks.IsEmpty) + { + // nobody ever reported a snapshot for this attempt, so nothing could be judged connected + reason = FullMeshCheckOutcomeReason.Timeout; } else { - outcome.mesh_complete = lstMissingConnections.Count == 0; - outcome.missing_connections = lstMissingConnections; + reason = FullMeshCheckOutcomeReason.MissingConnections; } - LastFullMeshConnectivityCheckOutcome = outcome.mesh_complete; + CompleteFullMeshConnectivityCheckLocked(bMeshCompleteFinal, lstFinalMissingConnections, reason); + } + } + } - // TODO_EFCORE: Later, these should really use lobby list instead of getting session from ID + // Sends FULL_MESH_CONNECTIVITY_CHECK_RESPONSE_COMPLETE_TO_HOST to the host and resets the + // pending-check state. Must only be called while holding m_LobbyGate. + private void CompleteFullMeshConnectivityCheckLocked(bool bMeshComplete, List lstMissingConnections, string reason) + { + WebSocketMessage_FullMeshConnectivityCheckOutcome outcome = new WebSocketMessage_FullMeshConnectivityCheckOutcome(); + outcome.msg_id = (int)EWebSocketMessageID.FULL_MESH_CONNECTIVITY_CHECK_RESPONSE_COMPLETE_TO_HOST; + outcome.mesh_complete = bMeshComplete; + outcome.missing_connections = lstMissingConnections; + outcome.reason = bMeshComplete ? FullMeshCheckOutcomeReason.None : reason; - // send to host - UserSession? hostSession = WebSocketManager.GetSessionFromUser(Owner, EUserSessionType.GameClient); // host should be a game client - if (hostSession != null) - { - byte[] bytesJSON = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(outcome)); - hostSession.QueueWebsocketSend(bytesJSON); - } + LastFullMeshConnectivityCheckOutcome = outcome.mesh_complete; - // reset state - PendingFullMeshConnectivityChecks = false; - TimeStartFullMeshChecks = -1; - m_TimeToRetryFullMeshChecks = -1; + // TODO_EFCORE: Later, these should really use lobby list instead of getting session from ID + + // Only ever answer the user who actually asked for this check, and only while they are + // still the owner. If they left and host migration promoted someone else, the new owner + // never asked for this check and must not have it land in their single callback slot as + // the answer to a question they didn't ask. + if (m_MeshCheckRequestingUserID == Owner) + { + UserSession? hostSession = WebSocketManager.GetSessionFromUser(Owner, EUserSessionType.GameClient); // host should be a game client + if (hostSession != null) + { + byte[] bytesJSON = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(outcome)); + hostSession.QueueWebsocketSend(bytesJSON); } } + + // reset state + PendingFullMeshConnectivityChecks = false; + TimeStartFullMeshChecks = -1; + m_TimeToRetryFullMeshChecks = -1; } public void AddPassword(string password) @@ -871,6 +984,13 @@ private void OnAfterPlayerLeftLocked(Int64 leavingUserID) // NOTE: By the time this is called, the member is no longer in the members list bool bNeedsHostMigrate = Owner == leavingUserID; + // A departure while a mesh check is pending can explain that check's eventual failure, + // so the outcome can say why instead of reporting a generic connectivity failure. + if (PendingFullMeshConnectivityChecks) + { + m_bMemberLeftDuringCurrentCheck = true; + } + // we need human members, not real members int numHumanMembers = GetNumberOfHumans(); @@ -1226,6 +1346,11 @@ public async Task AddMember(UserSession playerSession, string strDisplayNa Members[slotIndex] = newMember; TimeMemberLeft[playerSession.m_UserID] = DateTime.UnixEpoch; + // Membership just changed: any previously-passed mesh check no longer describes who is + // actually in the lobby. + ++MembershipVersion; + LastFullMeshConnectivityCheckOutcome = null; + // Lobby members leave public-room presence. playerSession.TryUpdateSessionNetworkRoom(-1); @@ -1340,6 +1465,20 @@ public async Task RemoveMember(LobbyMember member) Members[member.SlotIndex] = placeholderMember; TimeMemberLeft[member.UserID] = DateTime.UtcNow; + // Membership just changed: any previously-passed mesh check no longer describes who + // is actually in the lobby. + ++MembershipVersion; + LastFullMeshConnectivityCheckOutcome = null; + + // A quick match must never start on a stale "everyone joined/connected" verdict once + // someone has left. This is a direct static call (not an event) that only ever touches + // MatchmakingBucket's own lock, never this lobby's gate, so it is safe to make from in + // here. + if (LobbyType == ELobbyType.QuickMatch) + { + MatchmakingManager.InvalidateAutoStartForLobby(LobbyID); + } + OnAfterPlayerLeftLocked(member.UserID); return Task.FromResult(true); diff --git a/GenOnlineService/MatchmakingManager.cs b/GenOnlineService/MatchmakingManager.cs index 2a645d2..83f8e3e 100644 --- a/GenOnlineService/MatchmakingManager.cs +++ b/GenOnlineService/MatchmakingManager.cs @@ -694,6 +694,25 @@ internal void MarkPendingDeletion() } } + public bool OwnsLobby(Int64 lobbyID) + { + return lobbyID != -1 && m_LobbyID == lobbyID; + } + + // Same invalidation the existing RemovePlayer/PruneDeadMembers paths use, exposed for + // external hooks (a lobby-level leave, or a session going abandoned) that don't go through + // this bucket's own member list. + public void InvalidateAutoStart() + { + lock (m_StateLock) + { + if (m_bWaitingOnLobbyJoins || m_bHasStartedCountdown || m_bWaitingOnMeshConnectivityChecks) + { + m_bAutoStartInvalidated = true; + } + } + } + // TODO_EFCORE: Shared User data, and session<->websocket could be weakrefs public bool IsJoiningUserBlockedByOrHasBlockedAnyBucketMember(UserSession? joiningUserSession, Int64 joining_user) { @@ -1840,6 +1859,31 @@ public static void DestroyBucket(MatchmakingBucket bucket) m_bucketsPendingDeletion.Enqueue(bucket); } + // Hook from Lobby (a member left) or WebSocketManager (a session in a QM lobby went abandoned) + // into whichever bucket owns that lobby, so a stale "everyone connected" verdict can never be + // used to start a match that no longer holds the players it was formed with. Only ever touches + // MatchmakingBucket's own m_StateLock - never a Lobby's gate - so it is safe to call from + // anywhere, including from inside Lobby.RemoveMember's own gated callback. + public static void InvalidateAutoStartForLobby(Int64 lobbyID) + { + if (lobbyID == -1) + { + return; + } + + foreach (var bucketsForPlaylist in m_dictMatchmakingBuckets.Values) + { + foreach (MatchmakingBucket bucket in bucketsForPlaylist) + { + if (bucket.OwnsLobby(lobbyID)) + { + bucket.InvalidateAutoStart(); + return; + } + } + } + } + public static async Task RegisterPlayer(UserSession plr, UInt16 playlistID, List mapIndices, UInt32 exe_crc, UInt32 ini_crc, EKnownAnticheatID anticheatID) { // validate the request - a bad playlist or out of range map index from a client must never reach a bucket From 85a856ea406982b64b876721a775a7187f51be71 Mon Sep 17 00:00:00 2001 From: tintinhamans <5984296+tintinhamans@users.noreply.github.com> Date: Sat, 26 Sep 2026 23:17:10 +0200 Subject: [PATCH 4/8] feat(matchmaking): Verify quick matches before starting and referee their connectivity --- GenOnlineService/Constants.cs | 47 +- .../Controllers/Lobby/LobbyController.cs | 45 +- .../WebSocket/WebSocketController.cs | 4 - GenOnlineService/Discord.cs | 26 +- GenOnlineService/LobbyManager.cs | 172 +++---- GenOnlineService/MatchmakingManager.cs | 438 +++++++++++++----- GenOnlineService/Program.cs | 3 +- GenOnlineService/appsettings.json | 1 + 8 files changed, 401 insertions(+), 335 deletions(-) diff --git a/GenOnlineService/Constants.cs b/GenOnlineService/Constants.cs index fb1b503..d087b47 100644 --- a/GenOnlineService/Constants.cs +++ b/GenOnlineService/Constants.cs @@ -549,9 +549,7 @@ public static int GetNumberOfUsersOnline() return numSessions; } - // Pure re-check used by CheckForTimeouts right before actually clearing a snapshotted - // abandoned+expired entry: refuses unless it's still the SAME session object (a reconnect in - // between would have registered a new one) and it is STILL abandoned and expired. + // True only if currentSession is the same object as snapshotSession and still abandoned+expired. internal static bool ShouldStillClearAbandonedSession(UserSession? currentSession, UserSession snapshotSession) { if (!ReferenceEquals(currentSession, snapshotSession)) @@ -590,10 +588,6 @@ public static async Task CheckForTimeouts() foreach (var userData in lstCacheEntriesToDestroy) { - // A reconnect between the snapshot above and now would have replaced this user's - // session with a live one; only clear if the SAME session object is still registered - // and still abandoned+expired, so a fresh reconnect never has its live session torn - // down (kicked from its lobby, deregistered from matchmaking) by a stale sweep entry. UserSession? currentSession = GetSessionFromUser(userData.UserID, userData.SessionType); if (!ShouldStillClearAbandonedSession(currentSession, userData.Session)) { @@ -658,9 +652,6 @@ public static async Task DeleteSession(Int64 user_id, EUserSessionType sessionTy { sourceData.MarkAbandoned(); - // A quick match must never start believing this player is still connected. If - // they're in a QuickMatch lobby that's mid setup/countdown, invalidate its bucket's - // auto-start the same way a lobby-level leave would. MatchmakingManager.InvalidateAutoStartForLobby(sourceData.currentLobbyID); // If the player was in an active game when their connection dropped, record the @@ -1165,8 +1156,7 @@ public SharedUserData(Int64 ownerID, UserSocialContainer socialContainer, string } } - // Core:reconnect_grace_period_ms in appsettings.json, read on use. Mirrors the pattern used by - // FullMeshCheckSettings in LobbyManager.cs. + // Core:reconnect_grace_period_ms in appsettings.json. internal static class UserSessionSettings { internal static Int64 ReconnectGracePeriodMS => Get("reconnect_grace_period_ms", 30000); @@ -1266,10 +1256,6 @@ public UserSession(Int64 ownerID, EUserSessionType sessionType, KnownClients.EKn ACExeCRC = acExeCrcEntry.ExeCrcHash.ToUpper(); } - // store the game exe/ini CRCs registered at login (new clients only - see - // Helpers.RegisterInitialPlayerCRCsFromLoginPayload). Matchmaking registration - // (MatchmakingManager.RegisterPlayer) may still overwrite these from its own request - // body; that's fine as long as the values agree. if (Helpers.g_dictInitialGameCRCs.TryRemove(ownerID, out (UInt32 ExeCRC, UInt32 IniCRC, Int64 RegisteredAtTicks) gameCRCs)) { ExeCRC = gameCRCs.ExeCRC; @@ -1342,11 +1328,7 @@ public async Task CloseWebsocket(WebSocketCloseStatus rea public bool NeedsCleanup() { - // Grace period an abandoned (no live websocket) session gets before it's torn down, - // letting a brief disconnect reconnect instead of losing the slot. Configurable via - // Core:reconnect_grace_period_ms; defaults to 30 seconds, unchanged from before this - // was configurable (the old comment here said "5 minutes", which was wrong - 30000 is - // milliseconds, i.e. 30 seconds). + // Grace period before an abandoned session is torn down. Core:reconnect_grace_period_ms, default 30s. return Environment.TickCount64 - m_timeAbandoned >= UserSessionSettings.ReconnectGracePeriodMS; } @@ -3030,6 +3012,9 @@ public class WebSocketMessage_FullMeshConnectivityCheckResponseFromUser : WebSoc // peers the member is still negotiating with; absent from older clients public List connecting_map { get; set; } = new(); + + // user IDs this member has exhausted its connection attempts against; absent/empty from older clients + public List gave_up_user_ids { get; set; } = new(); } public class WebSocketMessage_Social_NewFriendRequest : WebSocketMessage @@ -3037,21 +3022,15 @@ public class WebSocketMessage_Social_NewFriendRequest : WebSocketMessage public string display_name { get; set; } = String.Empty; } - // Sent as FULL_MESH_CONNECTIVITY_CHECK_RESPONSE_COMPLETE_TO_HOST. The released client keeps a - // single callback slot for this and consumes the FIRST one it receives, so the server guarantees - // exactly one of these for every check the CURRENT owner is still waiting on - never more than - // one "in flight" answer. A check that gets superseded by a newer one, or whose requester is no - // longer the lobby's owner (e.g. they left and host migration promoted someone else), sends - // nothing at all: see Lobby.CompleteFullMeshConnectivityCheckLocked. + // FULL_MESH_CONNECTIVITY_CHECK_RESPONSE_COMPLETE_TO_HOST. Sent at most once per check, only to + // the current owner; a superseded or stale-requester check sends nothing (see + // Lobby.CompleteFullMeshConnectivityCheckLocked). public class WebSocketMessage_FullMeshConnectivityCheckOutcome: WebSocketMessage { public bool mesh_complete { get; set; } public List missing_connections { get; set; } = new(); - // "" when mesh_complete is true. Otherwise one of (see GenOnlineService.FullMeshCheckOutcomeReason): - // "missing_connections" - the check ran to completion with a real connection still missing - // "timeout" - nobody ever reported a connectivity snapshot before the window closed - // "member_left" - a member left the lobby while this check was pending + // "" if mesh_complete; else one of missing_connections/timeout/member_left/gave_up (FullMeshCheckOutcomeReason). public string reason { get; set; } = string.Empty; } @@ -3223,6 +3202,12 @@ public Int64 lobby_id { get; set; } + + // Same shape as GET lobby's "lobby" field. Optional; older clients ignore it. + public Lobby? lobby + { + get; set; + } } public class WebSocketMessage_MatchmakerStartGame : WebSocketMessage diff --git a/GenOnlineService/Controllers/Lobby/LobbyController.cs b/GenOnlineService/Controllers/Lobby/LobbyController.cs index d35b4c5..d26c7a4 100644 --- a/GenOnlineService/Controllers/Lobby/LobbyController.cs +++ b/GenOnlineService/Controllers/Lobby/LobbyController.cs @@ -467,13 +467,8 @@ public async Task Post(Int64 lobbyID) } } - // Every mutation below touches this lobby's Members, slot state, or ready state, - // so the whole field-update dispatch runs as one atomic operation under the - // per-lobby gate instead of each setter mutating state unguarded. - // A kick can't finish the actual removal in here: RemoveMember acquires this - // same lobby's gate, which is not reentrant, so ApplyLobbyFieldUpdateAsync only - // validates the target and hands the user ID back for PerformKickAsync to - // process once the gate below has been released. + // ApplyLobbyFieldUpdateAsync only validates a kick target; PerformKickAsync + // removes them after the gate releases (RemoveMember needs the gate too). Int64? kickedUserID = await lobby.RunExclusiveAsync(() => ApplyLobbyFieldUpdateAsync(lobby, SourceMember, field, data)); if (kickedUserID.HasValue) @@ -495,30 +490,19 @@ public async Task Post(Int64 lobbyID) return result; } - // Rejects any wire value that isn't a real EPlayerType member, so a malformed or - // out-of-range slot_state from HOST_ACTION_SET_SLOT_STATE can't be cast into an enum value - // the rest of the lobby code (SetPlayerSlotState, IsAI, serialization, ...) never expects. internal static bool TryParseSlotState(UInt16 rawValue, out EPlayerType slotState) { slotState = (EPlayerType)rawValue; return Enum.IsDefined(typeof(EPlayerType), slotState); } - // Rejects any wire value that isn't a real ELobbyUpdateField member, so an unknown/malformed - // field can't be cast into an enum value that then gets indexed straight into - // g_dictLobbyUpdatePermissionsTable (which previously threw KeyNotFoundException, silently - // swallowed by Post's outer catch, for anything out of range). internal static bool TryParseLobbyUpdateField(int rawValue, out ELobbyUpdateField field) { field = (ELobbyUpdateField)rawValue; return Enum.IsDefined(typeof(ELobbyUpdateField), field); } - // Pure decision for the join-time CRC check: only rejects when the joining session actually - // knows BOTH of its own game CRCs (non-zero) and at least one differs from the lobby's. A - // session with either CRC still at its default of 0 - the currently released client, or any - // client that hasn't gone through a path that reports them - is never rejected, since there - // is no real CRC data to compare. + // Rejects only when the joining session knows both its own game CRCs (non-zero) and at least one differs. internal static bool ShouldRejectJoinForCrcMismatch(UInt32 sessionExeCrc, UInt32 sessionIniCrc, UInt32 lobbyExeCrc, UInt32 lobbyIniCrc) { bool bSessionCrcsKnown = sessionExeCrc != 0 && sessionIniCrc != 0; @@ -530,12 +514,8 @@ internal static bool ShouldRejectJoinForCrcMismatch(UInt32 sessionExeCrc, UInt32 return sessionExeCrc != lobbyExeCrc || sessionIniCrc != lobbyIniCrc; } - // Runs the per-field lobby update dispatch. Must only be called from inside - // lobby.RunExclusiveAsync: every branch here mutates this lobby's Members, slot - // state/fields, or ready state, and previously ran completely unguarded. - // Returns the kicked user's ID when HOST_ACTION_KICK_USER validated a real target, so the - // caller can run the actual removal after releasing the gate; null for every other field - // (including a kick request with no valid target). + // Must be called from inside lobby.RunExclusiveAsync. Returns the kicked user's ID for + // HOST_ACTION_KICK_USER, else null. private async Task ApplyLobbyFieldUpdateAsync(Lobby lobby, LobbyMember SourceMember, ELobbyUpdateField field, Dictionary data) { // reset everyones ready states when anything changes (minus dummy actions) @@ -644,10 +624,7 @@ internal static bool ShouldRejectJoinForCrcMismatch(UInt32 sessionExeCrc, UInt32 { Int64 KickedUserID = data["userid"].GetInt64(); - // the target must actually be in THIS lobby, otherwise a host could wipe the - // TURN credentials / lobby state of any arbitrary online player. Checked here, - // under the gate, so it can't race a concurrent leave/kick of the same target; - // the actual removal runs after the gate is released (see PerformKickAsync). + // Target must be in this lobby, otherwise a host could wipe an arbitrary player's TURN/session state. if (lobby.GetMemberFromUserID(KickedUserID) != null) { return KickedUserID; @@ -806,9 +783,7 @@ internal static bool ShouldRejectJoinForCrcMismatch(UInt32 sessionExeCrc, UInt32 return null; } - // Runs the parts of a kick that RemoveMember itself performs (leaving the lobby, which - // acquires this same lobby's gate) plus TURN/session cleanup - all after - // ApplyLobbyFieldUpdateAsync's gate has been released, never from inside it. + // Must be called after the gate has released, never from inside ApplyLobbyFieldUpdateAsync. private async Task PerformKickAsync(Lobby lobby, Int64 kickedUserID) { // TODO: we should communicate the kick to the user... @@ -910,12 +885,6 @@ public async Task Put(Int64 lobbyID) if (playerSession != null) { - // Defense in depth: the released client already refuses client-side to join a - // lobby built from a different exe/ini, so a legitimate player never hits - // this. Only enforced when the joining session actually knows both of its own - // game CRCs (set at login by a new client, or by matchmaking registration - - // see Helpers.RegisterInitialPlayerCRCsFromLoginPayload); the currently - // released client never populates them, so it is never rejected here. if (ShouldRejectJoinForCrcMismatch(playerSession.ExeCRC, playerSession.IniCRC, lobby.ExeCRC, lobby.IniCRC)) { Response.StatusCode = (int)HttpStatusCode.Conflict; diff --git a/GenOnlineService/Controllers/WebSocket/WebSocketController.cs b/GenOnlineService/Controllers/WebSocket/WebSocketController.cs index bab3424..8803daa 100644 --- a/GenOnlineService/Controllers/WebSocket/WebSocketController.cs +++ b/GenOnlineService/Controllers/WebSocket/WebSocketController.cs @@ -673,8 +673,6 @@ private async Task ProcessWSMessage(UserWebSocketInstance sourceWS, UserSession Lobby? lobby = _lobbyManager.GetLobby(sourceUserSession.currentLobbyID); if (lobby != null) { - // Ready state is part of the lobby's mutable state, so it goes through the - // same per-lobby gate as slot/member mutations. await lobby.RunExclusiveAsync(() => { LobbyMember? member = lobby.GetMemberFromUserID(sourceUserSession.m_UserID); @@ -1034,8 +1032,6 @@ await lobby.RunExclusiveAsync(() => } // lock slots (more people joining when we're already doing connectivity checks won't help the situation) - // Awaited separately from StartFullMeshConnectivityCheck below: each is its own - // gated operation on the lobby, so neither ever nests inside the other's gate use. await lobbyInfo.CloseOpenSlots(); // mark lobby as in progress of full mesh connectivity checks diff --git a/GenOnlineService/Discord.cs b/GenOnlineService/Discord.cs index 5506f50..ed9fe04 100644 --- a/GenOnlineService/Discord.cs +++ b/GenOnlineService/Discord.cs @@ -56,10 +56,8 @@ public enum DiscordCommandParsingFlags public static class Helpers { - // A user who logs in but never opens a websocket (dropped connection, abandoned launcher, ...) - // would otherwise leak an entry here forever. RegisteredAtTicks (Environment.TickCount64) lets - // PruneExpiredLoginCRCs remove anything nobody ever consumed within c_LoginCrcEntryExpiryMS. - internal const Int64 c_LoginCrcEntryExpiryMS = 10 * 60 * 1000; // 10 minutes + // Expiry window for unclaimed login CRC entries (see PruneExpiredLoginCRCs). + internal const Int64 c_LoginCrcEntryExpiryMS = 10 * 60 * 1000; public static ConcurrentDictionary g_dictInitialExeCRCs = new(); public static void RegisterInitialPlayerExeCRC(Int64 user_id, string exe_crc) @@ -67,23 +65,18 @@ public static void RegisterInitialPlayerExeCRC(Int64 user_id, string exe_crc) g_dictInitialExeCRCs[user_id] = (exe_crc, Environment.TickCount64); } - // Handoff for the UInt32 game CRCs (the same ExeCRC/IniCRC meaning as lobby create/matchmaking - // bodies), separate from the AC .text-section hash above. Set at login, consumed once by - // UserSession's constructor when the websocket session is actually created. + // UInt32 game exe/ini CRC handoff from login to UserSession's constructor. public static ConcurrentDictionary g_dictInitialGameCRCs = new(); public static void RegisterInitialPlayerGameCRCs(Int64 user_id, UInt32 exeCrc, UInt32 iniCrc) { g_dictInitialGameCRCs[user_id] = (exeCrc, iniCrc, Environment.TickCount64); } - // Pure so the expiry boundary is unit-testable without touching the static dictionaries. internal static bool IsLoginCrcEntryExpired(Int64 registeredAtTicks, Int64 nowTicks) { return nowTicks - registeredAtTicks >= c_LoginCrcEntryExpiryMS; } - // Called from the existing 5s WebSocketManager.CheckForTimeouts sweep (Program.cs's - // timerCleanup), so entries for users who log in but never open a websocket don't leak forever. public static void PruneExpiredLoginCRCs() { Int64 nowTicks = Environment.TickCount64; @@ -105,8 +98,7 @@ public static void PruneExpiredLoginCRCs() } } - // exe_crc/ini_crc may arrive as a JSON number or a numeric string depending on the client; - // never throw on a malformed login payload. + // Accepts exe_crc/ini_crc as a JSON number or numeric string. public static bool TryParseUInt32Flexible(JsonElement element, out UInt32 value) { if (element.ValueKind == JsonValueKind.Number && element.TryGetUInt32(out value)) @@ -123,14 +115,8 @@ public static bool TryParseUInt32Flexible(JsonElement element, out UInt32 value) return false; } - // Login payload CRC handling. - // ac_exe_crc - SHA-256 hex hash of the exe's .text section, for anti-cheat. - // exe_crc - meaning depends on whether ac_exe_crc is present: - // - present (new client): exe_crc is the UInt32 game CRC (same value/meaning - // as the exe_crc field in lobby create / matchmaking bodies). - // - absent (currently released client): exe_crc IS the AC hash, exactly as - // before this was split into two fields, and no game exe CRC is known. - // ini_crc - UInt32 game CRC, sent by both old and new clients, always with the same meaning. + // ac_exe_crc: AC .text-section hash. exe_crc: game CRC if ac_exe_crc present, else the AC hash. + // ini_crc: game CRC, both client versions. // TODO: Remove the "ac_exe_crc absent" branch once every released client sends ac_exe_crc. public static void RegisterInitialPlayerCRCsFromLoginPayload(Int64 user_id, Dictionary data) { diff --git a/GenOnlineService/LobbyManager.cs b/GenOnlineService/LobbyManager.cs index 8ead040..19504cd 100644 --- a/GenOnlineService/LobbyManager.cs +++ b/GenOnlineService/LobbyManager.cs @@ -62,32 +62,17 @@ internal static bool ShouldRetry(bool meshComplete, bool hasLegacyResponse, int } } - // The set of "reason" values FULL_MESH_CONNECTIVITY_CHECK_RESPONSE_COMPLETE_TO_HOST can carry. - // Empty string ("") always means mesh_complete was true; every other outcome sets exactly one - // of these: - // - MissingConnections: the check ran to completion (attempts exhausted) with at least one - // real peer-to-peer connection still missing between members who are still in the lobby. - // - Timeout: the final attempt's window elapsed without a single member ever reporting a - // connectivity snapshot, so nothing could be judged connected. - // - MemberLeft: a member left the lobby while this check was pending, so the outcome may be - // explained by that departure rather than a real connectivity failure. - // - // A check that gets superseded (a newer StartFullMeshConnectivityCheck call arrives before this - // one finishes) or whose requester is no longer the lobby's owner (they left and host migration - // promoted someone else) never sends a COMPLETE_TO_HOST at all - see - // Lobby.CompleteFullMeshConnectivityCheckLocked. The released client keeps a single callback - // slot and consumes the FIRST completion it receives, so sending a stale/superseded answer - // would be consumed as the answer to whatever check the client is actually still waiting on and - // the real answer would then be silently dropped. The guarantee is therefore scoped to "the - // CURRENT owner gets exactly one completion for the check they are currently waiting on", not - // "every StartFullMeshConnectivityCheck call produces a message": a superseded check's own - // question is answered by the newer check's eventual completion instead. + // "reason" values for FULL_MESH_CONNECTIVITY_CHECK_RESPONSE_COMPLETE_TO_HOST. "" means + // mesh_complete. A superseded check, or one whose requester is no longer owner, sends nothing + // (see Lobby.CompleteFullMeshConnectivityCheckLocked): at most one completion per current owner. internal static class FullMeshCheckOutcomeReason { internal const string None = ""; internal const string MissingConnections = "missing_connections"; internal const string Timeout = "timeout"; internal const string MemberLeft = "member_left"; + // quick match only: a member reported giving up on a peer (gave_up_user_ids) + internal const string GaveUp = "gave_up"; } // Core:full_mesh_check_* in appsettings.json, read on use @@ -107,6 +92,9 @@ internal static class FullMeshCheckSettings // upper bound for a whole check, used as the clients' setup timeout internal static int MaxDurationMS => (AttemptWindowMS * MaxAttempts) + (RetryDelayMS * (MaxAttempts - 1)); + // single-window referee period for quick match; clients own connection retries (2 attempts x 10s GNS default) + internal static int QuickMatchRefereeWindowMS => Get("full_mesh_check_qm_referee_window_ms", 20000); + private static int Get(string key, int defaultValue) { int value = Program.g_Config?.GetSection("Core").GetValue(key, defaultValue) ?? defaultValue; @@ -133,38 +121,21 @@ public class Lobby [JsonIgnore] public Int64 TimeStartFullMeshChecks { get; private set; } = -1; - // Incremented under m_LobbyGate every time a human member is added or removed. A quick match - // must never start on the strength of a mesh check that passed for a different set of - // players than the lobby currently holds, so this - plus MembershipVersionAtLastCheckStart - - // lets the check outcome be tied to the exact membership it was run against. Any membership - // change also clears LastFullMeshConnectivityCheckOutcome outright, as a second, simpler line - // of defense. + // Incremented under m_LobbyGate on every human add/remove; ties a check's outcome to the + // membership it ran against. [JsonIgnore] public int MembershipVersion { get; private set; } = 0; - // MembershipVersion as of the moment the current/most recently started check began. + // MembershipVersion when the current/most recent check began. [JsonIgnore] public int MembershipVersionAtLastCheckStart { get; private set; } = -1; - // Single per-lobby exclusive gate for every mutation of this lobby's Members/Owner, slot - // state/fields, ready state, and mesh-check state. Everything that used to be split across - // g_SlotLock (slots) and m_FullMeshCheckLock (mesh-check fields) now goes through this one - // SemaphoreSlim, so there is exactly one lock to reason about and no lock-ordering hazard - // between the two. It is NOT reentrant: a callback passed to RunExclusiveAsync must never - // call back into RunExclusiveAsync on this same lobby, or it will deadlock - callers that are - // already inside a callback (e.g. DoHostMigration, called from RemoveMember's callback) must - // mutate state directly instead. Sending a websocket message from inside a callback is safe: - // QueueWebsocketSend only enqueues bytes onto an in-memory bounded channel (a non-blocking - // TryWrite) - it never performs real network I/O itself, so it cannot block while the gate is - // held. + // Exclusive gate for every mutation of Members/Owner, slot state/fields, ready state, and + // mesh-check state. Not reentrant: a callback must not call RunExclusiveAsync on this lobby again. private readonly SemaphoreSlim m_LobbyGate = new SemaphoreSlim(1, 1); #if DEBUG - // Debug-only reentrancy guard. AsyncLocal flows through the awaited async chain (and into - // fire-and-forget children too, since ExecutionContext is captured when they're created), so - // a callback that calls back into RunExclusiveAsync on this same lobby - the exact bug class - // HOST_ACTION_KICK_USER hit - throws immediately here instead of deadlocking (if awaited) or - // silently racing (if not). Cheap enough to leave on for tests; compiled out of Release. + // Reentrancy guard: throws instead of deadlocking if RunExclusiveAsync is re-entered. private readonly AsyncLocal m_bGateHeldInThisFlow = new(); #endif @@ -244,25 +215,29 @@ public async Task RunExclusiveAsync(Func> action) [JsonIgnore] private bool m_bCurrentAttemptHasLegacyResponse = false; - // Set while a check is pending if a member leaves the lobby before it completes, so the - // eventual outcome can report FullMeshCheckOutcomeReason.MemberLeft instead of a generic - // connectivity failure. Reset each time a new check starts. + // Set while a check is pending if a member leaves before it completes; reset on next check start. [JsonIgnore] private bool m_bMemberLeftDuringCurrentCheck = false; - // The user who was Owner when the current check started (both real call sites - the - // websocket host-requests-begin handler and quickmatch's TriggerFullMeshConnectivityChecks - - // only ever start a check while its requester is the current owner). The eventual outcome is - // only ever sent to this user, and only if they are still Owner: see - // CompleteFullMeshConnectivityCheckLocked. + // Set (quick match only) when any snapshot reports gave_up_user_ids; forces an immediate fail. + [JsonIgnore] + private bool m_bAnyMemberGaveUp = false; + + // Quick match referees a single window with no server-side retries or re-signalling; custom + // lobby host-triggered checks keep their existing attempts/retry behavior. + private bool IsQuickMatchRefereeMode => LobbyType == ELobbyType.QuickMatch; + + private int CurrentAttemptWindowMS => IsQuickMatchRefereeMode ? FullMeshCheckSettings.QuickMatchRefereeWindowMS : FullMeshCheckSettings.AttemptWindowMS; + + private int CurrentMaxAttempts => IsQuickMatchRefereeMode ? 1 : FullMeshCheckSettings.MaxAttempts; + + // Owner when the current check started; the outcome is only sent to this user while they still own it. [JsonIgnore] private Int64 m_MeshCheckRequestingUserID = -1; private static Int64 s_NextFullMeshCheckID = 0; - // Backing counter for LobbyMember.JoinSequence: per-lobby, starts at 1, only ever assigned - // from within AddMember (which already runs under m_LobbyGate), so a plain Interlocked - // increment is enough without adding another lock. + // Backing counter for LobbyMember.JoinSequence; per-lobby, starts at 1. private Int64 m_NextJoinSequence = 0; [JsonIgnore] @@ -283,8 +258,6 @@ public async Task RunExclusiveAsync(Func> action) ConcurrentDictionary m_dictProbe2_Received = new(); public void RegisterProbeSent_Type1(Int64 userID) { - // AddOrUpdate is atomic; the previous check-then-write on the dictionary indexer - // could lose an increment when two probes for the same user race. m_dictProbe1_Sent.AddOrUpdate(userID, 1, (_, count) => count + 1); } @@ -328,11 +301,6 @@ await RunExclusiveAsync(() => { if (PendingFullMeshConnectivityChecks) { - // A new check preempts whatever was still in flight. The old check's own question - // is answered by the new check's eventual completion instead of sending a stale - // COMPLETE_TO_HOST here: the released client has only one callback slot and would - // consume whichever answer arrives first, dropping the real one. So this discards - // the old check's state without sending anything. DiscardPendingFullMeshCheckLocked(); } @@ -341,19 +309,15 @@ await RunExclusiveAsync(() => m_TimeToRetryFullMeshChecks = -1; LastFullMeshConnectivityCheckOutcome = null; m_bMemberLeftDuringCurrentCheck = false; + m_bAnyMemberGaveUp = false; MembershipVersionAtLastCheckStart = MembershipVersion; - // Both real callers only ever start a check while they are the current owner (the - // websocket handler checks this explicitly; quickmatch's dummy host is set as Owner at - // lobby creation), so Owner at this instant is the requester the eventual outcome - // belongs to. m_MeshCheckRequestingUserID = Owner; BeginFullMeshConnectivityCheckAttempt(); return Task.CompletedTask; }); } - // Resets pending-check state without sending a COMPLETE_TO_HOST. Must only be called while - // holding m_LobbyGate. + // Must be called while holding m_LobbyGate. Does not send a completion. private void DiscardPendingFullMeshCheckLocked() { PendingFullMeshConnectivityChecks = false; @@ -393,7 +357,7 @@ private void LogFullMeshCheckAttempt(bool bMeshComplete, List $"{p.Item1}<->{p.Item2}")); Console.WriteLine("[Lobby {0}] Mesh check {1} attempt {2}/{3}: {4} after {5} ms with {6} humans{7}", - LobbyID, FullMeshCheckID, FullMeshCheckAttempt, FullMeshCheckSettings.MaxAttempts, + LobbyID, FullMeshCheckID, FullMeshCheckAttempt, CurrentMaxAttempts, bMeshComplete ? "complete" : "incomplete", elapsedMS, GetNumberOfHumans(), bMeshComplete ? "" : $", missing {strMissing}{(strConnecting.Length > 0 ? $" (still connecting {strConnecting})" : "")}"); } @@ -479,6 +443,11 @@ await RunExclusiveAsync(() => m_bCurrentAttemptHasLegacyResponse |= bLegacyResponse; FullMeshConnectivityChecks[sourceUser] = new ConcurrentList(response.connectivity_map); m_FullMeshConnecting[sourceUser] = new HashSet(response.connecting_map); + + if (IsQuickMatchRefereeMode && response.gave_up_user_ids.Count > 0) + { + m_bAnyMemberGaveUp = true; + } } ProcessPendingFullMeshConnectivityChecksInternal(); @@ -522,7 +491,8 @@ private void ProcessPendingFullMeshConnectivityChecksInternal() // judged as soon as anyone replies: members who haven't replied yet count as missing, and gaps before // the window closes are re-polled below, which also recovers replies lost to a reconnect - bool bWindowElapsed = (Environment.TickCount64 - TimeStartFullMeshChecks) >= FullMeshCheckSettings.AttemptWindowMS; + bool bWindowElapsed = (Environment.TickCount64 - TimeStartFullMeshChecks) >= CurrentAttemptWindowMS; + bool bGaveUpForcesCompletion = IsQuickMatchRefereeMode && m_bAnyMemberGaveUp; bDoneChecks = bWindowElapsed || !FullMeshConnectivityChecks.IsEmpty; List lstMissingConnections = new(); @@ -597,11 +567,11 @@ private void ProcessPendingFullMeshConnectivityChecksInternal() - bool bMeshComplete = bDisableMeshCheck || lstMissingConnections.Count == 0; + bool bMeshComplete = !bGaveUpForcesCompletion && (bDisableMeshCheck || lstMissingConnections.Count == 0); // members report a snapshot, so a gap before the window closes may be a connection still forming: // keep asking for fresh snapshots rather than re-signalling it early - if (!bMeshComplete && !bWindowElapsed) + if (!bMeshComplete && !bWindowElapsed && !bGaveUpForcesCompletion) { if (Environment.TickCount64 >= m_TimeNextFullMeshSnapshotRequest) { @@ -618,7 +588,7 @@ private void ProcessPendingFullMeshConnectivityChecksInternal() bMeshComplete, m_bCurrentAttemptHasLegacyResponse, FullMeshCheckAttempt, - FullMeshCheckSettings.MaxAttempts)) + CurrentMaxAttempts)) { ++FullMeshCheckAttempt; @@ -632,7 +602,7 @@ private void ProcessPendingFullMeshConnectivityChecksInternal() // inform host that we are done bool bMembershipChangedDuringCheck = MembershipVersion != MembershipVersionAtLastCheckStart; - bool bMeshCompleteFinal = !bMembershipChangedDuringCheck && (bDisableMeshCheck || lstMissingConnections.Count == 0); + bool bMeshCompleteFinal = !bGaveUpForcesCompletion && !bMembershipChangedDuringCheck && (bDisableMeshCheck || lstMissingConnections.Count == 0); List lstFinalMissingConnections = bDisableMeshCheck ? new List() : lstMissingConnections; @@ -642,15 +612,16 @@ private void ProcessPendingFullMeshConnectivityChecksInternal() { reason = FullMeshCheckOutcomeReason.None; } + else if (bGaveUpForcesCompletion) + { + reason = FullMeshCheckOutcomeReason.GaveUp; + } else if (bMembershipChangedDuringCheck || m_bMemberLeftDuringCurrentCheck) { - // A join also invalidates the check (not just a leave): either way the set of - // players this check was judged against is no longer the lobby's actual membership. reason = FullMeshCheckOutcomeReason.MemberLeft; } else if (FullMeshConnectivityChecks.IsEmpty) { - // nobody ever reported a snapshot for this attempt, so nothing could be judged connected reason = FullMeshCheckOutcomeReason.Timeout; } else @@ -663,8 +634,7 @@ private void ProcessPendingFullMeshConnectivityChecksInternal() } } - // Sends FULL_MESH_CONNECTIVITY_CHECK_RESPONSE_COMPLETE_TO_HOST to the host and resets the - // pending-check state. Must only be called while holding m_LobbyGate. + // Must be called while holding m_LobbyGate. private void CompleteFullMeshConnectivityCheckLocked(bool bMeshComplete, List lstMissingConnections, string reason) { WebSocketMessage_FullMeshConnectivityCheckOutcome outcome = new WebSocketMessage_FullMeshConnectivityCheckOutcome(); @@ -677,10 +647,6 @@ private void CompleteFullMeshConnectivityCheckLocked(bool bMeshComplete, List? OnLobbyNeedsDestroyed; - // Must only be called while holding m_LobbyGate (from within RemoveMember's callback): it reads - // and mutates Members/Owner and must observe RemoveMember's slot clear atomically with any - // concurrent join/leave/migration. + // Must be called while holding m_LobbyGate. private void OnAfterPlayerLeftLocked(Int64 leavingUserID) { // NOTE: By the time this is called, the member is no longer in the members list bool bNeedsHostMigrate = Owner == leavingUserID; - // A departure while a mesh check is pending can explain that check's eventual failure, - // so the outcome can say why instead of reporting a generic connectivity failure. if (PendingFullMeshConnectivityChecks) { m_bMemberLeftDuringCurrentCheck = true; @@ -1055,11 +1017,6 @@ public async Task FinalizeACChecks() } } - // Runs under the per-lobby gate: this mutates slot state and previously ran unguarded, - // racing against AddMember/RemoveMember/other slot updates. Sequential with (never nested - // inside) StartFullMeshConnectivityCheck's own gate use - callers await this first and then - // await StartFullMeshConnectivityCheck as a separate gated operation, so the gate is never - // re-entered. public async Task CloseOpenSlots() { await RunExclusiveAsync(() => @@ -1077,8 +1034,7 @@ await RunExclusiveAsync(() => }); } - // Must only be called while holding m_LobbyGate (currently only true from - // OnAfterPlayerLeftLocked, itself only called from within RemoveMember's callback). + // Must be called while holding m_LobbyGate. private void DoHostMigration() { Int64 oldOwner = Owner; @@ -1208,8 +1164,6 @@ public async Task Tick() public async Task AddMember(UserSession playerSession, string strDisplayName, UInt16 userPreferredPort, bool bHasMap, UserLobbyPreferences lobbyPrefs) { - // NOTE: AddMember runs inside the per-lobby gate, so timing + slot determination can no - // longer result in two concurrent joins picking the same slot. return await RunExclusiveAsync(async () => { if (State != ELobbyState.GAME_SETUP) @@ -1288,7 +1242,6 @@ public async Task AddMember(UserSession playerSession, string strDisplayNa strDisplayName = String.Format("{0} ({1})", strDisplayName, dupesSeen); } - // AddMember only runs inside RunExclusiveAsync, so this increment is already serialized. Int64 joinSequence = Interlocked.Increment(ref m_NextJoinSequence); // only apply lobby prefs if not QM @@ -1346,8 +1299,6 @@ public async Task AddMember(UserSession playerSession, string strDisplayNa Members[slotIndex] = newMember; TimeMemberLeft[playerSession.m_UserID] = DateTime.UnixEpoch; - // Membership just changed: any previously-passed mesh check no longer describes who is - // actually in the lobby. ++MembershipVersion; LastFullMeshConnectivityCheckOutcome = null; @@ -1447,11 +1398,6 @@ public void SendPeerTeardownToDepartingMember(LobbyMember departingMember) public async Task RemoveMember(LobbyMember member) { - // Matchmaking cancellation and the client's explicit lobby leave can arrive concurrently. - // Claim the slot once, and run host migration in the same critical section as the slot - // clear, so teardown, host migration, and destruction callbacks stay atomic and idempotent - // (previously DoHostMigration ran after this gate was released, so it could interleave - // with a concurrent AddMember/RemoveMember and observe a half-updated Members/Owner). bool bRemoved = await RunExclusiveAsync(() => { if (member.SlotIndex < 0 @@ -1465,15 +1411,9 @@ public async Task RemoveMember(LobbyMember member) Members[member.SlotIndex] = placeholderMember; TimeMemberLeft[member.UserID] = DateTime.UtcNow; - // Membership just changed: any previously-passed mesh check no longer describes who - // is actually in the lobby. ++MembershipVersion; LastFullMeshConnectivityCheckOutcome = null; - // A quick match must never start on a stale "everyone joined/connected" verdict once - // someone has left. This is a direct static call (not an event) that only ever touches - // MatchmakingBucket's own lock, never this lobby's gate, so it is safe to make from in - // here. if (LobbyType == ELobbyType.QuickMatch) { MatchmakingManager.InvalidateAutoStartForLobby(LobbyID); @@ -1853,11 +1793,7 @@ public void UpdateSlotIndex(UInt16 index) public string Region { get; private set; } = "Unknown"; public string MiddlewareUserID { get; private set; } = String.Empty; - // Per-lobby, monotonically increasing, assigned once when a human player is added - // (AddMember/CreateLobby) and never reassigned afterwards: host migration and slot moves - // only change SlotIndex/Owner on this same object, so JoinSequence is untouched. AI and - // open/closed placeholder slots keep the default of 0. A player who leaves and rejoins gets - // a new LobbyMember instance with a new, higher value. + // Per-lobby monotonic join order; 0 for AI/open/closed slots. public Int64 JoinSequence { get; private set; } = 0; [JsonIgnore] // cant serialize refs @@ -1996,11 +1932,7 @@ public class LobbyManager private Int64 m_NextLobbyID = 0; - // Concurrent CreateLobby calls (e.g. a custom lobby and a QuickMatch allocation racing) must never be - // handed the same ID. Interlocked.Increment returns the post-increment value, so subtracting 1 keeps - // the original "starts at 0" sequence while making the read-and-bump atomic. - // Internal (not private) so it is unit-testable via InternalsVisibleTo without needing the rest of - // CreateLobby's database dependencies. + // Atomic; sequence starts at 0. internal Int64 GenerateNextLobbyID() { return Interlocked.Increment(ref m_NextLobbyID) - 1; diff --git a/GenOnlineService/MatchmakingManager.cs b/GenOnlineService/MatchmakingManager.cs index 83f8e3e..07b1d46 100644 --- a/GenOnlineService/MatchmakingManager.cs +++ b/GenOnlineService/MatchmakingManager.cs @@ -153,9 +153,12 @@ public class Playlist public UInt16 PlaylistID { get; private set; } public string Name { get; private set; } public int MinPlayers { get; private set; } - public int DesiredPlayers { get; private set; } + public int MaxPlayers { get; private set; } public int MinSelectedMaps { get; private set; } + // Wire-compat alias for GET Playlists; MaxPlayers is the additive field. + public int DesiredPlayers => MaxPlayers; + public bool AllowTeams { get; private set; } public int TeamSize { get; private set; } public bool AllowArmySelection { get; private set; } @@ -163,13 +166,13 @@ public class Playlist public List Maps { get; private set; } public Playlist(UInt16 a_PlaylistID, string a_strName, - int a_MinPlayers, int a_DesiredPlayers, int a_MinSelectedMaps, bool a_bAllowTeams, int a_TeamSize, bool a_bAllowArmySelection, UInt16 a_gracePeriodAtMinPlayersMSec, List allowedMaps) + int a_MinPlayers, int a_MaxPlayers, int a_MinSelectedMaps, bool a_bAllowTeams, int a_TeamSize, bool a_bAllowArmySelection, UInt16 a_gracePeriodAtMinPlayersMSec, List allowedMaps) { PlaylistID = a_PlaylistID; Name = a_strName; MinPlayers = a_MinPlayers; MinSelectedMaps = a_MinSelectedMaps; - DesiredPlayers = a_DesiredPlayers; + MaxPlayers = a_MaxPlayers; AllowTeams = a_bAllowTeams; TeamSize = a_TeamSize; AllowArmySelection = a_bAllowArmySelection; @@ -180,6 +183,70 @@ public Playlist(UInt16 a_PlaylistID, string a_strName, static class MatchmakingManager { + // Min<=Max, every map's slot count within [Min,Max], every N in [Min,Max] has >=1 exact-N map. + internal static bool ValidatePlaylistMapSizes(Playlist playlist, out List errors) + { + errors = new List(); + + if (playlist.MinPlayers > playlist.MaxPlayers) + { + errors.Add($"MinPlayers ({playlist.MinPlayers}) > MaxPlayers ({playlist.MaxPlayers})"); + } + + foreach (PlaylistMap map in playlist.Maps) + { + if (map.MaxPlayers < playlist.MinPlayers || map.MaxPlayers > playlist.MaxPlayers) + { + errors.Add($"Map '{map.Name}' has {map.MaxPlayers} slots, outside [{playlist.MinPlayers},{playlist.MaxPlayers}]"); + } + } + + for (int n = playlist.MinPlayers; n <= playlist.MaxPlayers; n++) + { + bool bHasExactMap = false; + foreach (PlaylistMap map in playlist.Maps) + { + if (map.MaxPlayers == n) + { + bHasExactMap = true; + break; + } + } + + if (!bHasExactMap) + { + errors.Add($"No map with exactly {n} slots"); + } + } + + return errors.Count == 0; + } + + // Invalid playlists are excluded and logged rather than crashing startup. + public static void ValidatePlaylistsAtStartup() + { + List invalidPlaylistIDs = new(); + foreach (var kvPair in g_Playlists) + { + if (!ValidatePlaylistMapSizes(kvPair.Value, out List errors)) + { + Console.ForegroundColor = ConsoleColor.Red; + Console.WriteLine("[FATAL] Playlist {0} ('{1}') failed validation and is excluded from matchmaking:", kvPair.Key, kvPair.Value.Name); + foreach (string error in errors) + { + Console.WriteLine(" - {0}", error); + } + Console.ForegroundColor = ConsoleColor.Gray; + invalidPlaylistIDs.Add(kvPair.Key); + } + } + + foreach (UInt16 invalidID in invalidPlaylistIDs) + { + g_Playlists.Remove(invalidID); + } + } + // World Series 2026 Qualification in September requires the matchmaking to be // based off the monthly ELO. internal static int GetMatchmakingElo(PlayerStats stats) @@ -346,7 +413,7 @@ private bool IsPendingDeletion() public UInt16 PlaylistID { get; private set; } public int MinPlayers { get; private set; } - public int DesiredPlayers { get; private set; } + public int MaxPlayers { get; private set; } public UInt32 ExeCRC { get; private set; } public UInt32 IniCRC { get; private set; } @@ -380,19 +447,19 @@ private TimeSpan TimeSinceLastEloExpansion() return null; } - public void DetermineMap(out string strMapName, out string strMapPath) + // Selects a map with EXACTLY matchSize slots, never a different size. Returns false if none exists. + public bool TryDetermineMapForSize(int matchSize, out string strMapName, out string strMapPath) { - // If they are in this bucket, they had SOME map overlap with the bucket creator, now we need to find the common ground between everyone - - // TODO_QUICKMATCH: what if we cant find a suitable map? + strMapName = string.Empty; + strMapPath = string.Empty; - // first condense the map list doesn to a list that has mutually agreed upon maps/preferences from all participants - //var mapSetFromBucketCreator = new HashSet(lstMapIndices); + if (!MatchmakingManager.g_Playlists.TryGetValue(PlaylistID, out Playlist? playlist)) + { + return false; + } var perPlayerMapSet = new List>(); - var finalMapSet = new HashSet(lstMapIndices.ToList()); // we need to check intersection against this, so pre-populate it with the original bucket creation list, because that's the "biggest set" in theory - - // TODO_QUICKMATCH: Optimize this, it's inefficient + var finalMapSet = new HashSet(lstMapIndices.ToList()); foreach (MatchmakingBucketMember member in m_lstMembers) { @@ -403,97 +470,34 @@ public void DetermineMap(out string strMapName, out string strMapPath) } } - // Find shared values across all of perPlayerMapSet - if (perPlayerMapSet.Count > 0) + foreach (HashSet memberMapSet in perPlayerMapSet) { - for (int i = 0; i < perPlayerMapSet.Count; i++) - { - finalMapSet.IntersectWith(perPlayerMapSet[i]); - } + finalMapSet.IntersectWith(memberMapSet); } - // remove any maps that aren't big enough (mainly applies to FFA's where maps may be 6 players but bucket could be 8 players) - // NOTE: iterate a real copy, we are mutating finalMapSet below - var copyMapSetForIter = new HashSet(finalMapSet); - foreach (int mapIndex in copyMapSetForIter) - { - if (MatchmakingManager.g_Playlists.TryGetValue(PlaylistID, out Playlist? playlist)) - { - if (mapIndex >= 0 && mapIndex < playlist.Maps.Count) - { - if (playlist.Maps[mapIndex].MaxPlayers < CurrentMemberCount()) - { - finalMapSet.Remove(mapIndex); - } - } - } - } + List exactMatches = finalMapSet + .Where(mapIndex => mapIndex >= 0 && mapIndex < playlist.Maps.Count && playlist.Maps[mapIndex].MaxPlayers == matchSize) + .ToList(); - // Randomly select a map from the map list - if (finalMapSet.Count > 0) + if (exactMatches.Count > 0) { - // Get the playlist for this bucket - if (MatchmakingManager.g_Playlists.TryGetValue(PlaylistID, out Playlist? playlist)) - { - var finalMapIndices = finalMapSet.ToList(); - int selectedIndex = Random.Shared.Next(finalMapIndices.Count); - int mapIndex = finalMapIndices[selectedIndex]; - - // Defensive: ensure index is valid for playlist.Maps - if (mapIndex >= 0 && mapIndex < playlist.Maps.Count) - { - strMapName = playlist.Maps[mapIndex].Name; - strMapPath = playlist.Maps[mapIndex].Path; - return; - } - } - } - else - { - // pick a sensible default (biggest map in playlist), probably not what the players asked for, but we cant play on no map - Console.WriteLine("WARNING: No mutually agreed upon map found for matchmaking bucket, falling back to largest map in playlist"); - - if (MatchmakingManager.g_Playlists.TryGetValue(PlaylistID, out Playlist? playlist)) - { - int biggestCountSeen = 0; - PlaylistMap? mapToUse = null; - foreach (var map in playlist.Maps) - { - if (map.MaxPlayers > biggestCountSeen) - { - biggestCountSeen = map.MaxPlayers; - mapToUse = map; - } - } - - // TODO_QUICKMATCH: What if it's still null? don't think we can get into that state since we must have some kind of map in the playlist - if (mapToUse != null) - { - strMapName = mapToUse.Name; - strMapPath = mapToUse.Path; - return; - } - } + int mapIndex = exactMatches[Random.Shared.Next(exactMatches.Count)]; + strMapName = playlist.Maps[mapIndex].Name; + strMapPath = playlist.Maps[mapIndex].Path; + return true; } - // TODO_QUICKMATCH: What happens if you widen when already in a bucket? tell the user htey cant? you would need everyone to expand, or just expand for everyone? - - // Fallback: use first map from bucket creator's list if available - if (lstMapIndices.Count > 0 && MatchmakingManager.g_Playlists.TryGetValue(PlaylistID, out Playlist? fallbackPlaylist)) + List exactSizeMaps = playlist.Maps.Where(map => map.MaxPlayers == matchSize).ToList(); + if (exactSizeMaps.Count > 0) { - int fallbackIndex = lstMapIndices[0]; - if (fallbackIndex >= 0 && fallbackIndex < fallbackPlaylist.Maps.Count) - { - strMapName = fallbackPlaylist.Maps[fallbackIndex].Name; - strMapPath = fallbackPlaylist.Maps[fallbackIndex].Path; - return; - } + PlaylistMap chosenMap = exactSizeMaps[Random.Shared.Next(exactSizeMaps.Count)]; + strMapName = chosenMap.Name; + strMapPath = chosenMap.Path; + return true; } - // If no map found, set to empty - strMapName = string.Empty; - strMapPath = string.Empty; - + Console.WriteLine("WARNING: Playlist {0} has no exact-{1}-player map; a match cannot form at this size.", PlaylistID, matchSize); + return false; } public bool DoMapSelectionsIntersect(ConcurrentList lstRhs) @@ -609,7 +613,7 @@ public async Task MergeWithOtherBucket(MatchmakingBucket bucketToMerge) UserSession? session = member.GetAssociatedSession(); if (session != null) { - await SendMatchmakingMessage(session, String.Format("Your matchmaking bucket was merged with another bucket. Status: {0}/{1} players. ({2} required to start)", CurrentMemberCount(), DesiredPlayers, MinPlayers)); + await SendMatchmakingMessage(session, String.Format("Your matchmaking bucket was merged with another bucket. Status: {0}/{1} players. ({2} required to start)", CurrentMemberCount(), MaxPlayers, MinPlayers)); } } } @@ -699,9 +703,6 @@ public bool OwnsLobby(Int64 lobbyID) return lobbyID != -1 && m_LobbyID == lobbyID; } - // Same invalidation the existing RemovePlayer/PruneDeadMembers paths use, exposed for - // external hooks (a lobby-level leave, or a session going abandoned) that don't go through - // this bucket's own member list. public void InvalidateAutoStart() { lock (m_StateLock) @@ -770,7 +771,7 @@ public bool HasSpaceForUsers(int numUsers, UInt32 exe_crc, UInt32 ini_crc, EKnow return false; } - return numUsers <= (DesiredPlayers - m_lstMembers.Count); + return numUsers <= (MaxPlayers - m_lstMembers.Count); } public bool IsAvgEloWithinThreshold(int playerElo, int eloThreshold) @@ -843,7 +844,7 @@ public async Task Join(UserSession playerSession) UserSession? memberSession = member.GetAssociatedSession(); if (memberSession != null) { - await SendMatchmakingMessage(memberSession, String.Format("Status: {0}/{1} players. ({2} required to start)", CurrentMemberCount(), DesiredPlayers, MinPlayers)); + await SendMatchmakingMessage(memberSession, String.Format("Status: {0}/{1} players. ({2} required to start)", CurrentMemberCount(), MaxPlayers, MinPlayers)); } } @@ -854,11 +855,11 @@ public async Task Join(UserSession playerSession) return false; } - public MatchmakingBucket(UInt16 playlistID, UserSession owningSession, int minPlayers, int desiredPlayers, ConcurrentList mapIndices, UInt32 exe_crc, UInt32 ini_crc, EKnownAnticheatID anticheatID) + public MatchmakingBucket(UInt16 playlistID, UserSession owningSession, int minPlayers, int maxPlayers, ConcurrentList mapIndices, UInt32 exe_crc, UInt32 ini_crc, EKnownAnticheatID anticheatID) { PlaylistID = playlistID; MinPlayers = minPlayers; - DesiredPlayers = desiredPlayers; + MaxPlayers = maxPlayers; lstMapIndices = mapIndices; ExeCRC = exe_crc; IniCRC = ini_crc; @@ -867,6 +868,118 @@ public MatchmakingBucket(UInt16 playlistID, UserSession owningSession, int minPl m_lstMembers.Add(new MatchmakingBucketMember(owningSession)); } + internal void AddExistingMember(MatchmakingBucketMember member) + { + lock (m_StateLock) + { + m_lstMembers.Add(member); + } + } + + private static bool PlaylistHasExactSizeMap(Playlist playlist, int n) + { + foreach (PlaylistMap map in playlist.Maps) + { + if (map.MaxPlayers == n) + { + return true; + } + } + + return false; + } + + // Forms immediately at MaxPlayers; after the grace period, forms at the largest N in + // [MinPlayers, min(current, MaxPlayers)] that has an exact-N map. + internal bool TryDetermineFormationSize(Playlist playlist, bool bGraceExpired, out int matchSize) + { + int currentCount = CurrentMemberCount(); + + if (currentCount >= playlist.MaxPlayers) + { + matchSize = playlist.MaxPlayers; + return true; + } + + if (!bGraceExpired) + { + matchSize = 0; + return false; + } + + for (int n = Math.Min(currentCount, playlist.MaxPlayers); n >= playlist.MinPlayers; n--) + { + if (PlaylistHasExactSizeMap(playlist, n)) + { + matchSize = n; + return true; + } + } + + matchSize = 0; + return false; + } + + // Keeps the first keepCount members; moves the rest into a new bucket with the same grouping. + private async Task SplitOffExcessMembersToNewBucketAsync(int keepCount) + { + List excessMembers = new(); + lock (m_StateLock) + { + List all = m_lstMembers.ToList(); + for (int i = keepCount; i < all.Count; i++) + { + if (m_lstMembers.Remove(all[i])) + { + excessMembers.Add(all[i]); + } + } + } + + if (excessMembers.Count == 0) + { + return; + } + + // Pick the first excess member who still has a live session to own the new bucket - the + // constructor requires one. A dead session at index 0 must not lose every other (live) + // excess member; if nobody has a live session left, there's nothing to bucket or notify. + MatchmakingBucketMember? newBucketOwner = null; + foreach (MatchmakingBucketMember excessMember in excessMembers) + { + if (excessMember.GetAssociatedSession() != null) + { + newBucketOwner = excessMember; + break; + } + } + + if (newBucketOwner == null) + { + return; + } + + MatchmakingBucket newBucket = new MatchmakingBucket(PlaylistID, newBucketOwner.GetAssociatedSession()!, MinPlayers, MaxPlayers, new ConcurrentList(lstMapIndices.ToList()), ExeCRC, IniCRC, AnticheatID); + foreach (MatchmakingBucketMember excessMember in excessMembers) + { + if (excessMember != newBucketOwner && excessMember.GetAssociatedSession() != null) + { + newBucket.AddExistingMember(excessMember); + } + } + + MatchmakingManager.RegisterExistingBucket(PlaylistID, newBucket); + + foreach (MatchmakingBucketMember excessMember in excessMembers) + { + UserSession? excessSession = excessMember.GetAssociatedSession(); + if (excessSession != null) + { + await SendMatchmakingMessage(excessSession, "Still searching for more players..."); + } + } + } + public Int64 GetLobbyID() { return m_LobbyID; @@ -876,6 +989,9 @@ public Int64 GetLobbyID() Int64 m_StartTime = -1; Int64 m_timeStartedWaitingOnLobbyJoins = -1; + // Player count the match was formed with; the map's slot count must match this. + int m_MatchFormedSize = -1; + // how long we give everyone to actually connect to the QuickMatch lobby before we give up on the stragglers private const Int64 c_LobbyJoinTimeoutMSec = 45000; private const int c_GameStartCountdownMSec = 5000; @@ -935,7 +1051,7 @@ private async Task TriggerFullMeshConnectivityChecks(Lobby lobby) } // kept at or above the legacy threshold so short tuned checks don't show a start countdown on older clients - QueueSetupProgress(Math.Max(FullMeshCheckSettings.MaxDurationMS + c_SetupClientTimeoutMarginMSec, c_LegacyClientCountdownThresholdMSec)); + QueueSetupProgress(Math.Max(FullMeshCheckSettings.QuickMatchRefereeWindowMS + c_SetupClientTimeoutMarginMSec, c_LegacyClientCountdownThresholdMSec)); lobby.SendFullMeshConnectivityCheckRequestToMembers(); @@ -949,6 +1065,66 @@ private async Task TriggerFullMeshConnectivityChecks(Lobby lobby) } } + // Re-verifies the match is still valid to start: same players, all live, mesh check still valid. + internal bool VerifyMatchIsStillValid(Lobby lobby, out string failureReason) + { + List humanMembers = lobby.Members.Where(m => m.IsHuman()).ToList(); + + if (humanMembers.Count != m_MatchFormedSize) + { + failureReason = "the lobby no longer holds the number of players the match was formed with"; + return false; + } + + if (lobby.MaxPlayers != m_MatchFormedSize) + { + failureReason = "the lobby's slot count no longer matches the map's player count"; + return false; + } + + HashSet lobbyUserIDs = humanMembers.Select(m => m.UserID).ToHashSet(); + + HashSet bucketUserIDs = new(); + foreach (MatchmakingBucketMember bucketMember in m_lstMembers) + { + UserSession? bucketMemberSession = bucketMember.GetAssociatedSession(); + if (bucketMemberSession != null) + { + bucketUserIDs.Add(bucketMemberSession.m_UserID); + } + } + + if (!lobbyUserIDs.SetEquals(bucketUserIDs)) + { + failureReason = "the lobby's players no longer match the players the match was formed with"; + return false; + } + + foreach (LobbyMember member in humanMembers) + { + if (!member.GetSession().TryGetTarget(out UserSession? session) || session == null || session.IsAbandoned()) + { + failureReason = $"user {member.UserID}'s connection is no longer live"; + return false; + } + } + + if (lobby.MembershipVersion != lobby.MembershipVersionAtLastCheckStart) + { + failureReason = "lobby membership changed after the connectivity check"; + return false; + } + + if (lobby.LastFullMeshConnectivityCheckOutcome != true) + { + failureReason = "the last connectivity check did not report everyone connected"; + return false; + } + + failureReason = string.Empty; + return true; + } + private async Task AbortQuickMatchAutoStart(string reason) { List sessionsToRequeue = new(); @@ -1076,6 +1252,8 @@ public async Task Tick() if (!lobbyDuringMeshCheck.PendingFullMeshConnectivityChecks) { + bool bMatchStillValid = VerifyMatchIsStillValid(lobbyDuringMeshCheck, out string verifyFailureReason); + bool bStartCountdown; bool bAbortStart; bool bInvalidatedAtDecision; @@ -1090,7 +1268,7 @@ public async Task Tick() else { m_bWaitingOnMeshConnectivityChecks = false; - bStartCountdown = !bInvalidatedAtDecision && lobbyDuringMeshCheck.LastFullMeshConnectivityCheckOutcome == true; + bStartCountdown = !bInvalidatedAtDecision && bMatchStillValid; bAbortStart = !bStartCountdown; if (bStartCountdown) { @@ -1116,7 +1294,9 @@ public async Task Tick() { string reason = bInvalidatedAtDecision ? "QuickMatch auto-start was aborted because a player left during match setup." - : "QuickMatch auto-start was aborted because not all players were fully mesh-connected."; + : !bMatchStillValid + ? $"QuickMatch auto-start was aborted because the match is no longer valid: {verifyFailureReason}." + : "QuickMatch auto-start was aborted because not all players were fully mesh-connected."; await AbortQuickMatchAutoStart(reason); } } @@ -1138,6 +1318,8 @@ public async Task Tick() return; } + bool bMatchStillValidAtStart = VerifyMatchIsStillValid(lobbyAfterCountdown, out string startVerifyFailureReason); + bool bStartGame; bool bAbortStart; lock (m_StateLock) @@ -1151,7 +1333,7 @@ public async Task Tick() { m_bHasStartedCountdown = false; m_StartTime = -1; - bStartGame = !m_bAutoStartInvalidated; + bStartGame = !m_bAutoStartInvalidated && bMatchStillValidAtStart; bAbortStart = !bStartGame; if (bStartGame) { @@ -1167,7 +1349,10 @@ public async Task Tick() } else if (bAbortStart) { - await AbortQuickMatchAutoStart("QuickMatch auto-start was aborted because a player left during match setup."); + string abortReason = !bMatchStillValidAtStart + ? $"QuickMatch auto-start was aborted because the match is no longer valid: {startVerifyFailureReason}." + : "QuickMatch auto-start was aborted because a player left during match setup."; + await AbortQuickMatchAutoStart(abortReason); } return; @@ -1178,7 +1363,7 @@ public async Task Tick() if (!m_bWaitingOnLobbyJoins && !m_bHasStartedCountdown) { // must have a min player count - if (MinPlayers != DesiredPlayers) + if (MinPlayers != MaxPlayers) { // have we hit the min player count? start a timer // NOTE: >= not ==, a merge (or several joins in one tick) can jump straight past MinPlayers @@ -1233,12 +1418,19 @@ await SendMatchmakingMessage(memberSession, // did we hit the timer OR have enough players to start? bool bMinPlayersCountdownExpired = m_bReachedMinPlayers && (Environment.TickCount64 - m_timeReachedMinPlayers) > playlist.GracePeriodAtMinPlayersMSec; - if (bMinPlayersCountdownExpired || CurrentMemberCount() >= DesiredPlayers) + if (TryDetermineFormationSize(playlist, bMinPlayersCountdownExpired, out int formationSize)) { // reset min player countdown m_bReachedMinPlayers = false; m_timeReachedMinPlayers = -1; + if (formationSize < CurrentMemberCount()) + { + await SplitOffExcessMembersToNewBucketAsync(formationSize); + } + + m_MatchFormedSize = formationSize; + lock (m_StateLock) { m_bWaitingOnLobbyJoins = true; @@ -1275,20 +1467,25 @@ await SendMatchmakingMessage(memberSession, if (dummyHostUserData != null) { // make a lobby - DetermineMap(out string strMapName, out string strMapPath); + if (!TryDetermineMapForSize(m_MatchFormedSize, out string strMapName, out string strMapPath)) + { + Console.WriteLine("Matchmaking bucket {0} could not find a map for size {1}, skipping formation this tick", PlaylistID, m_MatchFormedSize); + return; + } using var scope = ServiceLocator.Services.CreateScope(); var factory = scope.ServiceProvider.GetRequiredService>(); await using var db = await factory.CreateDbContextAsync(); m_LobbyID = await lobbyManager.CreateLobby(db, dummyHostUser, dummyHostUserData.m_strDisplayName, "Quickmatch Lobby", strMapName, strMapPath + ".map", - true, playlist.DesiredPlayers, "", 12345, false, true, 10000, false, String.Empty, -5, false, Constants.g_DefaultCameraMaxHeight, dummyHostUser.ExeCRC, dummyHostUser.IniCRC, ELobbyType.QuickMatch, + true, m_MatchFormedSize, "", 12345, false, true, 10000, false, String.Empty, -5, false, Constants.g_DefaultCameraMaxHeight, dummyHostUser.ExeCRC, dummyHostUser.IniCRC, ELobbyType.QuickMatch, dummyHostUser.AnticheatID); // tell both to join our lobby WebSocketMessage_MatchmakerJoinLobby joinAction = new WebSocketMessage_MatchmakerJoinLobby(); joinAction.msg_id = (int)EWebSocketMessageID.MATCHMAKING_ACTION_JOIN_PREARRANGED_LOBBY; joinAction.lobby_id = m_LobbyID; + joinAction.lobby = lobbyManager.GetLobby(m_LobbyID); byte[] bytesJSON = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(joinAction)); foreach (MatchmakingBucketMember member in m_lstMembers) @@ -1328,7 +1525,7 @@ await SendMatchmakingMessage(memberSession, ActiveUserDataCache? memberSession = member.GetAssociatedSession(); if (memberSession != null) { - await SendMatchmakingMessage(memberSession, String.Format("A player has left and the starting countdown has been cancelled. Status: {0}/{1} players. ({2} required to start)", CurrentMemberCount(), DesiredPlayers, MinPlayers)); + await SendMatchmakingMessage(memberSession, String.Format("A player has left and the starting countdown has been cancelled. Status: {0}/{1} players. ({2} required to start)", CurrentMemberCount(), MaxPlayers, MinPlayers)); } } @@ -1761,14 +1958,14 @@ public static async Task Tick() // didnt find a bucket? make one if (bucketInUse == null) { - MatchmakingBucket newBucket = new MatchmakingBucket(playlist.PlaylistID, thisSession, playlist.MinPlayers, playlist.DesiredPlayers, thisSession.MatchmakingMapIndicies, thisSession.ExeCRC, thisSession.IniCRC, thisSession.AnticheatID); + MatchmakingBucket newBucket = new MatchmakingBucket(playlist.PlaylistID, thisSession, playlist.MinPlayers, playlist.MaxPlayers, thisSession.MatchmakingMapIndicies, thisSession.ExeCRC, thisSession.IniCRC, thisSession.AnticheatID); m_dictMatchmakingBuckets[thisSession.MatchmakingPlaylistID].Add(newBucket); bucketInUse = newBucket; } // send status to use await SendMatchmakingMessage(thisSession, String.Format("You are now matchmaking in playlist \"{0}\". There are currently {1} player(s) searching for a match in this playlist", playlist.Name, GetTotalQueuedPlayersInPlaylist(playlist.PlaylistID))); - await SendMatchmakingMessage(thisSession, String.Format("Status: {0}/{1} players. ({2} required to start)", bucketInUse.CurrentMemberCount(), bucketInUse.DesiredPlayers, bucketInUse.MinPlayers)); + await SendMatchmakingMessage(thisSession, String.Format("Status: {0}/{1} players. ({2} required to start)", bucketInUse.CurrentMemberCount(), bucketInUse.MaxPlayers, bucketInUse.MinPlayers)); // now remove us from lstSessions, this list is essentially people who need sorted into a bucket lstDestroy.Add(wrSession); @@ -1859,11 +2056,12 @@ public static void DestroyBucket(MatchmakingBucket bucket) m_bucketsPendingDeletion.Enqueue(bucket); } - // Hook from Lobby (a member left) or WebSocketManager (a session in a QM lobby went abandoned) - // into whichever bucket owns that lobby, so a stale "everyone connected" verdict can never be - // used to start a match that no longer holds the players it was formed with. Only ever touches - // MatchmakingBucket's own m_StateLock - never a Lobby's gate - so it is safe to call from - // anywhere, including from inside Lobby.RemoveMember's own gated callback. + internal static void RegisterExistingBucket(UInt16 playlistID, MatchmakingBucket bucket) + { + m_dictMatchmakingBuckets.GetOrAdd(playlistID, _ => new ConcurrentBag()).Add(bucket); + } + + // Safe to call from inside Lobby.RemoveMember's gated callback: only touches m_StateLock. public static void InvalidateAutoStartForLobby(Int64 lobbyID) { if (lobbyID == -1) @@ -2001,4 +2199,4 @@ public static async Task DeregisterPlayer(UserSession plr) await lobbyManager.LeaveAnyLobby(plr.m_UserID); } } -} +} diff --git a/GenOnlineService/Program.cs b/GenOnlineService/Program.cs index 750bfe9..0b0f16c 100644 --- a/GenOnlineService/Program.cs +++ b/GenOnlineService/Program.cs @@ -1614,6 +1614,7 @@ public static async Task Main(string[] args) app.MapControllers(); + MatchmakingManager.ValidatePlaylistsAtStartup(); // cleanup System.Timers.Timer timerCleanup = new System.Timers.Timer(5000); // 5s tick @@ -1633,8 +1634,6 @@ public static async Task Main(string[] args) PendingLoginManager.CleanupExpiredLogins(); - // Users who log in but never open a websocket would otherwise leak their - // registered login CRC entries forever. Helpers.PruneExpiredLoginCRCs(); } catch (Exception ex) diff --git a/GenOnlineService/appsettings.json b/GenOnlineService/appsettings.json index 3e791a3..74411d0 100644 --- a/GenOnlineService/appsettings.json +++ b/GenOnlineService/appsettings.json @@ -39,6 +39,7 @@ "full_mesh_check_snapshot_interval_ms": 1000, "full_mesh_check_retry_delay_ms": 3000, "full_mesh_check_max_attempts": 2, + "full_mesh_check_qm_referee_window_ms": 20000, "reconnect_grace_period_ms": 30000 }, "TURN": { From 12e031e0211e222d0d34bc944950d1312226b8c3 Mon Sep 17 00:00:00 2001 From: tintinhamans <5984296+tintinhamans@users.noreply.github.com> Date: Sun, 27 Sep 2026 02:17:48 +0200 Subject: [PATCH 5/8] feat(config): Let the service choose the client's ICE implementation --- .../ServiceConfig/ServiceConfigController.cs | 24 ++++++++++++++++++- GenOnlineService/appsettings.json | 5 +++- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/GenOnlineService/Controllers/ServiceConfig/ServiceConfigController.cs b/GenOnlineService/Controllers/ServiceConfig/ServiceConfigController.cs index 88aa960..d839075 100644 --- a/GenOnlineService/Controllers/ServiceConfig/ServiceConfigController.cs +++ b/GenOnlineService/Controllers/ServiceConfig/ServiceConfigController.cs @@ -19,10 +19,13 @@ using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Mvc; using System; +using System.Collections.Generic; using System.Net; using System.Net.WebSockets; +using System.Security.Claims; using System.Text; using System.Text.Json; +using System.Text.Json.Nodes; namespace GenOnlineService.Controllers { @@ -44,6 +47,25 @@ public ServiceConfigController() { string strFileData = await System.IO.File.ReadAllTextAsync(Path.Combine("data", "serviceconfig.json")); + JsonNode? configNode = JsonNode.Parse(strFileData); + if (configNode is JsonObject configObject) + { + // 0 = library default, 1 = native ICE, 2 = WebRTC + int iceImplementation = Program.g_Config?.GetSection("Core").GetValue("ice_implementation", 2) ?? 2; + + if (Int64.TryParse(this.User.FindFirst(ClaimTypes.NameIdentifier)?.Value, out Int64 userId)) + { + List lstTesterIDs = Program.g_Config?.GetSection("Core").GetSection("ice_implementation_testers").Get>() ?? new List(); + if (lstTesterIDs.Contains(userId)) + { + iceImplementation = Program.g_Config?.GetSection("Core").GetValue("ice_implementation_testers_value", 1) ?? 1; + } + } + + configObject["ice_implementation"] = iceImplementation; + strFileData = configObject.ToJsonString(); + } + Response.StatusCode = (int)HttpStatusCode.OK; return strFileData; } @@ -86,4 +108,4 @@ public AnticheatConfigController() } } } -} \ No newline at end of file +} diff --git a/GenOnlineService/appsettings.json b/GenOnlineService/appsettings.json index 74411d0..e164c27 100644 --- a/GenOnlineService/appsettings.json +++ b/GenOnlineService/appsettings.json @@ -40,7 +40,10 @@ "full_mesh_check_retry_delay_ms": 3000, "full_mesh_check_max_attempts": 2, "full_mesh_check_qm_referee_window_ms": 20000, - "reconnect_grace_period_ms": 30000 + "reconnect_grace_period_ms": 30000, + "ice_implementation": 2, + "ice_implementation_testers_value": 1, + "ice_implementation_testers": [] }, "TURN": { "key": null, From 80d162b7efbc5c9380f58a593710dd40fbca98d2 Mon Sep 17 00:00:00 2001 From: tintinhamans <5984296+tintinhamans@users.noreply.github.com> Date: Sun, 27 Sep 2026 03:18:22 +0200 Subject: [PATCH 6/8] feat(matchmaking): Explain map picks and show the skill range as the search widens --- .../Matchmaking/MatchmakingController.cs | 4 +- GenOnlineService/ELO.cs | 5 + GenOnlineService/MatchmakingManager.cs | 101 +++++++++++++++--- 3 files changed, 95 insertions(+), 15 deletions(-) diff --git a/GenOnlineService/Controllers/Matchmaking/MatchmakingController.cs b/GenOnlineService/Controllers/Matchmaking/MatchmakingController.cs index 1de844d..fc9dd68 100644 --- a/GenOnlineService/Controllers/Matchmaking/MatchmakingController.cs +++ b/GenOnlineService/Controllers/Matchmaking/MatchmakingController.cs @@ -99,7 +99,7 @@ public MatchmakingController(ILogger logger) [HttpPost("Widen")] [Authorize(Roles = "GameClient")] - public void Put_Widen() + public async Task Put_Widen() { // TODO_QUICKMATCH: What if a user widens after already being matched? We should probably tell them no // widen the search @@ -111,7 +111,7 @@ public void Put_Widen() if (playerSession != null) { - MatchmakingManager.PlayerWidenSearch(playerSession); + await MatchmakingManager.PlayerWidenSearch(playerSession); } } } diff --git a/GenOnlineService/ELO.cs b/GenOnlineService/ELO.cs index 73bba83..d07e667 100644 --- a/GenOnlineService/ELO.cs +++ b/GenOnlineService/ELO.cs @@ -29,6 +29,11 @@ public static class EloConfig public static int SecondsBetweenEloExpansionsInMatchmaking = 10; public static int HighEloThreshold = 2000; + + // Once a bucket's own widening range (eloExpansionIteration * its expansion value) exceeds this, + // it stops caring about Elo at all: merges involving it skip the threshold check, new players may + // join it regardless of Elo, and it stops sending per-step widening messages. + public static int MaxEloRangeBeforeSearchingAll = 1500; } public class EloData diff --git a/GenOnlineService/MatchmakingManager.cs b/GenOnlineService/MatchmakingManager.cs index 07b1d46..3ba60aa 100644 --- a/GenOnlineService/MatchmakingManager.cs +++ b/GenOnlineService/MatchmakingManager.cs @@ -260,7 +260,7 @@ internal static int GetMatchmakingElo(PlayerStats stats) : stats.EloRating; } - public static void PlayerWidenSearch(UserSession playerSession) + public static async Task PlayerWidenSearch(UserSession playerSession) { // NOTE: we dont check the state of the bucket here, but it doesn't really matter since expanding the maps after it started won't do anything anyway @@ -294,6 +294,8 @@ public static void PlayerWidenSearch(UserSession playerSession) // update our player too playerSession.MatchmakingMapIndicies = new ConcurrentList(lstAllMaps); + await SendMatchmakingMessage(playerSession, "Search widened: you can now be matched on any map in this playlist."); + // can't be in multiple buckets return; } @@ -312,6 +314,8 @@ public static void PlayerWidenSearch(UserSession playerSession) } playerSession.MatchmakingMapIndicies = new ConcurrentList(lstAllMaps); + + await SendMatchmakingMessage(playerSession, "Search widened: you can now be matched on any map in this playlist."); } } @@ -431,6 +435,20 @@ public void ExpandElo() DateTime m_CreationTime = DateTime.Now; DateTime m_LastELOExpansionTime = DateTime.Now; + // Once true, further widening steps for this bucket stop mattering: merges skip the Elo + // check, new players may join regardless of Elo, and the per-step widening message stops. + private bool m_bHasNotifiedSearchingAllSkillRanges = false; + + public int GetEloExpansionValueForThisBucket() + { + return GetAvgElo() >= EloConfig.HighEloThreshold ? EloConfig.EloExpansionValue_HighELO : EloConfig.EloExpansionValue_Standard; + } + + public bool HasExceededMaxEloRange() + { + return (eloExpansionIteration * GetEloExpansionValueForThisBucket()) > EloConfig.MaxEloRangeBeforeSearchingAll; + } + private TimeSpan TimeSinceLastEloExpansion() { TimeSpan timeDifference = DateTime.Now - m_LastELOExpansionTime; @@ -448,10 +466,13 @@ private TimeSpan TimeSinceLastEloExpansion() } // Selects a map with EXACTLY matchSize slots, never a different size. Returns false if none exists. - public bool TryDetermineMapForSize(int matchSize, out string strMapName, out string strMapPath) + // bWasFallback is true when no map in the group's shared selection has exactly matchSize slots, + // so a random exact-size map outside that selection was picked instead. + public bool TryDetermineMapForSize(int matchSize, out string strMapName, out string strMapPath, out bool bWasFallback) { strMapName = string.Empty; strMapPath = string.Empty; + bWasFallback = false; if (!MatchmakingManager.g_Playlists.TryGetValue(PlaylistID, out Playlist? playlist)) { @@ -493,6 +514,7 @@ public bool TryDetermineMapForSize(int matchSize, out string strMapName, out str PlaylistMap chosenMap = exactSizeMaps[Random.Shared.Next(exactSizeMaps.Count)]; strMapName = chosenMap.Name; strMapPath = chosenMap.Path; + bWasFallback = true; return true; } @@ -552,11 +574,15 @@ public bool CanMergeWithOtherBucket(MatchmakingBucket bucketToMerge) return false; } - // must be within the eloThreshold - int eloExpansionToUse = (bucketToMerge.GetAvgElo() >= EloConfig.HighEloThreshold || GetAvgElo() >= EloConfig.HighEloThreshold) ? EloConfig.EloExpansionValue_HighELO : EloConfig.EloExpansionValue_Standard; - if (!IsAvgEloWithinThreshold(bucketToMerge.GetAvgElo(), eloExpansionIteration * eloExpansionToUse)) + // must be within the eloThreshold, unless either side has widened past the point of caring + // about Elo at all - the long-waiting side just wants any match + if (!HasExceededMaxEloRange() && !bucketToMerge.HasExceededMaxEloRange()) { - return false; + int eloExpansionToUse = (bucketToMerge.GetAvgElo() >= EloConfig.HighEloThreshold || GetAvgElo() >= EloConfig.HighEloThreshold) ? EloConfig.EloExpansionValue_HighELO : EloConfig.EloExpansionValue_Standard; + if (!IsAvgEloWithinThreshold(bucketToMerge.GetAvgElo(), eloExpansionIteration * eloExpansionToUse)) + { + return false; + } } // cant be blocked by any participant (or have any participant blocked) @@ -1406,12 +1432,38 @@ await SendMatchmakingMessage(memberSession, // expand ExpandElo(); - foreach (MatchmakingBucketMember member in m_lstMembers) + if (HasExceededMaxEloRange()) { - UserSession? memberSession = member.GetAssociatedSession(); - if (memberSession != null) + // past this point Elo no longer gates merges or joins for this bucket - say so once and + // stop sending the per-step widening message, which no longer means anything + if (!m_bHasNotifiedSearchingAllSkillRanges) + { + m_bHasNotifiedSearchingAllSkillRanges = true; + + foreach (MatchmakingBucketMember member in m_lstMembers) + { + UserSession? memberSession = member.GetAssociatedSession(); + if (memberSession != null) + { + await SendMatchmakingMessage(memberSession, "Now searching all skill ranges..."); + } + } + } + } + else + { + // same threshold IsAvgEloWithinThreshold uses for merges at this step, but keyed off + // this bucket's own average (merges also factor in the other bucket's average; the + // message doesn't have another bucket to consider) + int eloRangeForMessage = eloExpansionIteration * GetEloExpansionValueForThisBucket(); + + foreach (MatchmakingBucketMember member in m_lstMembers) { - await SendMatchmakingMessage(memberSession, "Expanding search criteria to find more players..."); + UserSession? memberSession = member.GetAssociatedSession(); + if (memberSession != null) + { + await SendMatchmakingMessage(memberSession, $"Widening the skill range to ±{eloRangeForMessage} to find more players..."); + } } } } @@ -1467,12 +1519,34 @@ await SendMatchmakingMessage(memberSession, if (dummyHostUserData != null) { // make a lobby - if (!TryDetermineMapForSize(m_MatchFormedSize, out string strMapName, out string strMapPath)) + if (!TryDetermineMapForSize(m_MatchFormedSize, out string strMapName, out string strMapPath, out bool bMapWasFallback)) { Console.WriteLine("Matchmaking bucket {0} could not find a map for size {1}, skipping formation this tick", PlaylistID, m_MatchFormedSize); return; } + // Only speak up when there's something noteworthy, and never send both lines: the + // fallback note already states the match size, so it takes priority over the + // smaller-than-max note; when the map came from the group's own shared selection at + // full size, there's nothing worth telling the player. + string? strMapMessage = bMapWasFallback + ? $"None of your selected maps fit a {m_MatchFormedSize}-player match, so {strMapName} was picked." + : m_MatchFormedSize < MaxPlayers + ? $"Starting a {m_MatchFormedSize}-player match on {strMapName}." + : null; + + if (strMapMessage != null) + { + foreach (MatchmakingBucketMember mapMessageMember in m_lstMembers) + { + UserSession? mapMessageSession = mapMessageMember.GetAssociatedSession(); + if (mapMessageSession != null) + { + await SendMatchmakingMessage(mapMessageSession, strMapMessage); + } + } + } + using var scope = ServiceLocator.Services.CreateScope(); var factory = scope.ServiceProvider.GetRequiredService>(); await using var db = await factory.CreateDbContextAsync(); @@ -1931,10 +2005,11 @@ public static async Task Tick() continue; } - // must be within initial elo threshold for a join, otherwise we'll make a bucket and try to merge buckets using the elo iteration expansion algorithm + // must be within initial elo threshold for a join, otherwise we'll make a bucket and try to merge buckets using the elo iteration expansion algorithm - + // unless the bucket has already widened past the point of caring about Elo at all int matchmakingElo = MatchmakingManager.GetMatchmakingElo(thisSessionUserData.GameStats); int eloExpansionToUse = (mmBucket.GetAvgElo() >= EloConfig.HighEloThreshold || matchmakingElo >= EloConfig.HighEloThreshold) ? EloConfig.EloExpansionValue_HighELO : EloConfig.EloExpansionValue_Standard; - if (mmBucket.IsAvgEloWithinThreshold(matchmakingElo, eloExpansionToUse)) + if (mmBucket.HasExceededMaxEloRange() || mmBucket.IsAvgEloWithinThreshold(matchmakingElo, eloExpansionToUse)) { // TODO_MATCHMAKING: Squads if (mmBucket.HasSpaceForUsers(1, thisSession.ExeCRC, thisSession.IniCRC, thisSession.AnticheatID)) From fb8cd5fb4e0386e34ab4c0c31e071463669bbf93 Mon Sep 17 00:00:00 2001 From: tintinhamans <5984296+tintinhamans@users.noreply.github.com> Date: Sun, 27 Sep 2026 04:33:27 +0200 Subject: [PATCH 7/8] feat(lobby): Referee custom lobby start checks like quick match and show everyone who can't connect --- GenOnlineService/Constants.cs | 2 +- .../WebSocket/WebSocketController.cs | 2 +- GenOnlineService/LobbyManager.cs | 835 ++++++++++-------- GenOnlineService/MatchmakingManager.cs | 28 +- GenOnlineService/appsettings.json | 5 +- 5 files changed, 493 insertions(+), 379 deletions(-) diff --git a/GenOnlineService/Constants.cs b/GenOnlineService/Constants.cs index d087b47..f7bd338 100644 --- a/GenOnlineService/Constants.cs +++ b/GenOnlineService/Constants.cs @@ -3030,7 +3030,7 @@ public class WebSocketMessage_FullMeshConnectivityCheckOutcome: WebSocketMessage public bool mesh_complete { get; set; } public List missing_connections { get; set; } = new(); - // "" if mesh_complete; else one of missing_connections/timeout/member_left/gave_up (FullMeshCheckOutcomeReason). + // "" if mesh_complete; else one of missing_connections/timeout/membership_changed/gave_up (FullMeshCheckOutcomeReason). public string reason { get; set; } = string.Empty; } diff --git a/GenOnlineService/Controllers/WebSocket/WebSocketController.cs b/GenOnlineService/Controllers/WebSocket/WebSocketController.cs index 8803daa..73048f2 100644 --- a/GenOnlineService/Controllers/WebSocket/WebSocketController.cs +++ b/GenOnlineService/Controllers/WebSocket/WebSocketController.cs @@ -1032,7 +1032,7 @@ await lobby.RunExclusiveAsync(() => } // lock slots (more people joining when we're already doing connectivity checks won't help the situation) - await lobbyInfo.CloseOpenSlots(); + await lobbyInfo.CloseOpenSlots(true); // mark lobby as in progress of full mesh connectivity checks await lobbyInfo.StartFullMeshConnectivityCheck(); diff --git a/GenOnlineService/LobbyManager.cs b/GenOnlineService/LobbyManager.cs index 19504cd..4ddf01a 100644 --- a/GenOnlineService/LobbyManager.cs +++ b/GenOnlineService/LobbyManager.cs @@ -34,34 +34,28 @@ using System.Threading.Tasks; using System.Xml.Linq; -namespace GenOnlineService -{ - internal static class FullMeshCheckProtocol - { - // TODO: Remove the zero-valued response compatibility path when legacy - // clients are no longer supported. - internal static bool IsLegacyResponse(WebSocketMessage_FullMeshConnectivityCheckResponseFromUser response) - { - return response.mesh_check_id == 0 && response.attempt == 0; - } - - internal static bool MatchesCurrentAttempt( - WebSocketMessage_FullMeshConnectivityCheckResponseFromUser response, - Int64 currentCheckID, - int currentAttempt) - { - return IsLegacyResponse(response) - ? currentAttempt == 1 - : response.mesh_check_id == currentCheckID && response.attempt == currentAttempt; - } - - internal static bool ShouldRetry(bool meshComplete, bool hasLegacyResponse, int currentAttempt, int maxAttempts) - { - // TODO: Remove legacy retry suppression together with the legacy response path. - return !meshComplete && !hasLegacyResponse && currentAttempt < maxAttempts; - } - } - +namespace GenOnlineService +{ + internal static class FullMeshCheckProtocol + { + // TODO: Remove the zero-valued response compatibility path when legacy + // clients are no longer supported. + internal static bool IsLegacyResponse(WebSocketMessage_FullMeshConnectivityCheckResponseFromUser response) + { + return response.mesh_check_id == 0 && response.attempt == 0; + } + + internal static bool MatchesCurrentAttempt( + WebSocketMessage_FullMeshConnectivityCheckResponseFromUser response, + Int64 currentCheckID, + int currentAttempt) + { + return IsLegacyResponse(response) + ? currentAttempt == 1 + : response.mesh_check_id == currentCheckID && response.attempt == currentAttempt; + } + } + // "reason" values for FULL_MESH_CONNECTIVITY_CHECK_RESPONSE_COMPLETE_TO_HOST. "" means // mesh_complete. A superseded check, or one whose requester is no longer owner, sends nothing // (see Lobby.CompleteFullMeshConnectivityCheckLocked): at most one completion per current owner. @@ -70,39 +64,33 @@ internal static class FullMeshCheckOutcomeReason internal const string None = ""; internal const string MissingConnections = "missing_connections"; internal const string Timeout = "timeout"; - internal const string MemberLeft = "member_left"; - // quick match only: a member reported giving up on a peer (gave_up_user_ids) + // lobby membership changed while the check was pending - a join or a leave, not just a leave; + // MembershipVersion bumps on both, so this covers either. Neither the released client nor the + // in-progress client branch matches on this value (both only log it), so it's a plain rename + // rather than adding a separate "member_joined" value. + internal const string MembershipChanged = "membership_changed"; + // a member reported giving up on a peer (gave_up_user_ids) internal const string GaveUp = "gave_up"; } - // Core:full_mesh_check_* in appsettings.json, read on use - internal static class FullMeshCheckSettings - { - // how long an attempt waits for every connection before judging it - internal static int AttemptWindowMS => Get("full_mesh_check_attempt_window_ms", 8000); - - // how often an incomplete attempt asks members for a fresh snapshot - internal static int SnapshotIntervalMS => Get("full_mesh_check_snapshot_interval_ms", 1000); - - // time given to re-signalled connections before the next attempt - internal static int RetryDelayMS => Get("full_mesh_check_retry_delay_ms", 3000); - - internal static int MaxAttempts => Get("full_mesh_check_max_attempts", 2); - - // upper bound for a whole check, used as the clients' setup timeout - internal static int MaxDurationMS => (AttemptWindowMS * MaxAttempts) + (RetryDelayMS * (MaxAttempts - 1)); - - // single-window referee period for quick match; clients own connection retries (2 attempts x 10s GNS default) - internal static int QuickMatchRefereeWindowMS => Get("full_mesh_check_qm_referee_window_ms", 20000); - - private static int Get(string key, int defaultValue) - { - int value = Program.g_Config?.GetSection("Core").GetValue(key, defaultValue) ?? defaultValue; - return value > 0 ? value : defaultValue; - } - } - - public class Lobby + // Core:full_mesh_check_* in appsettings.json, read on use. Every check passes once the mesh is complete, + // fails once a member reports giving up, and times out as a backstop for clients that never report it. + internal static class FullMeshCheckSettings + { + // how often the window asks members for a fresh snapshot while it waits + internal static int SnapshotIntervalMS => Get("full_mesh_check_snapshot_interval_ms", 1000); + + // covers the clients' own retries (2 attempts x 10s GNS default) + internal static int RefereeWindowMS => Get("full_mesh_check_referee_window_ms", 20000); + + private static int Get(string key, int defaultValue) + { + int value = Program.g_Config?.GetSection("Core").GetValue(key, defaultValue) ?? defaultValue; + return value > 0 ? value : defaultValue; + } + } + + public class Lobby { public Int64 LobbyID { get; private set; } = -1; public Int64 Owner { get; private set; } = -1; @@ -133,7 +121,7 @@ public class Lobby // Exclusive gate for every mutation of Members/Owner, slot state/fields, ready state, and // mesh-check state. Not reentrant: a callback must not call RunExclusiveAsync on this lobby again. private readonly SemaphoreSlim m_LobbyGate = new SemaphoreSlim(1, 1); - + #if DEBUG // Reentrancy guard: throws instead of deadlocking if RunExclusiveAsync is re-entered. private readonly AsyncLocal m_bGateHeldInThisFlow = new(); @@ -193,54 +181,59 @@ public async Task RunExclusiveAsync(Func> action) } } - [JsonIgnore] - public bool? LastFullMeshConnectivityCheckOutcome { get; private set; } = null; + [JsonIgnore] + public bool? LastFullMeshConnectivityCheckOutcome { get; private set; } = null; + + // Pairs the last check found unconnected, so callers (custom lobby chat, quick match's + // requeue message) can tell members why it failed without re-deriving it themselves. + [JsonIgnore] + public List LastFullMeshConnectivityCheckMissingConnections { get; private set; } = new(); + + [JsonIgnore] + public int FullMeshCheckAttempt { get; private set; } = 0; + + [JsonIgnore] + public Int64 FullMeshCheckID { get; private set; } = 0; + + [JsonIgnore] + private Int64 m_TimeNextFullMeshSnapshotRequest = -1; + // per member, the peers they reported as still connecting in the current attempt [JsonIgnore] - public int FullMeshCheckAttempt { get; private set; } = 0; - - [JsonIgnore] - public Int64 FullMeshCheckID { get; private set; } = 0; - - [JsonIgnore] - private Int64 m_TimeToRetryFullMeshChecks = -1; - - [JsonIgnore] - private Int64 m_TimeNextFullMeshSnapshotRequest = -1; - - // per member, the peers they reported as still connecting in the current attempt - [JsonIgnore] - private Dictionary> m_FullMeshConnecting = new(); - - [JsonIgnore] - private bool m_bCurrentAttemptHasLegacyResponse = false; - + private Dictionary> m_FullMeshConnecting = new(); + // Set while a check is pending if a member leaves before it completes; reset on next check start. [JsonIgnore] private bool m_bMemberLeftDuringCurrentCheck = false; - // Set (quick match only) when any snapshot reports gave_up_user_ids; forces an immediate fail. + // any snapshot reported gave_up_user_ids [JsonIgnore] private bool m_bAnyMemberGaveUp = false; - // Quick match referees a single window with no server-side retries or re-signalling; custom - // lobby host-triggered checks keep their existing attempts/retry behavior. - private bool IsQuickMatchRefereeMode => LobbyType == ELobbyType.QuickMatch; + // slots a start check closed, reopened if it fails + [JsonIgnore] + private readonly List m_SlotsClosedByMeshCheck = new(); + + // legacy clients don't echo mesh_check_id/attempt, so their replies are matched by order + [JsonIgnore] + private readonly ConcurrentDictionary m_MeshRequestsOutstanding = new(); - private int CurrentAttemptWindowMS => IsQuickMatchRefereeMode ? FullMeshCheckSettings.QuickMatchRefereeWindowMS : FullMeshCheckSettings.AttemptWindowMS; + [JsonIgnore] + private readonly ConcurrentDictionary m_StaleLegacyMeshReplies = new(); - private int CurrentMaxAttempts => IsQuickMatchRefereeMode ? 1 : FullMeshCheckSettings.MaxAttempts; + // quick match reports check progress as matchmaking messages, not lobby announcements + private bool IsQuickMatch => LobbyType == ELobbyType.QuickMatch; // Owner when the current check started; the outcome is only sent to this user while they still own it. [JsonIgnore] private Int64 m_MeshCheckRequestingUserID = -1; - private static Int64 s_NextFullMeshCheckID = 0; - + private static Int64 s_NextFullMeshCheckID = 0; + // Backing counter for LobbyMember.JoinSequence; per-lobby, starts at 1. private Int64 m_NextJoinSequence = 0; - [JsonIgnore] + [JsonIgnore] public ConcurrentDictionary> FullMeshConnectivityChecks { get; set; } = new(); @@ -298,201 +291,214 @@ public async Task RegisterProbeResponse_Malformed_Type2(Int64 userID) public async Task StartFullMeshConnectivityCheck() { await RunExclusiveAsync(() => - { + { if (PendingFullMeshConnectivityChecks) { DiscardPendingFullMeshCheckLocked(); } - FullMeshCheckID = Interlocked.Increment(ref s_NextFullMeshCheckID); - FullMeshCheckAttempt = 1; - m_TimeToRetryFullMeshChecks = -1; - LastFullMeshConnectivityCheckOutcome = null; + // unanswered requests belong to earlier checks + m_StaleLegacyMeshReplies.Clear(); + foreach (var outstanding in m_MeshRequestsOutstanding) + { + m_StaleLegacyMeshReplies[outstanding.Key] = outstanding.Value; + } + + FullMeshCheckID = Interlocked.Increment(ref s_NextFullMeshCheckID); + FullMeshCheckAttempt = 1; + LastFullMeshConnectivityCheckOutcome = null; m_bMemberLeftDuringCurrentCheck = false; m_bAnyMemberGaveUp = false; MembershipVersionAtLastCheckStart = MembershipVersion; m_MeshCheckRequestingUserID = Owner; - BeginFullMeshConnectivityCheckAttempt(); + BeginFullMeshConnectivityCheckAttempt(); + + // custom lobbies only; quick match reports progress over its own matchmaking-status channel + if (!IsQuickMatch) + { + BroadcastLobbyAnnouncement("Checking connections between all players...", m_MeshCheckRequestingUserID); + } + return Task.CompletedTask; }); - } - + } + // Must be called while holding m_LobbyGate. Does not send a completion. private void DiscardPendingFullMeshCheckLocked() { PendingFullMeshConnectivityChecks = false; TimeStartFullMeshChecks = -1; - m_TimeToRetryFullMeshChecks = -1; LastFullMeshConnectivityCheckOutcome = null; } - private void BeginFullMeshConnectivityCheckAttempt() - { + private void BeginFullMeshConnectivityCheckAttempt() + { PendingFullMeshConnectivityChecks = true; - FullMeshConnectivityChecks = new(); - m_FullMeshConnecting = new(); - m_bCurrentAttemptHasLegacyResponse = false; - TimeStartFullMeshChecks = Environment.TickCount64; - m_TimeNextFullMeshSnapshotRequest = TimeStartFullMeshChecks + FullMeshCheckSettings.SnapshotIntervalMS; - } - - private bool IsFullMeshPairStillConnecting(Int64 userA, Int64 userB) - { - return (m_FullMeshConnecting.TryGetValue(userA, out HashSet? fromA) && fromA.Contains(userB)) - || (m_FullMeshConnecting.TryGetValue(userB, out HashSet? fromB) && fromB.Contains(userA)); - } - - // one line per judged attempt, so the window can be tuned from real connect times - private void LogFullMeshCheckAttempt(bool bMeshComplete, List lstMissingConnections) - { - Int64 elapsedMS = Environment.TickCount64 - TimeStartFullMeshChecks; - string strMissing = string.Join(", ", lstMissingConnections - .Select(c => (Math.Min(c.source_user_id, c.target_user_id), Math.Max(c.source_user_id, c.target_user_id))) - .Distinct() - .Select(p => $"{p.Item1}<->{p.Item2}")); - string strConnecting = string.Join(", ", lstMissingConnections - .Where(c => IsFullMeshPairStillConnecting(c.source_user_id, c.target_user_id)) - .Select(c => (Math.Min(c.source_user_id, c.target_user_id), Math.Max(c.source_user_id, c.target_user_id))) - .Distinct() - .Select(p => $"{p.Item1}<->{p.Item2}")); - - Console.WriteLine("[Lobby {0}] Mesh check {1} attempt {2}/{3}: {4} after {5} ms with {6} humans{7}", - LobbyID, FullMeshCheckID, FullMeshCheckAttempt, CurrentMaxAttempts, - bMeshComplete ? "complete" : "incomplete", elapsedMS, GetNumberOfHumans(), - bMeshComplete ? "" : $", missing {strMissing}{(strConnecting.Length > 0 ? $" (still connecting {strConnecting})" : "")}"); - } - - public void SendFullMeshConnectivityCheckRequestToMembers() - { - WebSocketMessage_FullMeshConnectivityCheckRequest startCommand = new WebSocketMessage_FullMeshConnectivityCheckRequest(); - startCommand.msg_id = (int)EWebSocketMessageID.FULL_MESH_CONNECTIVITY_CHECK_RESPONSE; - startCommand.mesh_check_id = FullMeshCheckID; - startCommand.attempt = FullMeshCheckAttempt; - byte[] bytesJSON = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(startCommand)); - - foreach (LobbyMember member in Members) - { - if (member.GetSession().TryGetTarget(out UserSession? session) && session != null) - { - session.QueueWebsocketSend(bytesJSON); - } - } - } - - // Re-issues signalling between the pairs that failed to connect. This is the same handshake a player - // gets when they join, which is why manually rejoining the lobby often repairs a broken mesh. - private void RestartSignallingForMissingConnections(List lstMissingConnections) - { - HashSet<(Int64, Int64)> alreadyResignalled = new(); - - foreach (MissingConnectionEntry missingConnection in lstMissingConnections) - { - Int64 lowUserID = Math.Min(missingConnection.source_user_id, missingConnection.target_user_id); - Int64 highUserID = Math.Max(missingConnection.source_user_id, missingConnection.target_user_id); - - if (!alreadyResignalled.Add((lowUserID, highUserID))) - { - continue; - } - - LobbyMember? sourceMember = GetMemberFromUserID(missingConnection.source_user_id); - LobbyMember? targetMember = GetMemberFromUserID(missingConnection.target_user_id); - - if (sourceMember == null || targetMember == null) - { - continue; - } - - Console.WriteLine("[Lobby {0}] Re-signalling {1} <-> {2} before mesh check retry", LobbyID, sourceMember.UserID, targetMember.UserID); - - SendStartSignallingToMember(sourceMember, targetMember); - SendStartSignallingToMember(targetMember, sourceMember); - } - } - - private void SendStartSignallingToMember(LobbyMember recipient, LobbyMember peer) - { - if (recipient.GetSession().TryGetTarget(out UserSession? recipientSession) && recipientSession != null) - { - WebSocketMessage_NetworkStartSignalling signallingMsg = new WebSocketMessage_NetworkStartSignalling(); - signallingMsg.msg_id = (int)EWebSocketMessageID.NETWORK_CONNECTION_START_SIGNALLING; - signallingMsg.lobby_id = LobbyID; - signallingMsg.user_id = peer.UserID; - signallingMsg.preferred_port = peer.Port; - signallingMsg.middleware_id = peer.MiddlewareUserID; - recipientSession.QueueWebsocketSend(Encoding.UTF8.GetBytes(JsonSerializer.Serialize(signallingMsg))); - } - } - + FullMeshConnectivityChecks = new(); + m_FullMeshConnecting = new(); + TimeStartFullMeshChecks = Environment.TickCount64; + m_TimeNextFullMeshSnapshotRequest = TimeStartFullMeshChecks + FullMeshCheckSettings.SnapshotIntervalMS; + } + + private bool IsFullMeshPairStillConnecting(Int64 userA, Int64 userB) + { + return (m_FullMeshConnecting.TryGetValue(userA, out HashSet? fromA) && fromA.Contains(userB)) + || (m_FullMeshConnecting.TryGetValue(userB, out HashSet? fromB) && fromB.Contains(userA)); + } + + // one line per judged check, so the window can be tuned from real connect times + private void LogFullMeshCheckAttempt(bool bMeshComplete, List lstMissingConnections) + { + Int64 elapsedMS = Environment.TickCount64 - TimeStartFullMeshChecks; + string strMissing = string.Join(", ", lstMissingConnections + .Select(c => (Math.Min(c.source_user_id, c.target_user_id), Math.Max(c.source_user_id, c.target_user_id))) + .Distinct() + .Select(p => $"{p.Item1}<->{p.Item2}")); + string strConnecting = string.Join(", ", lstMissingConnections + .Where(c => IsFullMeshPairStillConnecting(c.source_user_id, c.target_user_id)) + .Select(c => (Math.Min(c.source_user_id, c.target_user_id), Math.Max(c.source_user_id, c.target_user_id))) + .Distinct() + .Select(p => $"{p.Item1}<->{p.Item2}")); + + Console.WriteLine("[Lobby {0}] Mesh check {1}: {2} after {3} ms with {4} humans{5}", + LobbyID, FullMeshCheckID, + bMeshComplete ? "complete" : "incomplete", elapsedMS, GetNumberOfHumans(), + bMeshComplete ? "" : $", missing {strMissing}{(strConnecting.Length > 0 ? $" (still connecting {strConnecting})" : "")}"); + } + + public void SendFullMeshConnectivityCheckRequestToMembers() + { + WebSocketMessage_FullMeshConnectivityCheckRequest startCommand = new WebSocketMessage_FullMeshConnectivityCheckRequest(); + startCommand.msg_id = (int)EWebSocketMessageID.FULL_MESH_CONNECTIVITY_CHECK_RESPONSE; + startCommand.mesh_check_id = FullMeshCheckID; + startCommand.attempt = FullMeshCheckAttempt; + byte[] bytesJSON = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(startCommand)); + + foreach (LobbyMember member in Members) + { + if (member.GetSession().TryGetTarget(out UserSession? session) && session != null) + { + m_MeshRequestsOutstanding.AddOrUpdate(member.UserID, 1, (_, count) => count + 1); + session.QueueWebsocketSend(bytesJSON); + } + } + } + + // Reuses the released client's existing lobby chat/announcement display (LOBBY_CHAT_FROM_SERVER, + // announcement=true - the same path WOLGameSetupMenu already prints host/system lines from) for + // server-driven connectivity-check status, rather than a message ID older clients would ignore. + private void BroadcastLobbyAnnouncement(string message, Int64 excludeUserID) + { + WebSocketMessage_LobbyChatMessageOutbound outboundMsg = new WebSocketMessage_LobbyChatMessageOutbound(); + outboundMsg.msg_id = (int)EWebSocketMessageID.LOBBY_CHAT_FROM_SERVER; + outboundMsg.user_id = -2; // server/system line, not from a real player + outboundMsg.message = message; + outboundMsg.announcement = true; + outboundMsg.show_announcement_to_host = true; // irrelevant here: recipients are filtered below + byte[] bytesJSON = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(outboundMsg)); + + foreach (LobbyMember member in Members) + { + if (member.UserID == excludeUserID) + { + continue; + } + + if (member.GetSession().TryGetTarget(out UserSession? session) && session != null) + { + session.QueueWebsocketSend(bytesJSON); + } + } + } + + // One deduped "{A} can't connect to {B}" line per unique unordered pair. + public List BuildMissingConnectionMessages(List missingConnections) + { + HashSet<(Int64, Int64)> seenPairs = new(); + List messages = new(); + + foreach (MissingConnectionEntry entry in missingConnections) + { + Int64 lowUserID = Math.Min(entry.source_user_id, entry.target_user_id); + Int64 highUserID = Math.Max(entry.source_user_id, entry.target_user_id); + + if (!seenPairs.Add((lowUserID, highUserID))) + { + continue; + } + + string strA = GetMemberFromUserID(entry.source_user_id)?.DisplayName ?? entry.source_user_id.ToString(); + string strB = GetMemberFromUserID(entry.target_user_id)?.DisplayName ?? entry.target_user_id.ToString(); + + messages.Add($"{strA} can't connect to {strB}"); + } + + return messages; + } + public async Task StoreFullMeshConnectivityResponse(Int64 sourceUser, WebSocketMessage_FullMeshConnectivityCheckResponseFromUser response) - { + { await RunExclusiveAsync(() => - { - bool bLegacyResponse = FullMeshCheckProtocol.IsLegacyResponse(response); - bool bMatchesCurrentAttempt = FullMeshCheckProtocol.MatchesCurrentAttempt( - response, - FullMeshCheckID, - FullMeshCheckAttempt); - - LobbyMember? sourceMember = GetMemberFromUserID(sourceUser); - if (PendingFullMeshConnectivityChecks - && m_TimeToRetryFullMeshChecks == -1 - && sourceMember?.IsHuman() == true - && bMatchesCurrentAttempt) - { - m_bCurrentAttemptHasLegacyResponse |= bLegacyResponse; - FullMeshConnectivityChecks[sourceUser] = new ConcurrentList(response.connectivity_map); - m_FullMeshConnecting[sourceUser] = new HashSet(response.connecting_map); - - if (IsQuickMatchRefereeMode && response.gave_up_user_ids.Count > 0) + { + bool bMatchesCurrentAttempt = FullMeshCheckProtocol.MatchesCurrentAttempt( + response, + FullMeshCheckID, + FullMeshCheckAttempt); + + // a legacy reply to an earlier check's request + if (FullMeshCheckProtocol.IsLegacyResponse(response) + && m_StaleLegacyMeshReplies.TryGetValue(sourceUser, out int staleReplies) + && staleReplies > 0) + { + m_StaleLegacyMeshReplies[sourceUser] = staleReplies - 1; + bMatchesCurrentAttempt = false; + } + + m_MeshRequestsOutstanding.AddOrUpdate(sourceUser, 0, (_, count) => Math.Max(0, count - 1)); + + LobbyMember? sourceMember = GetMemberFromUserID(sourceUser); + if (PendingFullMeshConnectivityChecks + && sourceMember?.IsHuman() == true + && bMatchesCurrentAttempt) + { + FullMeshConnectivityChecks[sourceUser] = new ConcurrentList(response.connectivity_map); + m_FullMeshConnecting[sourceUser] = new HashSet(response.connecting_map); + + if (response.gave_up_user_ids.Count > 0) { m_bAnyMemberGaveUp = true; } - } - - ProcessPendingFullMeshConnectivityChecksInternal(); + } + + ProcessPendingFullMeshConnectivityChecksInternal(); return Task.CompletedTask; }); - } - + } + public async Task ProcessPendingFullMeshConnectivityChecks() - { + { await RunExclusiveAsync(() => - { - ProcessPendingFullMeshConnectivityChecksInternal(); + { + ProcessPendingFullMeshConnectivityChecksInternal(); return Task.CompletedTask; }); - } - + } + // Must only be called while holding m_LobbyGate (via RunExclusiveAsync). - private void ProcessPendingFullMeshConnectivityChecksInternal() - { - if (!PendingFullMeshConnectivityChecks) - { - return; - } - - // Give re-signalled connections time to establish before starting the retry. - if (m_TimeToRetryFullMeshChecks != -1) - { - if (Environment.TickCount64 < m_TimeToRetryFullMeshChecks) - { - return; - } - - m_TimeToRetryFullMeshChecks = -1; - BeginFullMeshConnectivityCheckAttempt(); - SendFullMeshConnectivityCheckRequestToMembers(); - return; - } - + private void ProcessPendingFullMeshConnectivityChecksInternal() + { + if (!PendingFullMeshConnectivityChecks) + { + return; + } + { bool bDoneChecks = false; // judged as soon as anyone replies: members who haven't replied yet count as missing, and gaps before // the window closes are re-polled below, which also recovers replies lost to a reconnect - bool bWindowElapsed = (Environment.TickCount64 - TimeStartFullMeshChecks) >= CurrentAttemptWindowMS; - bool bGaveUpForcesCompletion = IsQuickMatchRefereeMode && m_bAnyMemberGaveUp; + bool bWindowElapsed = (Environment.TickCount64 - TimeStartFullMeshChecks) >= FullMeshCheckSettings.RefereeWindowMS; + bool bGaveUpForcesCompletion = m_bAnyMemberGaveUp; bDoneChecks = bWindowElapsed || !FullMeshConnectivityChecks.IsEmpty; List lstMissingConnections = new(); @@ -536,24 +542,24 @@ private void ProcessPendingFullMeshConnectivityChecksInternal() } } - // A member that never reported cannot be assumed connected, so treat them as missing to everyone. - foreach (LobbyMember member in Members) - { - if (member.IsHuman() && !FullMeshConnectivityChecks.ContainsKey(member.UserID)) - { - foreach (LobbyMember otherMember in Members) - { - if (otherMember.IsHuman() && otherMember.UserID != member.UserID) - { - MissingConnectionEntry missingConnectionEntry = new(); - missingConnectionEntry.source_user_id = member.UserID; - missingConnectionEntry.target_user_id = otherMember.UserID; - lstMissingConnections.Add(missingConnectionEntry); - } - } - } - } - + // A member that never reported cannot be assumed connected, so treat them as missing to everyone. + foreach (LobbyMember member in Members) + { + if (member.IsHuman() && !FullMeshConnectivityChecks.ContainsKey(member.UserID)) + { + foreach (LobbyMember otherMember in Members) + { + if (otherMember.IsHuman() && otherMember.UserID != member.UserID) + { + MissingConnectionEntry missingConnectionEntry = new(); + missingConnectionEntry.source_user_id = member.UserID; + missingConnectionEntry.target_user_id = otherMember.UserID; + lstMissingConnections.Add(missingConnectionEntry); + } + } + } + } + bool bDisableMeshCheck = false; if (Program.g_Config != null) { @@ -568,38 +574,22 @@ private void ProcessPendingFullMeshConnectivityChecksInternal() bool bMeshComplete = !bGaveUpForcesCompletion && (bDisableMeshCheck || lstMissingConnections.Count == 0); - - // members report a snapshot, so a gap before the window closes may be a connection still forming: - // keep asking for fresh snapshots rather than re-signalling it early + + // members report a snapshot, so a gap before the window closes may be a connection still forming: + // keep asking for fresh snapshots rather than re-signalling it early if (!bMeshComplete && !bWindowElapsed && !bGaveUpForcesCompletion) - { - if (Environment.TickCount64 >= m_TimeNextFullMeshSnapshotRequest) - { - m_TimeNextFullMeshSnapshotRequest = Environment.TickCount64 + FullMeshCheckSettings.SnapshotIntervalMS; - SendFullMeshConnectivityCheckRequestToMembers(); - } - - return; - } - - LogFullMeshCheckAttempt(bMeshComplete, lstMissingConnections); - - if (FullMeshCheckProtocol.ShouldRetry( - bMeshComplete, - m_bCurrentAttemptHasLegacyResponse, - FullMeshCheckAttempt, - CurrentMaxAttempts)) - { - ++FullMeshCheckAttempt; - - // a pair still negotiating would be torn down by a re-signal; the next attempt re-checks it - RestartSignallingForMissingConnections(lstMissingConnections - .Where(c => !IsFullMeshPairStillConnecting(c.source_user_id, c.target_user_id)) - .ToList()); - m_TimeToRetryFullMeshChecks = Environment.TickCount64 + FullMeshCheckSettings.RetryDelayMS; - return; - } - + { + if (Environment.TickCount64 >= m_TimeNextFullMeshSnapshotRequest) + { + m_TimeNextFullMeshSnapshotRequest = Environment.TickCount64 + FullMeshCheckSettings.SnapshotIntervalMS; + SendFullMeshConnectivityCheckRequestToMembers(); + } + + return; + } + + LogFullMeshCheckAttempt(bMeshComplete, lstMissingConnections); + // inform host that we are done bool bMembershipChangedDuringCheck = MembershipVersion != MembershipVersionAtLastCheckStart; bool bMeshCompleteFinal = !bGaveUpForcesCompletion && !bMembershipChangedDuringCheck && (bDisableMeshCheck || lstMissingConnections.Count == 0); @@ -618,7 +608,7 @@ private void ProcessPendingFullMeshConnectivityChecksInternal() } else if (bMembershipChangedDuringCheck || m_bMemberLeftDuringCurrentCheck) { - reason = FullMeshCheckOutcomeReason.MemberLeft; + reason = FullMeshCheckOutcomeReason.MembershipChanged; } else if (FullMeshConnectivityChecks.IsEmpty) { @@ -633,7 +623,7 @@ private void ProcessPendingFullMeshConnectivityChecksInternal() } } } - + // Must be called while holding m_LobbyGate. private void CompleteFullMeshConnectivityCheckLocked(bool bMeshComplete, List lstMissingConnections, string reason) { @@ -644,6 +634,7 @@ private void CompleteFullMeshConnectivityCheckLocked(bool bMeshComplete, List 0) + { + foreach (string strMissingConnectionMessage in BuildMissingConnectionMessages(lstMissingConnections)) + { + BroadcastLobbyAnnouncement(strMissingConnectionMessage, m_MeshCheckRequestingUserID); + } + + // clients don't retry a pair the peer closed cleanly, so repair it for the next start attempt + RestartSignallingForMissingConnections(lstMissingConnections); + } + } + + if (!bMeshComplete && !IsQuickMatch) + { + ReopenSlotsClosedByMeshCheckLocked(); + } + + m_SlotsClosedByMeshCheck.Clear(); + // reset state PendingFullMeshConnectivityChecks = false; TimeStartFullMeshChecks = -1; - m_TimeToRetryFullMeshChecks = -1; + } + + // Must be called while holding m_LobbyGate. Only slots still closed. + private void ReopenSlotsClosedByMeshCheckLocked() + { + bool bReopened = false; + + foreach (UInt16 slotIndex in m_SlotsClosedByMeshCheck) + { + if (slotIndex < Members.Length && Members[slotIndex].SlotState == EPlayerType.SLOT_CLOSED) + { + Members[slotIndex].SetPlayerSlotState(EPlayerType.SLOT_OPEN); + bReopened = true; + } + } + + if (bReopened) + { + DirtyRetransmit(); + } + } + + // Repeats the join-time signalling for pairs that failed to connect. + // Must be called while holding m_LobbyGate. + private void RestartSignallingForMissingConnections(List lstMissingConnections) + { + HashSet<(Int64, Int64)> alreadyResignalled = new(); + + foreach (MissingConnectionEntry missingConnection in lstMissingConnections) + { + Int64 lowUserID = Math.Min(missingConnection.source_user_id, missingConnection.target_user_id); + Int64 highUserID = Math.Max(missingConnection.source_user_id, missingConnection.target_user_id); + + if (!alreadyResignalled.Add((lowUserID, highUserID))) + { + continue; + } + + // re-signalling would reset a pair still negotiating + if (IsFullMeshPairStillConnecting(lowUserID, highUserID)) + { + continue; + } + + LobbyMember? sourceMember = GetMemberFromUserID(missingConnection.source_user_id); + LobbyMember? targetMember = GetMemberFromUserID(missingConnection.target_user_id); + + if (sourceMember == null || targetMember == null) + { + continue; + } + + Console.WriteLine("[Lobby {0}] Re-signalling {1} <-> {2} after a failed mesh check", LobbyID, sourceMember.UserID, targetMember.UserID); + + SendStartSignallingToMember(sourceMember, targetMember); + SendStartSignallingToMember(targetMember, sourceMember); + } + } + + private void SendStartSignallingToMember(LobbyMember recipient, LobbyMember peer) + { + if (recipient.GetSession().TryGetTarget(out UserSession? recipientSession) && recipientSession != null) + { + WebSocketMessage_NetworkStartSignalling signallingMsg = new WebSocketMessage_NetworkStartSignalling(); + signallingMsg.msg_id = (int)EWebSocketMessageID.NETWORK_CONNECTION_START_SIGNALLING; + signallingMsg.lobby_id = LobbyID; + signallingMsg.user_id = peer.UserID; + signallingMsg.preferred_port = peer.Port; + signallingMsg.middleware_id = peer.MiddlewareUserID; + recipientSession.QueueWebsocketSend(Encoding.UTF8.GetBytes(JsonSerializer.Serialize(signallingMsg))); + } } public void AddPassword(string password) @@ -1017,7 +1107,7 @@ public async Task FinalizeACChecks() } } - public async Task CloseOpenSlots() + public async Task CloseOpenSlots(bool bReopenIfMeshCheckFails = false) { await RunExclusiveAsync(() => { @@ -1026,6 +1116,11 @@ await RunExclusiveAsync(() => if (member.SlotState == EPlayerType.SLOT_OPEN) { member.SetPlayerSlotState(EPlayerType.SLOT_CLOSED); + + if (bReopenIfMeshCheckFails && !IsQuickMatch) + { + m_SlotsClosedByMeshCheck.Add(member.SlotIndex); + } } } @@ -1080,8 +1175,8 @@ private void CalculateNextProbeTime(bool bIsFirstProbe) public async Task Tick() { - await ProcessPendingFullMeshConnectivityChecks(); - + await ProcessPendingFullMeshConnectivityChecks(); + if (m_NextProbe != 0 && Environment.TickCount64 >= m_NextProbe) { // send probe @@ -1165,13 +1260,13 @@ public async Task Tick() public async Task AddMember(UserSession playerSession, string strDisplayName, UInt16 userPreferredPort, bool bHasMap, UserLobbyPreferences lobbyPrefs) { return await RunExclusiveAsync(async () => - { - if (State != ELobbyState.GAME_SETUP) - { - return false; - } - - // NOTE: this must be inside the lock, otherwise two concurrent joins for the same user can both pass + { + if (State != ELobbyState.GAME_SETUP) + { + return false; + } + + // NOTE: this must be inside the lock, otherwise two concurrent joins for the same user can both pass // the check and end up occupying two slots LobbyMember? existingMember = GetMemberFromUserID(playerSession.m_UserID); if (existingMember != null) // we're already in this lobby @@ -1365,51 +1460,51 @@ public async Task AddMember(UserSession playerSession, string strDisplayNa }); } - public void SendPeerTeardownToDepartingMember(LobbyMember departingMember) - { - if (!departingMember.GetSession().TryGetTarget(out UserSession? departingSession) || departingSession == null) - { - return; - } - - foreach (LobbyMember remoteMember in Members) - { - if (remoteMember.SlotState != EPlayerType.SLOT_PLAYER || remoteMember.UserID == departingMember.UserID) - { - continue; - } - - WebSocketMessage_ACDeregisterPlayer remotePlayerAcMsg = new WebSocketMessage_ACDeregisterPlayer(); - remotePlayerAcMsg.msg_id = (int)EWebSocketMessageID.AC_DEREGISTER_PLAYER; - remotePlayerAcMsg.user_id = remoteMember.UserID; - remotePlayerAcMsg.mwid = remoteMember.MiddlewareUserID; - departingSession.QueueWebsocketSend(Encoding.UTF8.GetBytes(JsonSerializer.Serialize(remotePlayerAcMsg))); - - if (State != ELobbyState.INGAME) - { - WebSocketMessage_NetworkDisconnectPlayer remotePlayerMsg = new WebSocketMessage_NetworkDisconnectPlayer(); - remotePlayerMsg.msg_id = (int)EWebSocketMessageID.NETWORK_CONNECTION_DISCONNECT_PLAYER; - remotePlayerMsg.lobby_id = LobbyID; - remotePlayerMsg.user_id = remoteMember.UserID; - departingSession.QueueWebsocketSend(Encoding.UTF8.GetBytes(JsonSerializer.Serialize(remotePlayerMsg))); - } - } - } - - public async Task RemoveMember(LobbyMember member) - { + public void SendPeerTeardownToDepartingMember(LobbyMember departingMember) + { + if (!departingMember.GetSession().TryGetTarget(out UserSession? departingSession) || departingSession == null) + { + return; + } + + foreach (LobbyMember remoteMember in Members) + { + if (remoteMember.SlotState != EPlayerType.SLOT_PLAYER || remoteMember.UserID == departingMember.UserID) + { + continue; + } + + WebSocketMessage_ACDeregisterPlayer remotePlayerAcMsg = new WebSocketMessage_ACDeregisterPlayer(); + remotePlayerAcMsg.msg_id = (int)EWebSocketMessageID.AC_DEREGISTER_PLAYER; + remotePlayerAcMsg.user_id = remoteMember.UserID; + remotePlayerAcMsg.mwid = remoteMember.MiddlewareUserID; + departingSession.QueueWebsocketSend(Encoding.UTF8.GetBytes(JsonSerializer.Serialize(remotePlayerAcMsg))); + + if (State != ELobbyState.INGAME) + { + WebSocketMessage_NetworkDisconnectPlayer remotePlayerMsg = new WebSocketMessage_NetworkDisconnectPlayer(); + remotePlayerMsg.msg_id = (int)EWebSocketMessageID.NETWORK_CONNECTION_DISCONNECT_PLAYER; + remotePlayerMsg.lobby_id = LobbyID; + remotePlayerMsg.user_id = remoteMember.UserID; + departingSession.QueueWebsocketSend(Encoding.UTF8.GetBytes(JsonSerializer.Serialize(remotePlayerMsg))); + } + } + } + + public async Task RemoveMember(LobbyMember member) + { bool bRemoved = await RunExclusiveAsync(() => - { - if (member.SlotIndex < 0 - || member.SlotIndex >= Members.Length - || !ReferenceEquals(Members[member.SlotIndex], member)) - { + { + if (member.SlotIndex < 0 + || member.SlotIndex >= Members.Length + || !ReferenceEquals(Members[member.SlotIndex], member)) + { return Task.FromResult(false); - } - - LobbyMember placeholderMember = new LobbyMember(this, null, -1, String.Empty, String.Empty, 0, -1, -1, -1, EPlayerType.SLOT_OPEN, member.SlotIndex, true); - Members[member.SlotIndex] = placeholderMember; - TimeMemberLeft[member.UserID] = DateTime.UtcNow; + } + + LobbyMember placeholderMember = new LobbyMember(this, null, -1, String.Empty, String.Empty, 0, -1, -1, -1, EPlayerType.SLOT_OPEN, member.SlotIndex, true); + Members[member.SlotIndex] = placeholderMember; + TimeMemberLeft[member.UserID] = DateTime.UtcNow; ++MembershipVersion; LastFullMeshConnectivityCheckOutcome = null; @@ -1425,13 +1520,13 @@ public async Task RemoveMember(LobbyMember member) }); if (!bRemoved) - { + { return; - } - - // TODO_LOBBY: Optimize this - Int64 UserID = member.UserID; - Console.WriteLine("User {0} left lobby {1}", UserID, LobbyID); + } + + // TODO_LOBBY: Optimize this + Int64 UserID = member.UserID; + Console.WriteLine("User {0} left lobby {1}", UserID, LobbyID); // AC dergister WebSocketMessage_ACDeregisterPlayer remotePlayerAcMsg = new WebSocketMessage_ACDeregisterPlayer(); @@ -1670,8 +1765,8 @@ public bool HadAIAtStart() public async Task UpdateState(ELobbyState state) { await RunExclusiveAsync(() => - { - State = state; + { + State = state; return Task.CompletedTask; }); @@ -2245,22 +2340,22 @@ public async Task DeleteLobby(Lobby lobby) var factory = scope.ServiceProvider.GetRequiredService>(); await using var db = await factory.CreateDbContextAsync(); - if (lobby.State != ELobbyState.COMPLETE) - { - await lobby.UpdateState(ELobbyState.COMPLETE); - } - // Persist publication before removing the lobby. - await Database.MatchHistory.FinalizeAndScheduleExternalPublication(db, lobby); + if (lobby.State != ELobbyState.COMPLETE) + { + await lobby.UpdateState(ELobbyState.COMPLETE); + } + // Persist publication before removing the lobby. + await Database.MatchHistory.FinalizeAndScheduleExternalPublication(db, lobby); // delete bool bRemoved = m_dictLobbies.Remove(lobby.LobbyID, out _); if (bRemoved) { - WebSocketManager.QueueLobbyListUpdateForViewers(lobby.NetworkRoomID); - - lobby.OnLobbyNeedsDestroyed -= HandleLobbyNeedsDestroyed; - } + WebSocketManager.QueueLobbyListUpdateForViewers(lobby.NetworkRoomID); + + lobby.OnLobbyNeedsDestroyed -= HandleLobbyNeedsDestroyed; + } return bRemoved; } @@ -2278,4 +2373,4 @@ public bool IsUserInLobby(Lobby lobby, Int64 user_id) return member != null; } } -} +} diff --git a/GenOnlineService/MatchmakingManager.cs b/GenOnlineService/MatchmakingManager.cs index 3ba60aa..3d58421 100644 --- a/GenOnlineService/MatchmakingManager.cs +++ b/GenOnlineService/MatchmakingManager.cs @@ -1077,7 +1077,7 @@ private async Task TriggerFullMeshConnectivityChecks(Lobby lobby) } // kept at or above the legacy threshold so short tuned checks don't show a start countdown on older clients - QueueSetupProgress(Math.Max(FullMeshCheckSettings.QuickMatchRefereeWindowMS + c_SetupClientTimeoutMarginMSec, c_LegacyClientCountdownThresholdMSec)); + QueueSetupProgress(Math.Max(FullMeshCheckSettings.RefereeWindowMS + c_SetupClientTimeoutMarginMSec, c_LegacyClientCountdownThresholdMSec)); lobby.SendFullMeshConnectivityCheckRequestToMembers(); @@ -1087,6 +1087,7 @@ private async Task TriggerFullMeshConnectivityChecks(Lobby lobby) if (memberSession != null) { await SendMatchmakingMessage(memberSession, "Preparing match..."); + await SendMatchmakingMessage(memberSession, "Checking connections between all players..."); } } } @@ -1151,7 +1152,7 @@ internal bool VerifyMatchIsStillValid(Lobby lobby, out string failureReason) return true; } - private async Task AbortQuickMatchAutoStart(string reason) + private async Task AbortQuickMatchAutoStart(string reason, List? extraMessagesBeforeRequeue = null) { List sessionsToRequeue = new(); lock (m_StateLock) @@ -1208,6 +1209,15 @@ private async Task AbortQuickMatchAutoStart(string reason) if (await TryRequeueRegisteredPlayer(memberSession, requeueActionJSON)) { await SendMatchmakingMessage(memberSession, reason); + + if (extraMessagesBeforeRequeue != null) + { + foreach (string strExtraMessage in extraMessagesBeforeRequeue) + { + await SendMatchmakingMessage(memberSession, strExtraMessage); + } + } + await SendMatchmakingMessage(memberSession, "Re-queueing you into matchmaking..."); } } @@ -1312,6 +1322,7 @@ public async Task Tick() UserSession? memberSession = member.GetAssociatedSession(); if (memberSession != null) { + await SendMatchmakingMessage(memberSession, "All players are connected."); await SendMatchmakingMessage(memberSession, $"Starting game in {c_GameStartCountdownMSec / 1000} seconds."); } } @@ -1323,7 +1334,18 @@ public async Task Tick() : !bMatchStillValid ? $"QuickMatch auto-start was aborted because the match is no longer valid: {verifyFailureReason}." : "QuickMatch auto-start was aborted because not all players were fully mesh-connected."; - await AbortQuickMatchAutoStart(reason); + + // Only meaningful when the mesh check itself is why we're aborting - pull the pair + // data before the lobby gets torn down below. + List? missingConnectionMessages = null; + if (lobbyDuringMeshCheck.LastFullMeshConnectivityCheckOutcome == false + && lobbyDuringMeshCheck.LastFullMeshConnectivityCheckMissingConnections.Count > 0) + { + missingConnectionMessages = lobbyDuringMeshCheck.BuildMissingConnectionMessages( + lobbyDuringMeshCheck.LastFullMeshConnectivityCheckMissingConnections); + } + + await AbortQuickMatchAutoStart(reason, missingConnectionMessages); } } diff --git a/GenOnlineService/appsettings.json b/GenOnlineService/appsettings.json index e164c27..46dedc7 100644 --- a/GenOnlineService/appsettings.json +++ b/GenOnlineService/appsettings.json @@ -35,11 +35,8 @@ "cert_key_path": null, "cert_reload_interval_seconds": 30, "disable_full_mesh_check": false, - "full_mesh_check_attempt_window_ms": 8000, "full_mesh_check_snapshot_interval_ms": 1000, - "full_mesh_check_retry_delay_ms": 3000, - "full_mesh_check_max_attempts": 2, - "full_mesh_check_qm_referee_window_ms": 20000, + "full_mesh_check_referee_window_ms": 20000, "reconnect_grace_period_ms": 30000, "ice_implementation": 2, "ice_implementation_testers_value": 1, From 240d63be4b8a788f95ec056e5041a0c83b77dd1d Mon Sep 17 00:00:00 2001 From: tintinhamans <5984296+tintinhamans@users.noreply.github.com> Date: Tue, 29 Sep 2026 22:46:28 +0200 Subject: [PATCH 8/8] fix(lobby): Harden lobby membership and host actions --- .../Controllers/Lobby/LobbyController.cs | 38 ++++++++++------ GenOnlineService/LobbyManager.cs | 45 ++++++++++++++++++- 2 files changed, 68 insertions(+), 15 deletions(-) diff --git a/GenOnlineService/Controllers/Lobby/LobbyController.cs b/GenOnlineService/Controllers/Lobby/LobbyController.cs index d26c7a4..f762b52 100644 --- a/GenOnlineService/Controllers/Lobby/LobbyController.cs +++ b/GenOnlineService/Controllers/Lobby/LobbyController.cs @@ -245,19 +245,23 @@ public async Task Delete(Int64 lobbyID) } } - Console.WriteLine("[Source 1] User {0} Leave Any Lobby", user_id); - await _lobbyManager.LeaveAnyLobby(user_id); + // only the named lobby; a request for any other is a no-op + if (lobby != null && lobby.GetMemberFromUserID(user_id) != null) + { + Console.WriteLine("[Source 1] User {0} Leave Lobby {1}", user_id, lobbyID); + await _lobbyManager.LeaveSpecificLobby(user_id, lobbyID); - // cleanup TURN credentials - TURNCredentialManager.DeleteCredentialsForUser(user_id); + // cleanup TURN credentials + TURNCredentialManager.DeleteCredentialsForUser(user_id); - // clear our lobby ID - UserSession? sourceData = WebSocketManager.GetSessionFromUser(user_id, sessionType); + // clear our lobby ID + UserSession? sourceData = WebSocketManager.GetSessionFromUser(user_id, sessionType); - if (sourceData != null) - { - sourceData.UpdateSessionLobbyID(-1); - // NOTE: We dont update the match history match ID here, that is done by the match history service + if (sourceData != null) + { + sourceData.UpdateSessionLobbyID(-1); + // NOTE: We dont update the match history match ID here, that is done by the match history service + } } result.success = true; @@ -624,8 +628,8 @@ internal static bool ShouldRejectJoinForCrcMismatch(UInt32 sessionExeCrc, UInt32 { Int64 KickedUserID = data["userid"].GetInt64(); - // Target must be in this lobby, otherwise a host could wipe an arbitrary player's TURN/session state. - if (lobby.GetMemberFromUserID(KickedUserID) != null) + // Target must be in this lobby and not the host, otherwise a host could wipe an arbitrary player's TURN/session state. + if (KickedUserID != SourceMember.UserID && KickedUserID != lobby.Owner && lobby.GetMemberFromUserID(KickedUserID) != null) { return KickedUserID; } @@ -841,7 +845,7 @@ public async Task Put(Int64 lobbyID) if (user_id != -1 && SessionHelpers.SessionTypeHasAccessTo(sessionType, ESessionAccessType.Gameplay)) { UInt16 userPreferredPort = data["preferred_port"].GetUInt16(); - bool bHasMap = data["has_map"].GetBoolean(); + bool bHasMap = data.ContainsKey("has_map") && data["has_map"].GetBoolean(); // missing means no map EKnownAnticheatID anticheatID = (EKnownAnticheatID)data["anticheat_id"].GetInt32(); // does the lobby have a password? @@ -892,6 +896,14 @@ public async Task Put(Int64 lobbyID) return result; } + // don't strand them out of their current lobby; AddMember stays authoritative + if (lobby.GetMemberFromUserID(user_id) == null && !lobby.HasOpenSlot()) + { + Response.StatusCode = (int)HttpStatusCode.NotAcceptable; + result.success = false; + return result; + } + // leave any lobby await _lobbyManager.LeaveAnyLobby(user_id); diff --git a/GenOnlineService/LobbyManager.cs b/GenOnlineService/LobbyManager.cs index 4ddf01a..7ef376f 100644 --- a/GenOnlineService/LobbyManager.cs +++ b/GenOnlineService/LobbyManager.cs @@ -1145,8 +1145,11 @@ private void DoHostMigration() Owner = member.UserID; member.UpdateSlotIndex(0); - Members[0] = member; - Members[oldSlot] = new LobbyMember(this, null, -1, String.Empty, String.Empty, 0, -1, -1, -1, EPlayerType.SLOT_OPEN, oldSlot, true); + // one array swap, so lock-free readers never see the member twice + LobbyMember[] migratedMembers = (LobbyMember[])Members.Clone(); + migratedMembers[0] = member; + migratedMembers[oldSlot] = new LobbyMember(this, null, -1, String.Empty, String.Empty, 0, -1, -1, -1, EPlayerType.SLOT_OPEN, oldSlot, true); + Members = migratedMembers; member.SetReadyState(true); DirtyRetransmitLobbyList(); @@ -1571,6 +1574,38 @@ public async Task RemoveMember(LobbyMember member) DirtyRetransmitLobbyList(); } + // Advisory pre-check for joiners; AddMember re-checks under the gate. + public bool HasOpenSlot() + { + if (State != ELobbyState.GAME_SETUP) + { + return false; + } + + foreach (LobbyMember memberEntry in Members) + { + if (memberEntry.SlotState == EPlayerType.SLOT_OPEN) + { + return true; + } + } + + return false; + } + + public bool HasOtherHumans(Int64 excludedUserID) + { + foreach (LobbyMember memberEntry in Members) + { + if (memberEntry.SlotState == EPlayerType.SLOT_PLAYER && memberEntry.UserID != excludedUserID) + { + return true; + } + } + + return false; + } + public int GetNumberOfHumans() { int numHumanMembers = 0; @@ -2175,6 +2210,12 @@ public async Task CleanupUserLobbiesNotStarted(Int64 UserID) { if (ownedLobby.State == ELobbyState.GAME_SETUP) // only those in setup, in game games can continue { + // other humans keep the lobby; the leave below migrates the host + if (ownedLobby.GetMemberFromUserID(UserID) != null && ownedLobby.HasOtherHumans(UserID)) + { + continue; + } + await DeleteLobby(ownedLobby); } }