diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift index d5d691dd4c..c47b77842a 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift @@ -41,51 +41,6 @@ struct SnapshotQuality: Codable { var timing: SnapshotCaptureTiming? = nil } -enum SnapshotXCTestChannelPlanState: Equatable { - case normal - case deferredToIndependentBackend - case boundedXCTestProbe -} - -/// The XCTest accessibility channel penalty (#1105/#1156). While it is active, regular plans for its -/// bundle defer XCTest tiers. Each recovery interval, one deferred plan runs the bounded tree probe -/// first, inside that capture, so a screen that is light again can clear the penalty before it -/// expires (#3844). -struct SnapshotXCTestChannelPenalty: Equatable { - static let duration: TimeInterval = 120 - static let recoveryProbeInterval: TimeInterval = 30 - - private(set) var bundleId: String? - private(set) var until = Date.distantPast - private(set) var probeDueAt = Date.distantFuture - - /// A penalty armed without a bundle id applies to whatever target is current. - func isActive(for bundleId: String?, at now: Date) -> Bool { - guard now < until else { return false } - guard let penalized = self.bundleId else { return true } - return penalized == bundleId - } - - func isProbeDue(for bundleId: String?, at now: Date) -> Bool { - isActive(for: bundleId, at: now) && now >= probeDueAt - } - - mutating func arm(bundleId: String?, at now: Date) { - self.bundleId = bundleId - until = now.addingTimeInterval(Self.duration) - probeDueAt = now.addingTimeInterval(Self.recoveryProbeInterval) - } - - /// A started probe moves the next one a full interval out, whatever the probe finds. - mutating func claimProbe(at now: Date) { - probeDueAt = now.addingTimeInterval(Self.recoveryProbeInterval) - } - - mutating func clear() { - self = SnapshotXCTestChannelPenalty() - } -} - /// How one tier's bounded work ended. `deadlineExhausted` is a tier timeout: the tier stopped /// starting work it could not finish inside its own slice, so what it returns is a partial result /// it never completed collecting. The plan keeps that result only as the fallback and lets the next @@ -96,19 +51,14 @@ enum SnapshotTierOutcome: Equatable { case deadlineExhausted } +/// The plan one capture runs: ordered steps, the channel state the selection chose, and the pin — +/// if any — that narrowed it. struct EffectiveSnapshotCapturePlan { - let plan: [SnapshotBackendKind] + let steps: [SnapshotCaptureStep] let xCTestChannelState: SnapshotXCTestChannelPlanState - let treeCaptureSliceBudgetOverride: TimeInterval? /// Non-nil only when the plan was narrowed by an explicit internal backend preference. This /// keeps the quality marker tied to the plan decision rather than to an untrusted request field. let preferredBackend: SnapshotBackendKind? - - /// A deferred plan that starts with an XCTest tier is running the due recovery probe. - var leadsWithXCTestRecoveryProbe: Bool { - xCTestChannelState == .deferredToIndependentBackend - && plan.first?.usesXCTestAccessibilityChannel == true - } } /// What the plan runner does when every backend failed or stayed sparse. @@ -164,7 +114,6 @@ extension RunnerTests { /// but chained recovery tiers must never stack past the 30s main-thread watchdog: when the /// budget is spent, remaining tiers are skipped and the best payload so far is returned. static let snapshotPlanBudget: TimeInterval = 20 - static let penalizedXCTestProbeTreeSliceBudget: TimeInterval = 1 static let collapsedLeafMinimumSegments = 10 static func payloadNodeCount(_ payload: DataPayload?) -> Int { @@ -179,196 +128,6 @@ extension RunnerTests { /// trait instead of by someone remembering this line. static let rawDiagnosticPlan: [SnapshotBackendKind] = regularVisiblePlan.filter(\.supportsRawProjection) - // MARK: XCTest accessibility channel penalty (cross-attempt memory, #1105/#1156) - // - // On some deep/dynamic screens the XCTest bulk snapshot no longer fails fast with - // kAXErrorIllegalArgument (the #758 signature) — it grinds for many seconds first. One slow - // grind is tolerable; re-grinding on every subsequent capture of the same screen buries the - // main thread past the execution watchdog. After a slow, timed-out, or abandoned XCTest-backed - // capture, later plans for the same bundle use non-XCTest recovery tiers until the penalty expires. - - func penalizeSnapshotXCTestChannel(bundleId: String?, reason: String) { - snapshotXCTestChannelPenaltyLock.lock() - snapshotXCTestChannelPenalty.arm(bundleId: bundleId, at: Date()) - snapshotXCTestChannelPenaltyLock.unlock() - NSLog( - "AGENT_DEVICE_RUNNER_SNAPSHOT_XCTEST_CHANNEL_PENALIZED bundle=%@ reason=%@", - bundleId ?? "", - reason - ) - } - - func clearSnapshotXCTestChannelPenalty(reason: String) { - snapshotXCTestChannelPenaltyLock.lock() - let hadActivePenalty = Date() < snapshotXCTestChannelPenalty.until - snapshotXCTestChannelPenalty.clear() - snapshotXCTestChannelPenaltyLock.unlock() - if hadActivePenalty { - NSLog("AGENT_DEVICE_RUNNER_SNAPSHOT_XCTEST_CHANNEL_PENALTY_CLEARED reason=%@", reason) - } - } - - func isSnapshotXCTestChannelPenalized(bundleId: String?) -> Bool { - snapshotXCTestChannelPenaltyLock.lock() - defer { snapshotXCTestChannelPenaltyLock.unlock() } - return snapshotXCTestChannelPenalty.isActive(for: bundleId, at: Date()) - } - - func isSnapshotXCTestRecoveryProbeDue(bundleId: String?) -> Bool { - snapshotXCTestChannelPenaltyLock.lock() - defer { snapshotXCTestChannelPenaltyLock.unlock() } - return snapshotXCTestChannelPenalty.isProbeDue(for: bundleId, at: Date()) - } - - func claimSnapshotXCTestRecoveryProbe(bundleId: String?) { - snapshotXCTestChannelPenaltyLock.lock() - snapshotXCTestChannelPenalty.claimProbe(at: Date()) - snapshotXCTestChannelPenaltyLock.unlock() - NSLog("AGENT_DEVICE_RUNNER_SNAPSHOT_XCTEST_RECOVERY_PROBE bundle=%@", bundleId ?? "") - } - - /// The pre-seeded first-failure a penalized plan stamps into its verdict. The deferred case - /// uses the dedicated 'deferred' code: the breaker pre-selected the backend, nothing new - /// degraded on this capture, and the daemon keys warning suppression and the settle budget - /// reset off exactly that distinction. The bounded probe keeps 'budget': its short XCTest - /// slice genuinely constrains what this capture could read. - /// - /// `requestPinnedBackend` separates the two ways the plan can arrive at the same - /// deferred shape. A capture that ASKED for a private-AX-only reading (custom - /// actions) degraded nothing, so it seeds no failure at all — reporting slow - /// accessibility work would name a cause that does not exist. Its answer comes from - /// `snapshotPlanPreSelectionReason`. - static func xcTestChannelStateFirstFailure( - _ state: SnapshotXCTestChannelPlanState, - requestPinnedBackend: Bool = false - ) -> (reason: String, code: String)? { - switch state { - case .normal: - return nil - case .deferredToIndependentBackend: - if requestPinnedBackend { - return nil - } - return ( - "XCTest-backed snapshot tiers were deferred after recent slow accessibility work on this screen", - "deferred" - ) - case .boundedXCTestProbe: - return ( - "XCTest-backed snapshot tiers are running with a short recovery probe after recent slow accessibility work on this screen", - "budget" - ) - } - } - - /// The 'requested-backend' answer for a plan the request itself pinned: the recursive tree the - /// caller asked for, or the private AX the custom-actions reading requires. A pin names a plan - /// decision, not a failure, so it never enters the failure slot; `planVerdictReason` reports it - /// only while no tier failed on this capture. - static func snapshotPlanPreSelectionReason( - _ state: SnapshotXCTestChannelPlanState, - requestPinnedBackend: Bool = false, - preferredBackend: String? = nil - ) -> (reason: String, code: String)? { - if state == .normal && preferredBackend == SnapshotBackendKind.recursiveTree.rawValue { - return ( - "the recursive XCTest tree backend was explicitly selected for this capture", - "requested-backend" - ) - } - if state == .deferredToIndependentBackend && requestPinnedBackend { - return ( - "the private AX backend was selected because this capture asked for accessibility custom actions", - "requested-backend" - ) - } - return nil - } - - /// The reason a capture's verdict carries: the first tier failure when one exists, otherwise the - /// pre-selection the plan started with. Once an XCTest-backed tier ran behind a deferred plan, - /// it ran on the bounded probe's short slice, so the capture reports that constraint ('budget') - /// whether the tier recovered it or the plan ended sparse, rather than the pre-selection - /// ('deferred') the plan was seeded with. - static func planVerdictReason( - xCTestTierRan: Bool, - state: SnapshotXCTestChannelPlanState, - firstFailure: (reason: String, code: String)?, - preSelection: (reason: String, code: String)? - ) -> (reason: String, code: String)? { - guard xCTestTierRan, state == .deferredToIndependentBackend else { return firstFailure ?? preSelection } - return xcTestChannelStateFirstFailure(.boundedXCTestProbe) - } - - /// Pure gate: a capture is planned as penalized when the channel penalty is - /// active OR the daemon pinned the private-AX backend (same-backend evidence - /// probe) — both mean "do not enter XCTest tree work first, and stamp the - /// pre-selection as 'deferred' rather than a degradation". - static func snapshotXCTestChannelTreatedAsPenalized( - penalized: Bool, - preferredBackend: String? - ) -> Bool { - penalized || preferredBackend == SnapshotBackendKind.privateAX.rawValue - } - - /// Pure plan-reorder rule: an internal preferred backend pins the regular plan to that backend; - /// this is the only force seam used by same-backend evidence and conformance captures. A - /// penalized XCTest accessibility channel uses independent backends when the platform has one, - /// otherwise it keeps XCTest work on a short probe. A due recovery probe puts that short tree - /// ahead of the independent backend. The raw diagnostic plan keeps tree-first errors, and unknown - /// plans are left untouched. - static func effectiveSnapshotCapturePlan( - _ plan: [SnapshotBackendKind], - xCTestChannelPenalized: Bool, - xCTestRecoveryProbeDue: Bool = false, - availableBackends: Set = Set(SnapshotBackendKind.allCases), - preferredBackend: String? = nil - ) -> EffectiveSnapshotCapturePlan { - if - plan == Self.regularVisiblePlan, - let preferred = preferredBackend.flatMap(SnapshotBackendKind.init(rawValue:)), - preferred.isForceable, - availableBackends.contains(preferred) - { - return EffectiveSnapshotCapturePlan( - plan: [preferred], - xCTestChannelState: preferred == .privateAX ? .deferredToIndependentBackend : .normal, - treeCaptureSliceBudgetOverride: nil, - preferredBackend: preferred - ) - } - guard xCTestChannelPenalized, plan == Self.regularVisiblePlan else { - return EffectiveSnapshotCapturePlan( - plan: plan, - xCTestChannelState: .normal, - treeCaptureSliceBudgetOverride: nil, - preferredBackend: nil - ) - } - let availablePlan = plan.filter { availableBackends.contains($0) } - let recoveryPlan = availablePlan.filter { !$0.usesXCTestAccessibilityChannel } - if !recoveryPlan.isEmpty { - // The independent backend reads only the app it can match as the active AX application, so - // an out-of-process surface over the app (the Save Password sheet) leaves it empty for the - // whole penalty. The tree, on the bounded probe's short slice, stays behind it, or leads when - // the recovery probe is due; the query sweep does not, since its grind is what the penalty - // keeps off the main thread. - let boundedTree = availablePlan.filter { $0 == .recursiveTree } - return EffectiveSnapshotCapturePlan( - plan: xCTestRecoveryProbeDue ? boundedTree + recoveryPlan : recoveryPlan + boundedTree, - xCTestChannelState: .deferredToIndependentBackend, - treeCaptureSliceBudgetOverride: Self.penalizedXCTestProbeTreeSliceBudget, - preferredBackend: nil - ) - } - return EffectiveSnapshotCapturePlan( - plan: availablePlan.filter(\.usesXCTestAccessibilityChannel), - xCTestChannelState: .boundedXCTestProbe, - treeCaptureSliceBudgetOverride: Self.penalizedXCTestProbeTreeSliceBudget, - preferredBackend: nil - ) - } - func shouldSkipSnapshotBackendForAbandonedMainThreadWork(_ kind: SnapshotBackendKind) -> Bool { kind.usesXCTestAccessibilityChannel && hasAbandonedMainThreadWork() } @@ -390,20 +149,16 @@ extension RunnerTests { let suppressXCTestPenalty = snapshotXCTestPenaltyWarmupExemption.consume() var xCTestTierRan = false - // Reorder is iOS-only because hostile screens can make XCTest tree/query work grind while - // the app remains visually responsive. Simulators can avoid that channel through private AX; - // physical devices have no independent semantic backend yet, so they use a bounded probe. - // A daemon-preferred private-AX capture (same-backend evidence probe) takes the exact - // penalized route: privateAX-first plan, 'deferred' verdict — the backend was pre-selected - // deliberately, so no degradation warning should render for it. + // Reorder is iOS-only: physical devices have no independent semantic backend, so a penalized + // channel there uses the bounded probe instead of deferral. var xCTestChannelPenalized = false - var xCTestChannelPenalizedByBreaker = false + var breakerPenalized = false var xCTestRecoveryProbeDue = false #if os(iOS) - xCTestChannelPenalizedByBreaker = isSnapshotXCTestChannelPenalized(bundleId: target.bundleId) + breakerPenalized = isSnapshotXCTestChannelPenalized(bundleId: target.bundleId) xCTestRecoveryProbeDue = isSnapshotXCTestRecoveryProbeDue(bundleId: target.bundleId) xCTestChannelPenalized = Self.snapshotXCTestChannelTreatedAsPenalized( - penalized: xCTestChannelPenalizedByBreaker, + penalized: breakerPenalized, preferredBackend: options.preferredBackend ) #endif @@ -414,12 +169,11 @@ extension RunnerTests { availableBackends: Set(SnapshotBackendKind.allCases.filter(\.isAvailableOnCurrentPlatform)), preferredBackend: options.preferredBackend ) - let effectivePlan = effective.plan // A request pin answers through its own slot, never the failure slot: a tier that failed on // this capture owns the verdict reason there, and planVerdictReason falls back to the pin only // when nothing failed. Only a customActions-implied pin is request-pinned; the daemon's // same-backend evidence probe pins for its own reasons and keeps its 'deferred' seed. - let requestPinnedBackend = options.customActions && !xCTestChannelPenalizedByBreaker + let requestPinnedBackend = options.customActions && !breakerPenalized firstFailure = Self.xcTestChannelStateFirstFailure( effective.xCTestChannelState, requestPinnedBackend: requestPinnedBackend @@ -438,8 +192,9 @@ extension RunnerTests { NSLog("AGENT_DEVICE_RUNNER_SNAPSHOT_XCTEST_CHANNEL_PROBE_BOUNDED bundle=%@", target.bundleId ?? "") } - for kind in effectivePlan { - if kind != effectivePlan.first && Date() >= deadline { + for step in effective.steps { + let kind = step.backend + if kind != effective.steps.first?.backend && Date() >= deadline { NSLog("AGENT_DEVICE_RUNNER_SNAPSHOT_PLAN_BUDGET_EXHAUSTED skipped=%@", kind.rawValue) if firstFailure == nil { firstFailure = ("the capture plan ran out of its time budget", "budget") @@ -458,23 +213,21 @@ extension RunnerTests { } continue } - if kind == effectivePlan.first && effective.leadsWithXCTestRecoveryProbe { + // The claim is synchronous: a skipped step must leave the probe for the next plan. + if step.claimsPeriodicRecoveryProbe { claimSnapshotXCTestRecoveryProbe(bundleId: target.bundleId) } if kind.usesXCTestAccessibilityChannel { xCTestTierRan = true } let attempt = try captureWithBackend( - kind, + step, target: target, options: options, - deadline: deadline, - treeCaptureSliceBudgetOverride: effective.treeCaptureSliceBudgetOverride + deadline: deadline ) recordXCTestSnapshotBackendAttemptIfNeeded( - kind, attempt: attempt, bundleId: target.bundleId, - penaltySuppressed: suppressXCTestPenalty, - channelState: effective.xCTestChannelState + penaltySuppressed: suppressXCTestPenalty ) if case let .failed(failure, phase: _) = attempt.outcome { if Self.isAxSnapshotFailure(failure) { axFailure = failure } @@ -509,7 +262,7 @@ extension RunnerTests { continue } - let recovered = kind != effectivePlan.first || effective.xCTestChannelState != .normal + let recovered = kind != effective.steps.first?.backend || effective.xCTestChannelState != .normal let verdictReason = Self.planVerdictReason( xCTestTierRan: kind.usesXCTestAccessibilityChannel, state: effective.xCTestChannelState, @@ -560,7 +313,7 @@ extension RunnerTests { best.map { stampedSnapshotPayload($0.capture, backend: $0.kind, state: .sparse, reason: terminalReason) } ?? stampedSnapshotPayload( SnapshotBackendCapture(payload: sparseTruncatedSnapshotPayload(), effectiveDepth: nil), - backend: effectivePlan.last ?? plan.last ?? .recursiveTree, + backend: effective.steps.last?.backend ?? plan.last ?? .recursiveTree, state: .sparse, reason: terminalReason ) @@ -568,12 +321,12 @@ extension RunnerTests { } private func captureWithBackend( - _ kind: SnapshotBackendKind, + _ step: SnapshotCaptureStep, target: SnapshotCaptureTarget, options: PresentationOptions, - deadline: Date, - treeCaptureSliceBudgetOverride: TimeInterval? + deadline: Date ) throws -> SnapshotBackendAttempt { + let kind = step.backend let app = target.app let hint = SnapshotPresentation.captureHint(for: options) var timer = SnapshotPhaseTimer() @@ -591,7 +344,7 @@ extension RunnerTests { app: app, hint: hint, captureDeadline: deadline, - treeCaptureSliceBudgetOverride: treeCaptureSliceBudgetOverride + treeCaptureSliceBudgetOverride: step.treeCaptureSliceBudgetOverride ) else { return (nil, .completed) @@ -647,12 +400,14 @@ extension RunnerTests { tierOutcome = measured.1 } catch let failure as SnapshotCaptureFailure { return SnapshotBackendAttempt( + step: step, outcome: .failed(failure, phase: .acquisition), timing: timer.timing ) } guard let acquisition else { return SnapshotBackendAttempt( + step: step, outcome: .noCapture, timing: timer.timing, tierOutcome: tierOutcome @@ -688,12 +443,14 @@ extension RunnerTests { } } catch let failure as SnapshotPresentationFailure { return SnapshotBackendAttempt( + step: step, outcome: .failed(Self.snapshotCaptureFailure(for: failure), phase: .presentation), timing: timer.timing, tierOutcome: tierOutcome ) } catch let failure as SnapshotCaptureFailure { return SnapshotBackendAttempt( + step: step, outcome: .failed(failure, phase: .presentation), timing: timer.timing, tierOutcome: tierOutcome @@ -704,6 +461,7 @@ extension RunnerTests { capture.timing = timer.timing capture.keyboardBand = keyboardBand?.payload return SnapshotBackendAttempt( + step: step, outcome: .captured(capture), timing: timer.timing, tierOutcome: tierOutcome diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotChannelPolicy.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotChannelPolicy.swift new file mode 100644 index 0000000000..3f77bbb663 --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotChannelPolicy.swift @@ -0,0 +1,280 @@ +import Foundation + +// MARK: - XCTest accessibility channel policy (ADR 0004, #1105/#1156/#3896) + +/// One executed step of an effective plan. The mode decides which evidence the attempt may produce +/// and travels with it, so evidence cannot reach the recorder detached from its authority (#3896). +enum SnapshotCaptureStep { + /// A regular attempt: its acquisition facts may arm or renew the channel penalty. + case ordinary(SnapshotBackendKind) + /// A bounded attempt inside a penalized plan: it may never arm or renew the penalty. + case boundedRecovery(SnapshotBackendKind, claimsPeriodicProbe: Bool) + + /// The slice budget a bounded tree attempt gets while the channel is penalized. The query + /// sweep's slice has its own owner (`querySweepSliceDeadline`), which the penalty does not + /// shorten. + static let boundedTreeSliceBudget: TimeInterval = 1 + + var backend: SnapshotBackendKind { + switch self { + case let .ordinary(kind): return kind + case let .boundedRecovery(kind, _): return kind + } + } + + var treeCaptureSliceBudgetOverride: TimeInterval? { + guard backend == .recursiveTree, case .boundedRecovery = self else { return nil } + return Self.boundedTreeSliceBudget + } + + var claimsPeriodicRecoveryProbe: Bool { + if case let .boundedRecovery(_, claims) = self { return claims } + return false + } +} + +enum SnapshotXCTestChannelPlanState: Equatable { + case normal + case deferredToIndependentBackend + case boundedXCTestProbe +} + +/// The channel penalty (#1105/#1156): re-grinding a slow XCTest capture on every plan buries the +/// main thread past the watchdog, so while it is active regular plans defer XCTest tiers, and each +/// recovery interval one deferred plan runs the bounded tree probe first, inside that capture, so a +/// light screen clears the penalty early (#3844). It is the only expiry/due-time owner. +struct SnapshotXCTestChannelPenalty: Equatable { + static let duration: TimeInterval = 120 + static let recoveryProbeInterval: TimeInterval = 30 + + private(set) var bundleId: String? + private(set) var until = Date.distantPast + private(set) var probeDueAt = Date.distantFuture + + /// A penalty armed without a bundle id applies to whatever target is current. + func isActive(for bundleId: String?, at now: Date) -> Bool { + guard now < until else { return false } + guard let penalized = self.bundleId else { return true } + return penalized == bundleId + } + + func isProbeDue(for bundleId: String?, at now: Date) -> Bool { + isActive(for: bundleId, at: now) && now >= probeDueAt + } + + mutating func arm(bundleId: String?, at now: Date) { + self.bundleId = bundleId + until = now.addingTimeInterval(Self.duration) + probeDueAt = now.addingTimeInterval(Self.recoveryProbeInterval) + } + + /// A started probe moves the next one a full interval out, whatever the probe finds. + mutating func claimProbe(at now: Date) { + probeDueAt = now.addingTimeInterval(Self.recoveryProbeInterval) + } + + mutating func clear() { + self = SnapshotXCTestChannelPenalty() + } +} + +extension RunnerTests { + func penalizeSnapshotXCTestChannel(bundleId: String?, reason: String) { + snapshotXCTestChannelPenaltyLock.lock() + snapshotXCTestChannelPenalty.arm(bundleId: bundleId, at: Date()) + snapshotXCTestChannelPenaltyLock.unlock() + NSLog( + "AGENT_DEVICE_RUNNER_SNAPSHOT_XCTEST_CHANNEL_PENALIZED bundle=%@ reason=%@", + bundleId ?? "", + reason + ) + } + + func clearSnapshotXCTestChannelPenalty(reason: String) { + snapshotXCTestChannelPenaltyLock.lock() + let hadActivePenalty = Date() < snapshotXCTestChannelPenalty.until + snapshotXCTestChannelPenalty.clear() + snapshotXCTestChannelPenaltyLock.unlock() + if hadActivePenalty { + NSLog("AGENT_DEVICE_RUNNER_SNAPSHOT_XCTEST_CHANNEL_PENALTY_CLEARED reason=%@", reason) + } + } + + func isSnapshotXCTestChannelPenalized(bundleId: String?) -> Bool { + snapshotXCTestChannelPenaltyLock.lock() + defer { snapshotXCTestChannelPenaltyLock.unlock() } + return snapshotXCTestChannelPenalty.isActive(for: bundleId, at: Date()) + } + + func isSnapshotXCTestRecoveryProbeDue(bundleId: String?) -> Bool { + snapshotXCTestChannelPenaltyLock.lock() + defer { snapshotXCTestChannelPenaltyLock.unlock() } + return snapshotXCTestChannelPenalty.isProbeDue(for: bundleId, at: Date()) + } + + func claimSnapshotXCTestRecoveryProbe(bundleId: String?) { + snapshotXCTestChannelPenaltyLock.lock() + snapshotXCTestChannelPenalty.claimProbe(at: Date()) + snapshotXCTestChannelPenaltyLock.unlock() + NSLog("AGENT_DEVICE_RUNNER_SNAPSHOT_XCTEST_RECOVERY_PROBE bundle=%@", bundleId ?? "") + } + + /// Deferred uses the 'deferred' code, not a failure code: the daemon keys warning suppression + /// and the settle-budget reset off that distinction. The bounded probe keeps 'budget'. + static func xcTestChannelStateFirstFailure( + _ state: SnapshotXCTestChannelPlanState, + requestPinnedBackend: Bool = false + ) -> (reason: String, code: String)? { + switch state { + case .normal: + return nil + case .deferredToIndependentBackend: + if requestPinnedBackend { + return nil + } + return ( + "XCTest-backed snapshot tiers were deferred after recent slow accessibility work on this screen", + "deferred" + ) + case .boundedXCTestProbe: + return ( + "XCTest-backed snapshot tiers are running with a short recovery probe after recent slow accessibility work on this screen", + "budget" + ) + } + } + + /// A pin names a plan decision, not a failure, so it reports through this slot and never + /// through the failure slot (`planVerdictReason`). + static func snapshotPlanPreSelectionReason( + _ state: SnapshotXCTestChannelPlanState, + requestPinnedBackend: Bool = false, + preferredBackend: String? = nil + ) -> (reason: String, code: String)? { + if state == .normal && preferredBackend == SnapshotBackendKind.recursiveTree.rawValue { + return ( + "the recursive XCTest tree backend was explicitly selected for this capture", + "requested-backend" + ) + } + if state == .deferredToIndependentBackend && requestPinnedBackend { + return ( + "the private AX backend was selected because this capture asked for accessibility custom actions", + "requested-backend" + ) + } + return nil + } + + /// Once an XCTest tier ran behind a deferred plan, it ran on the probe's short slice, so the + /// capture reports 'budget' rather than the 'deferred' pre-selection it was seeded with. + static func planVerdictReason( + xCTestTierRan: Bool, + state: SnapshotXCTestChannelPlanState, + firstFailure: (reason: String, code: String)?, + preSelection: (reason: String, code: String)? + ) -> (reason: String, code: String)? { + guard xCTestTierRan, state == .deferredToIndependentBackend else { return firstFailure ?? preSelection } + return xcTestChannelStateFirstFailure(.boundedXCTestProbe) + } + + /// The daemon's private-AX pin takes the penalized route deliberately, not from degradation. + static func snapshotXCTestChannelTreatedAsPenalized( + penalized: Bool, + preferredBackend: String? + ) -> Bool { + penalized || preferredBackend == SnapshotBackendKind.privateAX.rawValue + } + + /// An internal preferred backend pins the regular plan. A penalized channel defers XCTest tiers + /// to independent backends where the platform has one (every step bounded recovery, so no + /// attempt can renew the penalty), otherwise it keeps XCTest work on a short probe. A due + /// recovery probe puts the bounded tree first, claiming the probe. Raw and unknown plans are + /// untouched. + static func effectiveSnapshotCapturePlan( + _ plan: [SnapshotBackendKind], + xCTestChannelPenalized: Bool, + xCTestRecoveryProbeDue: Bool = false, + availableBackends: Set = Set(SnapshotBackendKind.allCases), + preferredBackend: String? = nil + ) -> EffectiveSnapshotCapturePlan { + var ordered = plan + var state: SnapshotXCTestChannelPlanState = .normal + var preferred: SnapshotBackendKind? + var probeLeads = false + if + plan == Self.regularVisiblePlan, + let candidate = preferredBackend.flatMap(SnapshotBackendKind.init(rawValue:)), + candidate.isForceable, + availableBackends.contains(candidate) + { + ordered = [candidate] + preferred = candidate + state = candidate == .privateAX ? .deferredToIndependentBackend : .normal + } else if xCTestChannelPenalized, plan == Self.regularVisiblePlan { + let availablePlan = plan.filter { availableBackends.contains($0) } + let recoveryPlan = availablePlan.filter { !$0.usesXCTestAccessibilityChannel } + if !recoveryPlan.isEmpty { + // The independent backend reads only the app it matches as the active AX application, so an + // out-of-process surface leaves it empty for the whole penalty; the bounded tree stays + // behind it as the sparse fallback. The query sweep stays deferred: its grind is what the + // penalty keeps off the main thread. + let boundedTree = availablePlan.filter { $0 == .recursiveTree } + probeLeads = xCTestRecoveryProbeDue && !boundedTree.isEmpty + ordered = probeLeads ? boundedTree + recoveryPlan : recoveryPlan + boundedTree + state = .deferredToIndependentBackend + } else { + ordered = availablePlan.filter(\.usesXCTestAccessibilityChannel) + state = .boundedXCTestProbe + } + } + // A penalized state runs every tier as bounded recovery, so no attempt from it can arm or + // renew the penalty it is recovering from. Only the leading XCTest step of a probe-leading + // deferred plan claims the probe. + let steps = ordered.enumerated().map { offset, kind -> SnapshotCaptureStep in + guard state != .normal else { return .ordinary(kind) } + let claims = probeLeads && offset == 0 && kind.usesXCTestAccessibilityChannel + return .boundedRecovery(kind, claimsPeriodicProbe: claims) + } + return EffectiveSnapshotCapturePlan( + steps: steps, + xCTestChannelState: state, + preferredBackend: preferred + ) + } + + /// A bounded recovery attempt has no arming branch at all: that gate is what stops a probe from + /// renewing its own penalty (#3896). The breaker observes acquisition facts only; presentation + /// is a separate phase and cannot arm it. + static func snapshotXCTestPenaltyReason( + for attempt: SnapshotBackendAttempt, + slowThresholdMs: Double + ) -> String? { + guard case .ordinary(let kind) = attempt.step else { return nil } + guard kind.usesXCTestAccessibilityChannel else { return nil } + if case let .failed(failure, phase: .acquisition) = attempt.outcome, + failure.code == Self.xCTestSnapshotTimeoutCode + { + return "\(kind.rawValue)_backend_timeout" + } + if attempt.tierOutcome == .deadlineExhausted { + return "\(kind.rawValue)_backend_timeout" + } + guard attempt.timing.acquisitionMs > slowThresholdMs else { return nil } + return "slow_\(kind.rawValue)_capture_\(Int(attempt.timing.acquisitionMs))ms" + } + + /// Only an accepted, fast tree from a bounded recovery attempt clears the penalty early; an + /// ordinary tree capture can only arm it. + static func snapshotBoundedRecoveryRestoredChannel(_ attempt: SnapshotBackendAttempt) -> Bool { + guard case .boundedRecovery(.recursiveTree, _) = attempt.step, + attempt.timing.acquisitionMs <= SnapshotCaptureStep.boundedTreeSliceBudget * 1_000, + case let .captured(capture) = attempt.outcome + else { return false } + return Self.snapshotTierRejectionReason( + outcome: attempt.tierOutcome, + kind: .recursiveTree, + payload: capture.qualityPayload ?? capture.payload + ) == nil + } +} diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotTiming.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotTiming.swift index 86b8454926..62e6819a81 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotTiming.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotTiming.swift @@ -94,6 +94,7 @@ extension RunnerTests { case failed(SnapshotCaptureFailure, phase: SnapshotCapturePhase) } + let step: SnapshotCaptureStep /// The failure phase is part of the result, so penalty policy cannot infer it from duration /// or error text. let outcome: Outcome @@ -104,77 +105,34 @@ extension RunnerTests { let tierOutcome: SnapshotTierOutcome init( + step: SnapshotCaptureStep, outcome: Outcome, timing: SnapshotCaptureTiming, tierOutcome: SnapshotTierOutcome = .completed ) { + self.step = step self.outcome = outcome self.timing = timing self.tierOutcome = tierOutcome } } - /// The penalty breaker observes only acquisition facts. Presentation is a separate phase and - /// cannot arm the breaker, even when it is slower than the acquisition that produced the tree. - static func snapshotXCTestPenaltyReason( - kind: SnapshotBackendKind, - attempt: SnapshotBackendAttempt, - slowThresholdMs: Double, - channelState: SnapshotXCTestChannelPlanState = .normal - ) -> String? { - guard channelState == .normal, kind.usesXCTestAccessibilityChannel else { return nil } - if case let .failed(failure, phase: .acquisition) = attempt.outcome, - failure.code == Self.xCTestSnapshotTimeoutCode - { - return "\(kind.rawValue)_backend_timeout" - } - if attempt.tierOutcome == .deadlineExhausted { - return "\(kind.rawValue)_backend_timeout" - } - guard attempt.timing.acquisitionMs > slowThresholdMs else { return nil } - return "slow_\(kind.rawValue)_capture_\(Int(attempt.timing.acquisitionMs))ms" - } - - /// Only an accepted, fast tree proves that a bounded recovery probe restored the channel. - static func snapshotXCTestRecoveryProbeSucceeded( - kind: SnapshotBackendKind, - attempt: SnapshotBackendAttempt, - channelState: SnapshotXCTestChannelPlanState - ) -> Bool { - guard channelState != .normal, - kind == .recursiveTree, - attempt.timing.acquisitionMs <= Self.penalizedXCTestProbeTreeSliceBudget * 1_000, - case let .captured(capture) = attempt.outcome - else { return false } - return Self.snapshotTierRejectionReason( - outcome: attempt.tierOutcome, - kind: kind, - payload: capture.qualityPayload ?? capture.payload - ) == nil - } - + /// The warmup exemption suppresses both the arming and the clearing branch on the first plan + /// against a fresh process. func recordXCTestSnapshotBackendAttemptIfNeeded( - _ kind: SnapshotBackendKind, attempt: SnapshotBackendAttempt, bundleId: String?, - penaltySuppressed: Bool, - channelState: SnapshotXCTestChannelPlanState + penaltySuppressed: Bool ) { guard !penaltySuppressed else { return } - if Self.snapshotXCTestRecoveryProbeSucceeded( - kind: kind, - attempt: attempt, - channelState: channelState - ) { + if Self.snapshotBoundedRecoveryRestoredChannel(attempt) { clearSnapshotXCTestChannelPenalty(reason: "healthy_tree_recovery_probe") return } guard let reason = Self.snapshotXCTestPenaltyReason( - kind: kind, - attempt: attempt, - slowThresholdMs: snapshotXCTestSlowCaptureThreshold * 1_000, - channelState: channelState + for: attempt, + slowThresholdMs: snapshotXCTestSlowCaptureThreshold * 1_000 ) else { return } penalizeSnapshotXCTestChannel( diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotCapturePlanOccupancyTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotCapturePlanOccupancyTests.swift index 9f7e97ece6..9b5b7d9508 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotCapturePlanOccupancyTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotCapturePlanOccupancyTests.swift @@ -448,7 +448,7 @@ extension RunnerTests { XCTAssertEqual(quality.backend, SnapshotBackendKind.recursiveTree.rawValue) XCTAssertGreaterThan(payload.nodes?.count ?? 0, 1, "the bounded tree answers with a real tree") let timing = try XCTUnwrap(quality.timing) - guard timing.acquisitionMs <= Self.penalizedXCTestProbeTreeSliceBudget * 1_000 else { + guard timing.acquisitionMs <= SnapshotCaptureStep.boundedTreeSliceBudget * 1_000 else { throw XCTSkip("the accepted tree's total acquisition exceeded the fast-probe threshold on this host") } XCTAssertFalse( diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotCapturePlanTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotCapturePlanTests.swift index 34b827b91b..388bb94ea5 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotCapturePlanTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotCapturePlanTests.swift @@ -416,7 +416,7 @@ extension RunnerTests { xCTestChannelPenalized: treated, preferredBackend: options.preferredBackend ) - XCTAssertEqual(pinned.plan, [.privateAX]) + XCTAssertEqual(pinned.steps.map(\.backend), [.privateAX]) XCTAssertEqual(pinned.xCTestChannelState, .deferredToIndependentBackend) let raw = Self.effectiveSnapshotCapturePlan( @@ -424,7 +424,7 @@ extension RunnerTests { xCTestChannelPenalized: treated, preferredBackend: options.preferredBackend ) - XCTAssertEqual(raw.plan, Self.rawDiagnosticPlan) + XCTAssertEqual(raw.steps.map(\.backend), Self.rawDiagnosticPlan) // A command without the field decodes to no pin and a normal plan. let bare = try JSONDecoder().decode( @@ -441,7 +441,7 @@ extension RunnerTests { xCTestChannelPenalized: true, preferredBackend: SnapshotBackendKind.recursiveTree.rawValue ) - XCTAssertEqual(forced.plan, [.recursiveTree]) + XCTAssertEqual(forced.steps.map(\.backend), [.recursiveTree]) XCTAssertEqual(forced.xCTestChannelState, .normal) XCTAssertEqual( Self.snapshotPlanPreSelectionReason( @@ -475,7 +475,7 @@ extension RunnerTests { ), preferredBackend: "private-ax" ) - XCTAssertEqual(pinned.plan, [.privateAX]) + XCTAssertEqual(pinned.steps.map(\.backend), [.privateAX]) XCTAssertEqual(pinned.xCTestChannelState, .deferredToIndependentBackend) // The requested-backend rule covers this pin too: it answers through the pre-selection // slot and seeds no failure, so a failing private-AX tier owns the verdict reason. @@ -496,34 +496,38 @@ extension RunnerTests { Self.regularVisiblePlan, xCTestChannelPenalized: true ) - XCTAssertEqual(regular.plan, [.privateAX, .recursiveTree]) + XCTAssertEqual(regular.steps.map(\.backend), [.privateAX, .recursiveTree]) XCTAssertEqual(regular.xCTestChannelState, .deferredToIndependentBackend) - XCTAssertEqual(regular.treeCaptureSliceBudgetOverride, Self.penalizedXCTestProbeTreeSliceBudget) + XCTAssertEqual( + regular.steps.map(\.treeCaptureSliceBudgetOverride), + [nil, SnapshotCaptureStep.boundedTreeSliceBudget] + ) let unpenalized = Self.effectiveSnapshotCapturePlan( Self.regularVisiblePlan, xCTestChannelPenalized: false ) - XCTAssertEqual(unpenalized.plan, Self.regularVisiblePlan) + XCTAssertEqual(unpenalized.steps.map(\.backend), Self.regularVisiblePlan) XCTAssertEqual(unpenalized.xCTestChannelState, .normal) - XCTAssertNil(unpenalized.treeCaptureSliceBudgetOverride) + XCTAssertNil(unpenalized.steps.first?.treeCaptureSliceBudgetOverride) // The raw diagnostic plan preserves tree-first error propagation even under penalty. let raw = Self.effectiveSnapshotCapturePlan( Self.rawDiagnosticPlan, xCTestChannelPenalized: true ) - XCTAssertEqual(raw.plan, Self.rawDiagnosticPlan) + XCTAssertEqual(raw.steps.map(\.backend), Self.rawDiagnosticPlan) XCTAssertEqual(raw.xCTestChannelState, .normal) - XCTAssertNil(raw.treeCaptureSliceBudgetOverride) + XCTAssertNil(raw.steps.first?.treeCaptureSliceBudgetOverride) } func testPlanVerdictReasonNamesTheBoundedProbeOnceAnXCTestTierRanBehindPrivateAX() { + // The runner seeds a deferred plan's `deferred` into the failure slot, not the pre-selection. let deferred = Self.xcTestChannelStateFirstFailure(.deferredToIndependentBackend) XCTAssertEqual( Self.planVerdictReason( xCTestTierRan: true, state: .deferredToIndependentBackend, - firstFailure: deferred, preSelection: deferred + firstFailure: deferred, preSelection: nil )?.code, "budget" ) @@ -534,11 +538,11 @@ extension RunnerTests { let pin = Self.snapshotPlanPreSelectionReason( .normal, preferredBackend: SnapshotBackendKind.recursiveTree.rawValue) XCTAssertEqual(pin?.code, "requested-backend") - // Nothing failed: the pre-selection owns the verdict. + // Nothing failed on this capture: the breaker's seed owns the verdict from the failure slot. XCTAssertEqual( Self.planVerdictReason( xCTestTierRan: false, state: .deferredToIndependentBackend, - firstFailure: nil, preSelection: deferred + firstFailure: deferred, preSelection: nil )?.code, "deferred" ) @@ -568,11 +572,11 @@ extension RunnerTests { availableBackends: [.recursiveTree, .querySweep] ) - XCTAssertEqual(physicalDevicePlan.plan, [.recursiveTree, .querySweep]) + XCTAssertEqual(physicalDevicePlan.steps.map(\.backend), [.recursiveTree, .querySweep]) XCTAssertEqual(physicalDevicePlan.xCTestChannelState, .boundedXCTestProbe) XCTAssertEqual( - physicalDevicePlan.treeCaptureSliceBudgetOverride, - Self.penalizedXCTestProbeTreeSliceBudget + physicalDevicePlan.steps.map(\.treeCaptureSliceBudgetOverride), + [SnapshotCaptureStep.boundedTreeSliceBudget, nil] ) } @@ -657,75 +661,92 @@ extension RunnerTests { planTestNode(index: 1, type: "Button", label: "Open", hittable: true, parentIndex: 0), ], truncated: false) let healthy = SnapshotBackendCapture(payload: payload, effectiveDepth: nil) - let fast = SnapshotBackendAttempt( - outcome: .captured(healthy), - timing: SnapshotCaptureTiming(acquisitionMs: 100, presentationMs: 10) - ) - for state in [SnapshotXCTestChannelPlanState.deferredToIndependentBackend, .boundedXCTestProbe] { - XCTAssertTrue(Self.snapshotXCTestRecoveryProbeSucceeded( - kind: .recursiveTree, attempt: fast, channelState: state - )) - for kind in [SnapshotBackendKind.querySweep, .privateAX] { - XCTAssertFalse(Self.snapshotXCTestRecoveryProbeSucceeded( - kind: kind, attempt: fast, channelState: state - )) - } - XCTAssertTrue(Self.snapshotXCTestRecoveryProbeSucceeded( - kind: .recursiveTree, - attempt: SnapshotBackendAttempt(outcome: .captured(healthy), timing: SnapshotCaptureTiming(acquisitionMs: 1_000, presentationMs: 10)), - channelState: state + func bounded( + _ kind: SnapshotBackendKind, + outcome: SnapshotBackendAttempt.Outcome, + acquisitionMs: Double, + tierOutcome: SnapshotTierOutcome = .completed + ) -> SnapshotBackendAttempt { + SnapshotBackendAttempt( + step: .boundedRecovery(kind, claimsPeriodicProbe: false), + outcome: outcome, + timing: SnapshotCaptureTiming(acquisitionMs: acquisitionMs, presentationMs: 10), + tierOutcome: tierOutcome + ) + } + XCTAssertTrue(Self.snapshotBoundedRecoveryRestoredChannel( + bounded(.recursiveTree, outcome: .captured(healthy), acquisitionMs: 100) + )) + for kind in [SnapshotBackendKind.querySweep, .privateAX] { + XCTAssertFalse(Self.snapshotBoundedRecoveryRestoredChannel( + bounded(kind, outcome: .captured(healthy), acquisitionMs: 100) )) - let rejected = [ - SnapshotBackendAttempt(outcome: .captured(healthy), timing: fast.timing, tierOutcome: .deadlineExhausted), - SnapshotBackendAttempt(outcome: .noCapture, timing: fast.timing), - SnapshotBackendAttempt(outcome: .captured(healthy), timing: SnapshotCaptureTiming(acquisitionMs: 1_001, presentationMs: 0)), - SnapshotBackendAttempt(outcome: .captured(SnapshotBackendCapture( - payload: payload, effectiveDepth: nil, qualityPayload: DataPayload(nodes: [], truncated: false) - )), timing: fast.timing), - ] - for attempt in rejected { - XCTAssertFalse(Self.snapshotXCTestRecoveryProbeSucceeded( - kind: .recursiveTree, attempt: attempt, channelState: state - )) - } } - XCTAssertFalse(Self.snapshotXCTestRecoveryProbeSucceeded( - kind: .recursiveTree, attempt: fast, channelState: .normal + XCTAssertTrue(Self.snapshotBoundedRecoveryRestoredChannel( + bounded(.recursiveTree, outcome: .captured(healthy), acquisitionMs: 1_000) )) + let rejected = [ + bounded(.recursiveTree, outcome: .captured(healthy), acquisitionMs: 100, tierOutcome: .deadlineExhausted), + bounded(.recursiveTree, outcome: .noCapture, acquisitionMs: 100), + bounded(.recursiveTree, outcome: .captured(healthy), acquisitionMs: 1_001), + bounded(.recursiveTree, outcome: .captured(SnapshotBackendCapture( + payload: payload, effectiveDepth: nil, qualityPayload: DataPayload(nodes: [], truncated: false) + )), acquisitionMs: 100), + ] + for attempt in rejected { + XCTAssertFalse(Self.snapshotBoundedRecoveryRestoredChannel(attempt)) + } + // An ordinary accepted fast tree never clears the penalty; it is the attempt the arming + // branch reads, not a recovery signal. + XCTAssertFalse(Self.snapshotBoundedRecoveryRestoredChannel(SnapshotBackendAttempt( + step: .ordinary(.recursiveTree), + outcome: .captured(healthy), + timing: SnapshotCaptureTiming(acquisitionMs: 100, presentationMs: 10) + ))) } + /// The probe's claim authority belongs to the step that starts it, not to a plan position: + /// only the leading tree step of the due deferred Simulator plan claims the periodic probe. + /// The physical-device bounded plan probes on every capture without a claim, and pins and the + /// raw plan never gain the sweep or a claim. func testDueRecoveryProbeLeadsOnlyTheDeferredRegularPlanAndNeverAddsTheSweep() { let due = Self.effectiveSnapshotCapturePlan( Self.regularVisiblePlan, xCTestChannelPenalized: true, xCTestRecoveryProbeDue: true ) - XCTAssertEqual(due.plan, [.recursiveTree, .privateAX]) + XCTAssertEqual(due.steps.map(\.backend), [.recursiveTree, .privateAX]) XCTAssertEqual(due.xCTestChannelState, .deferredToIndependentBackend) - XCTAssertEqual(due.treeCaptureSliceBudgetOverride, Self.penalizedXCTestProbeTreeSliceBudget) - XCTAssertTrue(due.leadsWithXCTestRecoveryProbe) + XCTAssertEqual( + due.steps.map(\.treeCaptureSliceBudgetOverride), + [SnapshotCaptureStep.boundedTreeSliceBudget, nil] + ) + XCTAssertEqual(due.steps.map(\.claimsPeriodicRecoveryProbe), [true, false]) let notDue = Self.effectiveSnapshotCapturePlan(Self.regularVisiblePlan, xCTestChannelPenalized: true) - XCTAssertEqual(notDue.plan, [.privateAX, .recursiveTree]) - XCTAssertFalse(notDue.leadsWithXCTestRecoveryProbe) + XCTAssertEqual(notDue.steps.map(\.backend), [.privateAX, .recursiveTree]) + XCTAssertEqual(notDue.steps.map(\.claimsPeriodicRecoveryProbe), [false, false]) let physicalDevice = Self.effectiveSnapshotCapturePlan( Self.regularVisiblePlan, xCTestChannelPenalized: true, xCTestRecoveryProbeDue: true, availableBackends: [.recursiveTree, .querySweep] ) - XCTAssertEqual(physicalDevice.plan, [.recursiveTree, .querySweep]) - XCTAssertFalse(physicalDevice.leadsWithXCTestRecoveryProbe, "the bounded plan probes on every capture") + XCTAssertEqual(physicalDevice.steps.map(\.backend), [.recursiveTree, .querySweep]) + XCTAssertEqual( + physicalDevice.steps.map(\.claimsPeriodicRecoveryProbe), [false, false], + "the bounded plan probes on every capture" + ) for pinned in ["private-ax", "tree"] { let plan = Self.effectiveSnapshotCapturePlan( Self.regularVisiblePlan, xCTestChannelPenalized: true, xCTestRecoveryProbeDue: true, preferredBackend: pinned ) - XCTAssertEqual(plan.plan.count, 1) - XCTAssertFalse(plan.leadsWithXCTestRecoveryProbe) + XCTAssertEqual(plan.steps.count, 1) + XCTAssertEqual(plan.steps.map(\.claimsPeriodicRecoveryProbe), [false]) } let raw = Self.effectiveSnapshotCapturePlan( Self.rawDiagnosticPlan, xCTestChannelPenalized: true, xCTestRecoveryProbeDue: true ) - XCTAssertEqual(raw.plan, Self.rawDiagnosticPlan) - XCTAssertFalse(raw.leadsWithXCTestRecoveryProbe) + XCTAssertEqual(raw.steps.map(\.backend), Self.rawDiagnosticPlan) + XCTAssertEqual(raw.steps.map(\.claimsPeriodicRecoveryProbe), [false, false]) } func testChannelPenaltyProbeIsDueEachIntervalUntilItExpiresOrClears() { diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotTimingTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotTimingTests.swift index 9a5033934b..771f779aa3 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotTimingTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotTimingTests.swift @@ -5,25 +5,25 @@ import XCTest extension RunnerTests { func testXCTestPenaltyDecisionSeparatesAcquisitionAndPresentation() { let slowPresentation = SnapshotBackendAttempt( + step: .ordinary(.recursiveTree), outcome: .noCapture, timing: SnapshotCaptureTiming(acquisitionMs: 100, presentationMs: 4_000) ) XCTAssertNil( Self.snapshotXCTestPenaltyReason( - kind: .recursiveTree, - attempt: slowPresentation, + for: slowPresentation, slowThresholdMs: 3_000 ) ) let slowAcquisition = SnapshotBackendAttempt( + step: .ordinary(.recursiveTree), outcome: .noCapture, timing: SnapshotCaptureTiming(acquisitionMs: 3_001, presentationMs: 100) ) XCTAssertEqual( Self.snapshotXCTestPenaltyReason( - kind: .recursiveTree, - attempt: slowAcquisition, + for: slowAcquisition, slowThresholdMs: 3_000 ), "slow_tree_capture_3001ms" @@ -35,26 +35,26 @@ extension RunnerTests { hint: "test" ) let acquisitionFailure = SnapshotBackendAttempt( + step: .ordinary(.recursiveTree), outcome: .failed(timeout, phase: .acquisition), timing: SnapshotCaptureTiming(acquisitionMs: 100, presentationMs: 100) ) XCTAssertEqual( Self.snapshotXCTestPenaltyReason( - kind: .recursiveTree, - attempt: acquisitionFailure, + for: acquisitionFailure, slowThresholdMs: 3_000 ), "tree_backend_timeout" ) let presentationFailure = SnapshotBackendAttempt( + step: .ordinary(.recursiveTree), outcome: .failed(timeout, phase: .presentation), timing: SnapshotCaptureTiming(acquisitionMs: 100, presentationMs: 100) ) XCTAssertNil( Self.snapshotXCTestPenaltyReason( - kind: .recursiveTree, - attempt: presentationFailure, + for: presentationFailure, slowThresholdMs: 3_000 ) ) @@ -69,36 +69,41 @@ extension RunnerTests { effectiveDepth: nil ) let exhausted = SnapshotBackendAttempt( + step: .ordinary(.querySweep), outcome: .captured(capture), timing: SnapshotCaptureTiming(acquisitionMs: 1_000, presentationMs: 10), tierOutcome: .deadlineExhausted ) XCTAssertEqual( Self.snapshotXCTestPenaltyReason( - kind: .querySweep, - attempt: exhausted, + for: exhausted, slowThresholdMs: 3_000 ), "queries_backend_timeout" ) let completed = SnapshotBackendAttempt( + step: .ordinary(.querySweep), outcome: .captured(capture), timing: SnapshotCaptureTiming(acquisitionMs: 1_000, presentationMs: 10), tierOutcome: .completed ) XCTAssertNil( Self.snapshotXCTestPenaltyReason( - kind: .querySweep, - attempt: completed, + for: completed, slowThresholdMs: 3_000 ) ) + let unownedExhausted = SnapshotBackendAttempt( + step: .ordinary(.privateAX), + outcome: exhausted.outcome, + timing: exhausted.timing, + tierOutcome: exhausted.tierOutcome + ) XCTAssertNil( Self.snapshotXCTestPenaltyReason( - kind: .privateAX, - attempt: exhausted, + for: unownedExhausted, slowThresholdMs: 3_000 ), "a tier that owes nothing to the XCTest channel cannot penalize it" @@ -128,68 +133,84 @@ extension RunnerTests { code: Self.xCTestSnapshotTimeoutCode, message: "same timeout text", hint: "test" ) for outcome in [SnapshotBackendAttempt.Outcome.failed(timeout, phase: .acquisition), .noCapture] { - let short = SnapshotBackendAttempt( - outcome: outcome, - timing: SnapshotCaptureTiming(acquisitionMs: 1_000, presentationMs: 10), - tierOutcome: .deadlineExhausted - ) + let short = { (kind: SnapshotBackendKind) in + SnapshotBackendAttempt( + step: .ordinary(kind), + outcome: outcome, + timing: SnapshotCaptureTiming(acquisitionMs: 1_000, presentationMs: 10), + tierOutcome: .deadlineExhausted + ) + } XCTAssertEqual(Self.snapshotXCTestPenaltyReason( - kind: .querySweep, attempt: short, slowThresholdMs: 3_000 + for: short(.querySweep), slowThresholdMs: 3_000 ), "queries_backend_timeout") XCTAssertEqual(Self.snapshotXCTestPenaltyReason( - kind: .recursiveTree, attempt: short, slowThresholdMs: 3_000 + for: short(.recursiveTree), slowThresholdMs: 3_000 ), "tree_backend_timeout") - let slow = SnapshotBackendAttempt( - outcome: outcome, - timing: SnapshotCaptureTiming(acquisitionMs: 3_001, presentationMs: 10), - tierOutcome: .deadlineExhausted - ) + let slow = { (kind: SnapshotBackendKind) in + SnapshotBackendAttempt( + step: .ordinary(kind), + outcome: outcome, + timing: SnapshotCaptureTiming(acquisitionMs: 3_001, presentationMs: 10), + tierOutcome: .deadlineExhausted + ) + } XCTAssertEqual(Self.snapshotXCTestPenaltyReason( - kind: .querySweep, attempt: slow, slowThresholdMs: 3_000 + for: slow(.querySweep), slowThresholdMs: 3_000 ), "queries_backend_timeout") } let untyped = SnapshotBackendAttempt( + step: .ordinary(.recursiveTree), outcome: .failed(SnapshotCaptureFailure( code: "OTHER_FAILURE", message: "same timeout text", hint: "test" ), phase: .acquisition), timing: SnapshotCaptureTiming(acquisitionMs: 100, presentationMs: 0) ) XCTAssertNil(Self.snapshotXCTestPenaltyReason( - kind: .recursiveTree, attempt: untyped, slowThresholdMs: 3_000 + for: untyped, slowThresholdMs: 3_000 )) } + /// The shape #3896 removed and this refactor makes unreachable: an attempt produced by a + /// bounded-recovery step — the leading probe or the trailing bounded tier, on either the + /// Simulator deferred plan or the physical-device bounded plan — has no arming branch at all, + /// whatever its timing or failure. The attempt carries its step, so there is no channel-state + /// argument left to forget. Ordinary tree attempts must still penalize on the same facts. func testBoundedRecoveryAttemptsCannotRenewPenaltyOnEitherPlatformPlan() { let timeout = SnapshotCaptureFailure( code: Self.xCTestSnapshotTimeoutCode, message: "test timeout", hint: "test" ) - let attempts = [ - SnapshotBackendAttempt( - outcome: .failed(timeout, phase: .acquisition), - timing: SnapshotCaptureTiming(acquisitionMs: 1_000, presentationMs: 0) - ), - SnapshotBackendAttempt( - outcome: .noCapture, - timing: SnapshotCaptureTiming(acquisitionMs: 1_000, presentationMs: 0), - tierOutcome: .deadlineExhausted - ), - SnapshotBackendAttempt( - outcome: .noCapture, - timing: SnapshotCaptureTiming(acquisitionMs: 3_001, presentationMs: 0) - ), + let attempts: [( + outcome: SnapshotBackendAttempt.Outcome, timing: SnapshotCaptureTiming, tier: SnapshotTierOutcome + )] = [ + (.failed(timeout, phase: .acquisition), SnapshotCaptureTiming(acquisitionMs: 1_000, presentationMs: 0), .completed), + (.noCapture, SnapshotCaptureTiming(acquisitionMs: 1_000, presentationMs: 0), .deadlineExhausted), + (.noCapture, SnapshotCaptureTiming(acquisitionMs: 3_001, presentationMs: 0), .completed), ] - for state in [SnapshotXCTestChannelPlanState.deferredToIndependentBackend, .boundedXCTestProbe] { + for claimsProbe in [true, false] { for kind in [SnapshotBackendKind.recursiveTree, .querySweep] { for attempt in attempts { XCTAssertNil(Self.snapshotXCTestPenaltyReason( - kind: kind, attempt: attempt, slowThresholdMs: 3_000, channelState: state + for: SnapshotBackendAttempt( + step: .boundedRecovery(kind, claimsPeriodicProbe: claimsProbe), + outcome: attempt.outcome, + timing: attempt.timing, + tierOutcome: attempt.tier + ), + slowThresholdMs: 3_000 )) } } } for attempt in attempts { XCTAssertNotNil(Self.snapshotXCTestPenaltyReason( - kind: .recursiveTree, attempt: attempt, slowThresholdMs: 3_000 + for: SnapshotBackendAttempt( + step: .ordinary(.recursiveTree), + outcome: attempt.outcome, + timing: attempt.timing, + tierOutcome: attempt.tier + ), + slowThresholdMs: 3_000 ), "ordinary tree timeouts and slow acquisitions must still penalize") } }