Skip to content

Commit 97a2aff

Browse files
committed
TriggerTimreMixConcensus test
1 parent ac6bc63 commit 97a2aff

12 files changed

Lines changed: 303 additions & 7 deletions

run-trigger-timer.sh

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
#!/bin/bash
2+
3+
IMAGE=docker-registry.services.stellar-ops.com/dev/stellar-core:25.1.2-3047.7a0d9bcd2.jammy-do-not-use-in-prd-perftests
4+
5+
PROJECT="/mnt/xvdf/supercluster/src/App/App.fsproj"
6+
7+
# -- Drift distribution (uncomment one) --
8+
# No drift:
9+
#DRIFT_ARGS=""
10+
# Uniform drift in [-2000, +2000]ms:
11+
#DRIFT_ARGS="--uniform-drift=-2000,+2000 --drift-pct 70"
12+
# Bimodal drift: first half [-5000,-2000]ms, second half [+2000,+5000]ms:
13+
DRIFT_ARGS="--bimodal-drift=-5000,-2000,+2000,+5000 --drift-pct 70"
14+
15+
dotnet run --project $PROJECT clean --namespace=garand && dotnet run --project $PROJECT --configuration Release \
16+
-- mission TriggerTimerMixConsensus \
17+
--image=$IMAGE \
18+
--netdelay-image=docker-registry.services.stellar-ops.com/dev/sdf-netdelay:latest \
19+
--postgres-image=docker-registry.services.stellar-ops.com/dev/postgres:9.5.22 \
20+
--nginx-image=docker-registry.services.stellar-ops.com/dev/nginx:latest \
21+
--prometheus-exporter-image=docker-registry.services.stellar-ops.com/dev/stellar-core-prometheus-exporter:latest \
22+
--ingress-internal-domain=stellar-supercluster.kube001-ssc-eks.services.stellar-ops.com \
23+
--avoid-node-labels=purpose:ssc \
24+
--namespace=garand \
25+
--export-to-prometheus \
26+
--pubnet-data=/mnt/xvdf/supercluster/topologies/theoretical-max-tps.json \
27+
--trigger-timer-flag-pct 100 \
28+
$DRIFT_ARGS

src/App/Program.fs

Lines changed: 42 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -137,7 +137,12 @@ type MissionOptions
137137
maxBlockTimeMs: int,
138138
minBlockTimeMixedMode: string,
139139
minBlockTimeMixedClassicTxRate: int option,
140-
minBlockTimeMixedSorobanTxRate: int option
140+
minBlockTimeMixedSorobanTxRate: int option,
141+
triggerTimerFlagPct: int,
142+
uniformDrift: seq<int>,
143+
bimodalDrift: seq<int>,
144+
driftPct: int,
145+
disableTriggerTimer: bool
141146
) =
142147

143148
[<Option('k', "kubeconfig", HelpText = "Kubernetes config file", Required = false, Default = "~/.kube/config")>]
@@ -631,6 +636,36 @@ type MissionOptions
631636
Required = false)>]
632637
member self.MinBlockTimeMixedSorobanTxRate = minBlockTimeMixedSorobanTxRate
633638

639+
[<Option("trigger-timer-flag-pct",
640+
HelpText = "Percentage (0-100) of nodes with EXPERIMENTAL_TRIGGER_TIMER enabled",
641+
Required = false,
642+
Default = 100)>]
643+
member self.TriggerTimerFlagPct = triggerTimerFlagPct
644+
645+
[<Option("uniform-drift",
646+
Separator = ',',
647+
HelpText = "Uniform clock drift range in signed ms: --uniform-drift=lower,upper (e.g. --uniform-drift=-2000,+2000)",
648+
Required = false)>]
649+
member self.UniformDrift = uniformDrift
650+
651+
[<Option("bimodal-drift",
652+
Separator = ',',
653+
HelpText = "Bimodal clock drift ranges in signed ms: --bimodal-drift=min1,max1,min2,max2 (e.g. --bimodal-drift=-5000,-2000,+2000,+5000)",
654+
Required = false)>]
655+
member self.BimodalDrift = bimodalDrift
656+
657+
[<Option("drift-pct",
658+
HelpText = "Percentage (0-100) of nodes that receive clock drift",
659+
Required = false,
660+
Default = 0)>]
661+
member self.DriftPct = driftPct
662+
663+
[<Option("disable-trigger-timer",
664+
HelpText = "Disable EXPERIMENTAL_TRIGGER_TIMER on all nodes in MaxTPSClassic (default: enabled)",
665+
Required = false,
666+
Default = false)>]
667+
member self.DisableTriggerTimer = disableTriggerTimer
668+
634669
let splitLabel (lab: string) : (string * string option) =
635670
match lab.Split ':' |> Array.toList with
636671
| [ x ] -> x, None
@@ -869,7 +904,12 @@ let main argv =
869904
minBlockTimeMixedMode = mission.MinBlockTimeMixedMode
870905
minBlockTimeMixedClassicTxRate = mission.MinBlockTimeMixedClassicTxRate
871906
minBlockTimeMixedSorobanTxRate = mission.MinBlockTimeMixedSorobanTxRate
872-
runForMinBlockTime = false }
907+
runForMinBlockTime = false
908+
triggerTimerFlagPct = mission.TriggerTimerFlagPct
909+
uniformDrift = List.ofSeq mission.UniformDrift
910+
bimodalDrift = List.ofSeq mission.BimodalDrift
911+
driftPct = mission.DriftPct
912+
enableTriggerTimer = not mission.DisableTriggerTimer }
873913

874914
allMissions.[m] missionContext
875915

src/FSLibrary.Tests/Tests.fs

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -143,7 +143,12 @@ let ctx : MissionContext =
143143
minBlockTimeMixedMode = "mixed_pregen_sac_payment"
144144
minBlockTimeMixedClassicTxRate = None
145145
minBlockTimeMixedSorobanTxRate = None
146-
runForMinBlockTime = false }
146+
runForMinBlockTime = false
147+
triggerTimerFlagPct = 100
148+
uniformDrift = []
149+
bimodalDrift = []
150+
driftPct = 0
151+
enableTriggerTimer = true }
147152

148153
let netdata = __SOURCE_DIRECTORY__ + "/../../../data/public-network-data-2024-08-01.json"
149154
let pubkeys = __SOURCE_DIRECTORY__ + "/../../../data/tier1keys.json"

src/FSLibrary/FSLibrary.fsproj

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,7 @@
8181
<Compile Include="MinBlockTimeTest.fs" />
8282
<Compile Include="MissionMinBlockTimeClassic.fs" />
8383
<Compile Include="MissionMinBlockTimeMixed.fs" />
84+
<Compile Include="MissionTriggerTimerMixConsensus.fs" />
8485
<Compile Include="StellarMission.fs" />
8586
</ItemGroup>
8687
<ItemGroup>

src/FSLibrary/MaxTPSTest.fs

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -124,7 +124,12 @@ let upgradeSorobanLedgerLimits
124124
peer.WaitForLedgerMaxTxCount multiplier
125125

126126

127-
let maxTPSTest (context: MissionContext) (baseLoadGen: LoadGen) (setupCfg: LoadGen option) =
127+
let maxTPSTest
128+
(context: MissionContext)
129+
(baseLoadGen: LoadGen)
130+
(setupCfg: LoadGen option)
131+
(enableTriggerTimer: bool)
132+
=
128133
let allNodes =
129134
if context.pubnetData.IsSome then
130135
FullPubnetCoreSets context true false
@@ -133,6 +138,14 @@ let maxTPSTest (context: MissionContext) (baseLoadGen: LoadGen) (setupCfg: LoadG
133138
context.image
134139
(if context.flatQuorum.IsSome then context.flatQuorum.Value else false)
135140

141+
let allNodes =
142+
if enableTriggerTimer then
143+
allNodes
144+
|> List.map (fun (cs: CoreSet) ->
145+
{ cs with options = { cs.options with experimentalTriggerTimer = Some true } })
146+
else
147+
allNodes
148+
136149
// PayPregenerated requires node restart between failed iterations to ensure validity of the pregenerated transactions
137150
// However, large-scale simulation restarts can be slow, so for now only use the new mode on small networks
138151
let baseLoadGen =

src/FSLibrary/MissionMaxTPSClassic.fs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,4 +27,4 @@ let maxTPSClassic (context: MissionContext) =
2727
maxfeerate = None
2828
skiplowfeetxs = false }
2929

30-
maxTPSTest context baseLoadGen None
30+
maxTPSTest context baseLoadGen None context.enableTriggerTimer

src/FSLibrary/MissionMaxTPSMixed.fs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,4 +47,4 @@ let maxTPSMixed (baseContext: MissionContext) =
4747

4848
let invokeSetupCfg = { baseLoadGen with mode = SorobanInvokeSetup }
4949

50-
maxTPSTest context baseLoadGen (Some invokeSetupCfg)
50+
maxTPSTest context baseLoadGen (Some invokeSetupCfg) false
Lines changed: 178 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,178 @@
1+
// Copyright 2026 Stellar Development Foundation and contributors. Licensed
2+
// under the Apache License, Version 2.0. See the COPYING file at the root
3+
// of this distribution or at http://www.apache.org/licenses/LICENSE-2.0
4+
5+
// This mission tests the EXPERIMENTAL_TRIGGER_TIMER feature with a mix of
6+
// nodes that have it enabled vs disabled, under configurable clock drift
7+
// distributions. It uses generated pubnet topologies (--pubnet-data) and
8+
// overlays trigger timer and clock offset settings onto the CoreSets.
9+
//
10+
// CLI parameters:
11+
// --trigger-timer-flag-pct N percentage of nodes with the flag (0-100, default 100)
12+
// --drift-pct N percentage of nodes that drift (0-100, default 0)
13+
// --uniform-drift=lower,upper uniform random drift in [lower,upper] signed ms (e.g. -2000,+2000)
14+
// --bimodal-drift=m1,M1,m2,M2 first half in [m1,M1], second half in [m2,M2] signed ms
15+
16+
module MissionTriggerTimerMixConsensus
17+
18+
open Logging
19+
open StellarCoreHTTP
20+
open StellarCorePeer
21+
open StellarCoreSet
22+
open StellarFormation
23+
open StellarMissionContext
24+
open StellarNetworkData
25+
open StellarStatefulSets
26+
open StellarSupercluster
27+
28+
type ClockDriftDistribution =
29+
| NoDrift
30+
| UniformDrift of lower: int * upper: int
31+
| BimodalDrift of min1: int * max1: int * min2: int * max2: int
32+
33+
// Round ms to whole seconds, ceiling away from zero: 1500 -> 2, -800 -> -1
34+
let private ceilToSec (ms: int) =
35+
if ms >= 0 then (ms + 999) / 1000
36+
else -((abs ms + 999) / 1000)
37+
38+
// Drift suffix for a single offset: 0 -> "", 1500 -> "-p2", -800 -> "-m1"
39+
let private driftSuffix (ms: int) =
40+
let s = ceilToSec ms
41+
if s > 0 then sprintf "-p%d" s
42+
elif s < 0 then sprintf "-m%d" (abs s)
43+
else ""
44+
45+
// Build an annotated CoreSet name: append "-expr" if flag enabled, plus drift suffix
46+
let private annotateName (baseName: string) (flagEnabled: bool) (offsetMs: int) =
47+
let flagPart = if flagEnabled then "-expr" else ""
48+
CoreSetName(baseName + flagPart + driftSuffix offsetMs)
49+
50+
let private parseDrift (context: MissionContext) : ClockDriftDistribution =
51+
match context.uniformDrift, context.bimodalDrift with
52+
| [], [] -> NoDrift
53+
| [ lower; upper ], [] ->
54+
if upper < lower then
55+
failwith (sprintf "uniform-drift requires lower <= upper, got %d,%d" lower upper)
56+
UniformDrift(lower, upper)
57+
| [], [ min1; max1; min2; max2 ] ->
58+
if max1 < min1 then
59+
failwith (sprintf "bimodal-drift first range requires min <= max, got %d,%d" min1 max1)
60+
if max2 < min2 then
61+
failwith (sprintf "bimodal-drift second range requires min <= max, got %d,%d" min2 max2)
62+
BimodalDrift(min1, max1, min2, max2)
63+
| _ :: _, _ :: _ -> failwith "Cannot specify both --uniform-drift and --bimodal-drift"
64+
| u, [] -> failwith (sprintf "--uniform-drift requires exactly 2 values (lower,upper), got %d" u.Length)
65+
| [], b -> failwith (sprintf "--bimodal-drift requires exactly 4 values (min1,max1,min2,max2), got %d" b.Length)
66+
67+
let triggerTimerMixConsensus (baseContext: MissionContext) =
68+
let drift = parseDrift baseContext
69+
let flagPct = baseContext.triggerTimerFlagPct
70+
let driftPct = baseContext.driftPct
71+
72+
if flagPct < 0 || flagPct > 100 then
73+
failwith (sprintf "trigger-timer-flag-pct must be 0-100, got %d" flagPct)
74+
75+
if driftPct < 0 || driftPct > 100 then
76+
failwith (sprintf "drift-pct must be 0-100, got %d" driftPct)
77+
78+
let context =
79+
{ baseContext with
80+
numAccounts = 40000
81+
numTxs = 90000
82+
txRate = 150
83+
coreResources = MediumTestResources
84+
genesisTestAccountCount = Some 40000
85+
installNetworkDelay = Some(baseContext.installNetworkDelay |> Option.defaultValue true)
86+
maxConnections = Some(baseContext.maxConnections |> Option.defaultValue 65) }
87+
88+
let baseCoreSets = FullPubnetCoreSets context true false
89+
90+
let totalNodes =
91+
List.sumBy (fun (cs: CoreSet) -> cs.options.nodeCount) baseCoreSets
92+
93+
match drift with
94+
| NoDrift when driftPct > 0 ->
95+
failwith "drift-pct > 0 but no drift distribution specified (use --uniform-drift or --bimodal-drift)"
96+
| _ -> ()
97+
98+
LogInfo
99+
"TriggerTimerMixConsensus: %d total nodes, flag-pct=%d%%, drift-pct=%d%%"
100+
totalNodes
101+
flagPct
102+
driftPct
103+
104+
// Each node independently has a flagPct% chance of having the trigger
105+
// timer flag enabled, and a driftPct% chance of drifting. When drifting,
106+
// bimodal nodes have a 50/50 chance of being in the first or second group.
107+
let rng = System.Random(context.randomSeed)
108+
109+
let sampleFlag () = rng.Next(100) < flagPct
110+
111+
let sampleOffset () =
112+
match drift with
113+
| NoDrift -> 0
114+
| _ when rng.Next(100) >= driftPct -> 0
115+
| UniformDrift (lower, upper) -> rng.Next(lower, upper + 1)
116+
| BimodalDrift (min1, max1, min2, max2) ->
117+
if rng.Next(2) = 0 then rng.Next(min1, max1 + 1)
118+
else rng.Next(min2, max2 + 1)
119+
120+
// Walk through CoreSets, splitting each into single-node CoreSets so that
121+
// each node gets its own name with flag/drift annotation.
122+
let modifiedCoreSets =
123+
baseCoreSets
124+
|> List.collect (fun cs ->
125+
let nc = cs.options.nodeCount
126+
127+
[ for j in 0 .. nc - 1 do
128+
let flagEnabled = sampleFlag ()
129+
let offset = sampleOffset ()
130+
131+
let baseName =
132+
if nc > 1 then sprintf "%s-%d" cs.name.StringName j
133+
else cs.name.StringName
134+
135+
let annotatedName = annotateName baseName flagEnabled offset
136+
137+
LogInfo
138+
" Node %s: trigger_timer=%b, offset=%d"
139+
annotatedName.StringName
140+
flagEnabled
141+
offset
142+
143+
{ cs with
144+
name = annotatedName
145+
keys = [| cs.keys.[j] |]
146+
options =
147+
{ cs.options with
148+
nodeCount = 1
149+
nodeLocs =
150+
cs.options.nodeLocs
151+
|> Option.map (fun locs -> [ locs.[j] ])
152+
experimentalTriggerTimer = if flagEnabled then Some true else None
153+
clockOffsets = if offset <> 0 then Some [ offset ] else None } } ])
154+
155+
let tier1 =
156+
List.filter (fun (cs: CoreSet) -> cs.options.tier1 = Some true) modifiedCoreSets
157+
158+
let nonTier1 =
159+
List.filter (fun (cs: CoreSet) -> cs.options.tier1 <> Some true) modifiedCoreSets
160+
161+
context.Execute
162+
modifiedCoreSets
163+
None
164+
(fun (formation: StellarFormation) ->
165+
formation.WaitUntilConnected modifiedCoreSets
166+
formation.ManualClose tier1
167+
formation.WaitUntilSynced modifiedCoreSets
168+
169+
formation.UpgradeProtocolToLatest tier1
170+
formation.UpgradeMaxTxSetSize tier1 (context.txRate * 10)
171+
172+
let loadPeer =
173+
if nonTier1.Length > 0 then nonTier1.[0] else tier1.[0]
174+
175+
formation.RunLoadgen loadPeer context.GeneratePaymentLoad
176+
177+
formation.CheckNoErrorsAndPairwiseConsistency()
178+
formation.EnsureAllNodesInSync modifiedCoreSets)

src/FSLibrary/StellarCoreCfg.fs

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -169,6 +169,8 @@ type StellarCoreCfg =
169169
maxBatchWriteCount: int
170170
emitMeta: bool
171171
addArtificialDelayUsec: int option // optional delay for testing in microseconds
172+
experimentalTriggerTimer: bool option
173+
clockOffsetMs: int option
172174
surveyPhaseDuration: int option
173175
containerType: CoreContainerType
174176
skipHighCriticalValidatorChecks: bool }
@@ -272,6 +274,14 @@ type StellarCoreCfg =
272274
| None -> maybeAddGlobalDelay ()
273275
| Some sleep -> t.Add("ARTIFICIALLY_SLEEP_MAIN_THREAD_FOR_TESTING", sleep) |> ignore
274276

277+
match self.experimentalTriggerTimer with
278+
| Some v -> t.Add("EXPERIMENTAL_TRIGGER_TIMER", v) |> ignore
279+
| None -> ()
280+
281+
match self.clockOffsetMs with
282+
| Some offset -> t.Add("ARTIFICIALLY_SET_SYSTEM_CLOCK_OFFSET_FOR_TESTING", int64 offset) |> ignore
283+
| None -> ()
284+
275285
match self.network.missionContext.flowControlSendMoreBatchSize with
276286
| None -> ()
277287
| Some batchSize -> t.Add("FLOW_CONTROL_SEND_MORE_BATCH_SIZE", batchSize) |> ignore
@@ -642,6 +652,8 @@ type NetworkCfg with
642652
maxBatchWriteCount = opts.maxBatchWriteCount
643653
emitMeta = opts.emitMeta
644654
addArtificialDelayUsec = opts.addArtificialDelayUsec
655+
experimentalTriggerTimer = opts.experimentalTriggerTimer
656+
clockOffsetMs = None
645657
surveyPhaseDuration = opts.surveyPhaseDuration
646658
containerType = MainCoreContainer
647659
skipHighCriticalValidatorChecks = opts.skipHighCriticalValidatorChecks }
@@ -683,6 +695,14 @@ type NetworkCfg with
683695
maxBatchWriteCount = c.options.maxBatchWriteCount
684696
emitMeta = c.options.emitMeta
685697
addArtificialDelayUsec = c.options.addArtificialDelayUsec
698+
experimentalTriggerTimer = c.options.experimentalTriggerTimer
699+
clockOffsetMs =
700+
match c.options.clockOffsets with
701+
| Some offsets ->
702+
if offsets.Length <> c.options.nodeCount then
703+
failwith (sprintf "clockOffsets length %d does not match nodeCount %d" offsets.Length c.options.nodeCount)
704+
Some offsets.[i]
705+
| None -> None
686706
surveyPhaseDuration = c.options.surveyPhaseDuration
687707
containerType = ctype
688708
skipHighCriticalValidatorChecks = c.options.skipHighCriticalValidatorChecks }

0 commit comments

Comments
 (0)