diff --git a/benchmarks/android/app/build.gradle b/benchmarks/android/app/build.gradle index af90679f6..1c22d227c 100644 --- a/benchmarks/android/app/build.gradle +++ b/benchmarks/android/app/build.gradle @@ -63,6 +63,7 @@ react { * Set this to true to Run Proguard on Release builds to minify the Java bytecode. */ def enableProguardInReleaseBuilds = false +def flagsPrototypeEnabled = System.getenv('DD_FLAGS_KOTLIN_PROTOTYPE_PATH') != null /** * The preferred build flavor of JavaScriptCore (JSC) @@ -90,6 +91,7 @@ android { versionCode 1 versionName "1.0" buildConfigField("boolean", "IS_NEW_ARCHITECTURE_ENABLED", isNewArchitectureEnabled().toString()) + buildConfigField("boolean", "DD_FLAGS_PROTOTYPE_ENABLED", flagsPrototypeEnabled.toString()) } signingConfigs { debug { @@ -111,6 +113,11 @@ android { proguardFiles getDefaultProguardFile("proguard-android.txt"), "proguard-rules.pro" } } + if (flagsPrototypeEnabled) { + sourceSets.main.java.srcDir('src/flagsPrototype/java') + sourceSets.main.res.srcDir('src/flagsPrototype/res') + sourceSets.release.manifest.srcFile('src/flagsPrototype/AndroidManifest.xml') + } } repositories { @@ -118,6 +125,10 @@ repositories { } dependencies { + if (flagsPrototypeEnabled) { + implementation 'com.datadog.flags.benchmark:rules-evaluation-prototype:0.0.1-prototype' + implementation 'com.datadoghq:dd-sdk-android-flags:3.13.1' + } // The version of react-native is set by the React Native Gradle Plugin implementation("com.facebook.react:react-android") diff --git a/benchmarks/android/app/src/flagsPrototype/AndroidManifest.xml b/benchmarks/android/app/src/flagsPrototype/AndroidManifest.xml new file mode 100644 index 000000000..42dac5c00 --- /dev/null +++ b/benchmarks/android/app/src/flagsPrototype/AndroidManifest.xml @@ -0,0 +1,3 @@ + + + diff --git a/benchmarks/android/app/src/flagsPrototype/java/com/flagsbenchmark/FlagsBenchmarkModule.kt b/benchmarks/android/app/src/flagsPrototype/java/com/flagsbenchmark/FlagsBenchmarkModule.kt new file mode 100644 index 000000000..37ffce879 --- /dev/null +++ b/benchmarks/android/app/src/flagsPrototype/java/com/flagsbenchmark/FlagsBenchmarkModule.kt @@ -0,0 +1,162 @@ +/* + * Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0. + * This product includes software developed at Datadog (https://www.datadoghq.com/). + * Copyright 2026-Present Datadog, Inc. + */ + +package com.flagsbenchmark + +import android.os.Build +import android.os.Process +import com.benchmarkrunner.BuildConfig +import com.benchmarkvitals.NativeFlagsBenchmarkSpec +import com.datadog.flags.benchmark.FlagsBenchmark +import com.datadog.android.Datadog +import com.datadog.android._InternalProxy +import com.datadog.android.core.configuration.Configuration +import com.datadog.android.core.configuration.BatchSize +import com.datadog.android.core.configuration.UploadFrequency +import com.datadog.android.flags.Flags +import com.datadog.android.flags.FlagsClient +import com.datadog.android.flags.FlagsConfiguration +import com.datadog.android.privacy.TrackingConsent +import com.datadog.reactnative.DdFlagsImplementation +import com.facebook.react.bridge.Callback +import com.facebook.react.bridge.PromiseImpl +import com.facebook.react.bridge.ReadableArray +import com.facebook.react.bridge.Arguments +import com.facebook.react.bridge.Promise +import com.facebook.react.bridge.ReactApplicationContext +import com.facebook.react.bridge.ReadableMap +import com.facebook.react.bridge.WritableMap +import com.facebook.react.module.annotations.ReactModule +import org.json.JSONObject +import java.io.File +import java.util.UUID +import java.util.concurrent.Executors + +@ReactModule(name = FlagsBenchmarkModule.NAME) +class FlagsBenchmarkModule(private val context: ReactApplicationContext) : NativeFlagsBenchmarkSpec(context) { + private val cache = File(context.cacheDir, "flags-benchmark-${UUID.randomUUID()}.bin") + private val evaluator = FlagsBenchmark(cache) + private val lock = Any() + private val queue = Executors.newSingleThreadExecutor() + private val emulator = Build.HARDWARE in listOf("ranchu", "goldfish") + private var initialization: Map? = null + private val tracking by lazy { DdFlagsImplementation() } + + private fun settings(): Map { + val intent = requireNotNull(context.currentActivity?.intent) + val mode = intent.getStringExtra("flagsExperiment") ?: "" + if ((mode in listOf("none", "bridge-only", "exposures", "evaluations", "both") || mode.startsWith("batching-")) && initialization == null) { + check(!BuildConfig.DEBUG) { "Tracking requires a Release emulator build" } + val port = intent.getIntExtra("flagsSinkPort", 0) + require(port in 1..65535) { "Missing loopback collector port" } + val endpoint = "http://127.0.0.1:$port" + val config = Configuration.Builder("synthetic-benchmark-only", "benchmark", "", "flags-benchmark") + .setBatchSize(BatchSize.SMALL) + .setUploadFrequency(UploadFrequency.FREQUENT) + val core = requireNotNull(Datadog.initialize(context, _InternalProxy.allowClearTextHttp(config).build(), TrackingConsent.GRANTED)) + Datadog.clearAllData(core) + Flags.enable(FlagsConfiguration.Builder() + .trackExposures(mode == "exposures" || mode == "both" || mode.startsWith("batching-")) + .trackEvaluations(mode == "evaluations" || mode == "both" || mode.startsWith("batching-")) + .rumIntegrationEnabled(false) + .evaluationFlushInterval(1000) + .useCustomFlagEndpoint("$endpoint/config") + .useCustomExposureEndpoint("$endpoint/exposures") + .useCustomEvaluationEndpoint("$endpoint/evaluations") + .build(), core) + val client = FlagsClient.Builder("benchmark-preflight", core).build() + check(client.javaClass.simpleName == "DatadogFlagsClient") { "Native flags initialization failed" } + initialization = mapOf("coreType" to core.javaClass.simpleName, "clientType" to client.javaClass.simpleName) + } + return mapOf("mode" to mode, "initialization" to initialization, + "autorun" to (intent.getStringExtra("flagsRun") ?: "smoke")) + } + + private fun snapshot(): Map { + // Whole-process RSS, not evaluator heap or allocation peak. + val rss = File("/proc/${Process.myPid()}/status").readLines() + .first { it.startsWith("VmRSS:") }.trim().split(Regex("\\s+"))[1].toLong() * 1024 + return mapOf("residentBytes" to rss, "thermalState" to "not measured") + } + + override fun getName(): String = NAME + + override fun trackBatch(records: ReadableArray, promise: Promise) { + try { + check(emulator && !BuildConfig.DEBUG && initialization != null) + require(records.size() in 1..25) + for (index in 0 until records.size()) { + val record = requireNotNull(records.getMap(index)) + var resolved = false + var failure: String? = null + tracking.trackEvaluation( + requireNotNull(record.getString("client")), requireNotNull(record.getString("key")), + requireNotNull(record.getMap("flag")), requireNotNull(record.getString("targetingKey")), + requireNotNull(record.getMap("attributes")), + PromiseImpl(Callback { resolved = true }, Callback { args -> failure = args.contentToString() }) + ) + check(failure == null && resolved) { failure ?: "Tracking implementation did not acknowledge synchronously" } + } + promise.resolve(null) + } catch (exception: Exception) { + promise.reject("BATCH_TRACKING_FAILED", exception) + } + } + + override fun runSync(request: ReadableMap): WritableMap = Arguments.makeNativeMap(execute(request.toHashMap())) + + override fun runAsync(request: ReadableMap, promise: Promise) { + val input = request.toHashMap() + queue.execute { promise.resolve(Arguments.makeNativeMap(execute(input))) } + } + + private fun execute(request: Map): Map = synchronized(lock) { + try { + // Never run a benchmark on a physical Android device through this prototype. + check(emulator) { "Android benchmark is emulator-only" } + when (request["op"]) { + "metadata" -> mapOf( + "platform" to "android", "os" to "Android ${Build.VERSION.RELEASE} / API ${Build.VERSION.SDK_INT}", + "machine" to Build.MODEL, "abis" to Build.SUPPORTED_ABIS.toList(), + "simulator" to true, "nativeDebug" to BuildConfig.DEBUG, + "newArchitecture" to BuildConfig.IS_NEW_ARCHITECTURE_ENABLED, + "protobufJava" to "3.25.5", "kotlin" to KotlinVersion.CURRENT.toString(), "datadogFlags" to "3.13.1", + "nativeWorkload" to "client-protobuf boolean benchmark subset", + "vm" to System.getProperty("java.vm.version"), "thermalState" to "not measured" + ) + "experimentSettings" -> settings() + "experimentSnapshot" -> snapshot() + "experimentFlush" -> { + check(initialization != null) { "Native tracking not initialized" } + // Test-only drain after the aggregation timer; called on the worker queue. + Datadog._internalProxy().flushAndShutdownExecutors() + snapshot() + mapOf("initialization" to initialization) + } + "saveReport" -> { + val json = request["json"] as String + JSONObject(json) + val directory = requireNotNull(context.getExternalFilesDir(null)) + val report = File(directory, "flags-benchmark-result.json") + val temporary = File(directory, "flags-benchmark-result.tmp") + temporary.writeText(json) + check(temporary.renameTo(report)) + mapOf("path" to report.path) + } + else -> evaluator.run(request) + } + } catch (exception: Exception) { + mapOf("benchmarkError" to "${exception.javaClass.simpleName}: ${exception.message}") + } + } + + override fun invalidate() { + queue.execute { synchronized(lock) { cache.delete() } } + queue.shutdown() + super.invalidate() + } + + companion object { const val NAME = "FlagsBenchmark" } +} diff --git a/benchmarks/android/app/src/flagsPrototype/java/com/flagsbenchmark/FlagsBenchmarkPackage.kt b/benchmarks/android/app/src/flagsPrototype/java/com/flagsbenchmark/FlagsBenchmarkPackage.kt new file mode 100644 index 000000000..1016ab48d --- /dev/null +++ b/benchmarks/android/app/src/flagsPrototype/java/com/flagsbenchmark/FlagsBenchmarkPackage.kt @@ -0,0 +1,25 @@ +/* + * Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0. + * This product includes software developed at Datadog (https://www.datadoghq.com/). + * Copyright 2026-Present Datadog, Inc. + */ + +package com.flagsbenchmark + +import com.facebook.react.BaseReactPackage +import com.facebook.react.bridge.NativeModule +import com.facebook.react.bridge.ReactApplicationContext +import com.facebook.react.module.model.ReactModuleInfo +import com.facebook.react.module.model.ReactModuleInfoProvider + +class FlagsBenchmarkPackage : BaseReactPackage() { + override fun getModule(name: String, context: ReactApplicationContext): NativeModule? = + if (name == FlagsBenchmarkModule.NAME) FlagsBenchmarkModule(context) else null + + override fun getReactModuleInfoProvider() = ReactModuleInfoProvider { + mapOf(FlagsBenchmarkModule.NAME to ReactModuleInfo( + FlagsBenchmarkModule.NAME, FlagsBenchmarkModule::class.java.name, + false, false, false, true + )) + } +} diff --git a/benchmarks/android/app/src/flagsPrototype/res/xml/flags_benchmark_network_security.xml b/benchmarks/android/app/src/flagsPrototype/res/xml/flags_benchmark_network_security.xml new file mode 100644 index 000000000..033a2fc4e --- /dev/null +++ b/benchmarks/android/app/src/flagsPrototype/res/xml/flags_benchmark_network_security.xml @@ -0,0 +1,7 @@ + + + + + 127.0.0.1 + + diff --git a/benchmarks/android/app/src/main/java/com/benchmarkrunner/MainApplication.kt b/benchmarks/android/app/src/main/java/com/benchmarkrunner/MainApplication.kt index 84563b3d8..153738b6e 100644 --- a/benchmarks/android/app/src/main/java/com/benchmarkrunner/MainApplication.kt +++ b/benchmarks/android/app/src/main/java/com/benchmarkrunner/MainApplication.kt @@ -27,6 +27,11 @@ class MainApplication : Application(), ReactApplication { // Packages that cannot be autolinked yet can be added manually here, for example: // add(MyReactNativePackage()) add(BenchmarkVitalsPackage()) + if (BuildConfig.DD_FLAGS_PROTOTYPE_ENABLED) { + // Optional local prototype; reflection happens once, never on the evaluation path. + add(Class.forName("com.flagsbenchmark.FlagsBenchmarkPackage") + .getDeclaredConstructor().newInstance() as ReactPackage) + } } override fun getJSMainModuleName(): String = "index" diff --git a/benchmarks/android/settings.gradle b/benchmarks/android/settings.gradle index aa89e3f77..d6f4994f8 100644 --- a/benchmarks/android/settings.gradle +++ b/benchmarks/android/settings.gradle @@ -4,3 +4,8 @@ extensions.configure(com.facebook.react.ReactSettingsExtension){ ex -> ex.autoli rootProject.name = 'BenchmarkRunner' include ':app' includeBuild('../node_modules/@react-native/gradle-plugin') + +def flagsPrototype = System.getenv('DD_FLAGS_KOTLIN_PROTOTYPE_PATH') +if (flagsPrototype) { + includeBuild(flagsPrototype) +} diff --git a/benchmarks/flags-smoke.env b/benchmarks/flags-smoke.env new file mode 100644 index 000000000..beb035046 --- /dev/null +++ b/benchmarks/flags-smoke.env @@ -0,0 +1,4 @@ +# Headless correctness run only, not a performance report. +BENCH_SCENARIO=flags +BENCH_RUN_TYPE=baseline +BENCH_FLAGS_AUTORUN=smoke diff --git a/benchmarks/flags.env b/benchmarks/flags.env new file mode 100644 index 000000000..50e5d7e40 --- /dev/null +++ b/benchmarks/flags.env @@ -0,0 +1,3 @@ +# Synthetic benchmark only. No Datadog credentials or telemetry initialization. +BENCH_SCENARIO=flags +BENCH_RUN_TYPE=baseline diff --git a/benchmarks/flags.jest.config.js b/benchmarks/flags.jest.config.js new file mode 100644 index 000000000..6f3632e73 --- /dev/null +++ b/benchmarks/flags.jest.config.js @@ -0,0 +1,21 @@ +/* + * Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0. + * This product includes software developed at Datadog (https://www.datadoghq.com/). + * Copyright 2026-Present Datadog, Inc. + */ + +module.exports = { + watchman: false, + testEnvironment: 'node', + testMatch: ['/src/flags/**/*.test.ts'], + transform: { + '^.+\\.[jt]sx?$': [ + 'babel-jest', + { + babelrc: false, + configFile: false, + presets: ['module:@react-native/babel-preset'], + }, + ], + }, +}; diff --git a/benchmarks/ios/BenchmarkRunner.xcodeproj/project.pbxproj b/benchmarks/ios/BenchmarkRunner.xcodeproj/project.pbxproj index d845ae407..939a92a3c 100644 --- a/benchmarks/ios/BenchmarkRunner.xcodeproj/project.pbxproj +++ b/benchmarks/ios/BenchmarkRunner.xcodeproj/project.pbxproj @@ -17,6 +17,8 @@ 043403262DCB9CD700803AF8 /* BenchmarkVitals.mm in Sources */ = {isa = PBXBuildFile; fileRef = 043403242DCB9CD700803AF8 /* BenchmarkVitals.mm */; }; 04576C962DCBAF900078DCAB /* main.m in Sources */ = {isa = PBXBuildFile; fileRef = 04576C952DCBAF900078DCAB /* main.m */; }; 04726DD72F46402A008D1A48 /* TracerProvider.swift in Sources */ = {isa = PBXBuildFile; fileRef = 04726DD62F46402A008D1A48 /* TracerProvider.swift */; }; + 0BFD883AEE0A41CC521C5662 /* FlagsBenchmark.mm in Sources */ = {isa = PBXBuildFile; fileRef = 2FED7A81171836D1D57F12AB /* FlagsBenchmark.mm */; settings = {COMPILER_FLAGS = "-fmodules -fcxx-modules"; }; }; + B7A222C1BA8C487EA3BD6001 /* FlagsExperimentSupport.swift in Sources */ = {isa = PBXBuildFile; fileRef = B7A222C1BA8C487EA3BD6002 /* FlagsExperimentSupport.swift */; }; 0C80B921A6F3F58F76C31292 /* libPods-BenchmarkRunner.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 5DCACB8F33CDC322A6C60F78 /* libPods-BenchmarkRunner.a */; }; 13B07FBF1A68108700A75B9A /* Images.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = 13B07FB51A68108700A75B9A /* Images.xcassets */; }; 20A397FC280595A8A9851556 /* PrivacyInfo.xcprivacy in Resources */ = {isa = PBXBuildFile; fileRef = 13B07FB81A68108700A75B9A /* PrivacyInfo.xcprivacy */; }; @@ -40,11 +42,14 @@ 13B07FB51A68108700A75B9A /* Images.xcassets */ = {isa = PBXFileReference; lastKnownFileType = folder.assetcatalog; name = Images.xcassets; path = BenchmarkRunner/Images.xcassets; sourceTree = ""; }; 13B07FB61A68108700A75B9A /* Info.plist */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.plist.xml; name = Info.plist; path = BenchmarkRunner/Info.plist; sourceTree = ""; }; 13B07FB81A68108700A75B9A /* PrivacyInfo.xcprivacy */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.plist.xml; name = PrivacyInfo.xcprivacy; path = BenchmarkRunner/PrivacyInfo.xcprivacy; sourceTree = ""; }; + 2FED7A81171836D1D57F12AB /* FlagsBenchmark.mm */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.cpp.objcpp; path = FlagsBenchmark.mm; sourceTree = ""; }; + B7A222C1BA8C487EA3BD6002 /* FlagsExperimentSupport.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FlagsExperimentSupport.swift; sourceTree = ""; }; 3B4392A12AC88292D35C810B /* Pods-BenchmarkRunner.debug.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-BenchmarkRunner.debug.xcconfig"; path = "Target Support Files/Pods-BenchmarkRunner/Pods-BenchmarkRunner.debug.xcconfig"; sourceTree = ""; }; 5709B34CF0A7D63546082F79 /* Pods-BenchmarkRunner.release.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-BenchmarkRunner.release.xcconfig"; path = "Target Support Files/Pods-BenchmarkRunner/Pods-BenchmarkRunner.release.xcconfig"; sourceTree = ""; }; 5DCACB8F33CDC322A6C60F78 /* libPods-BenchmarkRunner.a */ = {isa = PBXFileReference; explicitFileType = archive.ar; includeInIndex = 0; path = "libPods-BenchmarkRunner.a"; sourceTree = BUILT_PRODUCTS_DIR; }; 761780EC2CA45674006654EE /* AppDelegate.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; name = AppDelegate.swift; path = BenchmarkRunner/AppDelegate.swift; sourceTree = ""; }; 81AB9BB72411601600AC10FF /* LaunchScreen.storyboard */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = file.storyboard; name = LaunchScreen.storyboard; path = BenchmarkRunner/LaunchScreen.storyboard; sourceTree = ""; }; + A3B9C892D474AFE597364EE6 /* FlagsBenchmark.h */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.c.h; path = FlagsBenchmark.h; sourceTree = ""; }; ED297162215061F000B7C4FE /* JavaScriptCore.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = JavaScriptCore.framework; path = System/Library/Frameworks/JavaScriptCore.framework; sourceTree = SDKROOT; }; /* End PBXFileReference section */ @@ -131,6 +136,7 @@ 83CBBA001A601CBA00E9B192 /* Products */, 2D16E6871FA4F8E400B85C8A /* Frameworks */, BBD78D7AC51CEA395F1C20DB /* Pods */, + BEF778B92318D45B0AE84BA5 /* FlagsBenchmark */, ); indentWidth = 2; sourceTree = ""; @@ -154,6 +160,17 @@ path = Pods; sourceTree = ""; }; + BEF778B92318D45B0AE84BA5 /* FlagsBenchmark */ = { + isa = PBXGroup; + children = ( + A3B9C892D474AFE597364EE6 /* FlagsBenchmark.h */, + 2FED7A81171836D1D57F12AB /* FlagsBenchmark.mm */, + B7A222C1BA8C487EA3BD6002 /* FlagsExperimentSupport.swift */, + ); + name = FlagsBenchmark; + path = FlagsBenchmark; + sourceTree = ""; + }; /* End PBXGroup section */ /* Begin PBXNativeTarget section */ @@ -313,6 +330,8 @@ 04576C962DCBAF900078DCAB /* main.m in Sources */, 761780ED2CA45674006654EE /* AppDelegate.swift in Sources */, 04726DD72F46402A008D1A48 /* TracerProvider.swift in Sources */, + 0BFD883AEE0A41CC521C5662 /* FlagsBenchmark.mm in Sources */, + B7A222C1BA8C487EA3BD6001 /* FlagsExperimentSupport.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; }; diff --git a/benchmarks/ios/BenchmarkRunner/AppDelegate.swift b/benchmarks/ios/BenchmarkRunner/AppDelegate.swift index 8ca5d31fd..335553e5a 100644 --- a/benchmarks/ios/BenchmarkRunner/AppDelegate.swift +++ b/benchmarks/ios/BenchmarkRunner/AppDelegate.swift @@ -22,6 +22,8 @@ class AppDelegate: RCTAppDelegate { // They will be passed down to the ViewController used by React Native. self.initialProps = [:] + FlagsExperimentSupport.prepare() + return super.application(application, didFinishLaunchingWithOptions: launchOptions) } diff --git a/benchmarks/ios/FlagsBenchmark/FlagsBenchmark.h b/benchmarks/ios/FlagsBenchmark/FlagsBenchmark.h new file mode 100644 index 000000000..3aff6e34d --- /dev/null +++ b/benchmarks/ios/FlagsBenchmark/FlagsBenchmark.h @@ -0,0 +1,15 @@ +/* + * Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0. + * This product includes software developed at Datadog (https://www.datadoghq.com/). + * Copyright 2026-Present Datadog, Inc. + */ + +#import +#ifdef RCT_NEW_ARCH_ENABLED +#import "BenchmarkVitalsSpec.h" +@interface FlagsBenchmark : NSObject +#else +#import +@interface FlagsBenchmark : NSObject +#endif +@end diff --git a/benchmarks/ios/FlagsBenchmark/FlagsBenchmark.mm b/benchmarks/ios/FlagsBenchmark/FlagsBenchmark.mm new file mode 100644 index 000000000..83c7701cf --- /dev/null +++ b/benchmarks/ios/FlagsBenchmark/FlagsBenchmark.mm @@ -0,0 +1,84 @@ +/* + * Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0. + * This product includes software developed at Datadog (https://www.datadoghq.com/). + * Copyright 2026-Present Datadog, Inc. + */ + +#import "FlagsBenchmark.h" +#import "BenchmarkRunner-Swift.h" +#ifdef DD_FLAGS_PROTOTYPE_ENABLED +@import RulesEvaluationPrototype; +#endif + +@implementation FlagsBenchmark { + dispatch_queue_t _queue; + NSLock *_lock; +#ifdef DD_FLAGS_PROTOTYPE_ENABLED + DDFlagsBenchmark *_engine; +#endif +} + +RCT_EXPORT_MODULE() + ++ (BOOL)requiresMainQueueSetup { return NO; } + +- (dispatch_queue_t)methodQueue { return [FlagsExperimentSupport trackingQueue]; } + +RCT_REMAP_METHOD(trackBatch, trackBatch:(NSArray *)records + resolve:(RCTPromiseResolveBlock)resolve reject:(RCTPromiseRejectBlock)reject) { + NSString *error = [FlagsExperimentSupport trackRecords:records]; + if (error) reject(@"BATCH_TRACKING_FAILED", error, nil); + else resolve(nil); +} + +- (instancetype)init { + if ((self = [super init])) { + _queue = dispatch_queue_create("com.datadog.flags.benchmark", DISPATCH_QUEUE_SERIAL); + _lock = [NSLock new]; +#ifdef DD_FLAGS_PROTOTYPE_ENABLED + _engine = [DDFlagsBenchmark new]; +#endif + } + return self; +} + +- (NSDictionary *)execute:(NSDictionary *)request { + if ([request[@"op"] isEqual:@"experimentSettings"]) return [FlagsExperimentSupport settings]; + if ([request[@"op"] isEqual:@"experimentSnapshot"]) return [FlagsExperimentSupport snapshot]; + if ([request[@"op"] isEqual:@"experimentFlush"]) return [FlagsExperimentSupport flush]; +#ifdef DD_FLAGS_PROTOTYPE_ENABLED + [_lock lock]; + NSDictionary *result = [_engine run:request]; + [_lock unlock]; + if ([request[@"op"] isEqual:@"metadata"]) { + NSMutableDictionary *metadata = [result mutableCopy]; +#ifdef RCT_NEW_ARCH_ENABLED + metadata[@"newArchitecture"] = @YES; +#else + metadata[@"newArchitecture"] = @NO; +#endif + return metadata; + } + return result; +#else + return @{ @"benchmarkError": @"Set DD_FLAGS_PROTOTYPE_PATH and run pod install" }; +#endif +} + +RCT_EXPORT_BLOCKING_SYNCHRONOUS_METHOD(runSync:(NSDictionary *)request) { + // Inline synchronous reads avoid adding an artificial dispatch hop to the comparison. + return [self execute:request]; +} + +RCT_REMAP_METHOD(runAsync, runAsync:(NSDictionary *)request + resolve:(RCTPromiseResolveBlock)resolve reject:(RCTPromiseRejectBlock)reject) { + dispatch_async(_queue, ^{ resolve([self execute:request]); }); +} + +#ifdef RCT_NEW_ARCH_ENABLED +- (std::shared_ptr)getTurboModule: + (const facebook::react::ObjCTurboModule::InitParams &)params { + return std::make_shared(params); +} +#endif +@end diff --git a/benchmarks/ios/FlagsBenchmark/FlagsExperimentSupport.swift b/benchmarks/ios/FlagsBenchmark/FlagsExperimentSupport.swift new file mode 100644 index 000000000..7c7970026 --- /dev/null +++ b/benchmarks/ios/FlagsBenchmark/FlagsExperimentSupport.swift @@ -0,0 +1,95 @@ +/* + * Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0. + * This product includes software developed at Datadog (https://www.datadoghq.com/). + * Copyright 2026-Present Datadog, Inc. + */ + +import Foundation +@_spi(Internal) import DatadogCore +import DatadogFlags +import DatadogSDKReactNative + +@objc(FlagsExperimentSupport) +public final class FlagsExperimentSupport: NSObject { + static let mode = ProcessInfo.processInfo.environment["DD_FLAGS_EXPERIMENT"] ?? "" + private static var initialization: [String: Any] = [:] + private static let tracking = DdFlagsImplementation() + + static func prepare() { + #if targetEnvironment(simulator) + guard ["none", "bridge-only", "exposures", "evaluations", "both"].contains(mode) || mode.hasPrefix("batching-") else { return } + guard let rawPort = ProcessInfo.processInfo.environment["DD_FLAGS_SINK_PORT"], + let port = UInt16(rawPort), port > 0 else { return } + let endpoint = "http://127.0.0.1:\(port)" + let core = Datadog.initialize( + with: Datadog.Configuration(clientToken: "synthetic-benchmark-only", env: "benchmark", + batchSize: .small, uploadFrequency: .frequent), + trackingConsent: .granted + ) + Flags.enable(with: .init( + customFlagsEndpoint: URL(string: "\(endpoint)/config"), + customExposureEndpoint: URL(string: "\(endpoint)/exposures"), + trackExposures: mode == "exposures" || mode == "both" || mode.hasPrefix("batching-"), + customEvaluationEndpoint: URL(string: "\(endpoint)/evaluations"), + trackEvaluations: mode == "evaluations" || mode == "both" || mode.hasPrefix("batching-"), + evaluationFlushInterval: 1, + rumIntegrationEnabled: false + )) + Datadog.clearAllData() + initialization = [ + "coreType": String(describing: type(of: core)), + "clientType": String(describing: type(of: FlagsClient.create(name: "benchmark-preflight"))), + ] + #endif + } + + @objc public static func settings() -> [String: Any] { ["mode": mode, "initialization": initialization] } + + @objc public static func trackingQueue() -> DispatchQueue { RNQueue.getSharedQueue() } + + @objc public static func trackRecords(_ records: NSArray) -> String? { + #if !targetEnvironment(simulator) || DEBUG + return "Batch tracking requires a Release simulator build" + #else + guard !initialization.isEmpty else { return "Native tracking not initialized" } + guard (1...25).contains(records.count) else { return "Invalid batch size" } + for raw in records { + guard let record = raw as? NSDictionary, + let client = record["client"] as? String, + let key = record["key"] as? String, + let flag = record["flag"] as? NSDictionary, + let target = record["targetingKey"] as? String, + let attributes = record["attributes"] as? NSDictionary else { return "Invalid tracking record" } + var resolved = false + var failure: String? + tracking.trackEvaluation(client, key: key, rawFlag: flag, targetingKey: target, attributes: attributes, + resolve: { _ in resolved = true }, reject: { code, message, _ in failure = "\(code): \(message)" }) + if let failure { return failure } + guard resolved else { return "Tracking implementation did not acknowledge synchronously" } + } + return nil + #endif + } + + @objc public static func snapshot() -> [String: Any] { + var info = mach_task_basic_info() + var count = mach_msg_type_number_t(MemoryLayout.size / MemoryLayout.size) + let status = withUnsafeMutablePointer(to: &info) { pointer in + pointer.withMemoryRebound(to: integer_t.self, capacity: Int(count)) { + task_info(mach_task_self_, task_flavor_t(MACH_TASK_BASIC_INFO), $0, &count) + } + } + return [ + "residentBytes": status == KERN_SUCCESS ? NSNumber(value: info.resident_size) : NSNull(), + "thermalState": ProcessInfo.processInfo.thermalState.rawValue, + ] + } + + @objc public static func flush() -> [String: Any] { + Datadog.flush() + var result = snapshot() + result["initialization"] = initialization + result["bridgeClientType"] = String(describing: type(of: FlagsClient.shared(named: "benchmark-0-repeated-0"))) + return result + } +} diff --git a/benchmarks/ios/Podfile b/benchmarks/ios/Podfile index 92841ceb1..42b740e6b 100644 --- a/benchmarks/ios/Podfile +++ b/benchmarks/ios/Podfile @@ -29,6 +29,11 @@ target 'BenchmarkRunner' do # OpenTelemetryApi is already pulled in transitively via DatadogTrace, but OpenTelemetrySdk is not. pod 'OpenTelemetry-Swift-Sdk', '~> 2.5.0' + # Opt-in experiment only; does not alter any shipping SDK podspec. + if ENV['DD_FLAGS_PROTOTYPE_PATH'] + pod 'RulesEvaluationPrototype', :path => ENV.fetch('DD_FLAGS_PROTOTYPE_PATH') + end + post_install do |installer| # https://github.com/facebook/react-native/blob/main/packages/react-native/scripts/react_native_pods.rb#L197-L202 react_native_post_install( diff --git a/benchmarks/ios/Podfile.lock b/benchmarks/ios/Podfile.lock index 577819ad4..346101308 100644 --- a/benchmarks/ios/Podfile.lock +++ b/benchmarks/ios/Podfile.lock @@ -1877,7 +1877,10 @@ PODS: - ReactCommon/turbomodule/bridging - ReactCommon/turbomodule/core - Yoga + - RulesEvaluationPrototype (0.0.1): + - SwiftProtobuf (= 1.38.1) - SocketRocket (0.7.1) + - SwiftProtobuf (1.38.1) - Yoga (0.0.0) DEPENDENCIES: @@ -1959,6 +1962,7 @@ DEPENDENCIES: - "RNCPicker (from `../node_modules/@react-native-picker/picker`)" - RNScreens (from `../node_modules/react-native-screens`) - RNSVG (from `../node_modules/react-native-svg`) + - RulesEvaluationPrototype (from `../../../dd-sdk-ios/DatadogFlags/Prototypes/RulesEvaluation`) - Yoga (from `../node_modules/react-native/ReactCommon/yoga`) SPEC REPOS: @@ -1976,6 +1980,7 @@ SPEC REPOS: - OpenTelemetry-Swift-Api - OpenTelemetry-Swift-Sdk - SocketRocket + - SwiftProtobuf EXTERNAL SOURCES: boost: @@ -2129,6 +2134,8 @@ EXTERNAL SOURCES: :path: "../node_modules/react-native-screens" RNSVG: :path: "../node_modules/react-native-svg" + RulesEvaluationPrototype: + :path: "../../../dd-sdk-ios/DatadogFlags/Prototypes/RulesEvaluation" Yoga: :path: "../node_modules/react-native/ReactCommon/yoga" @@ -2140,9 +2147,9 @@ SPEC CHECKSUMS: DatadogInternal: e5a4652ee266986e29c6b8491aff7f78dc3b731d DatadogLogs: 52792add5827042f619771a23656e2bfb6b0f4e7 DatadogRUM: b78d608e013f39c5679ed31ad97dad323c6113e8 - DatadogSDKReactNative: 09dad2b43643ba603e1d8bdb241847917ba6f15d - DatadogSDKReactNativeSessionReplay: 6c6ec0adf5645e7167a4a6b1a6d1ceb656c29c51 - DatadogSDKReactNativeWebView: 34b05bdd6f540bb48c2ed19a3744698075c1b463 + DatadogSDKReactNative: 2a1b6213a4756c19489f4dcc16972ec8b600471b + DatadogSDKReactNativeSessionReplay: 61c1c412a267d8379dacbcfd85f37c5f6556b566 + DatadogSDKReactNativeWebView: 661b7dfa95ca6c276535580604ffd8ad91e6d423 DatadogSessionReplay: 932b2077b1a5367de1450479d72093621b9fd618 DatadogTrace: 5ef84e254e2e1d337bc52e0cdf16396ca102aeb1 DatadogWebViewTracking: e0470157f55a83680a549d79ba2985e0d17465db @@ -2161,68 +2168,70 @@ SPEC CHECKSUMS: RCTTypeSafety: 48ad3c858926b1c46f46a81a58822b476e178e2c React: 3b5754191f1b65f1dbc52fbea7959c3d2d9e39c9 React-callinvoker: 6beeaf4c7db11b6cc953fac45f2c76e3fb125013 - React-Core: 88e817c42de035378cc71e009193b9a044d3f595 - React-CoreModules: dcf764d71efb4f75d38fcae8d4513b6729f49360 - React-cxxreact: 8cdcc937c5fbc406fe843a381102fd69440ca78a + React-Core: 8a10ac9de53373a3ecb5dfcbcf56df1d3dad0861 + React-CoreModules: af6999b35c7c01b0e12b59d27f3e054e13da43b1 + React-cxxreact: 833f00155ce8c2fda17f6d286f8eaeff2ececc69 React-debug: 440175830c448e7e53e61ebb8d8468c3256b645e - React-defaultsnativemodule: 4824bcd7b96ee2d75c28b1ca21f58976867f5535 - React-domnativemodule: a421118b475618961cf282e8ea85347cc9bb453c - React-Fabric: 6ac7de06009eb96b609a770b17abba6e460b5f45 - React-FabricComponents: e3bc2680a5a9a4917ff0c8d7f390688c30ef753c - React-FabricImage: 8bad558dec7478077974caa96acc79692d6b71f5 + React-defaultsnativemodule: a970effe18fe50bdbbb7115c3297f873b666d0d4 + React-domnativemodule: 45f886342a724e61531b18fba1859bb6782e5d62 + React-Fabric: 69f1881f2177a8512304a64157943548ab6df0cf + React-FabricComponents: f54111c8e2439fc273ab07483e3a7054ca1e75af + React-FabricImage: 9ad2619dfe8c386d79e8aaa87da6e8f018ab9592 React-featureflags: b9cf9b35baca1c7f20c06a104ffc325a02752faa - React-featureflagsnativemodule: dc93d81da9f41f7132e24455ec8b4b60802fd5b0 - React-graphics: aaa5a38bea15d7b895b210d95d554af45a07002a - React-hermes: 08ad9fb832d1b9faef391be17309aa6a69fad23b - React-idlecallbacksnativemodule: aacea33ef6c511a9781f9286cc7cdf93f39bba14 - React-ImageManager: c596c3b658c9c14607f9183ed0f635c8dd77987c - React-jserrorhandler: 987609b2f16b7d79d63fcd621bf0110dd7400b35 - React-jsi: afa286d7e0c102c2478dc420d4f8935e13c973fc - React-jsiexecutor: 08f5b512b4db9e2f147416d60a0a797576b9cfef - React-jsinspector: 5a94bcae66e3637711c4d96a00038ab9ec935bf5 - React-jsinspectortracing: a12589a0adbb2703cbc4380dabe9a58800810923 - React-jsitracing: 0b1a403d7757cec66b7dd8b308d04db85eef75f3 - React-logger: 304814ae37503c8eb54359851cc55bd4f936b39c - React-Mapbuffer: b588d1ca18d2ce626f868f04ab12d8b1f004f12c - React-microtasksnativemodule: 11831d070aa47755bb5739069eb04ec621fec548 - react-native-config: 3367df9c1f25bb96197007ec531c7087ed4554c3 - react-native-safe-area-context: 9b169299f9dc95f1d7fe1dd266fde53bd899cd0c - react-native-slider: 27263d134d55db948a4706f1e47d0ec88fb354dd - react-native-webview: be9957759cb73cb64f2ed5359e32a85f1f5bdff8 - React-NativeModulesApple: 79a4404ac301b40bec3b367879c5e9a9ce81683c - React-perflogger: 0ea25c109dba33d47dec36b2634bf7ea67c1a555 - React-performancetimeline: f74480de6efbcd8541c34317c0baedb433f27296 + React-featureflagsnativemodule: 7f1bc76d1d2c5bede5e753b8d188dbde7c59b12f + React-graphics: 069e0d0b31ed1e80feb023ad4f7e97f00e84f7b9 + React-hermes: 63df5ac5a944889c8758a6213b39ed825863adb7 + React-idlecallbacksnativemodule: 4c700bd7c0012adf904929075a79418b828b5ffc + React-ImageManager: 5d1ba8a7bae44ebba43fc93da64937c713d42941 + React-jserrorhandler: 0defd58f8bb797cdd0a820f733bf42d8bee708ce + React-jsi: 99d6207ec802ad73473a0dad3c9ad48cd98463f6 + React-jsiexecutor: 8c8097b4ba7e7f480582d6e6238b01be5dcc01c0 + React-jsinspector: ea148ec45bc7ff830e443383ea715f9780c15934 + React-jsinspectortracing: 46bb2841982f01e7b63eaab98140fa1de5b2a1db + React-jsitracing: c1063fc2233960d1c8322291e74bca51d25c10d7 + React-logger: 763728cf4eebc9c5dc9bfc3649e22295784f69f3 + React-Mapbuffer: 63278529b5cf531a7eaf8fc71244fabb062ca90c + React-microtasksnativemodule: 6a39463c32ce831c4c2aa8469273114d894b6be9 + react-native-config: 644074ab88db883fcfaa584f03520ec29589d7df + react-native-safe-area-context: afcc2e2b3e78ae8ef90d81e658aacee34ebc27ea + react-native-slider: 310d3f89edd6ca8344a974bfe83a29a3fbb60e5a + react-native-webview: 80ef603d1df42e24fdde765686fbb9b8a6ecd554 + React-NativeModulesApple: fd0545efbb7f936f78edd15a6564a72d2c34bb32 + React-perflogger: 5f8fa36a8e168fb355efe72099efe77213bc2ac6 + React-performancetimeline: 8c0ecfa1ae459cc5678a65f95ac3bf85644d6feb React-RCTActionSheet: 2ef95837e89b9b154f13cd8401f9054fc3076aff - React-RCTAnimation: 33d960d7f58a81779eea6dea47ad0364c67e1517 - React-RCTAppDelegate: 85c13403fd6f6b6cc630428d52bd8bd76a670dc9 - React-RCTBlob: 74c986a02d951931d2f6ed0e07ed5a7eb385bfc0 - React-RCTFabric: 384a8fea4f22fc0f21299d771971862883ba630a - React-RCTFBReactNativeSpec: eb1c3ec5149f76133593a516ff9d5efe32ebcecd - React-RCTImage: 2c58b5ddeb3c65e52f942bbe13ff9c59bd649b09 - React-RCTLinking: b6b14f8a3e62c02fc627ac4f3fb0c7bd941f907c - React-RCTNetwork: 1d050f2466c1541b339587d46f78d5eee218d626 - React-RCTSettings: 8148f6be0ccc0cfe6e313417ebf8a479caaa2146 - React-RCTText: 64114531ad1359e4e02a4a8af60df606dbbabc25 - React-RCTVibration: f4859417a7dd859b6bf18b1aba897e52beb72ef6 + React-RCTAnimation: 46abefd5acfda7e6629f9e153646deecc70babd2 + React-RCTAppDelegate: 7e58e0299e304cceee3f7019fa77bc6990f66b22 + React-RCTBlob: f68c63a801ef1d27e83c4011e3b083cc86a200d7 + React-RCTFabric: c59f41d0c4edbaac8baa232731ca09925ae4dda7 + React-RCTFBReactNativeSpec: 3240b9b8d792aa4be0fb85c9898fc183125ba8de + React-RCTImage: 34e0bba1507e55f1c614bd759eb91d9be48c8c5b + React-RCTLinking: a0b6c9f4871c18b0b81ea952f43e752718bd5f1d + React-RCTNetwork: bdafd661ac2b20d23b779e45bf7ac3e4c8bd1b60 + React-RCTSettings: 98aa5163796f43789314787b584a84eba47787a9 + React-RCTText: 424a274fc9015b29de89cf3cbcdf4dd85dd69f83 + React-RCTVibration: 92d9875a955b0adb34b4b773528fdbbbc5addd6c React-rendererconsistency: 5ac4164ec18cfdd76ed5f864dbfdc56a5a948bc9 - React-rendererdebug: 3dc1d97bbee0c0c13191e501a96ed9325bbd920e + React-rendererdebug: 710dbd7990e355852c786aa6bc7753f6028f357a React-rncore: 0bace3b991d8843bb5b57c5f2301ec6e9c94718b - React-RuntimeApple: 1e1e0a0c6086bc8c3b07e8f1a2f6ca99b50419a0 - React-RuntimeCore: d39322c59bef2a4b343fda663d20649f29f57fcc + React-RuntimeApple: 701ec44a8b5d863ee9b6a2b2447b6a26bb6805a1 + React-RuntimeCore: a82767065b9a936b05e209dc6987bc1ea9eb5d2d React-runtimeexecutor: 876dfc1d8daa819dfd039c40f78f277c5a3e66a6 - React-RuntimeHermes: 44f5f2baf039f249b31ea4f3e224484fd1731e0e - React-runtimescheduler: 3b3c5b50743bb8743ca49b9e5a70c2c385f156e1 + React-RuntimeHermes: e7a051fd91cab8849df56ac917022ef6064ad621 + React-runtimescheduler: c544141f2124ee3d5f3d5bf0d69f4029a61a68b0 React-timing: 1ee3572c398f5579c9df5bf76aacddf5683ff74e - React-utils: 0cfb7c7fb37d4e5f31cc18ffc7426be0ae6bf907 - ReactAppDependencyProvider: b48473fe434569ff8f6cb6ed4421217ebcbda878 - ReactCodegen: 653a0d8532d8c7dab50c391392044d98e20c9f79 - ReactCommon: 547db015202a80a5b3e7e041586ea54c4a087180 - RNCPicker: ffbd7b9fc7c1341929e61dbef6219f7860f57418 - RNScreens: 0f01bbed9bd8045a8d58e4b46993c28c7f498f3c - RNSVG: 082ae2874b288a96c95326c4f6029793f7009d4b + React-utils: 18703928768cb37e70cf2efff09def12d74a399e + ReactAppDependencyProvider: 4893bde33952f997a323eb1a1ee87a72764018ff + ReactCodegen: 6d003cb0cf0b773a52b3fbcc7984e5df1a3447a5 + ReactCommon: 865ebe76504a95e115b6229dd00a31e56d2d4bfe + RNCPicker: cfb51a08c6e10357d9a65832e791825b0747b483 + RNScreens: 790123c4a28783d80a342ce42e8c7381bed62db1 + RNSVG: 8ad5c94593e5e4a6e133e6934e2188e3c688d27a + RulesEvaluationPrototype: 5a25c0531ad96bdbae78052e57d6a2000c36f18d SocketRocket: d4aabe649be1e368d1318fdf28a022d714d65748 + SwiftProtobuf: 409d3aaec90e51b1705b1dd8065b56893450e77c Yoga: e14bad835e12b6c7e2260fc320bd00e0f4b45add -PODFILE CHECKSUM: d0574c1c0410627561bf4b1b55d0f1754283e022 +PODFILE CHECKSUM: e378cab35e06dc537c13c9dee54995cd3bb922ac COCOAPODS: 1.16.2 diff --git a/benchmarks/metro.config.js b/benchmarks/metro.config.js index 80b2747c5..93e30a5a7 100644 --- a/benchmarks/metro.config.js +++ b/benchmarks/metro.config.js @@ -34,6 +34,20 @@ const config = { // while it should be only one React instance in the runtime. exclusionList relies on the modules which are // declared as peer dependencies in the core package. resolver: { + // The optional flags benchmark uses Protobuf-ES package subpath exports. + unstable_enablePackageExports: true, + resolveRequest: (context, moduleName, platform) => { + // Keep the benchmark adapter and core on the same decoder instance. + // Metro 0.81 otherwise selects core's bundled legacy sub-entrypoint. + if (moduleName === '@datadog/flagging-core/rules-based' || + /^@bufbuild\/protobuf(?:\/|$)/.test(moduleName)) { + return { + type: 'sourceFile', + filePath: require.resolve(moduleName, { paths: [path.dirname(context.originModulePath)] }) + }; + } + return context.resolveRequest(context, moduleName, platform); + }, // Remove svg from asset extensions and add it to source extensions so Metro // sends it through the transformer rather than copying it as a static asset. assetExts: defaultConfig.resolver.assetExts.filter(ext => ext !== 'svg'), diff --git a/benchmarks/package.json b/benchmarks/package.json index d208e29fd..53993b556 100644 --- a/benchmarks/package.json +++ b/benchmarks/package.json @@ -7,7 +7,9 @@ "ios": "react-native run-ios", "lint": "eslint .", "start": "react-native start", - "test": "jest" + "test": "jest", + "test:flags": "jest --config flags.jest.config.js --runInBand", + "flags:compile-fixture": "tsc --outDir build/flags --target es2020 --module commonjs --moduleResolution node --skipLibCheck src/flags/fixtures.ts" }, "codegenConfig": { "name": "BenchmarkVitalsSpec", @@ -19,6 +21,8 @@ "library": "BenchmarkRunner" }, "dependencies": { + "@bufbuild/protobuf": "2.15.0", + "@datadog/flagging-core": "3.1.1", "@datadog/mobile-react-native": "workspace:packages/core", "@datadog/mobile-react-native-session-replay": "workspace:packages/react-native-session-replay", "@datadog/mobile-react-native-webview": "workspace:packages/react-native-webview", diff --git a/benchmarks/scripts/export-flags-fixture.cjs b/benchmarks/scripts/export-flags-fixture.cjs new file mode 100644 index 000000000..e560fd75b --- /dev/null +++ b/benchmarks/scripts/export-flags-fixture.cjs @@ -0,0 +1,65 @@ +/* + * Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0. + * This product includes software developed at Datadog (https://www.datadoghq.com/). + * Copyright 2026-Present Datadog, Inc. + */ + +// Run after flags:compile-fixture. The destination must be an explicit file path. +const fs = require('node:fs'); +const crypto = require('node:crypto'); +const { + configurationFromRulesBinary, +} = require('@datadog/flagging-core/rules-based'); +const {evaluateRulesBasedConfiguration} = require('@datadog/flagging-core'); +const {makeFixture, requests, comparable} = require('../build/flags/fixtures'); + +const destination = process.argv[2]; +if (!destination) + throw new Error('Usage: node scripts/export-flags-fixture.cjs '); +const logger = {debug() {}, info() {}, warn() {}, error() {}}; +const configurations = [0, 1].map(revision => { + const fixture = makeFixture(10, revision); + const configuration = configurationFromRulesBinary(fixture.bytes).rules + .response; + const cases = requests.map(request => ({ + request, + expected: comparable( + evaluateRulesBasedConfiguration( + configuration, + 'boolean', + request.flagKey, + false, + request.context, + logger, + ), + ), + })); + if (cases.some(({expected}) => expected.reason === 'ERROR')) + throw new Error('Invalid fixture'); + return { + revision, + protobufBase64: Buffer.from(fixture.bytes).toString('base64'), + protoJson: fixture.json, + sha256: crypto.createHash('sha256').update(fixture.bytes).digest('hex'), + cases, + }; +}); +fs.writeFileSync( + destination, + JSON.stringify( + { + generator: 'dd-sdk-reactnative/benchmarks/src/flags/fixtures.ts', + fixtureVersion: 1, + flaggingCore: '3.1.1', + protobufES: '2.15.0', + configurations, + }, + null, + 2, + ) + '\n', +); +console.log( + `Wrote ${ + configurations.length * requests.length + } reference cases to ${destination}`, +); diff --git a/benchmarks/scripts/run-flags-android.cjs b/benchmarks/scripts/run-flags-android.cjs new file mode 100644 index 000000000..086ada237 --- /dev/null +++ b/benchmarks/scripts/run-flags-android.cjs @@ -0,0 +1,228 @@ +/* + * Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0. + * This product includes software developed at Datadog (https://www.datadoghq.com/). + * Copyright 2026-Present Datadog, Inc. + */ + +const fs = require('node:fs'); +const path = require('node:path'); +const assert = require('node:assert/strict'); +const {execFileSync} = require('node:child_process'); +const {createHash} = require('node:crypto'); +const { + validateReport: validateFollowup, + createCollector, +} = require('./run-flags-followups.cjs'); +const trackingModes = [ + 'none', + 'bridge-only', + 'exposures', + 'evaluations', + 'both', + 'batching-none', + 'batching-current', + 'batching-control', + 'batching-buffered', +]; +const sha256 = data => createHash('sha256').update(data).digest('hex'); +const wait = ms => new Promise(resolve => setTimeout(resolve, ms)); + +function validateReport(report, mode) { + assert.equal(report.correctness, 'passed'); + assert.equal(report.metadata.platform, 'android'); + assert.equal(report.metadata.simulator, true); + assert.equal(report.metadata.debug, false); + assert.equal(report.metadata.nativeDebug, false); + assert.equal(report.metadata.hermes, true); + assert.equal(report.metadata.newArchitecture, true); + if (trackingModes.includes(mode)) { + assert.equal(report.mode, mode); + assert.equal(report.sink.initialization?.clientType, 'DatadogFlagsClient'); + if (!mode.startsWith('batching-')) { + assert.equal( + report.expected.exposures, + ['exposures', 'both'].includes(mode) ? 3006 : 0, + ); + assert.equal( + report.expected.evaluationCount, + ['evaluations', 'both'].includes(mode) ? 6012 : 0, + ); + } + return validateFollowup(report); + } + if (mode === 'hydration') { + assert.equal(report.mode, mode); + assert.equal(report.rows.length, 27); + for (const row of report.rows) assert.equal(row.total.count, 30); + return {hydrations: 810}; + } + const smoke = mode === 'smoke'; + assert.equal(report.metadata.smoke, smoke); + assert.equal(report.rows.length, smoke ? 9 : 95); + const timings = report.rows.filter(row => row.warmRead); + assert.equal(timings.length, smoke ? 6 : 90); + for (const row of timings) + assert.equal(row.warmRead.count, smoke ? 100 : 10000); + assert.equal(new Set(timings.map(row => row.mode)).size, 6); + return {placementRows: timings.length, timedReads: smoke ? 600 : 900000}; +} + +function sourceState(root, scope, output) { + const git = args => + execFileSync('git', args, {cwd: root, encoding: 'utf8'}).trim(); + const paths = new Set( + [ + ...git(['diff', '--name-only', 'HEAD', '--', scope]).split('\n'), + ...git(['ls-files', '--others', '--exclude-standard', '--', scope]).split( + '\n', + ), + ].filter(Boolean), + ); + const changedFiles = {}; + for (const relative of paths) { + const source = path.join(root, relative); + if (!fs.existsSync(source)) { + changedFiles[relative] = 'deleted'; + continue; + } + const data = fs.readFileSync(source); + changedFiles[relative] = sha256(data); + const destination = path.join(output, relative); + fs.mkdirSync(path.dirname(destination), {recursive: true}); + fs.writeFileSync(destination, data); + } + return { + head: git(['rev-parse', 'HEAD']), + branch: git(['branch', '--show-current']), + changedFiles, + }; +} + +async function main() { + const [serial, apk, output, androidRoot, mode = 'smoke'] = + process.argv.slice(2); + assert( + serial && apk && output && androidRoot, + 'Usage: node scripts/run-flags-android.cjs [smoke|full|hydration|none|bridge-only|exposures|evaluations|both|batching-none|batching-current|batching-control|batching-buffered]', + ); + assert( + ['smoke', 'full', 'hydration', ...trackingModes].includes(mode), + 'Unsupported experiment', + ); + assert.match(serial, /^emulator-\d+$/, 'Physical devices are not allowed'); + const adb = args => + execFileSync(process.env.ADB || 'adb', ['-s', serial, ...args], { + encoding: 'utf8', + maxBuffer: 32 * 1024 * 1024, + }).trim(); + assert.equal( + adb(['shell', 'getprop', 'ro.kernel.qemu']), + '1', + 'Target must be an emulator', + ); + assert( + !fs.existsSync(output), + 'Use a new result directory; existing evidence is never overwritten', + ); + fs.mkdirSync(output, {recursive: true}); + const rnRoot = path.resolve(__dirname, '../..'); + const manifest = { + recordedAt: new Date().toISOString(), + serial, + mode, + host: {platform: process.platform, arch: process.arch}, + device: adb(['shell', 'getprop']), + rn: sourceState(rnRoot, 'benchmarks', path.join(output, 'sources/rn')), + android: sourceState( + androidRoot, + 'prototypes/rules-evaluation', + path.join(output, 'sources/android'), + ), + yarnLockSha256: sha256(fs.readFileSync(path.join(rnRoot, 'yarn.lock'))), + apkSha256: sha256(fs.readFileSync(apk)), + validation: null, + }; + const bundle = 'com.benchmarkrunner'; + const reportPath = `/sdcard/Android/data/${bundle}/files/flags-benchmark-result.json`; + const collector = trackingModes.includes(mode) + ? await createCollector() + : null; + try { + adb(['install', '-r', apk]); + adb(['shell', 'am', 'force-stop', bundle]); + // The dedicated benchmark app contains only synthetic data. Start each tracking mode clean. + if (collector) { + adb(['shell', 'pm', 'clear', bundle]); + adb(['reverse', `tcp:${collector.port}`, `tcp:${collector.port}`]); + } + adb(['shell', 'rm', '-f', reportPath]); + const launchedAt = Date.now(); + adb([ + 'shell', + 'am', + 'start', + '-W', + '-n', + `${bundle}/.MainActivity`, + '--es', + 'flagsRun', + mode === 'full' ? 'full' : 'smoke', + ...(mode === 'hydration' || collector + ? ['--es', 'flagsExperiment', mode] + : []), + ...(collector ? ['--ei', 'flagsSinkPort', String(collector.port)] : []), + ]); + let report; + while (Date.now() - launchedAt < 15 * 60 * 1000) { + await wait(1000); + let candidate; + try { + candidate = JSON.parse(adb(['shell', 'cat', reportPath])); + } catch { + continue; + } + if (candidate.correctness === 'running') continue; + const completed = Date.parse( + candidate.completedAt || candidate.timestamp, + ); + assert( + completed >= launchedAt - 5000, + 'Stale report or emulator clock skew', + ); + report = candidate; + break; + } + assert(report, 'Timed out waiting for benchmark report'); + if (collector && report.sink) report.sink.requests = collector.requests; + const json = JSON.stringify(report, null, 2); + fs.writeFileSync(path.join(output, `${mode}.json`), json); + manifest.reportSha256 = sha256(json); + manifest.validation = validateReport(report, mode); + console.log(JSON.stringify(manifest.validation)); + } finally { + fs.writeFileSync( + path.join(output, 'manifest.json'), + JSON.stringify(manifest, null, 2), + ); + try { + fs.writeFileSync( + path.join(output, 'logcat.txt'), + adb(['logcat', '-d', '-t', '1500']), + ); + } finally { + try { + adb(['shell', 'am', 'force-stop', bundle]); + if (collector) adb(['reverse', '--remove', `tcp:${collector.port}`]); + } finally { + if (collector) await collector.close(); + } + } + } +} + +module.exports = {validateReport}; +if (require.main === module) + main().catch(error => { + console.error(error); + process.exitCode = 1; + }); diff --git a/benchmarks/scripts/run-flags-followups.cjs b/benchmarks/scripts/run-flags-followups.cjs new file mode 100644 index 000000000..185b7a6dd --- /dev/null +++ b/benchmarks/scripts/run-flags-followups.cjs @@ -0,0 +1,424 @@ +/* + * Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0. + * This product includes software developed at Datadog (https://www.datadoghq.com/). + * Copyright 2026-Present Datadog, Inc. + */ + +const fs = require('node:fs'); +const path = require('node:path'); +const {execFileSync} = require('node:child_process'); +const {createHash} = require('node:crypto'); +const {gunzipSync} = require('node:zlib'); +const assert = require('node:assert/strict'); +const http = require('node:http'); + +const sha256 = data => createHash('sha256').update(data).digest('hex'); +const command = (args, options = {}) => + execFileSync('xcrun', args, {encoding: 'utf8', ...options}).trim(); +const wait = ms => new Promise(resolve => setTimeout(resolve, ms)); +const batchingModes = [ + 'batching-none', + 'batching-current', + 'batching-control', + 'batching-buffered', +]; + +async function createCollector() { + const requests = []; + const server = http.createServer((request, response) => { + const chunks = []; + request.on('data', chunk => chunks.push(chunk)); + request.on('end', () => { + const body = Buffer.concat(chunks); + requests.push({ + receivedAt: Date.now(), + url: `http://127.0.0.1:${server.address().port}${request.url}`, + bodyBase64: body.toString('base64'), + bodyBytes: body.length, + contentEncoding: request.headers['content-encoding'] || '', + }); + response.writeHead(202); + response.end(); + }); + }); + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + return { + requests, + port: server.address().port, + async close() { + server.closeAllConnections(); + await new Promise(resolve => server.close(resolve)); + }, + }; +} + +function validateReport(report) { + assert.equal(report.correctness, 'passed'); + assert.equal(report.metadata.simulator, true); + assert.equal(report.metadata.debug, false); + assert.equal(report.metadata.nativeDebug, false); + if (report.mode === 'hydration') { + assert.equal(report.rows.length, 27); + for (const row of report.rows) assert.equal(row.total.count, 30); + return {hydrations: 810}; + } + const batching = batchingModes.includes(report.mode); + const expectedExposures = new Map(), + expectedEvaluations = new Map(); + if (batching) { + assert.equal(report.metadata.hermes, true); + assert.equal(report.metadata.newArchitecture, true); + assert.equal( + report.sink.initialization?.clientType, + report.metadata.platform === 'android' + ? 'DatadogFlagsClient' + : 'FlagsClient', + ); + const enabled = report.mode !== 'batching-none'; + const maxBatch = report.mode === 'batching-buffered' ? 25 : 1; + assert.deepEqual(report.expected, { + exposures: enabled ? 2772 : 0, + evaluationCount: enabled ? 5544 : 0, + }); + assert.deepEqual(report.policy, { + maxBatch, + maxWaitMs: maxBatch === 25 ? 50 : 0, + nativeDeduplication: true, + nativeAggregation: true, + }); + assert.equal(report.rows.length, 24); + const cases = new Set(); + for (const row of report.rows) { + const count = { + '10hz': 20, + '100hz': 200, + 'screen-burst': 200, + 'stress-burst': 500, + }[row.workload]; + assert( + count && + ['repeated', 'changing'].includes(row.pattern) && + [0, 1, 2].includes(row.repetition), + ); + const id = `${row.repetition}-${row.workload}-${row.pattern}`; + assert(!cases.has(id), 'Duplicate workload'); + cases.add(id); + assert.equal(row.client, `${report.mode}-${id}`); + assert.equal(row.count, count); + assert.equal(row.checksum, count); + assert.equal(row.caller.count, count); + assert.equal(row.callerMs.length, count); + assert.deepEqual(row.drained, {buffered: 0, inFlight: 0}); + assert.equal(row.bridgeCalls, row.batchSizes.length); + assert( + row.batchSizes.every( + size => Number.isInteger(size) && size > 0 && size <= maxBatch, + ), + ); + assert.equal( + row.batchSizes.reduce((sum, size) => sum + size, 0), + enabled ? count : 0, + ); + assert.equal(row.acknowledgementMs.length, enabled ? count : 0); + assert.equal(row.enqueueToSubmitMs.length, enabled ? count : 0); + assert.equal(row.submissionMs.length, row.bridgeCalls); + for (const field of [ + 'callerMs', + 'acknowledgementMs', + 'enqueueToSubmitMs', + 'submissionMs', + 'offCallerFlushMs', + 'timerDelayMs', + ]) { + assert( + row[field].every(value => Number.isFinite(value) && value >= 0), + `Invalid ${field}`, + ); + } + assert(row.timerDelayMs.length > 0); + assert(row.maxBufferedRecords <= maxBatch); + if (!enabled) continue; + expectedEvaluations.set(`${row.client}-warmup`, 1); + for (let i = 0; i < count; i++) { + const target = `${row.client}-${row.pattern === 'repeated' ? 0 : i}`; + expectedExposures.set(target, 1); + expectedEvaluations.set( + target, + (expectedEvaluations.get(target) || 0) + 1, + ); + } + } + } else { + assert.equal(report.rows.length, 12); + for (const row of report.rows) { + assert.equal(row.caller.count, 500); + assert.equal(row.checksum, 500); + assert.equal(row.pendingAtEnd, 0); + assert.equal(row.bridgeCalls, report.mode === 'none' ? 0 : 500); + } + } + const observedExposures = new Map(), + observedEvaluations = new Map(); + const countTarget = (counts, target, count) => { + assert.equal(typeof target, 'string'); + counts.set(target, (counts.get(target) || 0) + count); + }; + let exposures = 0, + evaluationCount = 0, + evaluationRecords = 0, + bodyBytes = 0; + for (const request of report.sink.requests) { + const url = new URL(request.url); + assert.equal(url.hostname, '127.0.0.1'); + let bytes = Buffer.from(request.bodyBase64, 'base64'); + bodyBytes += bytes.length; + if (request.contentEncoding === 'gzip') bytes = gunzipSync(bytes); + if (url.pathname === '/exposures') { + for (const line of bytes.toString('utf8').trim().split('\n')) { + const exposure = JSON.parse(line); + assert.equal(exposure.flag.key, 'flag-0'); + if (batching) countTarget(observedExposures, exposure.subject?.id, 1); + exposures++; + } + } else if (url.pathname === '/evaluations') { + const text = bytes.toString('utf8').trim(); + const evaluations = + report.metadata.platform === 'android' + ? text.split('\n').map(line => JSON.parse(line)) + : JSON.parse(text).flagEvaluations; + for (const evaluation of evaluations) { + assert.equal(evaluation.flag.key, 'flag-0'); + assert( + Number.isSafeInteger(evaluation.evaluation_count) && + evaluation.evaluation_count > 0, + ); + evaluationCount += evaluation.evaluation_count; + if (batching) + countTarget( + observedEvaluations, + evaluation.targeting_key, + evaluation.evaluation_count, + ); + evaluationRecords++; + } + } else throw new Error(`Unexpected request: ${url.pathname}`); + } + assert.equal( + exposures, + report.expected.exposures, + 'native exposure output/deduplication', + ); + assert.equal( + evaluationCount, + report.expected.evaluationCount, + 'native aggregation did not lose evaluations', + ); + if (batching) { + assert.deepEqual( + observedExposures, + expectedExposures, + 'exposures match each targeting key', + ); + assert.deepEqual( + observedEvaluations, + expectedEvaluations, + 'evaluations match each targeting key', + ); + } + return { + exposures, + evaluationCount, + evaluationRecords, + bodyBytes, + requests: report.sink.requests.length, + }; +} + +function sourceState(root, scopes, output) { + const git = args => + execFileSync('git', args, {cwd: root, encoding: 'utf8'}).trim(); + const changed = new Set( + [ + ...git(['diff', '--name-only', 'HEAD', '--', ...scopes]).split('\n'), + ...git([ + 'ls-files', + '--others', + '--exclude-standard', + '--', + ...scopes, + ]).split('\n'), + ].filter(Boolean), + ); + const files = {}; + for (const file of changed) { + const source = path.join(root, file); + if (!fs.existsSync(source)) { + files[file] = 'deleted'; + continue; + } + const data = fs.readFileSync(source); + files[file] = sha256(data); + const target = path.join(output, file); + fs.mkdirSync(path.dirname(target), {recursive: true}); + fs.writeFileSync(target, data); + } + return { + head: git(['rev-parse', 'HEAD']), + branch: git(['branch', '--show-current']), + changedFiles: files, + }; +} + +async function main() { + const [simulator, app, output, iosRoot, ...selectedModes] = + process.argv.slice(2); + if (!simulator || !app || !output || !iosRoot) + throw new Error( + 'Usage: node scripts/run-flags-followups.cjs ', + ); + const devices = JSON.parse(command(['simctl', 'list', 'devices', '--json'])); + assert( + Object.values(devices.devices) + .flat() + .some(device => device.udid === simulator && device.state === 'Booted'), + 'Boot the selected simulator first', + ); + assert( + !fs.existsSync(output), + 'Use a new result directory; existing evidence is never overwritten', + ); + fs.mkdirSync(output, {recursive: true}); + const rnRoot = path.resolve(__dirname, '../..'); + const manifest = { + recordedAt: new Date().toISOString(), + simulator, + rn: sourceState(rnRoot, ['benchmarks'], path.join(output, 'sources/rn')), + ios: sourceState( + iosRoot, + ['DatadogFlags/Prototypes/RulesEvaluation'], + path.join(output, 'sources/ios'), + ), + xcode: execFileSync('xcodebuild', ['-version'], {encoding: 'utf8'}).trim(), + podLockSha256: sha256( + fs.readFileSync(path.join(rnRoot, 'benchmarks/ios/Podfile.lock')), + ), + appExecutableSha256: sha256( + fs.readFileSync(path.join(app, 'BenchmarkRunner')), + ), + hermesBundleSha256: sha256( + fs.readFileSync(path.join(app, 'main.jsbundle')), + ), + reports: [], + }; + const bundle = execFileSync( + '/usr/libexec/PlistBuddy', + ['-c', 'Print :CFBundleIdentifier', path.join(app, 'Info.plist')], + {encoding: 'utf8'}, + ).trim(); + command(['simctl', 'install', simulator, app]); + const collector = await createCollector(); + try { + const modes = [ + 'hydration', + 'none', + 'bridge-only', + 'exposures', + 'evaluations', + 'both', + ...batchingModes, + ]; + assert( + selectedModes.every(mode => modes.includes(mode)), + 'Unknown experiment mode', + ); + for (const mode of selectedModes.length + ? selectedModes + : modes.filter(mode => !batchingModes.includes(mode))) { + collector.requests.length = 0; + const launchAt = Date.now(); + command( + [ + 'simctl', + 'launch', + '--terminate-running-process', + `--stdout=${path.resolve(output, `${mode}.stdout.log`)}`, + `--stderr=${path.resolve(output, `${mode}.stderr.log`)}`, + simulator, + bundle, + ], + { + env: { + ...process.env, + SIMCTL_CHILD_DD_FLAGS_EXPERIMENT: mode, + SIMCTL_CHILD_DD_FLAGS_SINK_PORT: String(collector.port), + }, + }, + ); + const data = command([ + 'simctl', + 'get_app_container', + simulator, + bundle, + 'data', + ]); + let report; + while (Date.now() - launchAt < 120000) { + await wait(500); + try { + const candidate = JSON.parse( + fs.readFileSync( + path.join(data, 'Documents/flags-benchmark-result.json'), + 'utf8', + ), + ); + if ( + candidate.correctness === 'failed' && + Date.parse(candidate.completedAt) >= launchAt + ) { + fs.writeFileSync( + path.join(output, `${mode}-failed.json`), + JSON.stringify(candidate, null, 2), + ); + throw new Error( + `App failed: ${candidate.error}\n${candidate.stack}`, + ); + } + if ( + candidate.mode === mode && + Date.parse(candidate.startedAt) >= launchAt && + candidate.correctness === 'passed' + ) { + report = candidate; + break; + } + } catch (error) { + if (error.message.startsWith('App failed:')) throw error; + } + } + assert(report, `Timed out waiting for ${mode}`); + if (mode !== 'hydration') report.sink.requests = collector.requests; + const json = JSON.stringify(report, null, 2); + const filename = `${mode}.json`; + // Preserve the report even when its event-count gate fails, for diagnosis. + fs.writeFileSync(path.join(output, filename), json); + const validation = validateReport(report); + manifest.reports.push({filename, sha256: sha256(json), validation}); + console.log(JSON.stringify({mode, validation})); + await wait(1000); + } + } finally { + fs.writeFileSync( + path.join(output, 'manifest.json'), + JSON.stringify(manifest, null, 2), + ); + command(['simctl', 'terminate', simulator, bundle]); + await collector.close(); + } +} + +module.exports = {validateReport, createCollector}; +if (require.main === module) + main().catch(error => { + console.error(error); + process.exitCode = 1; + }); diff --git a/benchmarks/scripts/summarize-flags-batching.cjs b/benchmarks/scripts/summarize-flags-batching.cjs new file mode 100644 index 000000000..0fbf64da4 --- /dev/null +++ b/benchmarks/scripts/summarize-flags-batching.cjs @@ -0,0 +1,113 @@ +/* + * Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0. + * This product includes software developed at Datadog (https://www.datadoghq.com/). + * Copyright 2026-Present Datadog, Inc. + */ + +const fs = require('node:fs'); +const path = require('node:path'); +const assert = require('node:assert/strict'); +const {createHash} = require('node:crypto'); +const {validateReport} = require('./run-flags-followups.cjs'); + +function files(directory) { + return fs.readdirSync(directory, {withFileTypes: true}).flatMap(entry => { + if (entry.name === 'sources' || entry.name.startsWith('diagnostic')) + return []; + const name = path.join(directory, entry.name); + return entry.isDirectory() + ? files(name) + : /^batching-.*\.json$/.test(entry.name) + ? [name] + : []; + }); +} + +function stats(samples) { + if (!samples.length) return null; + const sorted = [...samples].sort((a, b) => a - b); + const sum = samples.reduce((a, b) => a + b, 0); + return { + count: samples.length, + mean: sum / samples.length, + sum, + p50: sorted[Math.ceil(samples.length * 0.5) - 1], + p99: sorted[Math.ceil(samples.length * 0.99) - 1], + max: sorted[sorted.length - 1], + }; +} + +function summarize(root) { + return files(root) + .sort() + .map(file => { + const bytes = fs.readFileSync(file); + const report = JSON.parse(bytes); + const validation = validateReport(report); + const manifestPath = path.join(path.dirname(file), 'manifest.json'); + const manifest = JSON.parse(fs.readFileSync(manifestPath)); + const digest = createHash('sha256').update(bytes).digest('hex'); + const expected = + manifest.reportSha256 || + manifest.reports.find(item => item.filename === path.basename(file)) + ?.sha256; + assert.equal(digest, expected, 'Report hash does not match its manifest'); + const groups = new Map(); + for (const row of report.rows) { + const key = `${row.workload}/${row.pattern}`; + if (!groups.has(key)) groups.set(key, []); + groups.get(key).push(row); + } + return { + file: path.relative(root, file), + platform: report.metadata.platform, + mode: report.mode, + validation, + reportSha256: digest, + rows: [...groups].map(([workload, rows]) => { + assert.equal(rows.length, 3); + const samples = key => rows.flatMap(row => row[key]); + const caller = stats(samples('callerMs')); + const deferred = stats(samples('offCallerFlushMs')); + return { + workload, + callerMs: caller, + offCallerFlushMs: deferred, + // Does not include asynchronous acknowledgement handlers or RN internals. + timedJSWorkMsPerEvaluation: + (caller.sum + (deferred?.sum || 0)) / caller.count, + bridgeCalls: rows.map(row => row.bridgeCalls), + batchSizes: stats(samples('batchSizes')), + enqueueToSubmitMs: stats(samples('enqueueToSubmitMs')), + acknowledgementMs: stats(samples('acknowledgementMs')), + timerDelayMs: stats(samples('timerDelayMs')), + maxBufferedRecords: Math.max( + ...rows.map(row => row.maxBufferedRecords), + ), + maxRetainedRecords: Math.max( + ...rows.map(row => row.maxRetainedRecords), + ), + callerLoopMs: rows.map(row => row.callerLoopMs), + observedRatePerSecond: rows.map(row => + row.group === 1 && row.intervalMs > 0 + ? (1000 * (row.count - 1)) / row.callerLoopMs + : null, + ), + allAcknowledgedMs: rows.map(row => row.allAcknowledgedMs), + residentDeltaBytes: rows.map( + row => row.residentAfterBytes - row.residentBeforeBytes, + ), + }; + }), + }; + }); +} + +module.exports = {summarize}; +if (require.main === module) { + assert( + process.argv[2], + 'Usage: node summarize-flags-batching.cjs ', + ); + console.log(JSON.stringify(summarize(process.argv[2]), null, 2)); +} diff --git a/benchmarks/src/App.tsx b/benchmarks/src/App.tsx index 4a3bc0255..abc04234b 100644 --- a/benchmarks/src/App.tsx +++ b/benchmarks/src/App.tsx @@ -45,6 +45,10 @@ function App(): React.JSX.Element { switch (testConfig?.command) { case Command.Start: if (isValidScenario(testConfig?.scenario)) { + if (testConfig.scenario === Scenario.Flags) { + setTestConfig(testConfig); + return; + } const datadogConfig = getDatadogConfig(); testConfig.datadogConfig = datadogConfig; @@ -69,6 +73,10 @@ function App(): React.JSX.Element { } switch(testConfig?.scenario) { + case Scenario.Flags: { + const FlagsScenario = require('./flags/FlagsScenario').default; + return ; + } case Scenario.Default: return ; case Scenario.NavigationExample: diff --git a/benchmarks/src/flags/ANDROID_RESULTS.md b/benchmarks/src/flags/ANDROID_RESULTS.md new file mode 100644 index 000000000..c24aa6fd5 --- /dev/null +++ b/benchmarks/src/flags/ANDROID_RESULTS.md @@ -0,0 +1,204 @@ +# Android Emulator Benchmark Results + +Measured on 2026-09-29 for [FFL-3347](https://datadoghq.atlassian.net/browse/FFL-3347). +These are prototype architecture experiments, not shipping SDK performance guarantees. +Keep them separate from the [iOS results](RESULTS.md). + +## Takeaways + +- Repeated reads were faster for the RN caller when evaluation stayed in JS than + when every read crossed the synchronous or asynchronous Kotlin bridge. Both + full runs showed the same ordering. +- This does not mean that the JS evaluator itself is faster. The native-direct + control was faster, but excluded RN transport and used preconverted inputs. +- Native-only loading returned the first result sooner. For a JS evaluator, + native decoding followed by a ProtoJSON handoff cost more than transferring + bytes and decoding in JS. Parsing and repeated evaluation are separate choices. +- This supports the proposed JS-local repeated RN evaluation path, while retaining + native evaluation for native SDK clients and considering startup requirements + separately. +- Native tracking now passes the same experiments as iOS, including captured + exposure/evaluation totals in both mode orders. Each native tracking call still + costs JS time. These tests support separating tracking from evaluation, not a + claim that tracking is free or that native logging is faster than JS logging. + +## Environment and Validation + +- Host: Apple M3 Max, 64 GB RAM, macOS 26.6.2 (25G83). +- Android Emulator 37.1.11.0, AOSP API 35 / Android 15 ARM64 image, revision 2. + Pixel 6 profile, four virtual CPU cores, 4 GB guest RAM, 4 GB data partition. +- CPU acceleration: Apple's Hypervisor.Framework. Headless `-gpu auto` selected + software graphics (`swangle` / `lavapipe`); these are CPU/bridge measurements, + not graphics or UI frame-rate measurements. Snapshots were disabled. +- RN 0.78.2, New Architecture, Release/Hermes; core evaluator 3.1.1 and + Protobuf-ES 2.15.0. Kotlin 2.0.21, protobuf-java/protobuf-java-util 3.25.5. +- Gradle 8.12, JDK 17.0.18, build-tools 35.0.0, NDK 27.1.12297006, + CMake 3.22.1. Only `arm64-v8a` was built. R8/minification remained disabled, + matching the benchmark app's existing Release configuration. The benchmark + app uses its debug signing key with Release code. +- Release APK build, 28 RN harness tests, and eight Kotlin tests passed. +- Real-bridge smoke: six placements, 600 timed reads. Two full runs: 90 placement + rows and 900,000 timed reads each. Hydration: 27 rows and 810 samples. +- Both full runs used fresh app processes on the same emulator and the same APK. + No forced ART AOT/compilation-mode override was applied. Builds had finished + before timing; the iOS simulator was shut down. No physical phone was used. + +The correctness gates verify values, metadata, context A -> B -> A, configuration +replacement, and native control checksums against the JS reference. The native +engine covers a boolean benchmark subset, not full production UFC conformance. + +## Repeated Evaluation + +Each full run has five rotated repetitions across 10, 100, and 1,000 flags, with +10,000 timed reads and 1,000 warmup calls per placement/repetition/size. The hot +working set stays at four flags: static, string membership, compound numeric +targeting, and MD5 splits. It is not a scan of all flags in the configuration. + +100-flag configuration, microseconds. Each endpoint of a range is one full run's +median of five per-repetition percentiles. These are not pooled percentiles or +confidence intervals. + +| Decode / evaluation path | Read p50 us | Read p99 us | +| ------------------------------------ | ------------: | ------------: | +| JS / JS | 5.96-6.00 | 20.67-20.71 | +| Native / JS, ProtoJSON handoff | 5.96-6.04 | 20.92-23.00 | +| JS / native sync, ProtoJSON handoff | 14.08-14.54 | 34.79-45.50 | +| Native / native sync | 14.21-14.62 | 32.92-44.88 | +| JS / native async, ProtoJSON handoff | 190.71-199.71 | 388.63-388.71 | +| Native / native async | 188.04-199.63 | 383.54-412.46 | + +At 100 flags, the native-direct evaluation control had p50 values of 0.750 and +0.875 us. Native protobuf decoding had p50 values of 78.334 and 83.541 us, versus +2,161.458 and 2,216.083 us for the JS decoder control. Do not subtract control +percentiles from end-to-end percentiles or treat preconverted native inputs as +equivalent to a caller crossing the RN boundary. + +For 1,000 flags, in-memory installation p50 was 22.97-23.14 ms for JS decode, +60.23-60.33 ms for native decode followed by ProtoJSON transfer to JS, and +0.78 ms for native decode/install through the sync bridge. These installation +timers exclude preloading/base64 transfer, file I/O, and networking. + +## Warm Persisted Hydration + +The timer starts at the JS request to read a native cached file and ends when the +first correct static-flag result is usable in JS. All paths call native code +asynchronously. Files are written before timing and the OS cache is warm; this is +not cold app startup. Each cell is the median of three per-repetition p50 values, +with 30 samples per repetition. + +| Flags / protobuf bytes | Bytes as base64 -> JS decode/evaluate p50 ms | Native decode -> ProtoJSON -> JS evaluate p50 ms | Native decode/evaluate -> result p50 ms | +| ---------------------- | -------------------------------------------: | -----------------------------------------------: | --------------------------------------: | +| 10 / 869 | 1.352 | 2.022 | 0.797 | +| 100 / 8,269 | 4.315 | 7.183 | 0.898 | +| 1,000 / 86,084 | 30.653 | 61.058 | 1.712 | + +Returning the first result sooner does not eliminate the cost of later bridged +evaluations. Conversely, an async native file read does not make subsequent JS +parsing nonblocking. Thirty samples per row are insufficient for a stable p99. + +## Native Tracking + +The tracking extension uses the same JS workload as the iOS simulator: five +modes, repeated and changing contexts, 500-call uninterrupted bursts, yields +every 25 calls, and three rotated repetitions. Each mode has 6,000 timed +evaluations plus 12 native-client warmups when the bridge is enabled. A second +pass reverses the mode order. All 60,000 timed evaluations passed. + +The real `DdFlags.trackEvaluation` bridge calls native Datadog Flags 3.13.1. +Initialization is outside timing. RUM is disabled; uploads are small/frequent, +and evaluation aggregation runs every second. An `adb reverse` tunnel connects +emulator loopback to a collector bound to host `127.0.0.1`. The test-only final +drain runs on a worker thread. Each mode starts a fresh process with cleared +synthetic app data. No Datadog credentials, external intake, or phone is used. + +Verified totals in each pass: + +| Mode | Exposures | Evaluation count represented | Evaluation records | HTTP requests | +| ------------------------ | --------: | ---------------------------: | -----------------: | ------------: | +| No tracking | 0 | 0 | 0 | 0 | +| Bridge, loggers disabled | 0 | 0 | 0 | 0 | +| Exposures | 3,006 | 0 | 0 | 4 | +| Evaluations | 0 | 6,012 | 3,018 | 4 | +| Both | 3,006 | 6,012 | 3,018 | 8 | + +These exposure and evaluation totals match iOS. Android sends gzip-compressed +newline-delimited evaluation records; iOS uses a JSON envelope. Native batching +produces different request counts. Neither request count nor payload encoding +is a performance ranking between SDKs. + +Changing-context uninterrupted bursts, caller p50 / p99 in microseconds. Each +cell is the median of three per-repetition percentiles, not a pooled percentile: + +| Mode | Forward | Reverse order | +| ------------------------ | ---------------: | ---------------: | +| No tracking | 4.458 / 5.958 | 4.583 / 6.166 | +| Bridge, loggers disabled | 19.125 / 243.916 | 24.708 / 261.583 | +| Exposures | 17.041 / 314.584 | 20.834 / 253.625 | +| Evaluations | 16.625 / 240.917 | 20.500 / 199.584 | +| Both | 20.875 / 280.209 | 21.750 / 424.292 | + +Caller time includes JS evaluation, argument construction, bridge submission, +and measurement bookkeeping. It does not include waiting for native storage or +upload. Both passes show a cost for per-evaluation bridge submission. Variation +between modes and passes prevents ranking individual logger costs. These are +stress tests, not typical application rates or device latency guarantees. + +Uninterrupted tracking bursts reached 500 outstanding acknowledgements; yielding +every 25 calls limited that count to 25. No promises failed or remained pending. +This is not native queue depth or proof of a production backlog. We did not +compare native logging with a JS-only logger or measure RUM attribution. + +The tracking APK also passed the 600-read smoke test and all 810 hydration +samples. The RN harness now has 38 passing tests. Tracking evidence is retained +under `android-tracking-2026-09-29/forward/` and `reverse/`, with reports, captured +HTTP bodies, manifests, source snapshots, and hashes for all ten runs. The +summary is `android-tracking-2026-09-29/summary-statistics.json`. The APK SHA-256 is +`7f465cded748f0037ed538ff6d1c35cf7846ff0d453c54eff80a3ace2384f256`. +The earlier `diagnostic-both` run is excluded from timing tables: its initial +validator expected the iOS envelope and rejected Android's NDJSON. + +## Reproduction and Evidence + +Use the build and runner commands in [the README](README.md#android-emulator-extension). +The measured emulator command was: + +```sh +"$ANDROID_HOME/emulator/emulator" -avd flags-api35-arm64 -port 5554 \ + -memory 4096 -cores 4 -no-snapshot -no-boot-anim -no-audio -no-window -gpu auto +``` + +The original evidence is retained under +`mobile-flags-benchmark-results/android-emulator-2026-09-29/`. Successful reports, +filtered manifests, source snapshots, and summary JSON are also available in the +[recorded evidence archive](evidence/README.md). The original collection contains: + +- `smoke/`, `full/`, `full-repeat/`, and `hydration/`: reports, manifests, + changed-source copies/hashes, APK and lockfile hashes, device properties, logs. +- `environment.json` and `avd-config.ini`: host, toolchain, emulator settings, + and limitations. `summary-statistics.json`: all configuration sizes and controls. +- Baselines: RN `16f3a45e785a394e4f9f5227ce258f1825a49c6d` and Android + `9dee604d64065c0cd66381981bf73cd5b7ffeb2e`, plus each manifest's saved changes. + A baseline SHA alone does not identify the uncommitted measured prototype. +- APK SHA-256: `03db1a925d19ddfb8a1cf1402fd6875664ac5c41f32d551be855ce76ee718abd`. + All saved report hashes were verified. Documentation was updated afterward; + measured operations were not changed during the runs. + +Placement reports retain per-repetition summaries, not every individual sample. +Hydration reports also retain individual samples. The archive excludes build +logs, binaries, and full system-property dumps. The shared hydration metadata contains a generic loopback +sink label from the iOS tracking harness; Android hydration used no collector, +tracking, or network. This labeling limitation does not change its timed work. + +## Limits + +See the later [batching comparison](BATCHING_RESULTS.md) for matching iOS and +Android tests of per-evaluation delivery, a one-record wrapper control and a +25-record/50 ms JS buffer. Native tracking stays unchanged in that experiment. + +No physical Android, full evaluator conformance, cold +launch, shipping package size, or device energy conclusions are established. +Host/guest thermal state was not sampled. Process RSS is not isolated evaluator +heap or a true peak. ProtoJSON is only one possible transfer format, not a lower +bound for typed objects, optimized binary transfer, shared memory, or direct JSI. +Emulator timings and the unoptimized native prototype cannot establish absolute +customer-device latency or a universally fastest production design. diff --git a/benchmarks/src/flags/BATCHING_RESULTS.md b/benchmarks/src/flags/BATCHING_RESULTS.md new file mode 100644 index 000000000..1d9e16667 --- /dev/null +++ b/benchmarks/src/flags/BATCHING_RESULTS.md @@ -0,0 +1,163 @@ +# RN Tracking Delivery Comparison + +Measured September 29, 2026. This experiment compares one native call per +evaluation with JS-buffered calls on both bridges. Native exposure deduplication, +evaluation aggregation, storage and upload stay unchanged. It does not compare +native tracking with a JS-only logger. + +The existing bridge makes one `trackEvaluation` call per evaluation, including +when both loggers are enabled. This does not measure independently registered +hooks, which could submit separate calls. Repeat the measurements with the final +hook implementation. + +## Recommendation + +Keep native ownership of tracking for code reuse and maintenance. These results +do not establish that native logging is faster or more reliable than a JS logger. +Use asynchronous per-evaluation delivery as the initial proposal, and keep the +internal delivery method replaceable. Batching is a useful optimization for +larger bursts, not a performance requirement demonstrated at every rate. + +Do not claim batching has negligible benefits. It reduced the median total JS +loop time for 500 changing-context evaluations from 14.65/15.37 ms to 9.59/5.76 ms +on iOS, and from 22.15/21.75 ms to 7.26/7.43 ms on Android, in forward/reverse +passes. It also reduced measured JS work at 100 evaluations/second. The absolute +savings there were about 6.5/10.5 ms per second on iOS and 4.0/7.5 ms per second +on Android. These are estimates from measured work per evaluation, not CPU +utilization or frame-rate measurements. + +At 10 evaluations/second, buffering still made one bridge call per evaluation. +It provided no consistent reduction against the one-record control after +including deferred work. Records waited about 66 ms before submission with our +50 ms timer target. A production buffer would add backgrounding, consent, +shutdown, failure, queue-limit and timestamp requirements. Favor the simpler +delivery path initially unless expected application workloads justify that work. + +## Setup and Correctness + +- Release ARM64, Hermes, RN 0.78.2 with the new architecture, core 3.1.1. +- iPhone 17 Pro simulator, iOS 26.2; native DatadogFlags 3.16.0. +- Android API 35 ARM64 emulator, four cores and 4 GB RAM; native Flags 3.13.1. +- The same M3 Max host ran one simulator at a time, without concurrent builds. + Neither run used a physical phone. Absolute timings are not comparable across + platforms, native SDK versions or workloads. +- All modes use the same 100-flag synthetic configuration, decoded before + timing. The timed reads evaluate one boolean flag. Contexts either repeat or + change on every read. +- Modes: no tracking; existing `DdFlags.trackEvaluation`; prototype batch + bridge with one record; prototype batch bridge with up to 25 records or a + 50 ms timer. The one-record control isolates wrapper differences. +- Workloads: 20 evaluations at 10/second; 200 at 100/second; ten bursts of 20 + spaced 250 ms apart; and one uninterrupted burst of 500. Each has three + rotated repetitions. These rates are illustrative, not customer telemetry. +- Each mode starts a fresh process. Both mode orders were tested. All 16 runs + passed: 88,320 timed reads, plus native-client warmups outside timing. +- Each tracking run delivered exactly 2,772 exposures and 5,544 represented + evaluations, including 24 exposure-disabled warmups. The validator checks + every targeting key. Baselines sent no events. All pending calls and partial + batches were drained. There were no rejected native calls. +- Native uploads use a loopback collector, small/frequent uploads, a one-second + aggregation interval, and disabled RUM. iOS sent 121 requests per tracking + run; Android sent 24. These differences come from the native implementations, + not evidence that JS batching reduces network requests. + +## Measured JS Work + +Tables show changing contexts. Each cell is **forward / reverse** mean +microseconds per evaluation, pooled only across the three repetitions of that +workload and pass. Work includes evaluation, record construction, submission +inside the caller, and deferred flush work outside the caller. It excludes +asynchronous acknowledgement handlers and RN internals, so it is not a CPU +profile. The summary JSON also retains repeated-context results and individual +caller p50/p99, queue wait, acknowledgements, timer delay and memory snapshots. +The pacing and instrumentation differ from the earlier evaluator-only tests; +do not compare their absolute evaluation latencies with this table. + +### iOS Simulator + +| Workload | No tracking | Existing bridge | One-record control | Buffered | +| --- | ---: | ---: | ---: | ---: | +| 10/second | 112.5 / 124.7 | 349.7 / 377.4 | 305.6 / 254.3 | 330.5 / 270.2 | +| 100/second | 63.7 / 64.9 | 186.7 / 208.9 | 190.2 / 189.3 | 121.8 / 104.0 | +| 20-call bursts | 30.3 / 31.0 | 82.2 / 89.0 | 76.9 / 74.2 | 57.8 / 46.8 | +| 500-call burst | 11.1 / 12.8 | 30.0 / 30.1 | 26.6 / 24.2 | 18.0 / 11.5 | + +### Android Emulator + +| Workload | No tracking | Existing bridge | One-record control | Buffered | +| --- | ---: | ---: | ---: | ---: | +| 10/second | 96.7 / 93.1 | 277.2 / 262.1 | 292.2 / 224.3 | 291.7 / 287.5 | +| 100/second | 43.5 / 41.3 | 132.5 / 152.5 | 146.7 / 147.7 | 92.8 / 78.0 | +| 20-call bursts | 14.6 / 15.4 | 62.7 / 60.4 | 71.7 / 73.5 | 36.3 / 32.6 | +| 500-call burst | 8.0 / 8.3 | 41.0 / 41.8 | 32.4 / 35.5 | 15.1 / 15.7 | + +Buffering saved about 0.49/0.84 ms of measured JS work per 20 evaluations on +iOS, and 0.53/0.56 ms on Android. This includes deferred work; it is not a +measurement of an entire rendered screen or a proven frame-rate improvement. + +Repeated contexts also benefited in the 500-call stress burst. Median loop time +fell from 19.53/17.85 to 11.24/6.27 ms on iOS, and from 24.62/18.01 to +11.10/7.00 ms on Android. The observed 100/second rates across all modes and +contexts were 99.50-99.68 on iOS and 99.42-99.85 on Android. + +## Submission and Tail Costs + +For changing contexts, both platforms showed these bridge-call counts per case: + +| Workload | Existing bridge / control | Buffered | Buffered median wait before submission | +| --- | ---: | ---: | --- | +| 20 reads at 10/second | 20 | 20 | About 66 ms | +| 200 reads at 100/second | 200 | 29-30 | About 34-35 ms | +| Ten 20-call bursts | 200 | 10 | About 66 ms | +| One 500-call burst | 500 | 20 | About 0.07-0.11 ms | + +RN timers can run later than requested. Fixed-schedule pacing catches up after +late timers; observed rates are retained in the summary. The buffer's 50 ms +setting is not a delivery deadline. The final partial batch is drained at the +end of each case, so its wait can be shorter than the timer target. + +Batching moves some work to a timer and concentrates other work into the call +that fills a batch. On iOS's 500-call workload, caller p99 increased from +44.4/45.1 microseconds with the existing bridge to 234.0/152.2 with buffering, +even though the whole burst completed sooner. On Android, the corresponding +p99 values were 272.6/335.8 and 199.7/201.9 microseconds. Do not infer individual +call tails from average or whole-burst improvements. + +The 16 ms timer probe showed no consistent improvement across rates and both +orders. Its baseline already includes RN scheduling delays, and uninterrupted +bursts have very few timer samples. It is not a frame-rate test. One iOS +100/second existing-bridge pass had a 91 ms maximum delay that did not repeat in +the reverse pass; do not treat that outlier as an established tracking cost. + +The buffer held at most 25 unsent records, but all 500 records could remain +retained until acknowledgements were processed during the uninterrupted burst, +in both per-record and buffered modes. This is not a native queue-depth +measurement or a bound on the total delivery backlog. RSS includes the whole +app, native clients, sample arrays and SDK buffers; no isolated memory saving +was established. + +## Limitations and Evidence + +The prototype always drains before stopping. It does not test backgrounding, +crashes, consent changes, failed writes, bridge teardown or persistent retry. +Native code currently assigns event timestamps when records arrive; delaying +the call can delay those timestamps. A production design must preserve the +original evaluation time and context. No shipping hook API was implemented or +changed, and neither RUM nor a JS-only logger was benchmarked. + +Evidence is retained in `mobile-flags-benchmark-results/batching-2026-09-29/`: +`ios/forward`, `ios/reverse`, `android/forward`, `android/reverse`, and +`summary-statistics.json`. Reports contain raw samples and captured synthetic +requests. Manifests preserve report/binary hashes and changed-source copies. +All 16 report hashes and per-target output checks passed during summarization. +The code and [recorded evidence](evidence/README.md) are included in this prototype +branch. The archive contains the successful reports and measured source snapshots. + +Excluded diagnostics remain under `ios/diagnostic*`. One exposed the validator's +incorrect exposure field name. The first full pass exposed pacing capped near +60/second by an unconditional timer yield. Both were fixed before the final +passes. The pacing fix has a regression test with frame-quantized timers. + +Validation: 57 RN harness tests, targeted TypeScript, both Release builds and +the real-bridge runs passed. See [README.md](./README.md#per-evaluation-calls-versus-js-batching) +for reproduction commands. No additional physical-device test was performed. diff --git a/benchmarks/src/flags/FlagsScenario.tsx b/benchmarks/src/flags/FlagsScenario.tsx new file mode 100644 index 000000000..5692eb090 --- /dev/null +++ b/benchmarks/src/flags/FlagsScenario.tsx @@ -0,0 +1,118 @@ +/* + * Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0. + * This product includes software developed at Datadog (https://www.datadoghq.com/). + * Copyright 2026-Present Datadog, Inc. + */ + +import React, {useCallback, useEffect, useRef, useState} from 'react'; +import { + Button, + Platform, + SafeAreaView, + ScrollView, + Share, + StyleSheet, + Text, + View, +} from 'react-native'; +import Config from 'react-native-config'; +import {runBenchmarks, saveReport, sync} from './runner'; +import {experimentMode, runExperiment} from './experiments'; + +export default function FlagsScenario() { + const [busy, setBusy] = useState(false); + const [status, setStatus] = useState('Ready'); + const [report, setReport] = useState(''); + const running = useRef(false); + const run = useCallback(async (smoke: boolean) => { + if (running.current) return; + running.current = true; + setBusy(true); + setReport(''); + try { + saveReport( + JSON.stringify({ + correctness: 'running', + startedAt: new Date().toISOString(), + }), + ); + const result = experimentMode() + ? await runExperiment(setStatus) + : await runBenchmarks(setStatus, smoke); + const json = JSON.stringify(result, null, 2); + saveReport(json); + setReport(json); + setStatus('Completed; correctness checks passed'); + console.log(`FLAGS_BENCHMARK_RESULT ${JSON.stringify(result)}`); + } catch (error) { + setStatus(`Failed: ${String(error)}`); + const failure = JSON.stringify({ + correctness: 'failed', + completedAt: new Date().toISOString(), + error: String(error), + stack: error instanceof Error ? error.stack : undefined, + }); + try { + saveReport(failure); + } catch { + console.error(failure); + } + } finally { + running.current = false; + setBusy(false); + } + }, []); + useEffect(() => { + const androidRun = + Platform.OS === 'android' + ? sync({op: 'experimentSettings'}).autorun + : undefined; + if (androidRun === 'full') void run(false); + else if (Config.BENCH_FLAGS_AUTORUN === 'smoke' || androidRun === 'smoke') + void run(true); + }, [run]); + return ( + + + Flag evaluation benchmarks + + {__DEV__ ? 'Debug build: smoke testing only' : 'Release build'} + + + {status} + + +