Add telemetry events for unittest execution and discovery cycles

- Introduced new telemetry events: UNITTEST_RUN_DONE and UNITTEST_TREE_UPDATE.
- Enhanced existing telemetry events to capture additional properties such as mode, trigger, and duration.
- Updated the telemetry context for test discovery to include more detailed metrics.
- Refactored telemetry emission to ensure accurate reporting during test execution and discovery processes.

Author: eleanorjboyd

src/client/telemetry/constants.ts

@@ -48,7 +48,9 @@ export enum EventName {
     UNITTEST_DISCOVERY_DONE = 'UNITTEST.DISCOVERY.DONE',
     UNITTEST_RUN_STOP = 'UNITTEST.RUN.STOP',
     UNITTEST_RUN = 'UNITTEST.RUN',
+    UNITTEST_RUN_DONE = 'UNITTEST.RUN.DONE',
     UNITTEST_RUN_ALL_FAILED = 'UNITTEST.RUN_ALL_FAILED',
+    UNITTEST_TREE_UPDATE = 'UNITTEST.TREE.UPDATE',
     UNITTEST_DISABLED = 'UNITTEST.DISABLED',
 
     PYTHON_EXPERIMENTS_INIT_PERFORMANCE = 'PYTHON_EXPERIMENTS_INIT_PERFORMANCE',

src/client/telemetry/index.ts

@@ -2127,7 +2127,9 @@ export interface IEventNamePropertyMapping {
      */
     /* __GDPR__
        "unittest.discovery.trigger" : {
-          "trigger" : { "classification": "SystemMetaData", "purpose": "FeatureInsight", "owner": "eleanorjboyd" }
+          "trigger" : { "classification": "SystemMetaData", "purpose": "FeatureInsight", "owner": "eleanorjboyd" },
+          "filekind" : { "classification": "SystemMetaData", "purpose": "FeatureInsight", "owner": "eleanorjboyd" },
+          "mssincelasttrigger" : { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "owner": "eleanorjboyd", "isMeasurement": true }
        }
      */
     [EventName.UNITTEST_DISCOVERY_TRIGGER]: {
@@ -2142,6 +2144,16 @@ export interface IEventNamePropertyMapping {
          * interpreter    : Triggered by interpreter change.
          */
         trigger: 'auto' | 'ui' | 'commandpalette' | 'watching' | 'interpreter';
+        /**
+         * For 'auto' / 'watching' triggers, classifies the file whose change triggered discovery.
+         * Used to detect whether discovery is firing on non-test files (see #25866).
+         */
+        fileKind?: 'test' | 'conftest' | 'non-test' | 'unknown';
+        /**
+         * Milliseconds since the previous discovery trigger fired (any source).
+         * Helps detect chatty re-trigger storms.
+         */
+        msSinceLastTrigger?: number;
     };
     /**
      * Telemetry event sent with details about discovering tests
@@ -2165,7 +2177,13 @@ export interface IEventNamePropertyMapping {
     /* __GDPR__
        "unittest.discovery.done" : {
           "tool" : { "classification": "SystemMetaData", "purpose": "FeatureInsight", "owner": "eleanorjboyd" },
-          "failed" : { "classification": "SystemMetaData", "purpose": "FeatureInsight", "owner": "eleanorjboyd" }
+          "failed" : { "classification": "SystemMetaData", "purpose": "FeatureInsight", "owner": "eleanorjboyd" },
+          "mode" : { "classification": "SystemMetaData", "purpose": "FeatureInsight", "owner": "eleanorjboyd" },
+          "trigger" : { "classification": "SystemMetaData", "purpose": "FeatureInsight", "owner": "eleanorjboyd" },
+          "failurecategory" : { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "owner": "eleanorjboyd" },
+          "totaldurationms" : { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "owner": "eleanorjboyd", "isMeasurement": true },
+          "testcount" : { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "owner": "eleanorjboyd", "isMeasurement": true },
+          "exitcode" : { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "owner": "eleanorjboyd", "isMeasurement": true }
        }
      */
     [EventName.UNITTEST_DISCOVERY_DONE]: {
@@ -2181,6 +2199,39 @@ export interface IEventNamePropertyMapping {
          * @type {boolean}
          */
         failed: boolean;
+        /**
+         * Discovery code path: 'project' = Python Environments API / project-based;
+         * 'legacy' = single-workspace adapter fallback.
+         */
+        mode?: 'project' | 'legacy';
+        /**
+         * Source that triggered the discovery (mirrors UNITTEST_DISCOVERY_TRIGGER.trigger).
+         */
+        trigger?: 'auto' | 'ui' | 'commandpalette' | 'watching' | 'interpreter';
+        /**
+         * Coarse failure category. Only populated when `failed` is true.
+         */
+        failureCategory?:
+            | 'subprocess-exit-non-zero'
+            | 'pipe-error'
+            | 'pytest-collect-error'
+            | 'plugin-exception'
+            | 'timeout'
+            | 'env-resolution'
+            | 'cancelled'
+            | 'unknown';
+        /**
+         * Wall-clock duration of the discovery cycle in milliseconds.
+         */
+        totalDurationMs?: number;
+        /**
+         * Number of test items discovered (leaf nodes).
+         */
+        testCount?: number;
+        /**
+         * Subprocess exit code, when known (failed discoveries).
+         */
+        exitCode?: number;
     };
     /**
      * Telemetry event sent when cancelling discovering tests
@@ -2222,6 +2273,100 @@ export interface IEventNamePropertyMapping {
        "unittest.run.all_failed" : { "owner": "eleanorjboyd" }
      */
     [EventName.UNITTEST_RUN_ALL_FAILED]: never | undefined;
+    /**
+     * Telemetry event sent at the end of a test run, capturing duration and pipe health.
+     * Companion to UNITTEST_RUN (which is emitted at run start).
+     */
+    /* __GDPR__
+       "unittest.run.done" : {
+          "tool" : { "classification": "SystemMetaData", "purpose": "FeatureInsight", "owner": "eleanorjboyd" },
+          "debugging" : { "classification": "SystemMetaData", "purpose": "FeatureInsight", "owner": "eleanorjboyd" },
+          "mode" : { "classification": "SystemMetaData", "purpose": "FeatureInsight", "owner": "eleanorjboyd" },
+          "failed" : { "classification": "SystemMetaData", "purpose": "FeatureInsight", "owner": "eleanorjboyd" },
+          "failurecategory" : { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "owner": "eleanorjboyd" },
+          "pipeclosedearly" : { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "owner": "eleanorjboyd" },
+          "durationms" : { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "owner": "eleanorjboyd", "isMeasurement": true },
+          "requestedcount" : { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "owner": "eleanorjboyd", "isMeasurement": true },
+          "reportedcount" : { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "owner": "eleanorjboyd", "isMeasurement": true },
+          "missingcount" : { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "owner": "eleanorjboyd", "isMeasurement": true },
+          "exitcode" : { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "owner": "eleanorjboyd", "isMeasurement": true }
+       }
+     */
+    [EventName.UNITTEST_RUN_DONE]: {
+        tool: TestTool;
+        debugging: boolean;
+        mode: 'project' | 'legacy';
+        /**
+         * `true` if the run ended without reporting all requested results,
+         * or if the subprocess crashed / threw.
+         */
+        failed: boolean;
+        /**
+         * Coarse failure category when `failed` is true.
+         */
+        failureCategory?:
+            | 'pipe-cancelled'
+            | 'subprocess-crash'
+            | 'no-results'
+            | 'env-mismatch'
+            | 'cancelled'
+            | 'unknown';
+        /**
+         * `true` if the result pipe was disposed before the subprocess fully
+         * reported all requested test results (see #25872).
+         */
+        pipeClosedEarly?: boolean;
+        /**
+         * Wall-clock duration of the run in milliseconds.
+         */
+        durationMs?: number;
+        /**
+         * Number of test items the user asked to run.
+         */
+        requestedCount?: number;
+        /**
+         * Number of distinct test results reported back over the pipe.
+         */
+        reportedCount?: number;
+        /**
+         * requestedCount - reportedCount (signals #25892 "all skipped" pattern).
+         */
+        missingCount?: number;
+        /**
+         * Subprocess exit code when known.
+         */
+        exitCode?: number;
+    };
+    /**
+     * Telemetry event emitted after the test tree is updated with new discovery results.
+     * Used to detect full-rebuild-on-every-save pattern (see #25822, #25866).
+     */
+    /* __GDPR__
+       "unittest.tree.update" : {
+          "tool" : { "classification": "SystemMetaData", "purpose": "FeatureInsight", "owner": "eleanorjboyd" },
+          "mode" : { "classification": "SystemMetaData", "purpose": "FeatureInsight", "owner": "eleanorjboyd" },
+          "rebuiltfromscratch" : { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "owner": "eleanorjboyd" },
+          "beforecount" : { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "owner": "eleanorjboyd", "isMeasurement": true },
+          "aftercount" : { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "owner": "eleanorjboyd", "isMeasurement": true }
+       }
+     */
+    [EventName.UNITTEST_TREE_UPDATE]: {
+        tool: TestTool;
+        mode: 'project' | 'legacy';
+        /**
+         * `true` if the discovery handler cleared and rebuilt all test items rather
+         * than performing an incremental update.
+         */
+        rebuiltFromScratch: boolean;
+        /**
+         * Number of root test items in the controller before this update.
+         */
+        beforeCount: number;
+        /**
+         * Number of root test items in the controller after this update.
+         */
+        afterCount: number;
+    };
     /**
      * Telemetry event sent when testing is disabled for a workspace.
      */

src/client/testing/testController/common/projectTestExecution.ts

@@ -5,6 +5,7 @@ import { CancellationToken, FileCoverageDetail, TestItem, TestRun, TestRunProfil
 import { traceError, traceInfo, traceVerbose, traceWarn } from '../../../logging';
 import { sendTelemetryEvent } from '../../../telemetry';
 import { EventName } from '../../../telemetry/constants';
+import { StopWatch } from '../../../common/utils/stopWatch';
 import { IPythonExecutionFactory } from '../../../common/process/types';
 import { ITestDebugLauncher } from '../../common/types';
 import { ProjectAdapter } from './projectAdapter';
@@ -70,13 +71,35 @@ export async function executeTestsForProjects(
             debugging: isDebugMode,
         });
 
+        const stopWatch = new StopWatch();
+        let failed = false;
+        let failureCategory:
+            | 'pipe-cancelled'
+            | 'subprocess-crash'
+            | 'no-results'
+            | 'env-mismatch'
+            | 'cancelled'
+            | 'unknown'
+            | undefined;
         try {
             await executeTestsForProject(project, items, runInstance, request, deps);
         } catch (error) {
+            failed = true;
+            failureCategory = token.isCancellationRequested ? 'cancelled' : 'unknown';
             // Don't log cancellation as an error
             if (!token.isCancellationRequested) {
                 traceError(`[test-by-project] Execution failed for project ${project.projectName}:`, error);
             }
+        } finally {
+            sendTelemetryEvent(EventName.UNITTEST_RUN_DONE, undefined, {
+                tool: project.testProvider,
+                debugging: isDebugMode,
+                mode: 'project',
+                failed,
+                failureCategory,
+                durationMs: stopWatch.elapsedTime,
+                requestedCount: items.length,
+            });
         }
     });
 

src/client/testing/testController/common/resultResolver.ts

@@ -7,10 +7,29 @@ import { TestProvider } from '../../types';
 import { traceInfo } from '../../../logging';
 import { sendTelemetryEvent } from '../../../telemetry';
 import { EventName } from '../../../telemetry/constants';
+import { StopWatch } from '../../../common/utils/stopWatch';
 import { TestItemIndex } from './testItemIndex';
 import { TestDiscoveryHandler } from './testDiscoveryHandler';
 import { TestExecutionHandler } from './testExecutionHandler';
 import { TestCoverageHandler } from './testCoverageHandler';
+import { DiscoveredTestNode, DiscoveredTestItem } from './types';
+
+/**
+ * Trigger source label for the current discovery cycle (matches
+ * UNITTEST_DISCOVERY_TRIGGER.trigger values).
+ */
+export type DiscoveryTriggerKind = 'auto' | 'ui' | 'commandpalette' | 'watching' | 'interpreter';
+
+/**
+ * Per-cycle context the controller passes to the resolver so DISCOVERY_DONE can
+ * include trigger source, mode, and wall-clock duration without having to plumb
+ * these through every adapter call.
+ */
+export interface DiscoveryCycleContext {
+    mode: 'project' | 'legacy';
+    trigger?: DiscoveryTriggerKind;
+    stopWatch: StopWatch;
+}
 
 export class PythonResultResolver implements ITestResultResolver {
     testController: TestController;
@@ -39,6 +58,12 @@ export class PythonResultResolver implements ITestResultResolver {
      */
     private projectName?: string;
 
+    /**
+     * Per-cycle telemetry context set by the controller before invoking discovery.
+     * Consumed (and cleared) by resolveDiscovery to emit UNITTEST_DISCOVERY_DONE.
+     */
+    private discoveryCycle?: DiscoveryCycleContext;
+
     constructor(
         testController: TestController,
         testProvider: TestProvider,
@@ -75,6 +100,39 @@ export class PythonResultResolver implements ITestResultResolver {
         return this.projectId;
     }
 
+    /**
+     * Set per-discovery-cycle telemetry context. Called by the controller right
+     * before invoking the discovery adapter so resolveDiscovery / failure paths
+     * can include trigger, mode, and duration in UNITTEST_DISCOVERY_DONE.
+     */
+    public beginDiscoveryCycle(ctx: Omit<DiscoveryCycleContext, 'stopWatch'>): void {
+        this.discoveryCycle = { ...ctx, stopWatch: new StopWatch() };
+    }
+
+    /**
+     * Returns and clears the current discovery cycle context, if any.
+     */
+    private takeDiscoveryCycle(): DiscoveryCycleContext | undefined {
+        const cycle = this.discoveryCycle;
+        this.discoveryCycle = undefined;
+        return cycle;
+    }
+
+    /**
+     * Returns the current discovery cycle context without clearing it.
+     * Used by error paths that still want to clear via takeDiscoveryCycle.
+     */
+    public peekDiscoveryCycle(): DiscoveryCycleContext | undefined {
+        return this.discoveryCycle;
+    }
+
+    /**
+     * Clears the current discovery cycle context.
+     */
+    public clearDiscoveryCycle(): void {
+        this.discoveryCycle = undefined;
+    }
+
     public resolveDiscovery(payload: DiscoveredTestPayload, token?: CancellationToken): void {
         PythonResultResolver.discoveryHandler.processDiscovery(
             payload,
@@ -86,9 +144,15 @@ export class PythonResultResolver implements ITestResultResolver {
             this.projectId,
             this.projectName,
         );
+        const cycle = this.takeDiscoveryCycle();
+        const mode = cycle?.mode ?? (this.projectId ? 'project' : 'legacy');
         sendTelemetryEvent(EventName.UNITTEST_DISCOVERY_DONE, undefined, {
             tool: this.testProvider,
             failed: false,
+            mode,
+            trigger: cycle?.trigger,
+            totalDurationMs: cycle?.stopWatch.elapsedTime,
+            testCount: payload?.tests ? countDiscoveredTests(payload.tests) : 0,
         });
     }
 
@@ -135,3 +199,19 @@ export class PythonResultResolver implements ITestResultResolver {
         this.testItemIndex.cleanupStaleReferences(this.testController);
     }
 }
+
+/**
+ * Recursively counts leaf test items in a discovered test tree.
+ * Used to populate UNITTEST_DISCOVERY_DONE.testCount.
+ */
+function countDiscoveredTests(node: DiscoveredTestNode | DiscoveredTestItem): number {
+    if ((node as DiscoveredTestNode).children === undefined) {
+        // No children -> leaf (DiscoveredTestItem).
+        return 1;
+    }
+    let total = 0;
+    for (const child of (node as DiscoveredTestNode).children) {
+        total += countDiscoveredTests(child);
+    }
+    return total;
+}

src/client/testing/testController/common/testDiscoveryHandler.ts

@@ -6,6 +6,8 @@ import * as util from 'util';
 import { DiscoveredTestPayload } from './types';
 import { TestProvider } from '../../types';
 import { traceError, traceWarn } from '../../../logging';
+import { sendTelemetryEvent } from '../../../telemetry';
+import { EventName } from '../../../telemetry/constants';
 import { Testing } from '../../../common/utils/localize';
 import { createErrorTestItem } from './testItemUtilities';
 import { buildErrorNodeOptions, populateTestTree } from './utils';
@@ -54,6 +56,9 @@ export class TestDiscoveryHandler {
             // if any tests exist, they should be populated in the test tree, regardless of whether there were errors or not.
             // parse and insert test data.
 
+            // Snapshot root-item count before update for UNITTEST_TREE_UPDATE telemetry.
+            const beforeCount = testController.items.size;
+
             // Clear existing mappings before rebuilding test tree
             testItemIndex.clear();
 
@@ -73,6 +78,19 @@ export class TestDiscoveryHandler {
                 projectId,
                 projectName,
             );
+
+            // Emit TREE_UPDATE so we can quantify how often the tree is being rebuilt
+            // from scratch versus changing incrementally (see #25822, #25866).
+            // populateTestTree currently always rebuilds the subtree under the workspace/project root,
+            // so rebuiltFromScratch is true today; we keep the field so future incremental updates
+            // can flip it to false without a schema change.
+            sendTelemetryEvent(EventName.UNITTEST_TREE_UPDATE, undefined, {
+                tool: testProvider,
+                mode: projectId ? 'project' : 'legacy',
+                rebuiltFromScratch: true,
+                beforeCount,
+                afterCount: testController.items.size,
+            });
         }
     }