Skip to content

Commit 0f89c38

Browse files
Ma11hewThomasMa11hewThomas
committed
update test cases and examples for improved validation and reporting
1 parent a10846e commit 0f89c38

8 files changed

Lines changed: 473 additions & 88 deletions

File tree

‎.github/workflows/main.yaml‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -35,22 +35,22 @@ jobs:
3535
run: node dist/cli.js merge test-reports
3636

3737
- name: Test CLI - Validate
38-
run: node dist/cli.js validate ctrf/ctrf-report.json
38+
run: node dist/cli.js validate examples/minimal.json
3939

4040
- name: Test CLI - Validate Strict
41-
run: node dist/cli.js validate-strict ctrf/ctrf-report.json
41+
run: node dist/cli.js validate-strict examples/minimal.json
4242

4343
- name: Test CLI - Flaky
44-
run: node dist/cli.js flaky test-reports/ctrf-report-one.json
44+
run: node dist/cli.js flaky examples/with-retries.json
4545

4646
- name: Test CLI - Generate Test IDs
47-
run: node dist/cli.js generate-test-ids test-reports/ctrf-report-one.json
47+
run: node dist/cli.js generate-test-ids examples/minimal.json
4848

4949
- name: Test CLI - Generate Report ID
50-
run: node dist/cli.js generate-report-id test-reports/ctrf-report-one.json
50+
run: node dist/cli.js generate-report-id examples/minimal.json
5151

5252
- name: Test CLI - Add Insights
53-
run: node dist/cli.js add-insights test-reports
53+
run: node dist/cli.js add-insights examples
5454

5555
- name: Publish Test Report
5656
uses: ctrf-io/github-test-reporter@v1

‎examples/comprehensive.json‎

Lines changed: 253 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,253 @@
1+
{
2+
"reportFormat": "CTRF",
3+
"specVersion": "1.0.0",
4+
"reportId": "9d2c6a10-3f7a-4e22-9a8f-1a2b3c4d5e6f",
5+
"timestamp": "2025-11-24T12:00:00Z",
6+
"generatedBy": "example-ci",
7+
"extra": {
8+
"pipelineStage": "e2e",
9+
"trigger": "pull_request"
10+
},
11+
"results": {
12+
"tool": {
13+
"name": "example-runner",
14+
"version": "3.5.0",
15+
"extra": {
16+
"plugins": ["retry", "screenshots", "video"]
17+
}
18+
},
19+
"summary": {
20+
"tests": 3,
21+
"passed": 2,
22+
"failed": 1,
23+
"pending": 0,
24+
"skipped": 0,
25+
"other": 0,
26+
"flaky": 1,
27+
"suites": 2,
28+
"start": 1609459200000,
29+
"stop": 1609459220000,
30+
"duration": 20000
31+
},
32+
"tests": [
33+
{
34+
"id": "550e8400-e29b-41d4-a716-446655440000",
35+
"name": "user can login",
36+
"status": "passed",
37+
"duration": 3500,
38+
"start": 1609459200000,
39+
"stop": 1609459203500,
40+
"suite": ["Authentication", "Login"],
41+
"type": "e2e",
42+
"tags": ["smoke", "critical"],
43+
"filePath": "tests/auth/login.test.js",
44+
"browser": "chromium",
45+
"threadId": "worker-1",
46+
"insights": {
47+
"passRate": {
48+
"current": 1.0,
49+
"baseline": 0.95,
50+
"change": 0.05
51+
},
52+
"averageTestDuration": {
53+
"current": 3500,
54+
"baseline": 3200,
55+
"change": 300
56+
},
57+
"executedInRuns": 15
58+
}
59+
},
60+
{
61+
"id": "550e8400-e29b-41d4-a716-446655440001",
62+
"name": "user can checkout",
63+
"status": "passed",
64+
"duration": 5200,
65+
"start": 1609459203500,
66+
"stop": 1609459208700,
67+
"suite": ["Checkout"],
68+
"type": "e2e",
69+
"tags": ["smoke"],
70+
"filePath": "tests/checkout/checkout.test.js",
71+
"browser": "chromium",
72+
"threadId": "worker-2",
73+
"flaky": true,
74+
"retries": 1,
75+
"retryAttempts": [
76+
{
77+
"attempt": 1,
78+
"status": "failed",
79+
"duration": 5100,
80+
"start": 1609459203500,
81+
"stop": 1609459208600,
82+
"message": "Payment gateway timeout",
83+
"trace": "Error: Payment gateway timeout\n at processPayment (payment.js:23:10)",
84+
"attachments": [
85+
{
86+
"name": "attempt-1-screenshot.png",
87+
"contentType": "image/png",
88+
"path": "/artifacts/retry-1.png"
89+
}
90+
]
91+
},
92+
{
93+
"attempt": 2,
94+
"status": "passed",
95+
"duration": 5200,
96+
"start": 1609459208700,
97+
"stop": 1609459213900
98+
}
99+
],
100+
"insights": {
101+
"passRate": {
102+
"current": 1.0,
103+
"baseline": 0.9,
104+
"change": 0.1
105+
},
106+
"flakyRate": {
107+
"current": 0.2,
108+
"baseline": 0.3,
109+
"change": -0.1
110+
},
111+
"executedInRuns": 15
112+
}
113+
},
114+
{
115+
"id": "550e8400-e29b-41d4-a716-446655440002",
116+
"name": "admin dashboard loads",
117+
"status": "failed",
118+
"duration": 8000,
119+
"start": 1609459213900,
120+
"stop": 1609459221900,
121+
"suite": ["Admin", "Dashboard"],
122+
"type": "e2e",
123+
"tags": ["regression"],
124+
"filePath": "tests/admin/dashboard.test.js",
125+
"line": 42,
126+
"browser": "chromium",
127+
"threadId": "worker-1",
128+
"message": "Element not found: .dashboard-metrics",
129+
"trace": "Error: Element not found: .dashboard-metrics\n at waitForElement (test.js:45:10)\n at Object.<anonymous> (dashboard.test.js:42:5)",
130+
"snippet": "const metrics = await page.locator('.dashboard-metrics');\nawait expect(metrics).toBeVisible();",
131+
"screenshot": "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==",
132+
"stdout": [
133+
"Loading admin dashboard",
134+
"Checking authentication"
135+
],
136+
"stderr": [
137+
"Warning: Slow network detected",
138+
"Error: Timeout waiting for .dashboard-metrics"
139+
],
140+
"attachments": [
141+
{
142+
"name": "failure-screenshot.png",
143+
"contentType": "image/png",
144+
"path": "/artifacts/screenshots/admin-dashboard-fail.png"
145+
},
146+
{
147+
"name": "trace.zip",
148+
"contentType": "application/zip",
149+
"path": "/artifacts/traces/admin-dashboard.trace.zip"
150+
},
151+
{
152+
"name": "video.webm",
153+
"contentType": "video/webm",
154+
"path": "/artifacts/videos/admin-dashboard.webm"
155+
}
156+
],
157+
"parameters": {
158+
"userRole": "admin",
159+
"environment": "staging"
160+
},
161+
"steps": [
162+
{
163+
"name": "Navigate to dashboard",
164+
"status": "passed"
165+
},
166+
{
167+
"name": "Wait for metrics",
168+
"status": "failed"
169+
}
170+
],
171+
"insights": {
172+
"passRate": {
173+
"current": 0.0,
174+
"baseline": 0.8,
175+
"change": -0.8
176+
},
177+
"executedInRuns": 15
178+
}
179+
}
180+
],
181+
"environment": {
182+
"reportName": "PR #123 E2E Tests",
183+
"appName": "example-app",
184+
"appVersion": "2.5.0",
185+
"buildId": "build-456",
186+
"buildName": "PR #123",
187+
"buildNumber": 456,
188+
"buildUrl": "https://ci.example.com/builds/456",
189+
"repositoryName": "example-app",
190+
"repositoryUrl": "https://github.com/example/example-app",
191+
"commit": "a1b2c3d4e5f6",
192+
"branchName": "feature/new-dashboard",
193+
"osPlatform": "linux",
194+
"osRelease": "5.10.0",
195+
"osVersion": "Ubuntu 20.04",
196+
"testEnvironment": "staging",
197+
"healthy": false,
198+
"extra": {
199+
"nodeVersion": "18.0.0",
200+
"region": "us-west-2"
201+
}
202+
},
203+
"extra": {
204+
"shardIndex": 1,
205+
"totalShards": 4
206+
}
207+
},
208+
"insights": {
209+
"passRate": {
210+
"current": 0.67,
211+
"baseline": 0.85,
212+
"change": -0.18
213+
},
214+
"failRate": {
215+
"current": 0.33,
216+
"baseline": 0.15,
217+
"change": 0.18
218+
},
219+
"flakyRate": {
220+
"current": 0.33,
221+
"baseline": 0.2,
222+
"change": 0.13
223+
},
224+
"averageRunDuration": {
225+
"current": 20000,
226+
"baseline": 18000,
227+
"change": 2000
228+
},
229+
"p95RunDuration": {
230+
"current": 22000,
231+
"baseline": 20000,
232+
"change": 2000
233+
},
234+
"averageTestDuration": {
235+
"current": 5567,
236+
"baseline": 5100,
237+
"change": 467
238+
},
239+
"runsAnalyzed": 10
240+
},
241+
"baseline": {
242+
"reportId": "2f8c9a90-3e1a-4c3d-9b3a-8f0a9c123456",
243+
"timestamp": "2025-11-23T12:00:00Z",
244+
"source": "main",
245+
"buildNumber": 442,
246+
"buildName": "Nightly Build",
247+
"buildUrl": "https://ci.example.com/builds/442",
248+
"commit": "f6e5d4c3b2a1",
249+
"extra": {
250+
"branch": "main"
251+
}
252+
}
253+
}

‎examples/minimal.json‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
{
2+
"reportFormat": "CTRF",
3+
"specVersion": "1.0.0",
4+
"results": {
5+
"tool": {
6+
"name": "example-runner"
7+
},
8+
"summary": {
9+
"tests": 1,
10+
"passed": 1,
11+
"failed": 0,
12+
"pending": 0,
13+
"skipped": 0,
14+
"other": 0,
15+
"start": 1609459200000,
16+
"stop": 1609459201000
17+
},
18+
"tests": [
19+
{
20+
"name": "should pass",
21+
"status": "passed",
22+
"duration": 100
23+
}
24+
]
25+
}
26+
}

‎examples/with-diagnostics.json‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
{
2+
"reportFormat": "CTRF",
3+
"specVersion": "1.0.0",
4+
"results": {
5+
"tool": {
6+
"name": "example-runner",
7+
"version": "2.0.0"
8+
},
9+
"summary": {
10+
"tests": 1,
11+
"passed": 0,
12+
"failed": 1,
13+
"pending": 0,
14+
"skipped": 0,
15+
"other": 0,
16+
"start": 1609459200000,
17+
"stop": 1609459205000
18+
},
19+
"tests": [
20+
{
21+
"name": "should render homepage",
22+
"status": "failed",
23+
"duration": 5000,
24+
"message": "Expected element to be visible",
25+
"trace": "Error: Expected element to be visible\n at checkVisibility (test.js:45:10)\n at Object.<anonymous> (test.js:12:5)",
26+
"screenshot": "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==",
27+
"stdout": [
28+
"Starting test execution",
29+
"Loading page: https://example.com",
30+
"Waiting for element: .hero-section"
31+
],
32+
"stderr": [
33+
"Warning: Network slow",
34+
"Error: Element not found after 5000ms"
35+
],
36+
"attachments": [
37+
{
38+
"name": "full-page-screenshot.png",
39+
"contentType": "image/png",
40+
"path": "/artifacts/screenshots/test-1.png"
41+
},
42+
{
43+
"name": "console-logs.txt",
44+
"contentType": "text/plain",
45+
"path": "/artifacts/logs/console-1.txt"
46+
},
47+
{
48+
"name": "network-trace.har",
49+
"contentType": "application/json",
50+
"path": "/artifacts/traces/network-1.har"
51+
}
52+
]
53+
}
54+
]
55+
}
56+
}

0 commit comments

Comments
 (0)