Skip to content

feat(trace-viewer): show screencast frames during playback #42751

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Pavel Feldman (pavelfeldman) merged 1 commit into from
Sep 18, 2026
Merged

feat(trace-viewer): show screencast frames during playback #42751

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion packages/trace-viewer/src/ui/ariaModeView.css
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,8 @@
position: relative;
}

.aria-mode-screenshot img {
.aria-mode-screenshot img,
.aria-mode-screenshot .screencast-frame {
max-width: 100%;
max-height: 100%;
object-fit: contain;
Expand Down
30 changes: 18 additions & 12 deletions packages/trace-viewer/src/ui/ariaModeView.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ import { renderAriaSnapshotAsYaml } from '@isomorphic/ariaSnapshotRenderer';
import { clsx, useMeasure } from '@web/uiUtils';
import { PlaceholderPanel } from './placeholderPanel';

import type { ActionPhase, ActionTraceEvent, ScreenshotTraceEvent } from '@isomorphic/trace/trace';
import type { ActionPhase, ActionTraceEvent } from '@isomorphic/trace/trace';
import type { AriaNodeJSON, AriaSnapshotJSON } from '@isomorphic/ariaSnapshot';
import type { TraceModel } from '@isomorphic/trace/traceModel';

Expand Down Expand Up @@ -144,7 +144,8 @@ export const AriaModeView: React.FunctionComponent<{
target: AriaModeTarget | undefined,
point?: Point,
box?: Box,
}> = ({ model, target, point, box }) => {
screencastFrame?: React.ReactNode,
}> = ({ model, target, point, box, screencastFrame }) => {
const screenshot = model && target ? model.screenshotForCall(target.callId, target.phase) : undefined;
const ariaSnapshot = model && target ? model.ariaSnapshotForCall(target.callId, target.phase) : undefined;
const [lines, setLines] = React.useState<AriaSnapshotLine[]>([]);
Expand Down Expand Up @@ -172,11 +173,16 @@ export const AriaModeView: React.FunctionComponent<{
};
}, [model, ariaSnapshot]);

if (!screenshot && !ariaSnapshot)
if (!screenshot && !ariaSnapshot && !screencastFrame)
return <PlaceholderPanel text='No aria snapshot' />;

return <div className='aria-mode-view hbox'>
<AriaModeScreenshot model={model!} screenshot={screenshot} highlightedBox={highlightedBox} point={point} box={box} />
<AriaModeScreenshot
imageUrl={model && screenshot ? model.createRelativeUrl(`file/${screenshot.file}`) : undefined}
screencastFrame={screencastFrame}
highlightedBox={screencastFrame ? undefined : highlightedBox}
point={screencastFrame ? undefined : point}
box={screencastFrame ? undefined : box} />
<div className='aria-mode-snapshot vbox'>
{ariaSnapshot && <div className='aria-mode-lines' onMouseLeave={() => setHighlightedBox(undefined)}>
{lines.map((line, index) => <div
Expand All @@ -191,20 +197,20 @@ export const AriaModeView: React.FunctionComponent<{
};

const AriaModeScreenshot: React.FunctionComponent<{
model: TraceModel,
screenshot: ScreenshotTraceEvent | undefined,
imageUrl: string | undefined,
screencastFrame: React.ReactNode,
highlightedBox: Box | undefined,
point: Point | undefined,
box: Box | undefined,
}> = ({ model, screenshot, highlightedBox, point, box }) => {
}> = ({ imageUrl, screencastFrame, highlightedBox, point, box }) => {
const [measure, ref] = useMeasure<HTMLDivElement>();
const [naturalSize, setNaturalSize] = React.useState<{ width: number, height: number } | undefined>();

// Trace screenshots are taken with css scale, so image pixels match the aria box viewport
// coordinates. The image is scaled to fit into the available area, scale the boxes and
// the action point to match the rendered image.
let overlays: React.ReactNode;
if (screenshot && naturalSize && measure.width) {
if (imageUrl && naturalSize && measure.width) {
const padding = 10;
const availableWidth = measure.width - 2 * padding;
const availableHeight = measure.height - 2 * padding;
Expand All @@ -231,13 +237,13 @@ const AriaModeScreenshot: React.FunctionComponent<{
}

return <div ref={ref} className='aria-mode-screenshot'>
{screenshot && <img
key={screenshot.file}
src={model.createRelativeUrl(`file/${screenshot.file}`)}
{screencastFrame}
{!screencastFrame && imageUrl && <img
src={imageUrl}
alt='Screenshot'
onLoad={event => setNaturalSize({ width: event.currentTarget.naturalWidth, height: event.currentTarget.naturalHeight })}
/>}
{!screenshot && <PlaceholderPanel text='No screenshot' />}
{!screencastFrame && !imageUrl && <PlaceholderPanel text='No screenshot' />}
{overlays}
</div>;
};
73 changes: 46 additions & 27 deletions packages/trace-viewer/src/ui/filmStrip.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,10 @@ import * as React from 'react';
import { useMeasure, upperBound } from '@web/uiUtils';
import type { PageEntry } from '@isomorphic/trace/entries';
import { useTraceModel } from './traceModelContext';
import { useVideoThumbnails } from './videoThumbnails';
import type { VideoThumbnail } from './videoThumbnails';
import { useVideoSources } from './videoThumbnails';
import type { VideoSource, VideoThumbnail } from './videoThumbnails';
import { VideoFrame, lastVideoIndex } from './videoFrame';
import type * as trace from '@isomorphic/trace/trace';

export type FilmStripPreviewPoint = {
x: number;
Expand All @@ -41,7 +43,7 @@ export const FilmStrip: React.FunctionComponent<{
const lanesRef = React.useRef<HTMLDivElement>(null);

const videos = (model?.videos ?? []).map(video => ({ video, url: model!.createRelativeUrl(`file/${video.file}`) }));
const videoThumbnails = useVideoThumbnails(videos);
const videoSources = useVideoSources(videos);

let laneIndex = 0;
if (lanesRef.current && previewPoint) {
Expand All @@ -50,24 +52,30 @@ export const FilmStrip: React.FunctionComponent<{
}

const pageLanes = (model?.pages ?? []).filter(page => page.screencastFrames.length);
const videoLanes = videoLanesByPage(videos.map(({ video }) => video.pageId), videoThumbnails);
const videoLanes = videoLanesByPage(videos.map(({ video }) => video), videoSources);

let previewFrames: { timestamp: number, width: number, height: number, url: string }[] | undefined;
if (laneIndex < pageLanes.length)
previewFrames = model ? pageLanes[laneIndex]?.screencastFrames.map(frame => ({ ...frame, url: model.createRelativeUrl(`file/${frame.file}`) })) : undefined;
else
previewFrames = videoLanes[laneIndex - pageLanes.length];

let previewImage = undefined;
let previewSize = undefined;
if (previewPoint !== undefined && previewFrames && previewFrames.length) {
// The thumbnails only index the lanes, the preview renders the video itself.
let preview: React.ReactNode | undefined;
let previewSize: Size | undefined;
if (model && previewPoint !== undefined) {
const previewTime = boundaries.minimum + (boundaries.maximum - boundaries.minimum) * previewPoint.x / measure.width;
previewImage = previewFrames[upperBound(previewFrames, previewTime, timeComparator) - 1];
const fitInto = {
width: Math.min(800, (window.innerWidth / 2) | 0),
height: Math.min(800, (window.innerHeight / 2) | 0),
};
previewSize = previewImage ? inscribe({ width: previewImage.width, height: previewImage.height }, fitInto) : undefined;
const frames = pageLanes[laneIndex]?.screencastFrames;
const frame = frames?.[upperBound(frames, previewTime, timeComparator) - 1];
const lane = videoLanes[laneIndex - pageLanes.length];
const videoIndex = lane ? lastVideoIndex(lane.videos, previewTime) : -1;
const videoUrl = videoIndex === -1 ? undefined : lane.sources[videoIndex].url;
if (frame) {
previewSize = inscribe(frame, fitInto);
preview = <img src={model.createRelativeUrl(`file/${frame.file}`)} width={previewSize.width} height={previewSize.height} />;
} else if (videoUrl) {
const video = lane.videos[videoIndex];
previewSize = inscribe(video, fitInto);
preview = <VideoFrame url={videoUrl} startTime={video.timestamp} time={previewTime} width={previewSize.width} height={previewSize.height} />;
}
}

return <div className='film-strip' ref={ref}>
Expand All @@ -78,37 +86,48 @@ export const FilmStrip: React.FunctionComponent<{
width={measure.width}
key={index}
/>)}
{videoLanes.map((thumbnails, index) => <VideoFilmStripLane
{videoLanes.map((lane, index) => <VideoFilmStripLane
boundaries={boundaries}
thumbnails={thumbnails}
thumbnails={lane.thumbnails}
width={measure.width}
key={'video-' + index}
/>)}
</div>
{model && previewPoint && previewImage && previewSize &&
{previewPoint && preview && previewSize &&
<div className='film-strip-hover' style={{
top: measure.bottom + 5,
left: Math.min(previewPoint.x, measure.width - previewSize.width - 10),
width: previewSize.width,
height: previewSize.height,
}}>
<img src={previewImage.url} width={previewSize.width} height={previewSize.height} />
{preview}
</div>
}
</div>;
};

function videoLanesByPage(pageIds: string[], thumbnails: VideoThumbnail[][]): VideoThumbnail[][] {
const lanes = new Map<string, VideoThumbnail[]>();
pageIds.forEach((pageId, index) => {
let lane = lanes.get(pageId);
type VideoLane = {
videos: trace.VideoTraceEvent[];
sources: VideoSource[];
thumbnails: VideoThumbnail[];
};

function videoLanesByPage(videos: trace.VideoTraceEvent[], sources: VideoSource[]): VideoLane[] {
const lanes = new Map<string, VideoLane>();
videos.forEach((video, index) => {
let lane = lanes.get(video.pageId);
if (!lane) {
lane = [];
lanes.set(pageId, lane);
lane = { videos: [], sources: [], thumbnails: [] };
lanes.set(video.pageId, lane);
}
lane.push(...thumbnails[index]);
lane.videos.push(video);
lane.sources.push(sources[index]);
lane.thumbnails.push(...sources[index].thumbnails);
});
return [...lanes.values()].filter(lane => lane.thumbnails.length).map(lane => {
lane.thumbnails.sort((a, b) => a.timestamp - b.timestamp);
return lane;
});
return [...lanes.values()].filter(lane => lane.length).map(lane => lane.sort((a, b) => a.timestamp - b.timestamp));
}

const VideoFilmStripLane: React.FunctionComponent<{
Expand Down
Loading
Loading